From 492413504a9158dd83d619029f8a397f8bc3abfd Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Sat, 1 Jul 2023 16:45:06 +0900 Subject: [PATCH 01/14] Implement for CmdNamespaces --- .../pulsar/admin/cli/CmdNamespaces.java | 224 ++++++------------ .../admin/cli/utils/ValueValidationUtils.java | 51 ++++ ...ByteUnitIntegerAllowNegativeConverter.java | 41 ++++ .../converters/ByteUnitIntegerConverter.java | 45 ++++ .../converters/ByteUnitLongConverter.java | 44 ++++ .../cli/utils/converters/ByteUnitUtil.java | 62 +++++ .../OffloadedReadPriorityConverter.java | 45 ++++ .../converters/TimeUnitToMillisConverter.java | 48 ++++ .../TimeUnitToSecondsConverter.java | 46 ++++ .../cli/utils/converters/package-info.java | 19 ++ .../IntegerMaxValueLongValidator.java | 30 +++ .../validators/MinNegativeOneValidator.java | 30 +++ .../validators/NonNegativeValueValidator.java | 30 +++ .../validators/PositiveValueValidator.java | 31 +++ .../cli/utils/validators/package-info.java | 19 ++ 15 files changed, 611 insertions(+), 154 deletions(-) create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/ValueValidationUtils.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerAllowNegativeConverter.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerConverter.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitUtil.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/OffloadedReadPriorityConverter.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToMillisConverter.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToSecondsConverter.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/package-info.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/IntegerMaxValueLongValidator.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/MinNegativeOneValidator.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/NonNegativeValueValidator.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveValueValidator.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/package-info.java diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 6bd9617c37674..ff30106ede028 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -36,6 +36,14 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.admin.cli.utils.IOUtils; +import org.apache.pulsar.admin.cli.utils.converters.ByteUnitIntegerConverter; +import org.apache.pulsar.admin.cli.utils.converters.ByteUnitLongConverter; +import org.apache.pulsar.admin.cli.utils.converters.TimeUnitToMillisConverter; +import org.apache.pulsar.admin.cli.utils.converters.TimeUnitToSecondsConverter; +import org.apache.pulsar.admin.cli.utils.validators.IntegerMaxValueLongValidator; +import org.apache.pulsar.admin.cli.utils.validators.MinNegativeOneValidator; +import org.apache.pulsar.admin.cli.utils.validators.NonNegativeValueValidator; +import org.apache.pulsar.admin.cli.utils.validators.PositiveValueValidator; import org.apache.pulsar.client.admin.ListNamespaceTopicsOptions; import org.apache.pulsar.client.admin.Mode; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -402,25 +410,15 @@ private class SetMessageTTL extends CliCommand { @Parameter(names = { "--messageTTL", "-ttl" }, description = "Message TTL in seconds (or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w). " - + "When the value is set to `0`, TTL is disabled.", required = true) - private String messageTTLStr; + + "When the value is set to `0`, TTL is disabled.", required = true, + converter = TimeUnitToSecondsConverter.class, + validateValueWith = {IntegerMaxValueLongValidator.class, NonNegativeValueValidator.class}) + private Long messageTTLInSecond; @Override void run() throws PulsarAdminException { - long messageTTLInSecond; - try { - messageTTLInSecond = RelativeTimeUtil.parseRelativeTimeInSeconds(messageTTLStr); - } catch (IllegalArgumentException e) { - throw new ParameterException(e.getMessage()); - } - - if (messageTTLInSecond < 0 || messageTTLInSecond > Integer.MAX_VALUE) { - throw new ParameterException( - String.format("Message TTL cannot be negative or greater than %d seconds", Integer.MAX_VALUE)); - } - String namespace = validateNamespace(params); - getAdmin().namespaces().setNamespaceMessageTTL(namespace, (int) messageTTLInSecond); + getAdmin().namespaces().setNamespaceMessageTTL(namespace, messageTTLInSecond.intValue()); } } @@ -745,39 +743,27 @@ private class SetRetention extends CliCommand { + "For example, 100m, 3h, 2d, 5w. " + "If the time unit is not specified, the default unit is seconds. For example, " + "-t 120 sets retention to 2 minutes. " - + "0 means no retention and -1 means infinite time retention.", required = true) - private String retentionTimeStr; + + "0 means no retention and -1 means infinite time retention.", required = true, + converter = TimeUnitToSecondsConverter.class, + validateValueWith = MinNegativeOneValidator.class) + private Long retentionTimeInSec; @Parameter(names = { "--size", "-s" }, description = "Retention size limit with optional size unit suffix. " + "For example, 4096, 10M, 16G, 3T. The size unit suffix character can be k/K, m/M, g/G, or t/T. " + "If the size unit suffix is not specified, the default unit is bytes. " - + "0 or less than 1MB means no retention and -1 means infinite size retention", required = true) - private String limitStr; + + "0 or less than 1MB means no retention and -1 means infinite size retention", required = true, + converter = ByteUnitIntegerConverter.class) + private Integer sizeLimit; @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - long sizeLimit = validateSizeString(limitStr); - long retentionTimeInSec; - try { - retentionTimeInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(retentionTimeStr); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - - final int retentionTimeInMin; - if (retentionTimeInSec != -1) { - retentionTimeInMin = (int) TimeUnit.SECONDS.toMinutes(retentionTimeInSec); - } else { - retentionTimeInMin = -1; - } - - final int retentionSizeInMB; - if (sizeLimit != -1) { - retentionSizeInMB = (int) (sizeLimit / (1024 * 1024)); - } else { - retentionSizeInMB = -1; - } + final int retentionTimeInMin = retentionTimeInSec == -1 + ? retentionTimeInSec.intValue() + : (int) TimeUnit.SECONDS.toMinutes(retentionTimeInSec); + final int retentionSizeInMB = sizeLimit == -1 + ? sizeLimit + : (sizeLimit / (1024 * 1024)); getAdmin().namespaces() .setRetention(namespace, new RetentionPolicies(retentionTimeInMin, retentionSizeInMB)); } @@ -1265,13 +1251,18 @@ private class SetBacklogQuota extends CliCommand { @Parameter(description = "tenant/namespace", required = true) private java.util.List params; - @Parameter(names = { "-l", "--limit" }, description = "Size limit (eg: 10M, 16G)") - private String limitStr; + @Parameter(names = { "-l", "--limit" }, description = "Size limit (eg: 10M, 16G)", + required = true, + converter = ByteUnitLongConverter.class) + private Long limit; @Parameter(names = { "-lt", "--limitTime" }, description = "Time limit in second (or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w), " - + "non-positive number for disabling time limit.") - private String limitTimeStr = null; + + "non-positive number for disabling time limit.", + required = true, + converter = TimeUnitToSecondsConverter.class, + validateValueWith = PositiveValueValidator.class) + private Long limitTimeInSec; @Parameter(names = { "-p", "--policy" }, description = "Retention policy to enforce when the limit is reached. " + "Valid options are: [producer_request_hold, producer_exception, consumer_backlog_eviction]", @@ -1309,23 +1300,10 @@ void run() throws PulsarAdminException { BacklogQuota.Builder builder = BacklogQuota.builder().retentionPolicy(policy); if (backlogQuotaType == BacklogQuota.BacklogQuotaType.destination_storage) { // set quota by storage size - if (limitStr == null) { - throw new ParameterException("Quota type of 'destination_storage' needs a size limit"); - } - long limit = validateSizeString(limitStr); builder.limitSize(limit); } else { // set quota by time - if (limitTimeStr == null) { - throw new ParameterException("Quota type of 'message_age' needs a time limit"); - } - long limitTimeInSec; - try { - limitTimeInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(limitTimeStr); - } catch (IllegalArgumentException e) { - throw new ParameterException(e.getMessage()); - } - builder.limitTime((int) limitTimeInSec); + builder.limitTime(limitTimeInSec.intValue()); } getAdmin().namespaces().setBacklogQuota(namespace, builder.build(), backlogQuotaType); } @@ -1570,8 +1548,10 @@ private class SetInactiveTopicPolicies extends CliCommand { @Parameter(names = {"--max-inactive-duration", "-t"}, description = "Max duration of topic inactivity in " + "seconds, topics that are inactive for longer than this value will be deleted " - + "(eg: 1s, 10s, 1m, 5h, 3d)", required = true) - private String deleteInactiveTopicsMaxInactiveDuration; + + "(eg: 1s, 10s, 1m, 5h, 3d)", required = true, + converter = TimeUnitToSecondsConverter.class, + validateValueWith = PositiveValueValidator.class) + private Long maxInactiveDurationInSeconds; @Parameter(names = { "--delete-mode", "-m" }, description = "Mode of delete inactive topic, Valid options are: " + "[delete_when_no_subscriptions, delete_when_subscriptions_caught_up]", required = true) @@ -1580,14 +1560,6 @@ private class SetInactiveTopicPolicies extends CliCommand { @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - long maxInactiveDurationInSeconds; - try { - maxInactiveDurationInSeconds = TimeUnit.SECONDS.toSeconds( - RelativeTimeUtil.parseRelativeTimeInSeconds(deleteInactiveTopicsMaxInactiveDuration)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - if (enableDeleteWhileInactive == disableDeleteWhileInactive) { throw new ParameterException("Need to specify either enable-delete-while-inactive or " + "disable-delete-while-inactive"); @@ -1600,7 +1572,7 @@ void run() throws PulsarAdminException { + "delete_when_subscriptions_caught_up"); } getAdmin().namespaces().setInactiveTopicPolicies(namespace, new InactiveTopicPolicies(deleteMode, - (int) maxInactiveDurationInSeconds, enableDeleteWhileInactive)); + maxInactiveDurationInSeconds.intValue(), enableDeleteWhileInactive)); } } @@ -1617,20 +1589,14 @@ private class SetDelayedDelivery extends CliCommand { @Parameter(names = { "--time", "-t" }, description = "The tick time for when retrying on " + "delayed delivery messages, affecting the accuracy of the delivery time compared to " - + "the scheduled time. (eg: 1s, 10s, 1m, 5h, 3d)") - private String delayedDeliveryTimeStr = "1s"; + + "the scheduled time. (eg: 1s, 10s, 1m, 5h, 3d)", + converter = TimeUnitToMillisConverter.class, + validateValueWith = PositiveValueValidator.class) + private Long delayedDeliveryTimeInMills = 1000L; @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - long delayedDeliveryTimeInMills; - try { - delayedDeliveryTimeInMills = TimeUnit.SECONDS.toMillis( - RelativeTimeUtil.parseRelativeTimeInSeconds(delayedDeliveryTimeStr)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - if (enable == disable) { throw new ParameterException("Need to specify either --enable or --disable"); } @@ -1942,13 +1908,13 @@ private class SetCompactionThreshold extends CliCommand { @Parameter(names = { "--threshold", "-t" }, description = "Maximum number of bytes in a topic backlog before compaction is triggered " + "(eg: 10M, 16G, 3T). 0 disables automatic compaction", - required = true) - private String thresholdStr = "0"; + required = true, + converter = ByteUnitLongConverter.class) + private Long threshold = 0L; @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - long threshold = validateSizeString(thresholdStr); getAdmin().namespaces().setCompactionThreshold(namespace, threshold); } } @@ -1976,13 +1942,13 @@ private class SetOffloadThreshold extends CliCommand { + " -1 falls back to the cluster's namespace default." + " Negative values disable automatic offload." + " 0 triggers offloading as soon as possible.", - required = true) - private String thresholdStr = "-1"; + required = true, + converter = ByteUnitLongConverter.class) + private Long threshold = -1L; @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - long threshold = validateSizeString(thresholdStr); getAdmin().namespaces().setOffloadThreshold(namespace, threshold); } } @@ -2266,34 +2232,40 @@ private class SetOffloadPolicies extends CliCommand { names = {"--maxBlockSize", "-mbs"}, description = "Max block size (eg: 32M, 64M), default is 64MB" + "s3 and google-cloud-storage requires this parameter", - required = false) - private String maxBlockSizeStr; + required = false, + converter = ByteUnitIntegerConverter.class) + private Integer maxBlockSizeInBytes = OffloadPoliciesImpl.DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; @Parameter( names = {"--readBufferSize", "-rbs"}, description = "Read buffer size (eg: 1M, 5M), default is 1MB", - required = false) - private String readBufferSizeStr; + required = false, + converter = ByteUnitIntegerConverter.class) + private Integer readBufferSizeInBytes = OffloadPoliciesImpl.DEFAULT_READ_BUFFER_SIZE_IN_BYTES; @Parameter( names = {"--offloadAfterElapsed", "-oae"}, description = "Delay time in Millis for deleting the bookkeeper ledger after offload " + "(or seconds,minutes,hours,days,weeks eg: 10s, 100m, 3h, 2d, 5w).", - required = false) - private String offloadAfterElapsedStr; + required = false, + converter = TimeUnitToMillisConverter.class, + validateValueWith = PositiveValueValidator.class) + private Long offloadAfterElapsedInMillis = OffloadPoliciesImpl.DEFAULT_OFFLOAD_DELETION_LAG_IN_MILLIS; @Parameter( names = {"--offloadAfterThreshold", "-oat"}, description = "Offload after threshold size (eg: 1M, 5M)", - required = false) - private String offloadAfterThresholdStr; + required = false, + converter = ByteUnitLongConverter.class) + private Long offloadAfterThresholdInBytes = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_BYTES; @Parameter( names = {"--offloadAfterThresholdInSeconds", "-oats"}, description = "Offload after threshold seconds (or minutes,hours,days,weeks eg: 100m, 3h, 2d, 5w).", - required = false - ) - private String offloadAfterThresholdInSecondsStr; + required = false, + converter = TimeUnitToSecondsConverter.class, + validateValueWith = PositiveValueValidator.class) + private Long offloadThresholdInSeconds = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_SECONDS; @Parameter( names = {"--offloadedReadPriority", "-orp"}, @@ -2348,71 +2320,15 @@ void run() throws PulsarAdminException { + " if s3 offload enabled"); } - int maxBlockSizeInBytes = OffloadPoliciesImpl.DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; - if (StringUtils.isNotEmpty(maxBlockSizeStr)) { - long maxBlockSize = validateSizeString(maxBlockSizeStr); - if (positiveCheck("MaxBlockSize", maxBlockSize) - && maxValueCheck("MaxBlockSize", maxBlockSize, Integer.MAX_VALUE)) { - maxBlockSizeInBytes = Long.valueOf(maxBlockSize).intValue(); - } - } - - int readBufferSizeInBytes = OffloadPoliciesImpl.DEFAULT_READ_BUFFER_SIZE_IN_BYTES; - if (StringUtils.isNotEmpty(readBufferSizeStr)) { - long readBufferSize = validateSizeString(readBufferSizeStr); - if (positiveCheck("ReadBufferSize", readBufferSize) - && maxValueCheck("ReadBufferSize", readBufferSize, Integer.MAX_VALUE)) { - readBufferSizeInBytes = Long.valueOf(readBufferSize).intValue(); - } - } - - Long offloadAfterElapsedInMillis = OffloadPoliciesImpl.DEFAULT_OFFLOAD_DELETION_LAG_IN_MILLIS; - if (StringUtils.isNotEmpty(offloadAfterElapsedStr)) { - Long offloadAfterElapsed; - try { - offloadAfterElapsed = TimeUnit.SECONDS.toMillis( - RelativeTimeUtil.parseRelativeTimeInSeconds(offloadAfterElapsedStr)); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - if (positiveCheck("OffloadAfterElapsed", offloadAfterElapsed) - && maxValueCheck("OffloadAfterElapsed", offloadAfterElapsed, Long.MAX_VALUE)) { - offloadAfterElapsedInMillis = offloadAfterElapsed; - } - } - - Long offloadAfterThresholdInBytes = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_BYTES; - if (StringUtils.isNotEmpty(offloadAfterThresholdStr)) { - long offloadAfterThreshold = validateSizeString(offloadAfterThresholdStr); - if (maxValueCheck("OffloadAfterThreshold", offloadAfterThreshold, Long.MAX_VALUE)) { - offloadAfterThresholdInBytes = offloadAfterThreshold; - } - } - - Long offloadThresholdInSeconds = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_SECONDS; - if (StringUtils.isNotEmpty(offloadAfterThresholdInSecondsStr)) { - Long offloadThresholdInSeconds0; - try { - offloadThresholdInSeconds0 = TimeUnit.SECONDS.toSeconds( - RelativeTimeUtil.parseRelativeTimeInSeconds(offloadAfterThresholdInSecondsStr.trim())); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } - if (maxValueCheck("OffloadAfterThresholdInSeconds", offloadThresholdInSeconds0, Long.MAX_VALUE)) { - offloadThresholdInSeconds = offloadThresholdInSeconds0; - } - } - OffloadedReadPriority offloadedReadPriority = OffloadPoliciesImpl.DEFAULT_OFFLOADED_READ_PRIORITY; - if (this.offloadReadPriorityStr != null) { try { offloadedReadPriority = OffloadedReadPriority.fromString(this.offloadReadPriorityStr); } catch (Exception e) { throw new ParameterException("--offloadedReadPriority parameter must be one of " + Arrays.stream(OffloadedReadPriority.values()) - .map(OffloadedReadPriority::toString) - .collect(Collectors.joining(",")) + .map(OffloadedReadPriority::toString) + .collect(Collectors.joining(",")) + " but got: " + this.offloadReadPriorityStr, e); } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/ValueValidationUtils.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/ValueValidationUtils.java new file mode 100644 index 0000000000000..280ec774096f9 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/ValueValidationUtils.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils; + +import com.beust.jcommander.ParameterException; +import org.apache.commons.lang3.StringUtils; + +public class ValueValidationUtils { + private ValueValidationUtils() { + } + + public static void maxValueCheck(String paramName, long value, long maxValue) { + if (value > maxValue) { + throw new ParameterException(paramName + " cannot be bigger than <" + maxValue + ">!"); + } + } + + public static void positiveCheck(String paramName, long value) { + if (value <= 0) { + throw new ParameterException(paramName + " cannot be less than or equal to <0>!"); + } + } + + public static void emptyCheck(String paramName, String value) { + if (StringUtils.isEmpty(value)) { + throw new ParameterException("The value of " + paramName + " can't be empty"); + } + } + + public static void minValueCheck(String name, Long value, long min) { + if (value < min) { + throw new ParameterException(name + " cannot be less than <" + min + ">!"); + } + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerAllowNegativeConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerAllowNegativeConverter.java new file mode 100644 index 0000000000000..05229d932793c --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerAllowNegativeConverter.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.converters; + +import com.beust.jcommander.converters.BaseConverter; +import org.apache.pulsar.admin.cli.utils.ValueValidationUtils; + +public class ByteUnitIntegerAllowNegativeConverter extends BaseConverter { + + public ByteUnitIntegerAllowNegativeConverter(String optionName) { + super(optionName); + } + + @Override + public Integer convert(String argStr) { + return parseBytes(argStr).intValue(); + } + + Long parseBytes(String argStr) { + ValueValidationUtils.emptyCheck(getOptionName(), argStr); + long valueInBytes = ByteUnitUtil.validateSizeString(argStr); + ValueValidationUtils.maxValueCheck(getOptionName(), valueInBytes, Integer.MAX_VALUE); + return valueInBytes; + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerConverter.java new file mode 100644 index 0000000000000..0476c341b1639 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerConverter.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.converters; + +import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.emptyCheck; +import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.maxValueCheck; +import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.positiveCheck; +import static org.apache.pulsar.admin.cli.utils.converters.ByteUnitUtil.validateSizeString; +import com.beust.jcommander.converters.BaseConverter; + +public class ByteUnitIntegerConverter extends BaseConverter { + + public ByteUnitIntegerConverter(String optionName) { + super(optionName); + } + + @Override + public Integer convert(String argStr) { + return parseBytes(argStr).intValue(); + } + + Long parseBytes(String argStr) { + emptyCheck(getOptionName(), argStr); + long valueInBytes = validateSizeString(argStr); + maxValueCheck(getOptionName(), valueInBytes, Integer.MAX_VALUE); + positiveCheck(getOptionName(), valueInBytes); + return valueInBytes; + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java new file mode 100644 index 0000000000000..47a2f9f355526 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.converters; + +import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.emptyCheck; +import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.maxValueCheck; +import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.positiveCheck; +import com.beust.jcommander.converters.BaseConverter; + +public class ByteUnitLongConverter extends BaseConverter { + + public ByteUnitLongConverter(String optionName) { + super(optionName); + } + + @Override + public Long convert(String argStr) { + return parseBytes(argStr); + } + + Long parseBytes(String argStr) { + emptyCheck(getOptionName(), argStr); + long valueInBytes = ByteUnitUtil.validateSizeString(argStr); + maxValueCheck(getOptionName(), valueInBytes, Integer.MAX_VALUE); + positiveCheck(getOptionName(), valueInBytes); + return valueInBytes; + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitUtil.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitUtil.java new file mode 100644 index 0000000000000..5febb8fa4b700 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitUtil.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.converters; + +import com.beust.jcommander.ParameterException; +import com.google.common.collect.Sets; +import java.util.Set; + +class ByteUnitUtil { + + private static Set sizeUnit = Sets.newHashSet('k', 'K', 'm', 'M', 'g', 'G', 't', 'T'); + + static long validateSizeString(String s) { + char last = s.charAt(s.length() - 1); + String subStr = s.substring(0, s.length() - 1); + long size; + try { + size = sizeUnit.contains(last) + ? Long.parseLong(subStr) + : Long.parseLong(s); + } catch (IllegalArgumentException e) { + throw new ParameterException(String.format("Invalid size '%s'. Valid formats are: %s", + s, "(4096, 100K, 10M, 16G, 2T)")); + } + switch (last) { + case 'k': + case 'K': + return size * 1024; + + case 'm': + case 'M': + return size * 1024 * 1024; + + case 'g': + case 'G': + return size * 1024 * 1024 * 1024; + + case 't': + case 'T': + return size * 1024 * 1024 * 1024 * 1024; + + default: + return size; + } + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/OffloadedReadPriorityConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/OffloadedReadPriorityConverter.java new file mode 100644 index 0000000000000..90da9071a08f4 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/OffloadedReadPriorityConverter.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.converters; + +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.converters.BaseConverter; +import java.util.Arrays; +import java.util.stream.Collectors; +import org.apache.pulsar.common.policies.data.OffloadedReadPriority; + +public class OffloadedReadPriorityConverter extends BaseConverter { + + public OffloadedReadPriorityConverter(String optionName) { + super(optionName); + } + + @Override + public OffloadedReadPriority convert(String value) { + try { + return OffloadedReadPriority.fromString(value); + } catch (Exception e) { + throw new ParameterException("--offloadedReadPriority parameter must be one of " + + Arrays.stream(OffloadedReadPriority.values()) + .map(OffloadedReadPriority::toString) + .collect(Collectors.joining(",")) + + " but got: " + value, e); + } + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToMillisConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToMillisConverter.java new file mode 100644 index 0000000000000..b95c52a8b1299 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToMillisConverter.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.converters; + +import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.emptyCheck; +import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.maxValueCheck; +import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.positiveCheck; +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.converters.BaseConverter; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.common.util.RelativeTimeUtil; + +public class TimeUnitToMillisConverter extends BaseConverter { + + public TimeUnitToMillisConverter(String optionName) { + super(optionName); + } + + @Override + public Long convert(String str) { + emptyCheck(getOptionName(), str); + try { + long inputValue = TimeUnit.SECONDS.toMillis( + RelativeTimeUtil.parseRelativeTimeInSeconds(str.trim())); + positiveCheck(getOptionName(), inputValue); + maxValueCheck(getOptionName(), inputValue, Long.MAX_VALUE); + return inputValue; + } catch (IllegalArgumentException exception) { + throw new ParameterException("For input " + getOptionName() + ": " + exception.getMessage()); + } + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToSecondsConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToSecondsConverter.java new file mode 100644 index 0000000000000..f29912d325095 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToSecondsConverter.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.converters; + +import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.emptyCheck; +import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.maxValueCheck; +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.converters.BaseConverter; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.common.util.RelativeTimeUtil; + +public class TimeUnitToSecondsConverter extends BaseConverter { + + public TimeUnitToSecondsConverter(String optionName) { + super(optionName); + } + + @Override + public Long convert(String str) { + emptyCheck(getOptionName(), str); + try { + long inputValue = TimeUnit.SECONDS.toSeconds( + RelativeTimeUtil.parseRelativeTimeInSeconds(str.trim())); + maxValueCheck(getOptionName(), inputValue, Long.MAX_VALUE); + return inputValue; + } catch (IllegalArgumentException exception) { + throw new ParameterException("For input " + getOptionName() + ": " + exception.getMessage()); + } + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/package-info.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/package-info.java new file mode 100644 index 0000000000000..bac33be58dc21 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.converters; diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/IntegerMaxValueLongValidator.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/IntegerMaxValueLongValidator.java new file mode 100644 index 0000000000000..535a4c90859f0 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/IntegerMaxValueLongValidator.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.validators; + +import com.beust.jcommander.IValueValidator; +import com.beust.jcommander.ParameterException; +import org.apache.pulsar.admin.cli.utils.ValueValidationUtils; + +public class IntegerMaxValueLongValidator implements IValueValidator { + @Override + public void validate(String name, Long value) throws ParameterException { + ValueValidationUtils.maxValueCheck(name, value, Integer.MAX_VALUE); + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/MinNegativeOneValidator.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/MinNegativeOneValidator.java new file mode 100644 index 0000000000000..9952da445991c --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/MinNegativeOneValidator.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.validators; + +import com.beust.jcommander.IValueValidator; +import com.beust.jcommander.ParameterException; +import org.apache.pulsar.admin.cli.utils.ValueValidationUtils; + +public class MinNegativeOneValidator implements IValueValidator { + @Override + public void validate(String name, Long value) throws ParameterException { + ValueValidationUtils.minValueCheck(name, value, -1L); + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/NonNegativeValueValidator.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/NonNegativeValueValidator.java new file mode 100644 index 0000000000000..92542c4036fac --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/NonNegativeValueValidator.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.validators; + +import com.beust.jcommander.IValueValidator; +import com.beust.jcommander.ParameterException; +import org.apache.pulsar.admin.cli.utils.ValueValidationUtils; + +public class NonNegativeValueValidator implements IValueValidator { + @Override + public void validate(String name, Long value) throws ParameterException { + ValueValidationUtils.maxValueCheck(name, value, Long.MAX_VALUE); + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveValueValidator.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveValueValidator.java new file mode 100644 index 0000000000000..901903f74b7b1 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveValueValidator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.validators; + +import com.beust.jcommander.IValueValidator; +import com.beust.jcommander.ParameterException; +import org.apache.pulsar.admin.cli.utils.ValueValidationUtils; + +public class PositiveValueValidator implements IValueValidator { + + @Override + public void validate(String name, Long value) throws ParameterException { + ValueValidationUtils.positiveCheck(name, value); + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/package-info.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/package-info.java new file mode 100644 index 0000000000000..116c7d1b89b48 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.validators; From 90e51ed1491e42d8f9e8c63dfc2c31590c7736df Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Tue, 4 Jul 2023 21:00:38 +0900 Subject: [PATCH 02/14] Fix changes --- .../java/org/apache/pulsar/admin/cli/CmdNamespaces.java | 8 ++++++-- .../admin/cli/utils/converters/ByteUnitLongConverter.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index ff30106ede028..93a6f2a3e90f7 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -1252,14 +1252,12 @@ private class SetBacklogQuota extends CliCommand { private java.util.List params; @Parameter(names = { "-l", "--limit" }, description = "Size limit (eg: 10M, 16G)", - required = true, converter = ByteUnitLongConverter.class) private Long limit; @Parameter(names = { "-lt", "--limitTime" }, description = "Time limit in second (or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w), " + "non-positive number for disabling time limit.", - required = true, converter = TimeUnitToSecondsConverter.class, validateValueWith = PositiveValueValidator.class) private Long limitTimeInSec; @@ -1300,9 +1298,15 @@ void run() throws PulsarAdminException { BacklogQuota.Builder builder = BacklogQuota.builder().retentionPolicy(policy); if (backlogQuotaType == BacklogQuota.BacklogQuotaType.destination_storage) { // set quota by storage size + if (limit == null) { + throw new ParameterException("Quota type of 'destination_storage' needs a size limit"); + } builder.limitSize(limit); } else { // set quota by time + if (limitTimeInSec == null) { + throw new ParameterException("Quota type of 'message_age' needs a time limit"); + } builder.limitTime(limitTimeInSec.intValue()); } getAdmin().namespaces().setBacklogQuota(namespace, builder.build(), backlogQuotaType); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java index 47a2f9f355526..3f977e6dff863 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java @@ -37,7 +37,7 @@ public Long convert(String argStr) { Long parseBytes(String argStr) { emptyCheck(getOptionName(), argStr); long valueInBytes = ByteUnitUtil.validateSizeString(argStr); - maxValueCheck(getOptionName(), valueInBytes, Integer.MAX_VALUE); + maxValueCheck(getOptionName(), valueInBytes, Long.MAX_VALUE); positiveCheck(getOptionName(), valueInBytes); return valueInBytes; } From 4a2e2f7ec28b1e1cf6e262dce9ab36a99a0f7932 Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Wed, 5 Jul 2023 22:22:23 +0900 Subject: [PATCH 03/14] Fix coerced validation --- .../admin/cli/utils/converters/ByteUnitLongConverter.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java index 3f977e6dff863..d68a2587191e4 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java @@ -36,9 +36,6 @@ public Long convert(String argStr) { Long parseBytes(String argStr) { emptyCheck(getOptionName(), argStr); - long valueInBytes = ByteUnitUtil.validateSizeString(argStr); - maxValueCheck(getOptionName(), valueInBytes, Long.MAX_VALUE); - positiveCheck(getOptionName(), valueInBytes); - return valueInBytes; + return ByteUnitUtil.validateSizeString(argStr); } } From 94bb470482d795b095c83e99c7dc7455747c28c7 Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Wed, 5 Jul 2023 22:37:40 +0900 Subject: [PATCH 04/14] Update --- .../org/apache/pulsar/admin/cli/CmdNamespaces.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 82cfb002259b0..076ce9a032a77 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -73,7 +73,6 @@ import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; import org.apache.pulsar.common.policies.data.TopicType; -import org.apache.pulsar.common.util.RelativeTimeUtil; @Parameters(commandDescription = "Operations about namespaces") public class CmdNamespaces extends CmdBase { @@ -1982,18 +1981,13 @@ private class SetOffloadDeletionLag extends CliCommand { @Parameter(names = { "--lag", "-l" }, description = "Duration to wait after offloading a ledger segment, before deleting the copy of that" + " segment from cluster local storage. (eg: 10m, 5h, 3d, 2w).", - required = true) - private String lag = "-1"; + required = true, + converter = TimeUnitToSecondsConverter.class) + private Long lagInSec = -1L; @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - long lagInSec; - try { - lagInSec = RelativeTimeUtil.parseRelativeTimeInSeconds(lag); - } catch (IllegalArgumentException exception) { - throw new ParameterException(exception.getMessage()); - } getAdmin().namespaces().setOffloadDeleteLag(namespace, lagInSec, TimeUnit.SECONDS); } From 9914aee6ad3b0447bf38c685b716583f6501ca94 Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Thu, 6 Jul 2023 00:18:46 +0900 Subject: [PATCH 05/14] Revert "Merge remote-tracking branch 'upstream/master' into pip280-namespaces" This reverts commit 1ea46bb76601bba5a63cfc83f38ce50b8b71694a, reversing changes made to 90e51ed1491e42d8f9e8c63dfc2c31590c7736df. --- buildtools/pom.xml | 2 +- conf/bookkeeper.conf | 2 +- conf/broker.conf | 4 +- conf/proxy.conf | 2 +- conf/standalone.conf | 6 +- conf/zookeeper.conf | 2 +- distribution/server/pom.xml | 6 - .../server/src/assemble/LICENSE.bin.txt | 60 +++++---- .../shell/src/assemble/LICENSE.bin.txt | 8 +- pom.xml | 8 +- .../pulsar/broker/ServiceConfiguration.java | 2 +- pulsar-broker/pom.xml | 14 --- .../apache/pulsar/PulsarBrokerStarter.java | 2 +- .../pulsar/PulsarClusterMetadataSetup.java | 2 +- .../pulsar/PulsarClusterMetadataTeardown.java | 2 +- .../pulsar/PulsarInitialNamespaceSetup.java | 2 +- .../pulsar/PulsarStandaloneStarter.java | 2 +- ...arTransactionCoordinatorMetadataSetup.java | 4 +- .../apache/pulsar/PulsarVersionStarter.java | 2 +- .../broker/admin/impl/NamespacesBase.java | 22 ++-- .../admin/impl/PersistentTopicsBase.java | 17 ++- .../broker/admin/v2/PersistentTopics.java | 2 +- .../broker/intercept/BrokerInterceptors.java | 5 + .../extensions/ExtensibleLoadManagerImpl.java | 38 +++--- .../channel/ServiceUnitStateChannelImpl.java | 4 +- .../filter/AntiAffinityGroupPolicyFilter.java | 6 +- .../extensions/filter/BrokerFilter.java | 9 +- .../filter/BrokerIsolationPoliciesFilter.java | 19 +-- .../filter/BrokerLoadManagerClassFilter.java | 11 +- .../filter/BrokerMaxTopicCountFilter.java | 10 +- .../filter/BrokerVersionFilter.java | 17 ++- .../AntiAffinityGroupPolicyHelper.java | 11 +- .../policies/IsolationPoliciesHelper.java | 19 ++- .../extensions/scheduler/TransferShedder.java | 7 +- .../loadbalance/impl/LoadManagerShared.java | 114 ------------------ .../SimpleResourceAllocationPolicies.java | 15 --- .../service/AbstractBaseDispatcher.java | 2 +- .../pulsar/broker/service/ServerCnx.java | 2 +- .../broker/tools/GenerateDocsCommand.java | 2 +- .../pulsar/compaction/CompactedTopicImpl.java | 29 +++-- .../pulsar/compaction/CompactorTool.java | 2 +- .../utils/CmdGenerateDocumentation.java | 52 ++++++++ .../org/apache/pulsar/utils/CmdUtility.java | 86 +++++++++++++ .../utils/auth/tokens/TokensCliUtils.java | 2 +- .../pulsar/PulsarBrokerStarterTest.java | 2 +- .../broker/admin/PersistentTopicsTest.java | 13 ++ .../intercept/BrokerInterceptorTest.java | 25 ++++ ...tiAffinityNamespaceGroupExtensionTest.java | 4 +- .../ExtensibleLoadManagerImplTest.java | 36 +++--- .../BrokerIsolationPoliciesFilterTest.java | 33 +++-- .../BrokerLoadManagerClassFilterTest.java | 7 +- .../filter/BrokerMaxTopicCountFilterTest.java | 6 +- .../filter/BrokerVersionFilterTest.java | 29 ++--- .../scheduler/TransferShedderTest.java | 27 ++--- .../service/PublishRateLimiterTest.java | 9 -- .../transaction/TransactionConsumeTest.java | 60 --------- .../AuthenticatedProducerConsumerTest.java | 5 + .../pulsar/compaction/CompactorToolTest.java | 2 +- .../java/org/apache/pulsar/utils/CmdTest.java | 18 +-- pulsar-client-tools/pom.xml | 1 - .../pulsar/admin/cli/CmdNamespaces.java | 4 +- .../apache/pulsar/admin/cli/CmdTopics.java | 6 +- pulsar-client/pom.xml | 1 - .../pulsar/client/impl/ConnectionHandler.java | 13 +- .../impl/conf/CmdGenerateDocumentation.java | 51 ++++++++ pulsar-common/pom.xml | 7 +- .../util}/BaseGenerateDocumentation.java | 14 +-- .../pulsar/common/util}/CmdGenerateDocs.java | 17 +-- .../pulsar/common/util/SecurityUtility.java | 3 +- .../common/util}/CmdGenerateDocsTest.java | 4 +- pulsar-docs-tools/pom.xml | 51 -------- .../pulsar/docs/tools/package-info.java | 20 --- pulsar-functions/worker/pom.xml | 13 -- .../worker/FunctionWorkerStarter.java | 2 +- pulsar-proxy/pom.xml | 12 -- .../proxy/server/ProxyServiceStarter.java | 4 +- .../apache/pulsar/proxy/stats/ProxyStats.java | 2 + .../proxy/util/CmdGenerateDocumentation.java | 31 ++--- .../pulsar/proxy/util/package-info.java | 3 +- .../org/apache/pulsar/proxy/util/CmdTest.java | 58 +++++++++ pulsar-sql/presto-distribution/LICENSE | 32 ++--- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-websocket/pom.xml | 13 -- .../service/WebSocketServiceStarter.java | 2 +- .../ExtensibleLoadManagerTest.java | 3 +- 85 files changed, 613 insertions(+), 667 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdUtility.java rename pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentationTest.java => pulsar-broker/src/test/java/org/apache/pulsar/utils/CmdTest.java (83%) create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/CmdGenerateDocumentation.java rename {pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools => pulsar-common/src/main/java/org/apache/pulsar/common/util}/BaseGenerateDocumentation.java (97%) rename {pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools => pulsar-common/src/main/java/org/apache/pulsar/common/util}/CmdGenerateDocs.java (90%) rename {pulsar-docs-tools/src/test/java/org/apache/pulsar/docs/tools => pulsar-common/src/test/java/org/apache/pulsar/common/util}/CmdGenerateDocsTest.java (98%) delete mode 100644 pulsar-docs-tools/pom.xml delete mode 100644 pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/package-info.java create mode 100644 pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdTest.java diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 836e1c5cb5f0e..8d10f19a80ff8 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -49,7 +49,7 @@ 3.1.2 4.1.94.Final 4.2.3 - 32.1.1-jre + 32.0.0-jre 1.10.12 2.0 3.12.4 diff --git a/conf/bookkeeper.conf b/conf/bookkeeper.conf index 4241abe12eaa3..9661108acc041 100644 --- a/conf/bookkeeper.conf +++ b/conf/bookkeeper.conf @@ -243,7 +243,7 @@ enableBusyWait=false # The interval is specified in seconds. auditorPeriodicBookieCheckInterval=86400 -# The number of entries that a replication will re-replicate in parallel. +# The number of entries that a replication will rereplicate in parallel. rereplicationEntryBatchSize=100 # Auto-replication diff --git a/conf/broker.conf b/conf/broker.conf index 13789ce2eca96..2a0df8d3ba581 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -900,7 +900,7 @@ saslJaasServerRoleTokenSignerSecretPath= ### --- HTTP Server config --- ### -# If >0, it will reject all HTTP requests with bodies larger than the configured limit +# If >0, it will reject all HTTP requests with bodies larged than the configured limit httpMaxRequestSize=-1 # The maximum size in bytes of the request header. Larger headers will allow for more and/or larger cookies plus larger @@ -944,7 +944,7 @@ bookkeeperMetadataServiceUri= # Authentication plugin to use when connecting to bookies bookkeeperClientAuthenticationPlugin= -# BookKeeper auth plugin implementation specifics parameters name and values +# BookKeeper auth plugin implementatation specifics parameters name and values bookkeeperClientAuthenticationParametersName= bookkeeperClientAuthenticationParameters= diff --git a/conf/proxy.conf b/conf/proxy.conf index 2fda32abc1ba6..cfc1e47b7c445 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -152,7 +152,7 @@ authorizationEnabled=false # Authorization provider as a fully qualified class name authorizationProvider=org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider -# Whether client authorization credentials are forwarded to the broker for re-authorization. +# Whether client authorization credentials are forwared to the broker for re-authorization. # Authentication must be enabled via authenticationEnabled=true for this to take effect. forwardAuthorizationCredentials=false diff --git a/conf/standalone.conf b/conf/standalone.conf index 46f4efbcf9171..33010d12adb3a 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -269,7 +269,7 @@ dispatcherReadFailureBackoffMaxTimeInMs=60000 # The read failure backoff mandatory stop time in milliseconds. By default it is 0s. dispatcherReadFailureBackoffMandatoryStopTimeInMs=0 -# Precise dispatcher flow control according to history message number of each entry +# Precise dispathcer flow control according to history message number of each entry preciseDispatcherFlowControl=false # Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic @@ -581,7 +581,7 @@ tokenAudience= # Authentication plugin to use when connecting to bookies bookkeeperClientAuthenticationPlugin= -# BookKeeper auth plugin implementation specifics parameters name and values +# BookKeeper auth plugin implementatation specifics parameters name and values bookkeeperClientAuthenticationParametersName= bookkeeperClientAuthenticationParameters= @@ -1076,7 +1076,7 @@ journalSyncData=false # For each ledger dir, maximum disk space which can be used. # Default is 0.95f. i.e. 95% of disk can be used at most after which nothing will -# be written to that partition. If all ledger dir partitions are full, then bookie +# be written to that partition. If all ledger dir partions are full, then bookie # will turn to readonly mode if 'readOnlyModeEnabled=true' is set, else it will # shutdown. # Valid values should be in between 0 and 1 (exclusive). diff --git a/conf/zookeeper.conf b/conf/zookeeper.conf index db85e688e5c60..73240556d484c 100644 --- a/conf/zookeeper.conf +++ b/conf/zookeeper.conf @@ -63,7 +63,7 @@ forceSync=yes # Default: false sslQuorum=false -# Enable TLS Certificate reloading for Quorum and Server connections +# Enable TLS Certificate reloading for Quorum and Server connentions # Follows Pulsar's general default to reload these files. sslQuorumReloadCertFiles=true client.certReload=true diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 2702bace54e6a..5f946f888aec8 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -40,12 +40,6 @@ ${project.version} - - ${project.groupId} - pulsar-docs-tools - ${project.version} - - ${project.groupId} pulsar-proxy diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 3bbb5183b0f7f..be97087edb75a 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -265,15 +265,13 @@ The Apache Software License, Version 2.0 - com.google.code.gson-gson-2.8.9.jar - io.gsonfire-gson-fire-1.8.5.jar * Guava - - com.google.guava-guava-32.1.1-jre.jar + - com.google.guava-guava-32.0.0-jre.jar - com.google.guava-failureaccess-1.0.1.jar - com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar * J2ObjC Annotations -- com.google.j2objc-j2objc-annotations-1.3.jar * Netty Reactive Streams -- com.typesafe.netty-netty-reactive-streams-2.0.6.jar * Swagger - io.swagger-swagger-annotations-1.6.2.jar - - io.swagger-swagger-core-1.6.2.jar - - io.swagger-swagger-models-1.6.2.jar * DataSketches - com.yahoo.datasketches-memory-0.8.3.jar - com.yahoo.datasketches-sketches-core-0.8.3.jar @@ -346,34 +344,34 @@ The Apache Software License, Version 2.0 - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.16.2.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.2.jar - - org.apache.bookkeeper-bookkeeper-proto-4.16.2.jar - - org.apache.bookkeeper-bookkeeper-server-4.16.2.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.2.jar - - org.apache.bookkeeper-circe-checksum-4.16.2.jar - - org.apache.bookkeeper-cpu-affinity-4.16.2.jar - - org.apache.bookkeeper-statelib-4.16.2.jar - - org.apache.bookkeeper-stream-storage-api-4.16.2.jar - - org.apache.bookkeeper-stream-storage-common-4.16.2.jar - - org.apache.bookkeeper-stream-storage-java-client-4.16.2.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.16.2.jar - - org.apache.bookkeeper-stream-storage-proto-4.16.2.jar - - org.apache.bookkeeper-stream-storage-server-4.16.2.jar - - org.apache.bookkeeper-stream-storage-service-api-4.16.2.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.16.2.jar - - org.apache.bookkeeper.http-http-server-4.16.2.jar - - org.apache.bookkeeper.http-vertx-http-server-4.16.2.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.2.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.2.jar - - org.apache.distributedlog-distributedlog-common-4.16.2.jar - - org.apache.distributedlog-distributedlog-core-4.16.2-tests.jar - - org.apache.distributedlog-distributedlog-core-4.16.2.jar - - org.apache.distributedlog-distributedlog-protocol-4.16.2.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.2.jar - - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.2.jar - - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.2.jar - - org.apache.bookkeeper-native-io-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-common-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-proto-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-server-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.1.jar + - org.apache.bookkeeper-circe-checksum-4.16.1.jar + - org.apache.bookkeeper-cpu-affinity-4.16.1.jar + - org.apache.bookkeeper-statelib-4.16.1.jar + - org.apache.bookkeeper-stream-storage-api-4.16.1.jar + - org.apache.bookkeeper-stream-storage-common-4.16.1.jar + - org.apache.bookkeeper-stream-storage-java-client-4.16.1.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.16.1.jar + - org.apache.bookkeeper-stream-storage-proto-4.16.1.jar + - org.apache.bookkeeper-stream-storage-server-4.16.1.jar + - org.apache.bookkeeper-stream-storage-service-api-4.16.1.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.16.1.jar + - org.apache.bookkeeper.http-http-server-4.16.1.jar + - org.apache.bookkeeper.http-vertx-http-server-4.16.1.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.1.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.1.jar + - org.apache.distributedlog-distributedlog-common-4.16.1.jar + - org.apache.distributedlog-distributedlog-core-4.16.1-tests.jar + - org.apache.distributedlog-distributedlog-core-4.16.1.jar + - org.apache.distributedlog-distributedlog-protocol-4.16.1.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.1.jar + - org.apache.bookkeeper-native-io-4.16.1.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index dc8babb32f329..185b66f6cb416 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -326,7 +326,7 @@ The Apache Software License, Version 2.0 * Gson - gson-2.8.9.jar * Guava - - guava-32.1.1-jre.jar + - guava-32.0.0-jre.jar - failureaccess-1.0.1.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar * J2ObjC Annotations -- j2objc-annotations-1.3.jar @@ -386,9 +386,9 @@ The Apache Software License, Version 2.0 - log4j-web-2.18.0.jar * BookKeeper - - bookkeeper-common-allocator-4.16.2.jar - - cpu-affinity-4.16.2.jar - - circe-checksum-4.16.2.jar + - bookkeeper-common-allocator-4.16.1.jar + - cpu-affinity-4.16.1.jar + - circe-checksum-4.16.1.jar * AirCompressor - aircompressor-0.20.jar * AsyncHttpClient diff --git a/pom.xml b/pom.xml index 7e3538dc6a154..254609a6f07cb 100644 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.16.2 + 4.16.1 3.8.1 1.5.0 1.10.0 @@ -202,7 +202,7 @@ flexible messaging model and an intuitive client API. 2.10.2 3.3.5 2.4.16 - 32.1.1-jre + 32.0.0-jre 1.0 0.16.1 6.2.8 @@ -869,12 +869,14 @@ flexible messaging model and an intuitive client API. io.swagger swagger-core ${swagger.version} + provided io.swagger swagger-annotations ${swagger.version} + provided @@ -2219,7 +2221,6 @@ flexible messaging model and an intuitive client API. pulsar-client-tools-customcommand-example pulsar-client-tools-test pulsar-client-all - pulsar-docs-tools pulsar-websocket pulsar-proxy pulsar-testclient @@ -2282,7 +2283,6 @@ flexible messaging model and an intuitive client API. pulsar-client-tools pulsar-client-tools-customcommand-example pulsar-client-tools-test - pulsar-docs-tools pulsar-websocket pulsar-proxy pulsar-testclient diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 1a0ee2cfb756e..b41a562fbd788 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1356,7 +1356,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext( category = CATEGORY_SERVER, - doc = "Enable or disable the broker interceptor, which is only used for testing for now" + doc = "Enable or disable the broker interceptor" ) private boolean disableBrokerInterceptors = true; diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 4c39d7c4b79d0..e084d70c79ad6 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -283,18 +283,6 @@ guava - - ${project.groupId} - pulsar-docs-tools - ${project.version} - - - io.swagger - * - - - - com.beust jcommander @@ -303,7 +291,6 @@ io.swagger swagger-annotations - provided @@ -329,7 +316,6 @@ io.swagger swagger-core - provided diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java index 1b24c806e62cb..92fc8c5c9acfa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java @@ -55,9 +55,9 @@ import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.DirectMemoryUtils; import org.apache.pulsar.common.util.ShutdownUtil; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerService; import org.apache.pulsar.functions.worker.service.WorkerServiceLoader; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java index 96ebadb1ff4aa..9b757c55ccd1d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java @@ -44,8 +44,8 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.ShutdownUtil; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataTeardown.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataTeardown.java index 01a9eedcca357..cbbf909cdf7fa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataTeardown.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataTeardown.java @@ -33,8 +33,8 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.pulsar.broker.service.schema.SchemaStorageFormat.SchemaLocator; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.MetadataStoreFactory; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarInitialNamespaceSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarInitialNamespaceSetup.java index 22b38e59676ec..c4020546d1c6c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarInitialNamespaceSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarInitialNamespaceSetup.java @@ -23,7 +23,7 @@ import java.util.List; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; +import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.metadata.api.MetadataStore; /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java index 25320964fd62b..33dd58deac095 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java @@ -29,7 +29,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; +import org.apache.pulsar.common.util.CmdGenerateDocs; @Slf4j public class PulsarStandaloneStarter extends PulsarStandalone { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetup.java index 6aedfe13a5b50..78ce55f1b2ce2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetup.java @@ -23,11 +23,11 @@ import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; +import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; /** - * Set up the transaction coordinator metadata for a cluster, the setup will create pulsar/system namespace and create + * Setup the transaction coordinator metadata for a cluster, the setup will create pulsar/system namespace and create * partitioned topic for transaction coordinator assign. */ public class PulsarTransactionCoordinatorMetadataSetup { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java index 85a6c4156dbe4..7eee9d083fc29 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java @@ -20,7 +20,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; +import org.apache.pulsar.common.util.CmdGenerateDocs; /** * Pulsar version entry point. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 789a8e6dbdcb7..ff87f52b64a85 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -233,14 +233,14 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime })) .thenCompose(topics -> { List allTopics = topics.get(0); - Set allUserCreatedTopics = new HashSet<>(); + ArrayList allUserCreatedTopics = new ArrayList<>(); List allPartitionedTopics = topics.get(1); - Set allUserCreatedPartitionTopics = new HashSet<>(); + ArrayList allUserCreatedPartitionTopics = new ArrayList<>(); boolean hasNonSystemTopic = false; - Set allSystemTopics = new HashSet<>(); - Set allPartitionedSystemTopics = new HashSet<>(); - Set topicPolicy = new HashSet<>(); - Set partitionedTopicPolicy = new HashSet<>(); + List allSystemTopics = new ArrayList<>(); + List allPartitionedSystemTopics = new ArrayList<>(); + List topicPolicy = new ArrayList<>(); + List partitionedTopicPolicy = new ArrayList<>(); for (String topic : allTopics) { if (!pulsar().getBrokerService().isSystemTopic(TopicName.get(topic))) { hasNonSystemTopic = true; @@ -279,12 +279,6 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime return old; }); } - allUserCreatedTopics.removeIf(t -> - allPartitionedTopics.contains(TopicName.get(t).getPartitionedTopicName())); - allSystemTopics.removeIf(t -> - allPartitionedTopics.contains(TopicName.get(t).getPartitionedTopicName())); - topicPolicy.removeIf(t -> - allPartitionedTopics.contains(TopicName.get(t).getPartitionedTopicName())); return markDeleteFuture.thenCompose(__ -> internalDeleteTopicsAsync(allUserCreatedTopics)) .thenCompose(ignore -> @@ -354,7 +348,7 @@ private boolean isDeletedAlongWithUserCreatedTopic(String topic) { return topic.endsWith(SystemTopicNames.PENDING_ACK_STORE_SUFFIX); } - private CompletableFuture internalDeletePartitionedTopicsAsync(Set topicNames) { + private CompletableFuture internalDeletePartitionedTopicsAsync(List topicNames) { if (CollectionUtils.isEmpty(topicNames)) { return CompletableFuture.completedFuture(null); } @@ -368,7 +362,7 @@ private CompletableFuture internalDeletePartitionedTopicsAsync(Set return FutureUtil.waitForAll(futures); } - private CompletableFuture internalDeleteTopicsAsync(Set topicNames) { + private CompletableFuture internalDeleteTopicsAsync(List topicNames) { if (CollectionUtils.isEmpty(topicNames)) { return CompletableFuture.completedFuture(null); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 9878cc1592e9c..d8f8e7304d291 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -299,9 +299,20 @@ protected void internalGrantPermissionsOnTopic(final AsyncResponse asyncResponse // This operation should be reading from zookeeper and it should be allowed without having admin privileges validateAdminAccessForTenantAsync(namespaceName.getTenant()) .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync().thenCompose(unused1 -> - grantPermissionsAsync(topicName, role, actions) - .thenAccept(unused -> asyncResponse.resume(Response.noContent().build())))) - .exceptionally(ex -> { + getPartitionedTopicMetadataAsync(topicName, true, false) + .thenCompose(metadata -> { + int numPartitions = metadata.partitions; + CompletableFuture future = CompletableFuture.completedFuture(null); + if (numPartitions > 0) { + for (int i = 0; i < numPartitions; i++) { + TopicName topicNamePartition = topicName.getPartition(i); + future = future.thenCompose(unused -> grantPermissionsAsync(topicNamePartition, role, + actions)); + } + } + return future.thenCompose(unused -> grantPermissionsAsync(topicName, role, actions)) + .thenAccept(unused -> asyncResponse.resume(Response.noContent().build())); + }))).exceptionally(ex -> { Throwable realCause = FutureUtil.unwrapCompletionException(ex); log.error("[{}] Failed to get permissions for topic {}", clientAppId(), topicName, realCause); resumeAsyncResponseExceptionally(asyncResponse, realCause); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 1927d4b244aa4..0fac3a8e005fc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -1597,7 +1597,7 @@ public void createSubscription( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String topic, - @ApiParam(value = "Name of subscription to be created", required = true) + @ApiParam(value = "Subscription to create position on", required = true) @PathParam("subscriptionName") String encodedSubName, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java index cef3f0eb609a1..4ffd8732db94e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java @@ -59,6 +59,11 @@ public BrokerInterceptors(Map intercep * @return the collection of broker event interceptor */ public static BrokerInterceptor load(ServiceConfiguration conf) throws IOException { + if (conf.isDisableBrokerInterceptors()) { + log.info("Skip loading the broker interceptors when disableBrokerInterceptors is true"); + return null; + } + BrokerInterceptorDefinitions definitions = BrokerInterceptorUtils.searchForInterceptors(conf.getBrokerInterceptorsDirectory(), conf.getNarExtractionDirectory()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index f3c5e5dc2ee36..d0c55a6519159 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -34,7 +34,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -47,6 +46,7 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; @@ -477,10 +477,10 @@ public CompletableFuture> selectAsync(ServiceUnitId bundle, Set excludeBrokerSet) { BrokerRegistry brokerRegistry = getBrokerRegistry(); return brokerRegistry.getAvailableBrokerLookupDataAsync() - .thenComposeAsync(availableBrokers -> { + .thenCompose(availableBrokers -> { LoadManagerContext context = this.getContext(); - Map availableBrokerCandidates = new ConcurrentHashMap<>(availableBrokers); + Map availableBrokerCandidates = new HashMap<>(availableBrokers); if (!excludeBrokerSet.isEmpty()) { for (String exclude : excludeBrokerSet) { availableBrokerCandidates.remove(exclude); @@ -489,28 +489,24 @@ public CompletableFuture> selectAsync(ServiceUnitId bundle, // Filter out brokers that do not meet the rules. List filterPipeline = getBrokerFilterPipeline(); - ArrayList>> futures = - new ArrayList<>(filterPipeline.size()); for (final BrokerFilter filter : filterPipeline) { - CompletableFuture> future = - filter.filter(availableBrokerCandidates, bundle, context); - futures.add(future); - } - CompletableFuture> result = new CompletableFuture<>(); - FutureUtil.waitForAll(futures).whenComplete((__, ex) -> { - if (ex != null) { + try { + filter.filter(availableBrokerCandidates, bundle, context); + // Preserve the filter successes result. + availableBrokers.keySet().retainAll(availableBrokerCandidates.keySet()); + } catch (BrokerFilterException e) { // TODO: We may need to revisit this error case. - log.error("Failed to filter out brokers when select bundle: {}", bundle, ex); - } - if (availableBrokerCandidates.isEmpty()) { - result.complete(Optional.empty()); - return; + log.error("Failed to filter out brokers.", e); + availableBrokerCandidates = new HashMap<>(availableBrokers); } - Set candidateBrokers = availableBrokerCandidates.keySet(); + } + if (availableBrokerCandidates.isEmpty()) { + return CompletableFuture.completedFuture(Optional.empty()); + } + Set candidateBrokers = availableBrokerCandidates.keySet(); - result.complete(getBrokerSelectionStrategy().select(candidateBrokers, bundle, context)); - }); - return result; + return CompletableFuture.completedFuture( + getBrokerSelectionStrategy().select(candidateBrokers, bundle, context)); }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 717ff484fe772..70fb8f9ba6f89 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -619,7 +619,7 @@ public CompletableFuture publishSplitEventAsync(Split split) { private void handle(String serviceUnit, ServiceUnitStateData data) { long totalHandledRequests = getHandlerTotalCounter(data).incrementAndGet(); - if (debug()) { + if (log.isDebugEnabled()) { log.info("{} received a handle request for serviceUnit:{}, data:{}. totalHandledRequests:{}", lookupServiceAddress, serviceUnit, data, totalHandledRequests); } @@ -708,7 +708,7 @@ private void handleSkippedEvent(String serviceUnit) { var getOwnerRequest = getOwnerRequests.get(serviceUnit); if (getOwnerRequest != null) { var data = tableview.get(serviceUnit); - if (data != null && data.state() == Owned) { + if (data.state() == Owned) { getOwnerRequest.complete(data.dstBroker()); getOwnerRequests.remove(serviceUnit); stateChangeListeners.notify(serviceUnit, data, null); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java index df08ba96b9d7a..462f8f0e3597a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; -import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper; @@ -39,9 +38,10 @@ public AntiAffinityGroupPolicyFilter(AntiAffinityGroupPolicyHelper helper) { } @Override - public CompletableFuture> filter( + public Map filter( Map brokers, ServiceUnitId serviceUnitId, LoadManagerContext context) { - return helper.filterAsync(brokers, serviceUnitId.toString()); + helper.filter(brokers, serviceUnitId.toString()); + return brokers; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java index 046cae48914f8..d9cbfdc391ed4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; -import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -42,8 +42,9 @@ public interface BrokerFilter { * @param context The load manager context. * @return Filtered broker list. */ - CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context); + Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) + throws BrokerFilterException; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java index 0aa1dda437af7..eeb0d9d3a3309 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java @@ -19,8 +19,9 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; -import java.util.concurrent.CompletableFuture; +import java.util.Set; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; @@ -44,13 +45,13 @@ public String name() { } @Override - public CompletableFuture> filter(Map availableBrokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { - return isolationPoliciesHelper.applyIsolationPoliciesAsync(availableBrokers, serviceUnit) - .thenApply(brokerCandidateCache -> { - availableBrokers.keySet().retainAll(brokerCandidateCache); - return availableBrokers; - }); + public Map filter(Map availableBrokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) + throws BrokerFilterException { + Set brokerCandidateCache = + isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, serviceUnit); + availableBrokers.keySet().retainAll(brokerCandidateCache); + return availableBrokers; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java index 6504da047a001..07109b277ae98 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java @@ -20,7 +20,7 @@ import java.util.Map; import java.util.Objects; -import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -34,12 +34,13 @@ public String name() { } @Override - public CompletableFuture> filter( + public Map filter( Map brokers, ServiceUnitId serviceUnit, - LoadManagerContext context) { + LoadManagerContext context) + throws BrokerFilterException { if (brokers.isEmpty()) { - return CompletableFuture.completedFuture(brokers); + return brokers; } brokers.entrySet().removeIf(entry -> { BrokerLookupData v = entry.getValue(); @@ -47,6 +48,6 @@ public CompletableFuture> filter( return !Objects.equals(v.getLoadManagerClassName(), context.brokerConfiguration().getLoadManagerClassName()); }); - return CompletableFuture.completedFuture(brokers); + return brokers; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java index 044dcc83a7835..0bceae36bb8c2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java @@ -20,7 +20,7 @@ import java.util.Map; import java.util.Optional; -import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; @@ -36,9 +36,9 @@ public String name() { } @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { int loadBalancerBrokerMaxTopics = context.brokerConfiguration().getLoadBalancerBrokerMaxTopics(); brokers.keySet().removeIf(broker -> { Optional brokerLoadDataOpt = context.brokerLoadDataStore().get(broker); @@ -46,6 +46,6 @@ public CompletableFuture> filter(Map= loadBalancerBrokerMaxTopics; }); - return CompletableFuture.completedFuture(brokers); + return brokers; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java index 6e8ce5e58f2bb..7420fcc211309 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java @@ -21,14 +21,13 @@ import com.github.zafarkhaja.semver.Version; import java.util.Iterator; import java.util.Map; -import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterBadVersionException; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.naming.ServiceUnitId; -import org.apache.pulsar.common.util.FutureUtil; /** * Filter by broker version. @@ -47,12 +46,13 @@ public class BrokerVersionFilter implements BrokerFilter { * */ @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) + throws BrokerFilterException { ServiceConfiguration conf = context.brokerConfiguration(); if (!conf.isPreferLaterVersions() || brokers.isEmpty()) { - return CompletableFuture.completedFuture(brokers); + return brokers; } Version latestVersion; @@ -63,8 +63,7 @@ public CompletableFuture> filter(Map> filter(Map> filterAsync(Map brokers, - String bundle) { - return LoadManagerShared.filterAntiAffinityGroupOwnedBrokersAsync(pulsar, bundle, - brokers.keySet(), channel.getOwnershipEntrySet(), brokerToFailureDomainMap) - .thenApply(__ -> brokers); + public void filter( + Map brokers, String bundle) { + LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, bundle, + brokers.keySet(), + channel.getOwnershipEntrySet(), brokerToFailureDomainMap); } public boolean hasAntiAffinityGroupPolicy(String bundle) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java index 468552db541ec..67dc702cc0c9f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java @@ -18,9 +18,10 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.policies; +import io.netty.util.concurrent.FastThreadLocal; +import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; @@ -37,9 +38,18 @@ public IsolationPoliciesHelper(SimpleResourceAllocationPolicies policies) { this.policies = policies; } - public CompletableFuture> applyIsolationPoliciesAsync(Map availableBrokers, - ServiceUnitId serviceUnit) { - return LoadManagerShared.applyNamespacePoliciesAsync(serviceUnit, policies, + private static final FastThreadLocal> localBrokerCandidateCache = new FastThreadLocal<>() { + @Override + protected Set initialValue() { + return new HashSet<>(); + } + }; + + public Set applyIsolationPolicies(Map availableBrokers, + ServiceUnitId serviceUnit) { + Set brokerCandidateCache = localBrokerCandidateCache.get(); + brokerCandidateCache.clear(); + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, availableBrokers.keySet(), new LoadManagerShared.BrokerTopicLoadingPredicate() { @Override public boolean isEnablePersistentTopics(String brokerUrl) { @@ -53,6 +63,7 @@ public boolean isEnableNonPersistentTopics(String brokerUrl) { return lookupData != null && lookupData.nonPersistentTopicsEnabled(); } }); + return brokerCandidateCache; } public boolean hasIsolationPolicy(NamespaceName namespaceName) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index 7bb16bac124e9..07d521a28afa7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -47,6 +47,7 @@ import lombok.experimental.Accessors; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; @@ -719,10 +720,8 @@ private boolean isTransferable(LoadManagerContext context, Map candidates = new HashMap<>(availableBrokers); for (var filter : brokerFilterPipeline) { try { - filter.filter(candidates, namespaceBundle, context) - .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), - TimeUnit.SECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { + filter.filter(candidates, namespaceBundle, context); + } catch (BrokerFilterException e) { log.error("Failed to filter brokers with filter: {}", filter.getClass().getName(), e); return false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java index 5f2e4b1f25d8b..33f346adbe0c5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java @@ -181,104 +181,6 @@ public static void applyNamespacePolicies(final ServiceUnitId serviceUnit, } } - public static CompletableFuture> applyNamespacePoliciesAsync( - final ServiceUnitId serviceUnit, final SimpleResourceAllocationPolicies policies, - final Set availableBrokers, final BrokerTopicLoadingPredicate brokerTopicLoadingPredicate) { - NamespaceName namespace = serviceUnit.getNamespaceObject(); - return policies.areIsolationPoliciesPresentAsync(namespace).thenApply(isIsolationPoliciesPresent -> { - final Set brokerCandidateCache = new HashSet<>(); - Set primariesCache = localPrimariesCache.get(); - primariesCache.clear(); - - Set secondaryCache = localSecondaryCache.get(); - secondaryCache.clear(); - boolean isNonPersistentTopic = (serviceUnit instanceof NamespaceBundle) - ? ((NamespaceBundle) serviceUnit).hasNonPersistentTopic() : false; - if (isIsolationPoliciesPresent) { - if (LOG.isDebugEnabled()) { - LOG.debug("Isolation Policies Present for namespace - [{}]", namespace.toString()); - } - } - for (final String broker : availableBrokers) { - final String brokerUrlString = String.format("http://%s", broker); - URL brokerUrl; - try { - brokerUrl = new URL(brokerUrlString); - } catch (MalformedURLException e) { - LOG.error("Unable to parse brokerUrl from ResourceUnitId", e); - continue; - } - // todo: in future check if the resource unit has resources to take the namespace - if (isIsolationPoliciesPresent) { - // note: serviceUnitID is namespace name and ResourceID is brokerName - if (policies.isPrimaryBroker(namespace, brokerUrl.getHost())) { - primariesCache.add(broker); - if (LOG.isDebugEnabled()) { - LOG.debug("Added Primary Broker - [{}] as possible Candidates for" - + " namespace - [{}] with policies", brokerUrl.getHost(), namespace.toString()); - } - } else if (policies.isSecondaryBroker(namespace, brokerUrl.getHost())) { - secondaryCache.add(broker); - if (LOG.isDebugEnabled()) { - LOG.debug( - "Added Shared Broker - [{}] as possible " - + "Candidates for namespace - [{}] with policies", - brokerUrl.getHost(), namespace.toString()); - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Skipping Broker - [{}] not primary broker and not shared" - + " for namespace - [{}] ", brokerUrl.getHost(), namespace.toString()); - } - - } - } else { - // non-persistent topic can be assigned to only those brokers that enabled for non-persistent topic - if (isNonPersistentTopic - && !brokerTopicLoadingPredicate.isEnableNonPersistentTopics(brokerUrlString)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Filter broker- [{}] because it doesn't support non-persistent namespace - [{}]", - brokerUrl.getHost(), namespace.toString()); - } - } else if (!isNonPersistentTopic - && !brokerTopicLoadingPredicate.isEnablePersistentTopics(brokerUrlString)) { - // persistent topic can be assigned to only brokers that enabled for persistent-topic - if (LOG.isDebugEnabled()) { - LOG.debug("Filter broker- [{}] because broker only supports non-persistent " - + "namespace - [{}]", brokerUrl.getHost(), namespace.toString()); - } - } else if (policies.isSharedBroker(brokerUrl.getHost())) { - secondaryCache.add(broker); - if (LOG.isDebugEnabled()) { - LOG.debug("Added Shared Broker - [{}] as possible Candidates for namespace - [{}]", - brokerUrl.getHost(), namespace.toString()); - } - } - } - } - if (isIsolationPoliciesPresent) { - brokerCandidateCache.addAll(primariesCache); - if (policies.shouldFailoverToSecondaries(namespace, primariesCache.size())) { - if (LOG.isDebugEnabled()) { - LOG.debug( - "Not enough of primaries [{}] available for namespace - [{}], " - + "adding shared [{}] as possible candidate owners", - primariesCache.size(), namespace.toString(), secondaryCache.size()); - } - brokerCandidateCache.addAll(secondaryCache); - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug( - "Policies not present for namespace - [{}] so only " - + "considering shared [{}] brokers for possible owner", - namespace.toString(), secondaryCache.size()); - } - brokerCandidateCache.addAll(secondaryCache); - } - return brokerCandidateCache; - }); - } /** * Using the given bundles, populate the namespace to bundle range map. * @@ -506,22 +408,6 @@ public static void filterAntiAffinityGroupOwnedBrokers( } } - public static CompletableFuture filterAntiAffinityGroupOwnedBrokersAsync( - final PulsarService pulsar, final String assignedBundleName, - final Set candidates, - Set> bundleOwnershipData, - Map brokerToDomainMap - ) { - if (candidates.isEmpty()) { - return CompletableFuture.completedFuture(null); - } - final String namespaceName = getNamespaceNameFromBundleName(assignedBundleName); - return getAntiAffinityNamespaceOwnedBrokers(pulsar, namespaceName, bundleOwnershipData) - .thenAccept(brokerToAntiAffinityNamespaceCount -> - filterAntiAffinityGroupOwnedBrokers(pulsar, candidates, - brokerToDomainMap, brokerToAntiAffinityNamespaceCount)); - } - /** * It computes least number of namespace owned by any of the domain and then it filters out all the domains that own * namespaces more than this count. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java index 25fbc152b1163..4a1577a4e28b6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java @@ -20,7 +20,6 @@ import java.util.Map; import java.util.Optional; -import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.LoadReport; import org.apache.pulsar.broker.loadbalance.ResourceUnit; @@ -54,20 +53,6 @@ private Optional getIsolationPolicies(String cluster } } - private CompletableFuture> getIsolationPoliciesAsync(String clusterName) { - return this.pulsar.getPulsarResources().getNamespaceResources() - .getIsolationPolicies().getIsolationDataPoliciesAsync(clusterName); - } - - public CompletableFuture areIsolationPoliciesPresentAsync(NamespaceName namespace) { - return getIsolationPoliciesAsync(pulsar.getConfiguration().getClusterName()) - .thenApply(policies -> { - return policies.filter(isolationPolicies -> - isolationPolicies.getPolicyByNamespace(namespace) != null) - .isPresent(); - }); - } - public boolean areIsolationPoliciesPresent(NamespaceName namespace) { try { Optional policies = diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index 50f05f80e3f8b..437a6527e8538 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -171,7 +171,7 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i entry.release(); continue; } - if (msgMetadata != null && msgMetadata.hasTxnidMostBits() + if (!isReplayRead && msgMetadata != null && msgMetadata.hasTxnidMostBits() && msgMetadata.hasTxnidLeastBits()) { if (Markers.isTxnMarker(msgMetadata)) { // because consumer can receive message is smaller than maxReadPosition, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 0be78ca4ecfe1..98c0e5b497998 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1191,7 +1191,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { "Consumer that failed is already present on the connection"); } else { Consumer consumer = existingConsumerFuture.getNow(null); - log.warn("[{}] Consumer with the same id is already created:" + log.info("[{}] Consumer with the same id is already created:" + " consumerId={}, consumer={}", remoteAddress, consumerId, consumer); commandSender.sendSuccessResponse(requestId); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/GenerateDocsCommand.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/GenerateDocsCommand.java index b020b4bfd8bd1..1819a56baa12a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/GenerateDocsCommand.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/GenerateDocsCommand.java @@ -26,7 +26,7 @@ import org.apache.bookkeeper.tools.framework.CliCommand; import org.apache.bookkeeper.tools.framework.CliFlags; import org.apache.bookkeeper.tools.framework.CliSpec; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; +import org.apache.pulsar.common.util.CmdGenerateDocs; /** * The command to generate documents of broker-tool. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java index 1f117338ea074..e2d3de9c19a29 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java @@ -60,7 +60,7 @@ public class CompactedTopicImpl implements CompactedTopic { private final BookKeeper bk; - private volatile PositionImpl compactionHorizon = null; + private PositionImpl compactionHorizon = null; private volatile CompletableFuture compactedTopicContext = null; public CompactedTopicImpl(BookKeeper bk) { @@ -91,6 +91,7 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, int numberOfEntriesToRead, boolean isFirstRead, ReadEntriesCallback callback, Consumer consumer) { + synchronized (this) { PositionImpl cursorPosition; if (isFirstRead && MessageId.earliest.equals(consumer.getStartMessageId())){ cursorPosition = PositionImpl.EARLIEST; @@ -100,11 +101,8 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, // TODO: redeliver epoch link https://github.com/apache/pulsar/issues/13690 ReadEntriesCtx readEntriesCtx = ReadEntriesCtx.create(consumer, DEFAULT_CONSUMER_EPOCH); - - final PositionImpl currentCompactionHorizon = compactionHorizon; - - if (currentCompactionHorizon == null - || currentCompactionHorizon.compareTo(cursorPosition) < 0) { + if (compactionHorizon == null + || compactionHorizon.compareTo(cursorPosition) < 0) { cursor.asyncReadEntriesOrWait(numberOfEntriesToRead, callback, readEntriesCtx, PositionImpl.LATEST); } else { compactedTopicContext.thenCompose( @@ -112,13 +110,23 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, .thenCompose((startPoint) -> { // do not need to read the compaction ledger if it is empty. // the cursor just needs to be set to the compaction horizon - if (startPoint == COMPACT_LEDGER_EMPTY || startPoint == NEWER_THAN_COMPACTED) { - cursor.seek(currentCompactionHorizon.getNext()); + if (startPoint == COMPACT_LEDGER_EMPTY) { + cursor.seek(compactionHorizon.getNext()); callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx); return CompletableFuture.completedFuture(null); + } + if (startPoint == NEWER_THAN_COMPACTED && compactionHorizon.compareTo(cursorPosition) < 0) { + cursor.asyncReadEntriesOrWait(numberOfEntriesToRead, callback, readEntriesCtx, + PositionImpl.LATEST); + return CompletableFuture.completedFuture(null); } else { long endPoint = Math.min(context.ledger.getLastAddConfirmed(), startPoint + numberOfEntriesToRead); + if (startPoint == NEWER_THAN_COMPACTED) { + cursor.seek(compactionHorizon.getNext()); + callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx); + return CompletableFuture.completedFuture(null); + } return readEntries(context.ledger, startPoint, endPoint) .thenAccept((entries) -> { Entry lastEntry = entries.get(entries.size() - 1); @@ -134,7 +142,7 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, })) .exceptionally((exception) -> { if (exception.getCause() instanceof NoSuchElementException) { - cursor.seek(currentCompactionHorizon.getNext()); + cursor.seek(compactionHorizon.getNext()); callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx); } else { callback.readEntriesFailed(new ManagedLedgerException(exception), readEntriesCtx); @@ -142,6 +150,7 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, return null; }); } + } } static CompletableFuture findStartPoint(PositionImpl p, @@ -309,7 +318,7 @@ private static int comparePositionAndMessageId(PositionImpl p, MessageIdData m) .compare(p.getEntryId(), m.getEntryId()).result(); } - public Optional getCompactionHorizon() { + public synchronized Optional getCompactionHorizon() { return Optional.ofNullable(this.compactionHorizon); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java index 83ff790228108..2539c306500a2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java @@ -41,8 +41,8 @@ import org.apache.pulsar.client.api.SizeUnit; import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.netty.EventLoopUtil; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java new file mode 100644 index 0000000000000..4a8639c060603 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.utils; + +import com.beust.jcommander.Parameters; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.common.util.BaseGenerateDocumentation; +import org.apache.pulsar.websocket.service.WebSocketProxyConfiguration; + +@Data +@Parameters(commandDescription = "Generate documentation automatically.") +@Slf4j +public class CmdGenerateDocumentation extends BaseGenerateDocumentation { + + @Override + public String generateDocumentByClassName(String className) throws Exception { + StringBuilder sb = new StringBuilder(); + if (ServiceConfiguration.class.getName().equals(className)) { + return generateDocByFieldContext(className, "Broker", sb); + } else if (ClientConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Client", sb); + } else if (WebSocketProxyConfiguration.class.getName().equals(className)) { + return generateDocByFieldContext(className, "WebSocket", sb); + } + + return "Class [" + className + "] not found"; + } + + public static void main(String[] args) throws Exception { + CmdGenerateDocumentation generateDocumentation = new CmdGenerateDocumentation(); + generateDocumentation.run(args); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdUtility.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdUtility.java new file mode 100644 index 0000000000000..f0f88562953e5 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdUtility.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.utils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Writer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CmdUtility { + private static final Logger LOG = LoggerFactory.getLogger(CmdUtility.class); + private static final Charset UTF_8 = StandardCharsets.UTF_8; + + /** + * Executes the specified string command in a separate process. STDOUT and STDERR output will be buffered to the + * given writer ( {@link Writer}) argument. + * + * @param writer + * stdout and stderr output + * @param command + * a specified system command + * @return exitValue 0: success, Non-zero: failure + * @throws IOException + */ + public static int exec(Writer writer, String... command) throws IOException { + if (LOG.isDebugEnabled()) { + StringBuilder sb = new StringBuilder(); + for (String str : command) { + sb.append(str).append(' '); + } + LOG.debug("command={}", sb); + } + + ProcessBuilder pb = new ProcessBuilder(command); + pb.redirectErrorStream(true); + Process proc = null; + BufferedReader reader = null; + try { + proc = pb.start(); + reader = new BufferedReader(new InputStreamReader(proc.getInputStream(), UTF_8)); + String line = null; + while ((line = reader.readLine()) != null) { + if (writer != null) { + writer.write(line); + writer.write('\n'); + } + } + LOG.debug("sending the command to the host"); + int exitValue = proc.waitFor(); + if (LOG.isDebugEnabled()) { + LOG.debug("command exit value={}", exitValue); + } + return exitValue; + } catch (InterruptedException e) { + throw new RuntimeException(e); + } finally { + if (reader != null) { + reader.close(); + } + if (proc != null) { + proc.destroy(); + } + } + } + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java index fa91ef79d4067..2477e1ad2bb1b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java @@ -45,8 +45,8 @@ import javax.crypto.SecretKey; import lombok.Cleanup; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.RelativeTimeUtil; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; public class TokensCliUtils { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java index 1bc3bd26f1294..c6945169512b5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java @@ -38,7 +38,7 @@ import java.lang.reflect.Method; import java.util.Arrays; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; +import org.apache.pulsar.common.util.CmdGenerateDocs; import org.testng.annotations.Test; @Test(groups = "broker") diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 83ed63bf0d95d..a5de86b08df2b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -930,6 +930,19 @@ public void testGrantPartitionedTopic() { verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Map> permissions = (Map>) responseCaptor.getValue(); Assert.assertEquals(permissions.get(role), expectActions); + TopicName topicName = TopicName.get(TopicDomain.persistent.value(), testTenant, testNamespace, + partitionedTopicName); + for (int i = 0; i < numPartitions; i++) { + TopicName partition = topicName.getPartition(i); + response = mock(AsyncResponse.class); + responseCaptor = ArgumentCaptor.forClass(Response.class); + persistentTopics.getPermissionsOnTopic(response, testTenant, testNamespace, + partition.getEncodedLocalName()); + verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); + Map> partitionPermissions = + (Map>) responseCaptor.getValue(); + Assert.assertEquals(partitionPermissions.get(role), expectActions); + } } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java index d1cf91635f992..c612104f8bf1b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java @@ -20,9 +20,12 @@ import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; + import java.io.IOException; import java.util.HashMap; import java.util.HashSet; @@ -36,6 +39,7 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -307,4 +311,25 @@ public void requestInterceptorFailedTest() { } } + @Test + public void testLoadWhenDisableBrokerInterceptorsIsTrue() throws IOException { + ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class); + serviceConfiguration.setDisableBrokerInterceptors(true); + BrokerInterceptor brokerInterceptor = BrokerInterceptors.load(serviceConfiguration); + assertNull(brokerInterceptor); + + verify(serviceConfiguration, times(1)).isDisableBrokerInterceptors(); + verify(serviceConfiguration, times(0)).getBrokerInterceptorsDirectory(); + } + + @Test + public void testLoadWhenDisableBrokerInterceptorsIsFalse() throws IOException { + ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class); + serviceConfiguration.setDisableBrokerInterceptors(false); + BrokerInterceptor brokerInterceptor = BrokerInterceptors.load(serviceConfiguration); + assertNull(brokerInterceptor); + + verify(serviceConfiguration, times(1)).isDisableBrokerInterceptors(); + verify(serviceConfiguration, times(1)).getBrokerInterceptorsDirectory(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java index b766c7de5a5f9..014e6fa3c7a04 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java @@ -131,7 +131,7 @@ public void testAntiAffinityGroupPolicyFilter() var expected = new HashMap<>(brokers); var actual = antiAffinityGroupPolicyFilter.filter( - brokers, namespaceBundle, context).get(); + brokers, namespaceBundle, context); assertEquals(actual, expected); doReturn(antiAffinityEnabledNameSpace + "/" + bundle).when(namespaceBundle).toString(); @@ -142,7 +142,7 @@ public void testAntiAffinityGroupPolicyFilter() .get(5, TimeUnit.SECONDS).get(); expected.remove(srcBroker); actual = antiAffinityGroupPolicyFilter.filter( - brokers, namespaceBundle, context).get(); + brokers, namespaceBundle, context); assertEquals(actual, expected); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 4d15a5a5e2725..d348347515f03 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -106,7 +106,6 @@ import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.stats.Metrics; -import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; import org.awaitility.Awaitility; @@ -268,11 +267,11 @@ public String name() { } @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { brokers.remove(pulsar1.getLookupServiceAddress()); - return CompletableFuture.completedFuture(brokers); + return brokers; } })).when(primaryLoadManager).getBrokerFilterPipeline(); @@ -289,11 +288,11 @@ public void testFilterHasException() throws Exception { doReturn(List.of(new MockBrokerFilter() { @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { - brokers.remove(brokers.keySet().iterator().next()); - return FutureUtil.failedFuture(new BrokerFilterException("Test")); + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { + brokers.clear(); + throw new BrokerFilterException("Test"); } })).when(primaryLoadManager).getBrokerFilterPipeline(); @@ -532,18 +531,19 @@ public void testMoreThenOneFilter() throws Exception { String lookupServiceAddress1 = pulsar1.getLookupServiceAddress(); doReturn(List.of(new MockBrokerFilter() { @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { brokers.remove(lookupServiceAddress1); - return CompletableFuture.completedFuture(brokers); + return brokers; } },new MockBrokerFilter() { @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { - return FutureUtil.failedFuture(new BrokerFilterException("Test")); + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { + brokers.clear(); + throw new BrokerFilterException("Test"); } })).when(primaryLoadManager).getBrokerFilterPipeline(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java index b8d5d98fd4c08..c2c534f72e9db 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java @@ -31,9 +31,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; - import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; @@ -70,7 +67,7 @@ public class BrokerIsolationPoliciesFilterTest { */ @Test public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() - throws IllegalAccessException, BrokerFilterException, ExecutionException, InterruptedException { + throws IllegalAccessException, BrokerFilterException { var namespace = "my-tenant/my-ns"; NamespaceName namespaceName = NamespaceName.get(namespace); @@ -86,18 +83,18 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker Map result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()).get(); + "broker3", getLookupData())), namespaceName, getContext()); assertEquals(result.keySet(), Set.of("broker1")); // b. available-brokers: broker2, broker3 => result: broker2 result = filter.filter(new HashMap<>(Map.of( "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()).get(); + "broker3", getLookupData())), namespaceName, getContext()); assertEquals(result.keySet(), Set.of("broker2")); // c. available-brokers: broker3 => result: NULL result = filter.filter(new HashMap<>(Map.of( - "broker3", getLookupData())), namespaceName, getContext()).get(); + "broker3", getLookupData())), namespaceName, getContext()); assertTrue(result.isEmpty()); // 2. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 2 @@ -107,24 +104,24 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()).get(); + "broker3", getLookupData())), namespaceName, getContext()); assertEquals(result.keySet(), Set.of("broker1", "broker2")); // b. available-brokers: broker2, broker3 => result: broker2 result = filter.filter(new HashMap<>(Map.of( "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()).get(); + "broker3", getLookupData())), namespaceName, getContext()); assertEquals(result.keySet(), Set.of("broker2")); // c. available-brokers: broker3 => result: NULL result = filter.filter(new HashMap<>(Map.of( - "broker3", getLookupData())), namespaceName, getContext()).get(); + "broker3", getLookupData())), namespaceName, getContext()); assertTrue(result.isEmpty()); } @Test public void testFilterWithPersistentOrNonPersistentDisabled() - throws IllegalAccessException, BrokerFilterException, ExecutionException, InterruptedException { + throws IllegalAccessException, BrokerFilterException { var namespace = "my-tenant/my-ns"; NamespaceName namespaceName = NamespaceName.get(namespace); NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); @@ -132,8 +129,7 @@ public void testFilterWithPersistentOrNonPersistentDisabled() doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); var policies = mock(SimpleResourceAllocationPolicies.class); - doReturn(CompletableFuture.completedFuture(false)) - .when(policies).areIsolationPoliciesPresentAsync(eq(namespaceName)); + doReturn(false).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); doReturn(true).when(policies).isSharedBroker(any()); IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(policies); @@ -144,14 +140,14 @@ public void testFilterWithPersistentOrNonPersistentDisabled() Map result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()).get(); + "broker3", getLookupData())), namespaceBundle, getContext()); assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(true, false), "broker2", getLookupData(true, false), - "broker3", getLookupData())), namespaceBundle, getContext()).get(); + "broker3", getLookupData())), namespaceBundle, getContext()); assertEquals(result.keySet(), Set.of("broker3")); doReturn(false).when(namespaceBundle).hasNonPersistentTopic(); @@ -159,13 +155,13 @@ public void testFilterWithPersistentOrNonPersistentDisabled() result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()).get(); + "broker3", getLookupData())), namespaceBundle, getContext()); assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(false, true), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()).get(); + "broker3", getLookupData())), namespaceBundle, getContext()); assertEquals(result.keySet(), Set.of("broker2", "broker3")); } @@ -176,8 +172,7 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, Set shared, int min_limit) { reset(policies); - doReturn(CompletableFuture.completedFuture(true)) - .when(policies).areIsolationPoliciesPresentAsync(eq(namespaceName)); + doReturn(true).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); doReturn(false).when(policies).isPrimaryBroker(eq(namespaceName), any()); doReturn(false).when(policies).isSecondaryBroker(eq(namespaceName), any()); doReturn(false).when(policies).isSharedBroker(any()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java index 6b35bf71a0317..4aef87cf63aa8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java @@ -27,7 +27,6 @@ import org.testng.annotations.Test; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ExecutionException; /** @@ -36,7 +35,7 @@ public class BrokerLoadManagerClassFilterTest extends BrokerFilterTestBase { @Test - public void test() throws BrokerFilterException, ExecutionException, InterruptedException { + public void test() throws BrokerFilterException { LoadManagerContext context = getContext(); context.brokerConfiguration().setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); BrokerLoadManagerClassFilter filter = new BrokerLoadManagerClassFilter(); @@ -49,14 +48,14 @@ public void test() throws BrokerFilterException, ExecutionException, Interrupted "broker5", getLookupData("3.0.0", null) ); - Map result = filter.filter(new HashMap<>(originalBrokers), null, context).get(); + Map result = filter.filter(new HashMap<>(originalBrokers), null, context); assertEquals(result, Map.of( "broker1", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()), "broker2", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()) )); context.brokerConfiguration().setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); - result = filter.filter(new HashMap<>(originalBrokers), null, context).get(); + result = filter.filter(new HashMap<>(originalBrokers), null, context); assertEquals(result, Map.of( "broker3", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java index a6214304bd201..da13a9526a881 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ExecutionException; import static org.testng.Assert.assertEquals; @@ -39,7 +38,7 @@ public class BrokerMaxTopicCountFilterTest extends BrokerFilterTestBase { @Test - public void test() throws IllegalAccessException, BrokerFilterException, ExecutionException, InterruptedException { + public void test() throws IllegalAccessException, BrokerFilterException { LoadManagerContext context = getContext(); LoadDataStore store = context.brokerLoadDataStore(); BrokerLoadData maxTopicLoadData = new BrokerLoadData(); @@ -59,8 +58,7 @@ public void test() throws IllegalAccessException, BrokerFilterException, Executi "broker3", getLookupData(), "broker4", getLookupData() ); - Map result = - filter.filter(new HashMap<>(originalBrokers), null, context).get(); + Map result = filter.filter(new HashMap<>(originalBrokers), null, context); assertEquals(result, Map.of( "broker2", getLookupData(), "broker4", getLookupData() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java index 4a1a45439384e..cafd8f0ea7a4c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java @@ -21,11 +21,9 @@ import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ExecutionException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterBadVersionException; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; @@ -41,14 +39,14 @@ public class BrokerVersionFilterTest extends BrokerFilterTestBase { @Test - public void testFilterEmptyBrokerList() throws BrokerFilterException, ExecutionException, InterruptedException { + public void testFilterEmptyBrokerList() throws BrokerFilterException { BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(new HashMap<>(), null, getContext()).get(); + Map result = brokerVersionFilter.filter(new HashMap<>(), null, getContext()); assertTrue(result.isEmpty()); } @Test - public void testDisabledFilter() throws BrokerFilterException, ExecutionException, InterruptedException { + public void testDisabledFilter() throws BrokerFilterException { LoadManagerContext context = getContext(); ServiceConfiguration configuration = new ServiceConfiguration(); configuration.setPreferLaterVersions(false); @@ -60,12 +58,12 @@ public void testDisabledFilter() throws BrokerFilterException, ExecutionExceptio ); Map brokers = new HashMap<>(originalBrokers); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(brokers, null, context).get(); + Map result = brokerVersionFilter.filter(brokers, null, context); assertEquals(result, originalBrokers); } @Test - public void testFilter() throws BrokerFilterException, ExecutionException, InterruptedException { + public void testFilter() throws BrokerFilterException { Map originalBrokers = Map.of( "localhost:6650", getLookupData("2.10.0"), "localhost:6651", getLookupData("2.10.1"), @@ -74,7 +72,7 @@ public void testFilter() throws BrokerFilterException, ExecutionException, Inter ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); Map result = brokerVersionFilter.filter( - new HashMap<>(originalBrokers), null, getContext()).get(); + new HashMap<>(originalBrokers), null, getContext()); assertEquals(result, Map.of( "localhost:6651", getLookupData("2.10.1"), "localhost:6652", getLookupData("2.10.1"), @@ -87,7 +85,7 @@ public void testFilter() throws BrokerFilterException, ExecutionException, Inter "localhost:6652", getLookupData("2.10.1"), "localhost:6653", getLookupData("2.10.1") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()).get(); + result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); assertEquals(result, Map.of( "localhost:6652", getLookupData("2.10.1"), @@ -101,24 +99,19 @@ public void testFilter() throws BrokerFilterException, ExecutionException, Inter "localhost:6653", getLookupData("2.10.2-SNAPSHOT") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()).get(); + result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); assertEquals(result, Map.of( "localhost:6653", getLookupData("2.10.2-SNAPSHOT") )); } - @Test - public void testInvalidVersionString() { + @Test(expectedExceptions = BrokerFilterBadVersionException.class) + public void testInvalidVersionString() throws BrokerFilterException { Map originalBrokers = Map.of( "localhost:6650", getLookupData("xxx") ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - try { - brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()).get(); - fail(); - } catch (Exception ex) { - assertEquals(ex.getCause().getClass(), BrokerFilterBadVersionException.class); - } + brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index 3dd41a92e682c..af5890fcacbb2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -634,8 +634,7 @@ public void testBundlesWithIsolationPolicies() { // test setLoadBalancerSheddingBundlesWithPoliciesEnabled=false; - doReturn(CompletableFuture.completedFuture(true)) - .when(allocationPoliciesSpy).areIsolationPoliciesPresentAsync(any()); + doReturn(true).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any()); ctx.brokerConfiguration().setLoadBalancerTransferEnabled(true); ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(false); res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -680,14 +679,12 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); doReturn(true).when(namespaceBundle).hasNonPersistentTopic(); doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); - doReturn(CompletableFuture.completedFuture(false)) - .when(policies).areIsolationPoliciesPresentAsync(any()); + doReturn(false).when(policies).areIsolationPoliciesPresent(any()); doReturn(false).when(policies).isPrimaryBroker(any(), any()); doReturn(false).when(policies).isSecondaryBroker(any(), any()); doReturn(true).when(policies).isSharedBroker(any()); - doReturn(CompletableFuture.completedFuture(true)) - .when(policies).areIsolationPoliciesPresentAsync(eq(namespaceName)); + doReturn(true).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); primary.forEach(broker -> { doReturn(true).when(policies).isPrimaryBroker(eq(namespaceName), eq(broker)); @@ -726,8 +723,8 @@ public void testBundlesWithAntiAffinityGroup() throws MetadataStoreException { doAnswer(invocationOnMock -> { Map brokers = invocationOnMock.getArgument(0); brokers.clear(); - return CompletableFuture.completedFuture(brokers); - }).when(antiAffinityGroupPolicyHelper).filterAsync(any(), any()); + return brokers; + }).when(antiAffinityGroupPolicyHelper).filter(any(), any()); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); assertTrue(res.isEmpty()); @@ -740,11 +737,11 @@ public void testBundlesWithAntiAffinityGroup() throws MetadataStoreException { String bundle = invocationOnMock.getArgument(1, String.class); if (bundle.equalsIgnoreCase(bundleE1)) { - return CompletableFuture.completedFuture(brokers); + return brokers; } brokers.clear(); - return CompletableFuture.completedFuture(brokers); - }).when(antiAffinityGroupPolicyHelper).filterAsync(any(), any()); + return brokers; + }).when(antiAffinityGroupPolicyHelper).filter(any(), any()); var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected2 = new HashSet<>(); expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), @@ -764,10 +761,10 @@ public String name() { } @Override - public CompletableFuture> filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) { - return FutureUtil.failedFuture(new BrokerFilterException("test")); + public Map filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) throws BrokerFilterException { + throw new BrokerFilterException("test"); } }; filters.add(filter); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java index f3cb25e789f08..b934ced08c5db 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java @@ -21,7 +21,6 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; import org.apache.pulsar.common.util.RateLimiter; -import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -52,14 +51,6 @@ public void setup() throws Exception { publishRateLimiter = new PublishRateLimiterImpl(policies, CLUSTER_NAME); } - @AfterMethod - public void cleanup() throws Exception { - policies.publishMaxMessageRate.clear(); - policies.publishMaxMessageRate = null; - precisePublishLimiter.close(); - publishRateLimiter.close(); - } - @Test public void testPublishRateLimiterImplExceed() throws Exception { // increment not exceed diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java index 0e17719aca7e7..78846fb75922a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java @@ -22,7 +22,6 @@ import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; -import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -33,9 +32,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.Topic; -import org.apache.pulsar.broker.service.persistent.MessageRedeliveryController; -import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; -import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -228,62 +224,6 @@ public void sortedTest() throws Exception { log.info("TransactionConsumeTest sortedTest finish."); } - @Test - public void testMessageRedelivery() throws Exception { - int transactionMessageCnt = 10; - String subName = "shared-test"; - - @Cleanup - Consumer sharedConsumer = pulsarClient.newConsumer() - .topic(CONSUME_TOPIC) - .subscriptionName(subName) - .subscriptionType(SubscriptionType.Shared) - .subscribe(); - - Awaitility.await().until(sharedConsumer::isConnected); - - long mostSigBits = 2L; - long leastSigBits = 5L; - TxnID txnID = new TxnID(mostSigBits, leastSigBits); - - // produce batch message with txn and then abort - PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() - .getTopic(CONSUME_TOPIC, false).get().get(); - - List sendMessageList = new ArrayList<>(); - List messageIdDataList = appendTransactionMessages(txnID, persistentTopic, transactionMessageCnt, sendMessageList); - - persistentTopic.endTxn(txnID, TxnAction.ABORT_VALUE, 0L).get(); - log.info("Abort txn."); - - // redeliver transaction messages to shared consumer - PersistentSubscription subRef = persistentTopic.getSubscription(subName); - PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef - .getDispatcher(); - Field redeliveryMessagesField = PersistentDispatcherMultipleConsumers.class - .getDeclaredField("redeliveryMessages"); - redeliveryMessagesField.setAccessible(true); - MessageRedeliveryController redeliveryMessages = new MessageRedeliveryController(true); - - final Field totalAvailablePermitsField = PersistentDispatcherMultipleConsumers.class - .getDeclaredField("totalAvailablePermits"); - totalAvailablePermitsField.setAccessible(true); - totalAvailablePermitsField.set(dispatcher, 1000); - - for (MessageIdData messageIdData : messageIdDataList) { - redeliveryMessages.add(messageIdData.getLedgerId(), messageIdData.getEntryId()); - } - - redeliveryMessagesField.set(dispatcher, redeliveryMessages); - dispatcher.readMoreEntries(); - - // shared consumer should not receive the redelivered aborted transaction messages - Message message = sharedConsumer.receive(5, TimeUnit.SECONDS); - Assert.assertNull(message); - - log.info("TransactionConsumeTest testMessageRedelivery finish."); - } - private void sendNormalMessages(Producer producer, int startMsgCnt, int messageCnt, List sendMessageList) throws PulsarClientException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java index 8189f8e86b5b3..75ae91f18a305 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java @@ -396,6 +396,11 @@ public void testDeleteAuthenticationPoliciesOfTopic() throws Exception { Awaitility.await().untilAsserted(() -> { assertTrue(pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get("p1/ns1")) .get().auth_policies.getTopicAuthentication().containsKey(partitionedTopic)); + for (int i = 0; i < numPartitions; i++) { + assertTrue(pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get("p1/ns1")) + .get().auth_policies.getTopicAuthentication() + .containsKey(TopicName.get(partitionedTopic).getPartition(i).toString())); + } }); admin.topics().deletePartitionedTopic("persistent://p1/ns1/partitioned-topic"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java index fb8d6566d9a0d..b1f653fae012a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java @@ -35,7 +35,7 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; +import org.apache.pulsar.common.util.CmdGenerateDocs; import org.testng.annotations.Test; /** diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/utils/CmdTest.java similarity index 83% rename from pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentationTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/utils/CmdTest.java index b76463e862f50..a571a1cd0c92d 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/utils/CmdTest.java @@ -16,22 +16,17 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.pulsar.utils; -package org.apache.pulsar.proxy.util; - +import static org.testng.Assert.assertTrue; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Field; import org.apache.pulsar.common.configuration.FieldContext; import org.testng.annotations.Test; -import static org.testng.Assert.assertTrue; -public class CmdGenerateDocumentationTest { - @Test - public void cmdParserProxyConfigurationTest() throws Exception { - String value = generateDoc("org.apache.pulsar.proxy.server.ProxyConfiguration"); - assertTrue(value.contains("Pulsar proxy")); - } +@Test(groups = "utils") +public class CmdTest { @Test public void cmdParserTest() throws Exception { @@ -48,14 +43,14 @@ public void cmdParserClientTest() throws Exception { generateDoc("org.apache.pulsar.client.impl.conf.ClientConfigurationData"); } - private String generateDoc(String clazz) throws Exception { + private void generateDoc(String clazz) throws Exception { PrintStream oldStream = System.out; try (ByteArrayOutputStream baoStream = new ByteArrayOutputStream(2048); PrintStream cacheStream = new PrintStream(baoStream);) { System.setOut(cacheStream); CmdGenerateDocumentation.main(("-c " + clazz).split(" ")); String message = baoStream.toString(); - Class cls = Class.forName(clazz); + Class cls = Class.forName(clazz); Field[] fields = cls.getDeclaredFields(); for (Field field : fields) { field.setAccessible(true); @@ -65,7 +60,6 @@ private String generateDoc(String clazz) throws Exception { } assertTrue(message.indexOf(field.getName()) > 0); } - return message; } finally { System.setOut(oldStream); } diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 16a30f8d63040..025189b7bbfa5 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -107,7 +107,6 @@ io.swagger swagger-core - provided diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 076ce9a032a77..4bc499322466a 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -2291,14 +2291,14 @@ public boolean isS3Driver(String driver) { public boolean positiveCheck(String paramName, long value) { if (value <= 0) { - throw new ParameterException(paramName + " cannot be less than or equal to 0!"); + throw new ParameterException(paramName + " is not be negative or 0!"); } return true; } public boolean maxValueCheck(String paramName, long value, long maxValue) { if (value > maxValue) { - throw new ParameterException(paramName + " cannot be greater than " + maxValue + "!"); + throw new ParameterException(paramName + " is not bigger than " + maxValue + "!"); } return true; } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index 31d9648babcff..b07ad0056617e 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -1004,7 +1004,7 @@ private class CreateSubscription extends CliCommand { private java.util.List params; @Parameter(names = { "-s", - "--subscription" }, description = "Name of subscription to be created", required = true) + "--subscription" }, description = "Subscription to reset position on", required = true) private String subscriptionName; @Parameter(names = { "-m" , "--messageId" }, description = "messageId where to create the subscription. " @@ -2198,14 +2198,14 @@ public boolean isS3Driver(String driver) { public boolean positiveCheck(String paramName, long value) { if (value <= 0) { - throw new ParameterException(paramName + " cannot be less than or equal to 0!"); + throw new ParameterException(paramName + " is not be negative or 0!"); } return true; } public boolean maxValueCheck(String paramName, long value, long maxValue) { if (value > maxValue) { - throw new ParameterException(paramName + " cannot be greater than " + maxValue + "!"); + throw new ParameterException(paramName + " is not bigger than " + maxValue + "!"); } return true; } diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 0e45e73636612..db2297100eeb2 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -76,7 +76,6 @@ io.swagger swagger-annotations - provided diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java index 263507dac1dc6..365abce3b90d2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java @@ -64,12 +64,6 @@ protected ConnectionHandler(HandlerState state, Backoff backoff, Connection conn } protected void grabCnx() { - if (!duringConnect.compareAndSet(false, true)) { - log.info("[{}] [{}] Skip grabbing the connection since there is a pending connection", - state.topic, state.getHandlerName()); - return; - } - if (CLIENT_CNX_UPDATER.get(this) != null) { log.warn("[{}] [{}] Client cnx already set, ignoring reconnection request", state.topic, state.getHandlerName()); @@ -82,6 +76,11 @@ protected void grabCnx() { state.topic, state.getHandlerName(), state.getState()); return; } + if (!duringConnect.compareAndSet(false, true)) { + log.info("[{}] [{}] Skip grabbing the connection since there is a pending connection", + state.topic, state.getHandlerName()); + return; + } try { CompletableFuture cnxFuture; @@ -124,8 +123,8 @@ private Void handleConnectionError(Throwable exception) { } void reconnectLater(Throwable exception) { - CLIENT_CNX_UPDATER.set(this, null); duringConnect.set(false); + CLIENT_CNX_UPDATER.set(this, null); if (!isValidStateForReconnection()) { log.info("[{}] [{}] Ignoring reconnection request (state: {})", state.topic, state.getHandlerName(), state.getState()); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/CmdGenerateDocumentation.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/CmdGenerateDocumentation.java new file mode 100644 index 0000000000000..ae6210b978edc --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/CmdGenerateDocumentation.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl.conf; + +import com.beust.jcommander.Parameters; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.util.BaseGenerateDocumentation; + +@Data +@Parameters(commandDescription = "Generate documentation automatically.") +@Slf4j +public class CmdGenerateDocumentation extends BaseGenerateDocumentation { + + @Override + public String generateDocumentByClassName(String className) throws Exception { + StringBuilder sb = new StringBuilder(); + if (ClientConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Client", sb); + } else if (ProducerConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Producer", sb); + } else if (ConsumerConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Consumer", sb); + } else if (ReaderConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Reader", sb); + } + + return "Class [" + className + "] not found"; + } + + public static void main(String[] args) throws Exception { + CmdGenerateDocumentation generateDocumentation = new CmdGenerateDocumentation(); + generateDocumentation.run(args); + } +} diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 8c99638878f53..cfc9cba7ede7f 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -50,7 +50,6 @@ io.swagger swagger-annotations - provided @@ -227,12 +226,16 @@ snappy-java test - com.google.code.gson gson + + com.beust + jcommander + + org.awaitility awaitility diff --git a/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/BaseGenerateDocumentation.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/BaseGenerateDocumentation.java similarity index 97% rename from pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/BaseGenerateDocumentation.java rename to pulsar-common/src/main/java/org/apache/pulsar/common/util/BaseGenerateDocumentation.java index db6178a7fda4d..b88ed197e8f2e 100644 --- a/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/BaseGenerateDocumentation.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/BaseGenerateDocumentation.java @@ -16,10 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.docs.tools; +package org.apache.pulsar.common.util; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; import io.swagger.annotations.ApiModelProperty; import java.io.Serializable; import java.lang.annotation.Annotation; @@ -30,12 +31,15 @@ import java.util.List; import java.util.function.Predicate; import java.util.stream.Collectors; +import lombok.Data; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.MethodUtils; import org.apache.commons.lang3.tuple.Pair; +@Data +@Parameters(commandDescription = "Generate documentation automatically.") @Slf4j public abstract class BaseGenerateDocumentation { @@ -182,9 +186,7 @@ public int compare(Pair o1, Pair clazz = Class.forName(className); Object obj = clazz.getDeclaredConstructor().newInstance(); Field[] fields = clazz.getDeclaredFields(); @@ -216,9 +218,7 @@ protected String generateDocByFieldContext(String className, String type) throws return sb.toString(); } - protected String generateDocByApiModelProperty(String className, String type) throws Exception { - final StringBuilder sb = new StringBuilder(); - + protected String generateDocByApiModelProperty(String className, String type, StringBuilder sb) throws Exception { Class clazz = Class.forName(className); Object obj = clazz.getDeclaredConstructor().newInstance(); Field[] fields = clazz.getDeclaredFields(); diff --git a/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/CmdGenerateDocs.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java similarity index 90% rename from pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/CmdGenerateDocs.java rename to pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java index 8f784c1eca1fa..f41e53a80e7f3 100644 --- a/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/CmdGenerateDocs.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java @@ -16,14 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.docs.tools; +package org.apache.pulsar.common.util; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterDescription; import com.beust.jcommander.Parameters; import java.util.ArrayList; -import java.util.Comparator; import java.util.List; import java.util.Map; import lombok.Getter; @@ -146,15 +145,11 @@ private String generateDocument(String module, JCommander commander) { sb.append("|Flag|Description|Default|\n"); sb.append("|---|---|---|\n"); List options = cmd.getParameters(); - options.stream().sorted(Comparator.comparing(ParameterDescription::getLongestName)) - .forEach((option) -> - sb.append("| `") - .append(option.getNames()) - .append("` | ") - .append(option.getDescription().replace("\n", " ")) - .append("|") - .append(option.getDefault()).append("|\n") - ); + options.forEach((option) -> + sb.append("| `").append(option.getNames()) + .append("` | ").append(option.getDescription().replace("\n", " ")) + .append("|").append(option.getDefault()).append("|\n") + ); } return sb.toString(); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java index 8dfd478cc8fbe..f0023ce5a42dd 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java @@ -130,8 +130,7 @@ private static Provider loadConscryptProvider() { log.warn("Conscrypt isn't available for {} {}. Using JDK default security provider.", System.getProperty("os.name"), System.getProperty("os.arch")); } else { - log.warn("Conscrypt isn't available. Using JDK default security provider." - + " Cause : {}, Reason : {}", e.getCause(), e.getMessage()); + log.warn("Conscrypt isn't available. Using JDK default security provider.", e); } return null; } diff --git a/pulsar-docs-tools/src/test/java/org/apache/pulsar/docs/tools/CmdGenerateDocsTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/CmdGenerateDocsTest.java similarity index 98% rename from pulsar-docs-tools/src/test/java/org/apache/pulsar/docs/tools/CmdGenerateDocsTest.java rename to pulsar-common/src/test/java/org/apache/pulsar/common/util/CmdGenerateDocsTest.java index 3f96ddaef591a..68d8a02f48673 100644 --- a/pulsar-docs-tools/src/test/java/org/apache/pulsar/docs/tools/CmdGenerateDocsTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/CmdGenerateDocsTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.docs.tools; +package org.apache.pulsar.common.util; import static org.testng.Assert.assertEquals; import com.beust.jcommander.Parameter; @@ -86,8 +86,8 @@ public void testGenerateDocs() { + "\n" + "|Flag|Description|Default|\n" + "|---|---|---|\n" - + "| `-h, --help` | Show this help message|false|\n" + "| `-n, --name` | Name|null|\n" + + "| `-h, --help` | Show this help message|false|\n" + System.lineSeparator(); assertEquals(rightMsg, message); } finally { diff --git a/pulsar-docs-tools/pom.xml b/pulsar-docs-tools/pom.xml deleted file mode 100644 index 9074ad0d6c591..0000000000000 --- a/pulsar-docs-tools/pom.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - 4.0.0 - - - org.apache.pulsar - pulsar - 3.1.0-SNAPSHOT - - - pulsar-docs-tools - Pulsar Documentation Generators - - - - io.swagger - swagger-annotations - - - io.swagger - swagger-core - - - com.beust - jcommander - - - - diff --git a/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/package-info.java b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/package-info.java deleted file mode 100644 index d971fc8ee6b11..0000000000000 --- a/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/package-info.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.pulsar.docs.tools; diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index c6db572f57d7e..da7c83689c1c9 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -129,22 +129,9 @@ commons-io - - ${project.groupId} - pulsar-docs-tools - ${project.version} - - - io.swagger - * - - - - io.swagger swagger-core - provided com.fasterxml.jackson.core diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionWorkerStarter.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionWorkerStarter.java index 679ce1db70d97..fa988a0927bd6 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionWorkerStarter.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionWorkerStarter.java @@ -22,8 +22,8 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.ShutdownUtil; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; /** * A starter to start function worker. diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index de1eb9a16c05b..a607b9e835f8e 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -49,12 +49,6 @@ ${project.version} - - ${project.groupId} - pulsar-docs-tools - ${project.version} - - ${project.groupId} pulsar-websocket @@ -66,12 +60,6 @@ commons-lang3 - - io.swagger - swagger-annotations - provided - - org.eclipse.jetty jetty-server diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index d8bb048710770..beee9f1a4f763 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -46,9 +46,9 @@ import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.configuration.VipStatus; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.DirectMemoryUtils; import org.apache.pulsar.common.util.ShutdownUtil; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.proxy.stats.ProxyStats; import org.apache.pulsar.websocket.WebSocketConsumerServlet; import org.apache.pulsar.websocket.WebSocketPingPongServlet; @@ -61,8 +61,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; + /** * Starts an instance of the Pulsar ProxyService. + * */ public class ProxyServiceStarter { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java index afa2e12dabb4f..27e61c90e9e26 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java @@ -38,6 +38,8 @@ import javax.ws.rs.core.Response.Status; import org.apache.pulsar.proxy.server.ProxyService; + + @Path("/") @Api(value = "/proxy-stats", description = "Stats for proxy", tags = "proxy-stats", hidden = true) @Produces(MediaType.APPLICATION_JSON) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentation.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentation.java index a9164b6d95393..a1c7e30a25ed6 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentation.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentation.java @@ -16,41 +16,26 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.pulsar.proxy.util; +import com.beust.jcommander.Parameters; +import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; -import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; -import org.apache.pulsar.client.impl.conf.ReaderConfigurationData; -import org.apache.pulsar.docs.tools.BaseGenerateDocumentation; +import org.apache.pulsar.common.util.BaseGenerateDocumentation; import org.apache.pulsar.proxy.server.ProxyConfiguration; -import org.apache.pulsar.websocket.service.WebSocketProxyConfiguration; +@Data +@Parameters(commandDescription = "Generate documentation automatically.") @Slf4j public class CmdGenerateDocumentation extends BaseGenerateDocumentation { @Override public String generateDocumentByClassName(String className) throws Exception { + StringBuilder sb = new StringBuilder(); if (ProxyConfiguration.class.getName().equals(className)) { - return generateDocByFieldContext(className, "Pulsar proxy"); - } else if (ServiceConfiguration.class.getName().equals(className)) { - return generateDocByFieldContext(className, "Broker"); - } else if (ClientConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Client"); - } else if (WebSocketProxyConfiguration.class.getName().equals(className)) { - return generateDocByFieldContext(className, "WebSocket"); - } else if (ProducerConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Producer"); - } else if (ConsumerConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Consumer"); - } else if (ReaderConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Reader"); - } else { - return "Class [" + className + "] not found"; + return generateDocByFieldContext(className, "Pulsar proxy", sb); } + return "Class [" + className + "] not found"; } public static void main(String[] args) throws Exception { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/package-info.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/package-info.java index 85284c0dd229a..1501a16f55813 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/package-info.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/package-info.java @@ -16,5 +16,4 @@ * specific language governing permissions and limitations * under the License. */ - -package org.apache.pulsar.proxy.util; +package org.apache.pulsar.proxy.util; \ No newline at end of file diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdTest.java new file mode 100644 index 0000000000000..59e1a43310d67 --- /dev/null +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.proxy.util; + +import static org.testng.Assert.assertTrue; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.lang.reflect.Field; +import org.apache.pulsar.common.configuration.FieldContext; +import org.testng.annotations.Test; + +public class CmdTest { + + @Test + public void cmdParserProxyConfigurationTest() throws Exception { + String value = generateDoc("org.apache.pulsar.proxy.server.ProxyConfiguration"); + assertTrue(value.contains("Pulsar proxy")); + } + + private String generateDoc(String clazz) throws Exception { + PrintStream oldStream = System.out; + try (ByteArrayOutputStream baoStream = new ByteArrayOutputStream(2048); + PrintStream cacheStream = new PrintStream(baoStream);) { + System.setOut(cacheStream); + CmdGenerateDocumentation.main(("-c " + clazz).split(" ")); + String message = baoStream.toString(); + Class cls = Class.forName(clazz); + Field[] fields = cls.getDeclaredFields(); + for (Field field : fields) { + field.setAccessible(true); + FieldContext fieldContext = field.getAnnotation(FieldContext.class); + if (fieldContext == null) { + continue; + } + assertTrue(message.indexOf(field.getName()) > 0); + } + return message; + } finally { + System.setOut(oldStream); + } + } +} diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 9afd6c98bc2ff..15f20ae7e4ad6 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -221,7 +221,7 @@ The Apache Software License, Version 2.0 - jackson-module-jaxb-annotations-2.14.2.jar - jackson-module-jsonSchema-2.14.2.jar * Guava - - guava-32.1.1-jre.jar + - guava-32.0.0-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - failureaccess-1.0.1.jar * Google Guice @@ -430,21 +430,21 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.16.2.jar - - bookkeeper-common-allocator-4.16.2.jar - - bookkeeper-proto-4.16.2.jar - - bookkeeper-server-4.16.2.jar - - bookkeeper-stats-api-4.16.2.jar - - bookkeeper-tools-framework-4.16.2.jar - - circe-checksum-4.16.2.jar - - codahale-metrics-provider-4.16.2.jar - - cpu-affinity-4.16.2.jar - - http-server-4.16.2.jar - - prometheus-metrics-provider-4.16.2.jar - - codahale-metrics-provider-4.16.2.jar - - bookkeeper-slogger-api-4.16.2.jar - - bookkeeper-slogger-slf4j-4.16.2.jar - - native-io-4.16.2.jar + - bookkeeper-common-4.16.1.jar + - bookkeeper-common-allocator-4.16.1.jar + - bookkeeper-proto-4.16.1.jar + - bookkeeper-server-4.16.1.jar + - bookkeeper-stats-api-4.16.1.jar + - bookkeeper-tools-framework-4.16.1.jar + - circe-checksum-4.16.1.jar + - codahale-metrics-provider-4.16.1.jar + - cpu-affinity-4.16.1.jar + - http-server-4.16.1.jar + - prometheus-metrics-provider-4.16.1.jar + - codahale-metrics-provider-4.16.1.jar + - bookkeeper-slogger-api-4.16.1.jar + - bookkeeper-slogger-slf4j-4.16.1.jar + - native-io-4.16.1.jar * Apache Commons - commons-cli-1.5.0.jar - commons-codec-1.15.jar diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 1ac764a3c64d9..8335aa3603f63 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -37,7 +37,7 @@ 2.6 0.0.12 3.0.5 - 32.1.1-jre + 32.0.0-jre 2.12.1 2.5.1 4.0.1 diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index bf8f38f2da1af..f742c286f0b68 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -52,18 +52,6 @@ test - - ${project.groupId} - pulsar-docs-tools - ${project.version} - - - io.swagger - * - - - - org.apache.commons commons-lang3 @@ -92,7 +80,6 @@ io.swagger swagger-core - provided diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java index 530f0796d488d..fbcecc0642e34 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java @@ -26,8 +26,8 @@ import com.beust.jcommander.Parameter; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.configuration.VipStatus; +import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.ShutdownUtil; -import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.websocket.WebSocketConsumerServlet; import org.apache.pulsar.websocket.WebSocketPingPongServlet; import org.apache.pulsar.websocket.WebSocketProducerServlet; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java index 51dc9d7dd72da..86228d25a793f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -261,7 +261,8 @@ public void testStopBroker() throws Exception { assertNotEquals(broker1, broker); } - @Test(timeOut = 40 * 1000) + // TODO: This test is very flaky and it's disabled for now to unblock CI + @Test(timeOut = 40 * 1000, enabled = false) public void testAntiaffinityPolicy() throws PulsarAdminException { final String namespaceAntiAffinityGroup = "my-anti-affinity-filter"; final String antiAffinityEnabledNameSpace = DEFAULT_TENANT + "/my-ns-filter" + nsSuffix; From 82a17c59b01369102f699ee98bceea4b87e79ed2 Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Thu, 6 Jul 2023 00:25:47 +0900 Subject: [PATCH 06/14] Fix name and checkstyle --- .../org/apache/pulsar/admin/cli/CmdNamespaces.java | 10 +++++----- ...LongConverter.java => ByteUnitToLongConverter.java} | 6 ++---- 2 files changed, 7 insertions(+), 9 deletions(-) rename pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/{ByteUnitLongConverter.java => ByteUnitToLongConverter.java} (81%) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 4bc499322466a..42278bf55b71d 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -37,7 +37,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.admin.cli.utils.IOUtils; import org.apache.pulsar.admin.cli.utils.converters.ByteUnitIntegerConverter; -import org.apache.pulsar.admin.cli.utils.converters.ByteUnitLongConverter; +import org.apache.pulsar.admin.cli.utils.converters.ByteUnitToLongConverter; import org.apache.pulsar.admin.cli.utils.converters.TimeUnitToMillisConverter; import org.apache.pulsar.admin.cli.utils.converters.TimeUnitToSecondsConverter; import org.apache.pulsar.admin.cli.utils.validators.IntegerMaxValueLongValidator; @@ -1251,7 +1251,7 @@ private class SetBacklogQuota extends CliCommand { private java.util.List params; @Parameter(names = { "-l", "--limit" }, description = "Size limit (eg: 10M, 16G)", - converter = ByteUnitLongConverter.class) + converter = ByteUnitToLongConverter.class) private Long limit; @Parameter(names = { "-lt", "--limitTime" }, @@ -1912,7 +1912,7 @@ private class SetCompactionThreshold extends CliCommand { description = "Maximum number of bytes in a topic backlog before compaction is triggered " + "(eg: 10M, 16G, 3T). 0 disables automatic compaction", required = true, - converter = ByteUnitLongConverter.class) + converter = ByteUnitToLongConverter.class) private Long threshold = 0L; @Override @@ -1946,7 +1946,7 @@ private class SetOffloadThreshold extends CliCommand { + " Negative values disable automatic offload." + " 0 triggers offloading as soon as possible.", required = true, - converter = ByteUnitLongConverter.class) + converter = ByteUnitToLongConverter.class) private Long threshold = -1L; @Override @@ -2254,7 +2254,7 @@ private class SetOffloadPolicies extends CliCommand { names = {"--offloadAfterThreshold", "-oat"}, description = "Offload after threshold size (eg: 1M, 5M)", required = false, - converter = ByteUnitLongConverter.class) + converter = ByteUnitToLongConverter.class) private Long offloadAfterThresholdInBytes = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_BYTES; @Parameter( diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitToLongConverter.java similarity index 81% rename from pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java rename to pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitToLongConverter.java index d68a2587191e4..e401d1503b2cf 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitLongConverter.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitToLongConverter.java @@ -19,13 +19,11 @@ package org.apache.pulsar.admin.cli.utils.converters; import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.emptyCheck; -import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.maxValueCheck; -import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.positiveCheck; import com.beust.jcommander.converters.BaseConverter; -public class ByteUnitLongConverter extends BaseConverter { +public class ByteUnitToLongConverter extends BaseConverter { - public ByteUnitLongConverter(String optionName) { + public ByteUnitToLongConverter(String optionName) { super(optionName); } From 7e2ae5baa3d3d838db342d1d3c2f366e18c0ca36 Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Thu, 6 Jul 2023 00:33:59 +0900 Subject: [PATCH 07/14] Clean up test --- ...ByteUnitIntegerAllowNegativeConverter.java | 41 ----------------- .../converters/ByteUnitIntegerConverter.java | 2 - .../OffloadedReadPriorityConverter.java | 45 ------------------- 3 files changed, 88 deletions(-) delete mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerAllowNegativeConverter.java delete mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/OffloadedReadPriorityConverter.java diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerAllowNegativeConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerAllowNegativeConverter.java deleted file mode 100644 index 05229d932793c..0000000000000 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerAllowNegativeConverter.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.admin.cli.utils.converters; - -import com.beust.jcommander.converters.BaseConverter; -import org.apache.pulsar.admin.cli.utils.ValueValidationUtils; - -public class ByteUnitIntegerAllowNegativeConverter extends BaseConverter { - - public ByteUnitIntegerAllowNegativeConverter(String optionName) { - super(optionName); - } - - @Override - public Integer convert(String argStr) { - return parseBytes(argStr).intValue(); - } - - Long parseBytes(String argStr) { - ValueValidationUtils.emptyCheck(getOptionName(), argStr); - long valueInBytes = ByteUnitUtil.validateSizeString(argStr); - ValueValidationUtils.maxValueCheck(getOptionName(), valueInBytes, Integer.MAX_VALUE); - return valueInBytes; - } -} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerConverter.java index 0476c341b1639..f8080541c6776 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerConverter.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerConverter.java @@ -38,8 +38,6 @@ public Integer convert(String argStr) { Long parseBytes(String argStr) { emptyCheck(getOptionName(), argStr); long valueInBytes = validateSizeString(argStr); - maxValueCheck(getOptionName(), valueInBytes, Integer.MAX_VALUE); - positiveCheck(getOptionName(), valueInBytes); return valueInBytes; } } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/OffloadedReadPriorityConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/OffloadedReadPriorityConverter.java deleted file mode 100644 index 90da9071a08f4..0000000000000 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/OffloadedReadPriorityConverter.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.admin.cli.utils.converters; - -import com.beust.jcommander.ParameterException; -import com.beust.jcommander.converters.BaseConverter; -import java.util.Arrays; -import java.util.stream.Collectors; -import org.apache.pulsar.common.policies.data.OffloadedReadPriority; - -public class OffloadedReadPriorityConverter extends BaseConverter { - - public OffloadedReadPriorityConverter(String optionName) { - super(optionName); - } - - @Override - public OffloadedReadPriority convert(String value) { - try { - return OffloadedReadPriority.fromString(value); - } catch (Exception e) { - throw new ParameterException("--offloadedReadPriority parameter must be one of " - + Arrays.stream(OffloadedReadPriority.values()) - .map(OffloadedReadPriority::toString) - .collect(Collectors.joining(",")) - + " but got: " + value, e); - } - } -} From 72f1bfe009ad42fcf9d81bfaf8e357e5eba38c43 Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Thu, 6 Jul 2023 00:47:02 +0900 Subject: [PATCH 08/14] Fix tests and validations --- .../pulsar/admin/cli/CmdNamespaces.java | 44 ++++++------------- .../converters/ByteUnitIntegerConverter.java | 2 - .../converters/TimeUnitToMillisConverter.java | 7 +-- .../TimeUnitToSecondsConverter.java | 5 +-- .../validators/NonNegativeValueValidator.java | 2 +- 5 files changed, 17 insertions(+), 43 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 42278bf55b71d..84e121151c5e2 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -757,12 +757,12 @@ private class SetRetention extends CliCommand { @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - final int retentionTimeInMin = retentionTimeInSec == -1 - ? retentionTimeInSec.intValue() - : (int) TimeUnit.SECONDS.toMinutes(retentionTimeInSec); - final int retentionSizeInMB = sizeLimit == -1 - ? sizeLimit - : (sizeLimit / (1024 * 1024)); + final int retentionTimeInMin = retentionTimeInSec != -1 + ? (int) TimeUnit.SECONDS.toMinutes(retentionTimeInSec) + : retentionTimeInSec.intValue(); + final int retentionSizeInMB = sizeLimit != -1 + ? (int) (sizeLimit / (1024 * 1024)) + : sizeLimit; getAdmin().namespaces() .setRetention(namespace, new RetentionPolicies(retentionTimeInMin, retentionSizeInMB)); } @@ -1257,8 +1257,7 @@ private class SetBacklogQuota extends CliCommand { @Parameter(names = { "-lt", "--limitTime" }, description = "Time limit in second (or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w), " + "non-positive number for disabling time limit.", - converter = TimeUnitToSecondsConverter.class, - validateValueWith = PositiveValueValidator.class) + converter = TimeUnitToSecondsConverter.class) private Long limitTimeInSec; @Parameter(names = { "-p", "--policy" }, description = "Retention policy to enforce when the limit is reached. " @@ -1552,8 +1551,7 @@ private class SetInactiveTopicPolicies extends CliCommand { @Parameter(names = {"--max-inactive-duration", "-t"}, description = "Max duration of topic inactivity in " + "seconds, topics that are inactive for longer than this value will be deleted " + "(eg: 1s, 10s, 1m, 5h, 3d)", required = true, - converter = TimeUnitToSecondsConverter.class, - validateValueWith = PositiveValueValidator.class) + converter = TimeUnitToSecondsConverter.class) private Long maxInactiveDurationInSeconds; @Parameter(names = { "--delete-mode", "-m" }, description = "Mode of delete inactive topic, Valid options are: " @@ -1593,8 +1591,7 @@ private class SetDelayedDelivery extends CliCommand { @Parameter(names = { "--time", "-t" }, description = "The tick time for when retrying on " + "delayed delivery messages, affecting the accuracy of the delivery time compared to " + "the scheduled time. (eg: 1s, 10s, 1m, 5h, 3d)", - converter = TimeUnitToMillisConverter.class, - validateValueWith = PositiveValueValidator.class) + converter = TimeUnitToMillisConverter.class) private Long delayedDeliveryTimeInMills = 1000L; @Override @@ -2231,14 +2228,16 @@ private class SetOffloadPolicies extends CliCommand { description = "Max block size (eg: 32M, 64M), default is 64MB" + "s3 and google-cloud-storage requires this parameter", required = false, - converter = ByteUnitIntegerConverter.class) + converter = ByteUnitIntegerConverter.class, + validateValueWith = {PositiveValueValidator.class, IntegerMaxValueLongValidator.class}) private Integer maxBlockSizeInBytes = OffloadPoliciesImpl.DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; @Parameter( names = {"--readBufferSize", "-rbs"}, description = "Read buffer size (eg: 1M, 5M), default is 1MB", required = false, - converter = ByteUnitIntegerConverter.class) + converter = ByteUnitIntegerConverter.class, + validateValueWith = {PositiveValueValidator.class, IntegerMaxValueLongValidator.class}) private Integer readBufferSizeInBytes = OffloadPoliciesImpl.DEFAULT_READ_BUFFER_SIZE_IN_BYTES; @Parameter( @@ -2261,8 +2260,7 @@ private class SetOffloadPolicies extends CliCommand { names = {"--offloadAfterThresholdInSeconds", "-oats"}, description = "Offload after threshold seconds (or minutes,hours,days,weeks eg: 100m, 3h, 2d, 5w).", required = false, - converter = TimeUnitToSecondsConverter.class, - validateValueWith = PositiveValueValidator.class) + converter = TimeUnitToSecondsConverter.class) private Long offloadThresholdInSeconds = OffloadPoliciesImpl.DEFAULT_OFFLOAD_THRESHOLD_IN_SECONDS; @Parameter( @@ -2289,20 +2287,6 @@ public boolean isS3Driver(String driver) { return driver.equalsIgnoreCase(driverNames.get(0)) || driver.equalsIgnoreCase(driverNames.get(1)); } - public boolean positiveCheck(String paramName, long value) { - if (value <= 0) { - throw new ParameterException(paramName + " is not be negative or 0!"); - } - return true; - } - - public boolean maxValueCheck(String paramName, long value, long maxValue) { - if (value > maxValue) { - throw new ParameterException(paramName + " is not bigger than " + maxValue + "!"); - } - return true; - } - @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerConverter.java index f8080541c6776..84252c5f88b5f 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerConverter.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/ByteUnitIntegerConverter.java @@ -19,8 +19,6 @@ package org.apache.pulsar.admin.cli.utils.converters; import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.emptyCheck; -import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.maxValueCheck; -import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.positiveCheck; import static org.apache.pulsar.admin.cli.utils.converters.ByteUnitUtil.validateSizeString; import com.beust.jcommander.converters.BaseConverter; diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToMillisConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToMillisConverter.java index b95c52a8b1299..43fe01b13d247 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToMillisConverter.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToMillisConverter.java @@ -19,8 +19,6 @@ package org.apache.pulsar.admin.cli.utils.converters; import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.emptyCheck; -import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.maxValueCheck; -import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.positiveCheck; import com.beust.jcommander.ParameterException; import com.beust.jcommander.converters.BaseConverter; import java.util.concurrent.TimeUnit; @@ -36,11 +34,8 @@ public TimeUnitToMillisConverter(String optionName) { public Long convert(String str) { emptyCheck(getOptionName(), str); try { - long inputValue = TimeUnit.SECONDS.toMillis( + return TimeUnit.SECONDS.toMillis( RelativeTimeUtil.parseRelativeTimeInSeconds(str.trim())); - positiveCheck(getOptionName(), inputValue); - maxValueCheck(getOptionName(), inputValue, Long.MAX_VALUE); - return inputValue; } catch (IllegalArgumentException exception) { throw new ParameterException("For input " + getOptionName() + ": " + exception.getMessage()); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToSecondsConverter.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToSecondsConverter.java index f29912d325095..cd0aba22b2b1e 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToSecondsConverter.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/converters/TimeUnitToSecondsConverter.java @@ -19,7 +19,6 @@ package org.apache.pulsar.admin.cli.utils.converters; import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.emptyCheck; -import static org.apache.pulsar.admin.cli.utils.ValueValidationUtils.maxValueCheck; import com.beust.jcommander.ParameterException; import com.beust.jcommander.converters.BaseConverter; import java.util.concurrent.TimeUnit; @@ -35,10 +34,8 @@ public TimeUnitToSecondsConverter(String optionName) { public Long convert(String str) { emptyCheck(getOptionName(), str); try { - long inputValue = TimeUnit.SECONDS.toSeconds( + return TimeUnit.SECONDS.toSeconds( RelativeTimeUtil.parseRelativeTimeInSeconds(str.trim())); - maxValueCheck(getOptionName(), inputValue, Long.MAX_VALUE); - return inputValue; } catch (IllegalArgumentException exception) { throw new ParameterException("For input " + getOptionName() + ": " + exception.getMessage()); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/NonNegativeValueValidator.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/NonNegativeValueValidator.java index 92542c4036fac..c6ad0a327ac28 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/NonNegativeValueValidator.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/NonNegativeValueValidator.java @@ -25,6 +25,6 @@ public class NonNegativeValueValidator implements IValueValidator { @Override public void validate(String name, Long value) throws ParameterException { - ValueValidationUtils.maxValueCheck(name, value, Long.MAX_VALUE); + ValueValidationUtils.minValueCheck(name, value, 0L); } } From 26d33482235e283c80b041186b66f856e9258fd9 Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Thu, 6 Jul 2023 07:53:57 +0900 Subject: [PATCH 09/14] Temp change --- pulsar-client-tools/pom.xml | 9 +++++++++ .../java/org/apache/pulsar/admin/cli/CmdNamespaces.java | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 025189b7bbfa5..f7a48b9b97736 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -38,6 +38,10 @@ jcommander compile + + com.google.guava + guava + ${project.groupId} pulsar-client-admin-api @@ -136,6 +140,11 @@ org.apache.commons commons-text + + com.google.guava + guava + 32.0.0-jre + diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 84e121151c5e2..bf302fcffec00 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -27,6 +27,7 @@ import com.google.common.collect.Sets; import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -2274,7 +2275,7 @@ private class SetOffloadPolicies extends CliCommand { ) private String offloadReadPriorityStr; - public final List driverNames = OffloadPoliciesImpl.DRIVER_NAMES; + public final List driverNames = Collections.singletonList(OffloadPoliciesImpl.DRIVER_NAMES); public boolean driverSupported(String driver) { return driverNames.stream().anyMatch(d -> d.equalsIgnoreCase(driver)); From d528acdad97034f3351deefd9c0df47451e283b5 Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Thu, 6 Jul 2023 07:54:43 +0900 Subject: [PATCH 10/14] Revert revert merge upstream --- buildtools/pom.xml | 2 +- conf/bookkeeper.conf | 2 +- conf/broker.conf | 4 +- conf/proxy.conf | 2 +- conf/standalone.conf | 6 +- conf/zookeeper.conf | 2 +- distribution/server/pom.xml | 6 + .../server/src/assemble/LICENSE.bin.txt | 60 ++++----- .../shell/src/assemble/LICENSE.bin.txt | 8 +- pom.xml | 8 +- .../pulsar/broker/ServiceConfiguration.java | 2 +- pulsar-broker/pom.xml | 14 +++ .../apache/pulsar/PulsarBrokerStarter.java | 2 +- .../pulsar/PulsarClusterMetadataSetup.java | 2 +- .../pulsar/PulsarClusterMetadataTeardown.java | 2 +- .../pulsar/PulsarInitialNamespaceSetup.java | 2 +- .../pulsar/PulsarStandaloneStarter.java | 2 +- ...arTransactionCoordinatorMetadataSetup.java | 4 +- .../apache/pulsar/PulsarVersionStarter.java | 2 +- .../broker/admin/impl/NamespacesBase.java | 22 ++-- .../admin/impl/PersistentTopicsBase.java | 17 +-- .../broker/admin/v2/PersistentTopics.java | 2 +- .../broker/intercept/BrokerInterceptors.java | 5 - .../extensions/ExtensibleLoadManagerImpl.java | 38 +++--- .../channel/ServiceUnitStateChannelImpl.java | 4 +- .../filter/AntiAffinityGroupPolicyFilter.java | 6 +- .../extensions/filter/BrokerFilter.java | 9 +- .../filter/BrokerIsolationPoliciesFilter.java | 19 ++- .../filter/BrokerLoadManagerClassFilter.java | 11 +- .../filter/BrokerMaxTopicCountFilter.java | 10 +- .../filter/BrokerVersionFilter.java | 17 +-- .../AntiAffinityGroupPolicyHelper.java | 11 +- .../policies/IsolationPoliciesHelper.java | 19 +-- .../extensions/scheduler/TransferShedder.java | 7 +- .../loadbalance/impl/LoadManagerShared.java | 114 ++++++++++++++++++ .../SimpleResourceAllocationPolicies.java | 15 +++ .../service/AbstractBaseDispatcher.java | 2 +- .../pulsar/broker/service/ServerCnx.java | 2 +- .../broker/tools/GenerateDocsCommand.java | 2 +- .../pulsar/compaction/CompactedTopicImpl.java | 29 ++--- .../pulsar/compaction/CompactorTool.java | 2 +- .../utils/CmdGenerateDocumentation.java | 52 -------- .../org/apache/pulsar/utils/CmdUtility.java | 86 ------------- .../utils/auth/tokens/TokensCliUtils.java | 2 +- .../pulsar/PulsarBrokerStarterTest.java | 2 +- .../broker/admin/PersistentTopicsTest.java | 13 -- .../intercept/BrokerInterceptorTest.java | 25 ---- ...tiAffinityNamespaceGroupExtensionTest.java | 4 +- .../ExtensibleLoadManagerImplTest.java | 36 +++--- .../BrokerIsolationPoliciesFilterTest.java | 33 ++--- .../BrokerLoadManagerClassFilterTest.java | 7 +- .../filter/BrokerMaxTopicCountFilterTest.java | 6 +- .../filter/BrokerVersionFilterTest.java | 29 +++-- .../scheduler/TransferShedderTest.java | 27 +++-- .../service/PublishRateLimiterTest.java | 9 ++ .../transaction/TransactionConsumeTest.java | 60 +++++++++ .../AuthenticatedProducerConsumerTest.java | 5 - .../pulsar/compaction/CompactorToolTest.java | 2 +- pulsar-client-tools/pom.xml | 1 + .../pulsar/admin/cli/CmdNamespaces.java | 14 +++ .../apache/pulsar/admin/cli/CmdTopics.java | 6 +- pulsar-client/pom.xml | 1 + .../pulsar/client/impl/ConnectionHandler.java | 13 +- .../impl/conf/CmdGenerateDocumentation.java | 51 -------- pulsar-common/pom.xml | 7 +- .../pulsar/common/util/SecurityUtility.java | 3 +- pulsar-docs-tools/pom.xml | 51 ++++++++ .../tools}/BaseGenerateDocumentation.java | 14 +-- .../pulsar/docs/tools}/CmdGenerateDocs.java | 17 ++- .../pulsar/docs/tools/package-info.java | 20 +++ .../docs/tools}/CmdGenerateDocsTest.java | 4 +- pulsar-functions/worker/pom.xml | 13 ++ .../worker/FunctionWorkerStarter.java | 2 +- pulsar-proxy/pom.xml | 12 ++ .../proxy/server/ProxyServiceStarter.java | 4 +- .../apache/pulsar/proxy/stats/ProxyStats.java | 2 - .../proxy/util/CmdGenerateDocumentation.java | 31 +++-- .../pulsar/proxy/util/package-info.java | 3 +- .../util/CmdGenerateDocumentationTest.java | 18 ++- .../org/apache/pulsar/proxy/util/CmdTest.java | 58 --------- pulsar-sql/presto-distribution/LICENSE | 32 ++--- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-websocket/pom.xml | 13 ++ .../service/WebSocketServiceStarter.java | 2 +- .../ExtensibleLoadManagerTest.java | 3 +- 85 files changed, 679 insertions(+), 611 deletions(-) delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdUtility.java delete mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/CmdGenerateDocumentation.java create mode 100644 pulsar-docs-tools/pom.xml rename {pulsar-common/src/main/java/org/apache/pulsar/common/util => pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools}/BaseGenerateDocumentation.java (97%) rename {pulsar-common/src/main/java/org/apache/pulsar/common/util => pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools}/CmdGenerateDocs.java (90%) create mode 100644 pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/package-info.java rename {pulsar-common/src/test/java/org/apache/pulsar/common/util => pulsar-docs-tools/src/test/java/org/apache/pulsar/docs/tools}/CmdGenerateDocsTest.java (98%) rename pulsar-broker/src/test/java/org/apache/pulsar/utils/CmdTest.java => pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentationTest.java (83%) delete mode 100644 pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdTest.java diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 8d10f19a80ff8..836e1c5cb5f0e 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -49,7 +49,7 @@ 3.1.2 4.1.94.Final 4.2.3 - 32.0.0-jre + 32.1.1-jre 1.10.12 2.0 3.12.4 diff --git a/conf/bookkeeper.conf b/conf/bookkeeper.conf index 9661108acc041..4241abe12eaa3 100644 --- a/conf/bookkeeper.conf +++ b/conf/bookkeeper.conf @@ -243,7 +243,7 @@ enableBusyWait=false # The interval is specified in seconds. auditorPeriodicBookieCheckInterval=86400 -# The number of entries that a replication will rereplicate in parallel. +# The number of entries that a replication will re-replicate in parallel. rereplicationEntryBatchSize=100 # Auto-replication diff --git a/conf/broker.conf b/conf/broker.conf index 2a0df8d3ba581..13789ce2eca96 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -900,7 +900,7 @@ saslJaasServerRoleTokenSignerSecretPath= ### --- HTTP Server config --- ### -# If >0, it will reject all HTTP requests with bodies larged than the configured limit +# If >0, it will reject all HTTP requests with bodies larger than the configured limit httpMaxRequestSize=-1 # The maximum size in bytes of the request header. Larger headers will allow for more and/or larger cookies plus larger @@ -944,7 +944,7 @@ bookkeeperMetadataServiceUri= # Authentication plugin to use when connecting to bookies bookkeeperClientAuthenticationPlugin= -# BookKeeper auth plugin implementatation specifics parameters name and values +# BookKeeper auth plugin implementation specifics parameters name and values bookkeeperClientAuthenticationParametersName= bookkeeperClientAuthenticationParameters= diff --git a/conf/proxy.conf b/conf/proxy.conf index cfc1e47b7c445..2fda32abc1ba6 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -152,7 +152,7 @@ authorizationEnabled=false # Authorization provider as a fully qualified class name authorizationProvider=org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider -# Whether client authorization credentials are forwared to the broker for re-authorization. +# Whether client authorization credentials are forwarded to the broker for re-authorization. # Authentication must be enabled via authenticationEnabled=true for this to take effect. forwardAuthorizationCredentials=false diff --git a/conf/standalone.conf b/conf/standalone.conf index 33010d12adb3a..46f4efbcf9171 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -269,7 +269,7 @@ dispatcherReadFailureBackoffMaxTimeInMs=60000 # The read failure backoff mandatory stop time in milliseconds. By default it is 0s. dispatcherReadFailureBackoffMandatoryStopTimeInMs=0 -# Precise dispathcer flow control according to history message number of each entry +# Precise dispatcher flow control according to history message number of each entry preciseDispatcherFlowControl=false # Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic @@ -581,7 +581,7 @@ tokenAudience= # Authentication plugin to use when connecting to bookies bookkeeperClientAuthenticationPlugin= -# BookKeeper auth plugin implementatation specifics parameters name and values +# BookKeeper auth plugin implementation specifics parameters name and values bookkeeperClientAuthenticationParametersName= bookkeeperClientAuthenticationParameters= @@ -1076,7 +1076,7 @@ journalSyncData=false # For each ledger dir, maximum disk space which can be used. # Default is 0.95f. i.e. 95% of disk can be used at most after which nothing will -# be written to that partition. If all ledger dir partions are full, then bookie +# be written to that partition. If all ledger dir partitions are full, then bookie # will turn to readonly mode if 'readOnlyModeEnabled=true' is set, else it will # shutdown. # Valid values should be in between 0 and 1 (exclusive). diff --git a/conf/zookeeper.conf b/conf/zookeeper.conf index 73240556d484c..db85e688e5c60 100644 --- a/conf/zookeeper.conf +++ b/conf/zookeeper.conf @@ -63,7 +63,7 @@ forceSync=yes # Default: false sslQuorum=false -# Enable TLS Certificate reloading for Quorum and Server connentions +# Enable TLS Certificate reloading for Quorum and Server connections # Follows Pulsar's general default to reload these files. sslQuorumReloadCertFiles=true client.certReload=true diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 5f946f888aec8..2702bace54e6a 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -40,6 +40,12 @@ ${project.version} + + ${project.groupId} + pulsar-docs-tools + ${project.version} + + ${project.groupId} pulsar-proxy diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index be97087edb75a..3bbb5183b0f7f 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -265,13 +265,15 @@ The Apache Software License, Version 2.0 - com.google.code.gson-gson-2.8.9.jar - io.gsonfire-gson-fire-1.8.5.jar * Guava - - com.google.guava-guava-32.0.0-jre.jar + - com.google.guava-guava-32.1.1-jre.jar - com.google.guava-failureaccess-1.0.1.jar - com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar * J2ObjC Annotations -- com.google.j2objc-j2objc-annotations-1.3.jar * Netty Reactive Streams -- com.typesafe.netty-netty-reactive-streams-2.0.6.jar * Swagger - io.swagger-swagger-annotations-1.6.2.jar + - io.swagger-swagger-core-1.6.2.jar + - io.swagger-swagger-models-1.6.2.jar * DataSketches - com.yahoo.datasketches-memory-0.8.3.jar - com.yahoo.datasketches-sketches-core-0.8.3.jar @@ -344,34 +346,34 @@ The Apache Software License, Version 2.0 - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-proto-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-server-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.1.jar - - org.apache.bookkeeper-circe-checksum-4.16.1.jar - - org.apache.bookkeeper-cpu-affinity-4.16.1.jar - - org.apache.bookkeeper-statelib-4.16.1.jar - - org.apache.bookkeeper-stream-storage-api-4.16.1.jar - - org.apache.bookkeeper-stream-storage-common-4.16.1.jar - - org.apache.bookkeeper-stream-storage-java-client-4.16.1.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.16.1.jar - - org.apache.bookkeeper-stream-storage-proto-4.16.1.jar - - org.apache.bookkeeper-stream-storage-server-4.16.1.jar - - org.apache.bookkeeper-stream-storage-service-api-4.16.1.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.16.1.jar - - org.apache.bookkeeper.http-http-server-4.16.1.jar - - org.apache.bookkeeper.http-vertx-http-server-4.16.1.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.1.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.1.jar - - org.apache.distributedlog-distributedlog-common-4.16.1.jar - - org.apache.distributedlog-distributedlog-core-4.16.1-tests.jar - - org.apache.distributedlog-distributedlog-core-4.16.1.jar - - org.apache.distributedlog-distributedlog-protocol-4.16.1.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.1.jar - - org.apache.bookkeeper-native-io-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-common-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-proto-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-server-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.2.jar + - org.apache.bookkeeper-circe-checksum-4.16.2.jar + - org.apache.bookkeeper-cpu-affinity-4.16.2.jar + - org.apache.bookkeeper-statelib-4.16.2.jar + - org.apache.bookkeeper-stream-storage-api-4.16.2.jar + - org.apache.bookkeeper-stream-storage-common-4.16.2.jar + - org.apache.bookkeeper-stream-storage-java-client-4.16.2.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.16.2.jar + - org.apache.bookkeeper-stream-storage-proto-4.16.2.jar + - org.apache.bookkeeper-stream-storage-server-4.16.2.jar + - org.apache.bookkeeper-stream-storage-service-api-4.16.2.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.16.2.jar + - org.apache.bookkeeper.http-http-server-4.16.2.jar + - org.apache.bookkeeper.http-vertx-http-server-4.16.2.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.2.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.2.jar + - org.apache.distributedlog-distributedlog-common-4.16.2.jar + - org.apache.distributedlog-distributedlog-core-4.16.2-tests.jar + - org.apache.distributedlog-distributedlog-core-4.16.2.jar + - org.apache.distributedlog-distributedlog-protocol-4.16.2.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.2.jar + - org.apache.bookkeeper-native-io-4.16.2.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 185b66f6cb416..dc8babb32f329 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -326,7 +326,7 @@ The Apache Software License, Version 2.0 * Gson - gson-2.8.9.jar * Guava - - guava-32.0.0-jre.jar + - guava-32.1.1-jre.jar - failureaccess-1.0.1.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar * J2ObjC Annotations -- j2objc-annotations-1.3.jar @@ -386,9 +386,9 @@ The Apache Software License, Version 2.0 - log4j-web-2.18.0.jar * BookKeeper - - bookkeeper-common-allocator-4.16.1.jar - - cpu-affinity-4.16.1.jar - - circe-checksum-4.16.1.jar + - bookkeeper-common-allocator-4.16.2.jar + - cpu-affinity-4.16.2.jar + - circe-checksum-4.16.2.jar * AirCompressor - aircompressor-0.20.jar * AsyncHttpClient diff --git a/pom.xml b/pom.xml index 254609a6f07cb..7e3538dc6a154 100644 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.16.1 + 4.16.2 3.8.1 1.5.0 1.10.0 @@ -202,7 +202,7 @@ flexible messaging model and an intuitive client API. 2.10.2 3.3.5 2.4.16 - 32.0.0-jre + 32.1.1-jre 1.0 0.16.1 6.2.8 @@ -869,14 +869,12 @@ flexible messaging model and an intuitive client API. io.swagger swagger-core ${swagger.version} - provided io.swagger swagger-annotations ${swagger.version} - provided @@ -2221,6 +2219,7 @@ flexible messaging model and an intuitive client API. pulsar-client-tools-customcommand-example pulsar-client-tools-test pulsar-client-all + pulsar-docs-tools pulsar-websocket pulsar-proxy pulsar-testclient @@ -2283,6 +2282,7 @@ flexible messaging model and an intuitive client API. pulsar-client-tools pulsar-client-tools-customcommand-example pulsar-client-tools-test + pulsar-docs-tools pulsar-websocket pulsar-proxy pulsar-testclient diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index b41a562fbd788..1a0ee2cfb756e 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1356,7 +1356,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext( category = CATEGORY_SERVER, - doc = "Enable or disable the broker interceptor" + doc = "Enable or disable the broker interceptor, which is only used for testing for now" ) private boolean disableBrokerInterceptors = true; diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index e084d70c79ad6..4c39d7c4b79d0 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -283,6 +283,18 @@ guava + + ${project.groupId} + pulsar-docs-tools + ${project.version} + + + io.swagger + * + + + + com.beust jcommander @@ -291,6 +303,7 @@ io.swagger swagger-annotations + provided @@ -316,6 +329,7 @@ io.swagger swagger-core + provided diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java index 92fc8c5c9acfa..1b24c806e62cb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java @@ -55,9 +55,9 @@ import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.DirectMemoryUtils; import org.apache.pulsar.common.util.ShutdownUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerService; import org.apache.pulsar.functions.worker.service.WorkerServiceLoader; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java index 9b757c55ccd1d..96ebadb1ff4aa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java @@ -44,8 +44,8 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.ShutdownUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataTeardown.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataTeardown.java index cbbf909cdf7fa..01a9eedcca357 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataTeardown.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataTeardown.java @@ -33,8 +33,8 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.pulsar.broker.service.schema.SchemaStorageFormat.SchemaLocator; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.MetadataStoreFactory; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarInitialNamespaceSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarInitialNamespaceSetup.java index c4020546d1c6c..22b38e59676ec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarInitialNamespaceSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarInitialNamespaceSetup.java @@ -23,7 +23,7 @@ import java.util.List; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.metadata.api.MetadataStore; /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java index 33dd58deac095..25320964fd62b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java @@ -29,7 +29,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; @Slf4j public class PulsarStandaloneStarter extends PulsarStandalone { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetup.java index 78ce55f1b2ce2..6aedfe13a5b50 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarTransactionCoordinatorMetadataSetup.java @@ -23,11 +23,11 @@ import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; /** - * Setup the transaction coordinator metadata for a cluster, the setup will create pulsar/system namespace and create + * Set up the transaction coordinator metadata for a cluster, the setup will create pulsar/system namespace and create * partitioned topic for transaction coordinator assign. */ public class PulsarTransactionCoordinatorMetadataSetup { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java index 7eee9d083fc29..85a6c4156dbe4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java @@ -20,7 +20,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; /** * Pulsar version entry point. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index ff87f52b64a85..789a8e6dbdcb7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -233,14 +233,14 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime })) .thenCompose(topics -> { List allTopics = topics.get(0); - ArrayList allUserCreatedTopics = new ArrayList<>(); + Set allUserCreatedTopics = new HashSet<>(); List allPartitionedTopics = topics.get(1); - ArrayList allUserCreatedPartitionTopics = new ArrayList<>(); + Set allUserCreatedPartitionTopics = new HashSet<>(); boolean hasNonSystemTopic = false; - List allSystemTopics = new ArrayList<>(); - List allPartitionedSystemTopics = new ArrayList<>(); - List topicPolicy = new ArrayList<>(); - List partitionedTopicPolicy = new ArrayList<>(); + Set allSystemTopics = new HashSet<>(); + Set allPartitionedSystemTopics = new HashSet<>(); + Set topicPolicy = new HashSet<>(); + Set partitionedTopicPolicy = new HashSet<>(); for (String topic : allTopics) { if (!pulsar().getBrokerService().isSystemTopic(TopicName.get(topic))) { hasNonSystemTopic = true; @@ -279,6 +279,12 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime return old; }); } + allUserCreatedTopics.removeIf(t -> + allPartitionedTopics.contains(TopicName.get(t).getPartitionedTopicName())); + allSystemTopics.removeIf(t -> + allPartitionedTopics.contains(TopicName.get(t).getPartitionedTopicName())); + topicPolicy.removeIf(t -> + allPartitionedTopics.contains(TopicName.get(t).getPartitionedTopicName())); return markDeleteFuture.thenCompose(__ -> internalDeleteTopicsAsync(allUserCreatedTopics)) .thenCompose(ignore -> @@ -348,7 +354,7 @@ private boolean isDeletedAlongWithUserCreatedTopic(String topic) { return topic.endsWith(SystemTopicNames.PENDING_ACK_STORE_SUFFIX); } - private CompletableFuture internalDeletePartitionedTopicsAsync(List topicNames) { + private CompletableFuture internalDeletePartitionedTopicsAsync(Set topicNames) { if (CollectionUtils.isEmpty(topicNames)) { return CompletableFuture.completedFuture(null); } @@ -362,7 +368,7 @@ private CompletableFuture internalDeletePartitionedTopicsAsync(List internalDeleteTopicsAsync(List topicNames) { + private CompletableFuture internalDeleteTopicsAsync(Set topicNames) { if (CollectionUtils.isEmpty(topicNames)) { return CompletableFuture.completedFuture(null); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index d8f8e7304d291..9878cc1592e9c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -299,20 +299,9 @@ protected void internalGrantPermissionsOnTopic(final AsyncResponse asyncResponse // This operation should be reading from zookeeper and it should be allowed without having admin privileges validateAdminAccessForTenantAsync(namespaceName.getTenant()) .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync().thenCompose(unused1 -> - getPartitionedTopicMetadataAsync(topicName, true, false) - .thenCompose(metadata -> { - int numPartitions = metadata.partitions; - CompletableFuture future = CompletableFuture.completedFuture(null); - if (numPartitions > 0) { - for (int i = 0; i < numPartitions; i++) { - TopicName topicNamePartition = topicName.getPartition(i); - future = future.thenCompose(unused -> grantPermissionsAsync(topicNamePartition, role, - actions)); - } - } - return future.thenCompose(unused -> grantPermissionsAsync(topicName, role, actions)) - .thenAccept(unused -> asyncResponse.resume(Response.noContent().build())); - }))).exceptionally(ex -> { + grantPermissionsAsync(topicName, role, actions) + .thenAccept(unused -> asyncResponse.resume(Response.noContent().build())))) + .exceptionally(ex -> { Throwable realCause = FutureUtil.unwrapCompletionException(ex); log.error("[{}] Failed to get permissions for topic {}", clientAppId(), topicName, realCause); resumeAsyncResponseExceptionally(asyncResponse, realCause); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 0fac3a8e005fc..1927d4b244aa4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -1597,7 +1597,7 @@ public void createSubscription( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String topic, - @ApiParam(value = "Subscription to create position on", required = true) + @ApiParam(value = "Name of subscription to be created", required = true) @PathParam("subscriptionName") String encodedSubName, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java index 4ffd8732db94e..cef3f0eb609a1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java @@ -59,11 +59,6 @@ public BrokerInterceptors(Map intercep * @return the collection of broker event interceptor */ public static BrokerInterceptor load(ServiceConfiguration conf) throws IOException { - if (conf.isDisableBrokerInterceptors()) { - log.info("Skip loading the broker interceptors when disableBrokerInterceptors is true"); - return null; - } - BrokerInterceptorDefinitions definitions = BrokerInterceptorUtils.searchForInterceptors(conf.getBrokerInterceptorsDirectory(), conf.getNarExtractionDirectory()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index d0c55a6519159..f3c5e5dc2ee36 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -34,6 +34,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -46,7 +47,6 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; @@ -477,10 +477,10 @@ public CompletableFuture> selectAsync(ServiceUnitId bundle, Set excludeBrokerSet) { BrokerRegistry brokerRegistry = getBrokerRegistry(); return brokerRegistry.getAvailableBrokerLookupDataAsync() - .thenCompose(availableBrokers -> { + .thenComposeAsync(availableBrokers -> { LoadManagerContext context = this.getContext(); - Map availableBrokerCandidates = new HashMap<>(availableBrokers); + Map availableBrokerCandidates = new ConcurrentHashMap<>(availableBrokers); if (!excludeBrokerSet.isEmpty()) { for (String exclude : excludeBrokerSet) { availableBrokerCandidates.remove(exclude); @@ -489,24 +489,28 @@ public CompletableFuture> selectAsync(ServiceUnitId bundle, // Filter out brokers that do not meet the rules. List filterPipeline = getBrokerFilterPipeline(); + ArrayList>> futures = + new ArrayList<>(filterPipeline.size()); for (final BrokerFilter filter : filterPipeline) { - try { - filter.filter(availableBrokerCandidates, bundle, context); - // Preserve the filter successes result. - availableBrokers.keySet().retainAll(availableBrokerCandidates.keySet()); - } catch (BrokerFilterException e) { + CompletableFuture> future = + filter.filter(availableBrokerCandidates, bundle, context); + futures.add(future); + } + CompletableFuture> result = new CompletableFuture<>(); + FutureUtil.waitForAll(futures).whenComplete((__, ex) -> { + if (ex != null) { // TODO: We may need to revisit this error case. - log.error("Failed to filter out brokers.", e); - availableBrokerCandidates = new HashMap<>(availableBrokers); + log.error("Failed to filter out brokers when select bundle: {}", bundle, ex); } - } - if (availableBrokerCandidates.isEmpty()) { - return CompletableFuture.completedFuture(Optional.empty()); - } - Set candidateBrokers = availableBrokerCandidates.keySet(); + if (availableBrokerCandidates.isEmpty()) { + result.complete(Optional.empty()); + return; + } + Set candidateBrokers = availableBrokerCandidates.keySet(); - return CompletableFuture.completedFuture( - getBrokerSelectionStrategy().select(candidateBrokers, bundle, context)); + result.complete(getBrokerSelectionStrategy().select(candidateBrokers, bundle, context)); + }); + return result; }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 70fb8f9ba6f89..717ff484fe772 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -619,7 +619,7 @@ public CompletableFuture publishSplitEventAsync(Split split) { private void handle(String serviceUnit, ServiceUnitStateData data) { long totalHandledRequests = getHandlerTotalCounter(data).incrementAndGet(); - if (log.isDebugEnabled()) { + if (debug()) { log.info("{} received a handle request for serviceUnit:{}, data:{}. totalHandledRequests:{}", lookupServiceAddress, serviceUnit, data, totalHandledRequests); } @@ -708,7 +708,7 @@ private void handleSkippedEvent(String serviceUnit) { var getOwnerRequest = getOwnerRequests.get(serviceUnit); if (getOwnerRequest != null) { var data = tableview.get(serviceUnit); - if (data.state() == Owned) { + if (data != null && data.state() == Owned) { getOwnerRequest.complete(data.dstBroker()); getOwnerRequests.remove(serviceUnit); stateChangeListeners.notify(serviceUnit, data, null); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java index 462f8f0e3597a..df08ba96b9d7a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper; @@ -38,10 +39,9 @@ public AntiAffinityGroupPolicyFilter(AntiAffinityGroupPolicyHelper helper) { } @Override - public Map filter( + public CompletableFuture> filter( Map brokers, ServiceUnitId serviceUnitId, LoadManagerContext context) { - helper.filter(brokers, serviceUnitId.toString()); - return brokers; + return helper.filterAsync(brokers, serviceUnitId.toString()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java index d9cbfdc391ed4..046cae48914f8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -42,9 +42,8 @@ public interface BrokerFilter { * @param context The load manager context. * @return Filtered broker list. */ - Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) - throws BrokerFilterException; + CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java index eeb0d9d3a3309..0aa1dda437af7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java @@ -19,9 +19,8 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; -import java.util.Set; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; @@ -45,13 +44,13 @@ public String name() { } @Override - public Map filter(Map availableBrokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) - throws BrokerFilterException { - Set brokerCandidateCache = - isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, serviceUnit); - availableBrokers.keySet().retainAll(brokerCandidateCache); - return availableBrokers; + public CompletableFuture> filter(Map availableBrokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + return isolationPoliciesHelper.applyIsolationPoliciesAsync(availableBrokers, serviceUnit) + .thenApply(brokerCandidateCache -> { + availableBrokers.keySet().retainAll(brokerCandidateCache); + return availableBrokers; + }); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java index 07109b277ae98..6504da047a001 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java @@ -20,7 +20,7 @@ import java.util.Map; import java.util.Objects; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -34,13 +34,12 @@ public String name() { } @Override - public Map filter( + public CompletableFuture> filter( Map brokers, ServiceUnitId serviceUnit, - LoadManagerContext context) - throws BrokerFilterException { + LoadManagerContext context) { if (brokers.isEmpty()) { - return brokers; + return CompletableFuture.completedFuture(brokers); } brokers.entrySet().removeIf(entry -> { BrokerLookupData v = entry.getValue(); @@ -48,6 +47,6 @@ public Map filter( return !Objects.equals(v.getLoadManagerClassName(), context.brokerConfiguration().getLoadManagerClassName()); }); - return brokers; + return CompletableFuture.completedFuture(brokers); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java index 0bceae36bb8c2..044dcc83a7835 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java @@ -20,7 +20,7 @@ import java.util.Map; import java.util.Optional; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; @@ -36,9 +36,9 @@ public String name() { } @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { int loadBalancerBrokerMaxTopics = context.brokerConfiguration().getLoadBalancerBrokerMaxTopics(); brokers.keySet().removeIf(broker -> { Optional brokerLoadDataOpt = context.brokerLoadDataStore().get(broker); @@ -46,6 +46,6 @@ public Map filter(Map broker // TODO: The broker load data might be delayed, so the max topic check might not accurate. return topics >= loadBalancerBrokerMaxTopics; }); - return brokers; + return CompletableFuture.completedFuture(brokers); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java index 7420fcc211309..6e8ce5e58f2bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java @@ -21,13 +21,14 @@ import com.github.zafarkhaja.semver.Version; import java.util.Iterator; import java.util.Map; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterBadVersionException; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.naming.ServiceUnitId; +import org.apache.pulsar.common.util.FutureUtil; /** * Filter by broker version. @@ -46,13 +47,12 @@ public class BrokerVersionFilter implements BrokerFilter { * */ @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) - throws BrokerFilterException { + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { ServiceConfiguration conf = context.brokerConfiguration(); if (!conf.isPreferLaterVersions() || brokers.isEmpty()) { - return brokers; + return CompletableFuture.completedFuture(brokers); } Version latestVersion; @@ -63,7 +63,8 @@ public Map filter(Map broker } } catch (Exception ex) { log.warn("Disabling PreferLaterVersions feature; reason: " + ex.getMessage()); - throw new BrokerFilterBadVersionException("Cannot determine newest broker version: " + ex.getMessage()); + return FutureUtil.failedFuture( + new BrokerFilterBadVersionException("Cannot determine newest broker version: " + ex.getMessage())); } int numBrokersLatestVersion = 0; @@ -88,7 +89,7 @@ public Map filter(Map broker if (numBrokersOlderVersion == 0) { log.info("All {} brokers are running the latest version [{}]", numBrokersLatestVersion, latestVersion); } - return brokers; + return CompletableFuture.completedFuture(brokers); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java index 44360bc77d83f..3781c9c95f6a7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java @@ -20,6 +20,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; @@ -41,11 +42,11 @@ public AntiAffinityGroupPolicyHelper(PulsarService pulsar, this.channel = channel; } - public void filter( - Map brokers, String bundle) { - LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, bundle, - brokers.keySet(), - channel.getOwnershipEntrySet(), brokerToFailureDomainMap); + public CompletableFuture> filterAsync(Map brokers, + String bundle) { + return LoadManagerShared.filterAntiAffinityGroupOwnedBrokersAsync(pulsar, bundle, + brokers.keySet(), channel.getOwnershipEntrySet(), brokerToFailureDomainMap) + .thenApply(__ -> brokers); } public boolean hasAntiAffinityGroupPolicy(String bundle) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java index 67dc702cc0c9f..468552db541ec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java @@ -18,10 +18,9 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.policies; -import io.netty.util.concurrent.FastThreadLocal; -import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; @@ -38,18 +37,9 @@ public IsolationPoliciesHelper(SimpleResourceAllocationPolicies policies) { this.policies = policies; } - private static final FastThreadLocal> localBrokerCandidateCache = new FastThreadLocal<>() { - @Override - protected Set initialValue() { - return new HashSet<>(); - } - }; - - public Set applyIsolationPolicies(Map availableBrokers, - ServiceUnitId serviceUnit) { - Set brokerCandidateCache = localBrokerCandidateCache.get(); - brokerCandidateCache.clear(); - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + public CompletableFuture> applyIsolationPoliciesAsync(Map availableBrokers, + ServiceUnitId serviceUnit) { + return LoadManagerShared.applyNamespacePoliciesAsync(serviceUnit, policies, availableBrokers.keySet(), new LoadManagerShared.BrokerTopicLoadingPredicate() { @Override public boolean isEnablePersistentTopics(String brokerUrl) { @@ -63,7 +53,6 @@ public boolean isEnableNonPersistentTopics(String brokerUrl) { return lookupData != null && lookupData.nonPersistentTopicsEnabled(); } }); - return brokerCandidateCache; } public boolean hasIsolationPolicy(NamespaceName namespaceName) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index 07d521a28afa7..7bb16bac124e9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -47,7 +47,6 @@ import lombok.experimental.Accessors; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; @@ -720,8 +719,10 @@ private boolean isTransferable(LoadManagerContext context, Map candidates = new HashMap<>(availableBrokers); for (var filter : brokerFilterPipeline) { try { - filter.filter(candidates, namespaceBundle, context); - } catch (BrokerFilterException e) { + filter.filter(candidates, namespaceBundle, context) + .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), + TimeUnit.SECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { log.error("Failed to filter brokers with filter: {}", filter.getClass().getName(), e); return false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java index 33f346adbe0c5..5f2e4b1f25d8b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java @@ -181,6 +181,104 @@ public static void applyNamespacePolicies(final ServiceUnitId serviceUnit, } } + public static CompletableFuture> applyNamespacePoliciesAsync( + final ServiceUnitId serviceUnit, final SimpleResourceAllocationPolicies policies, + final Set availableBrokers, final BrokerTopicLoadingPredicate brokerTopicLoadingPredicate) { + NamespaceName namespace = serviceUnit.getNamespaceObject(); + return policies.areIsolationPoliciesPresentAsync(namespace).thenApply(isIsolationPoliciesPresent -> { + final Set brokerCandidateCache = new HashSet<>(); + Set primariesCache = localPrimariesCache.get(); + primariesCache.clear(); + + Set secondaryCache = localSecondaryCache.get(); + secondaryCache.clear(); + boolean isNonPersistentTopic = (serviceUnit instanceof NamespaceBundle) + ? ((NamespaceBundle) serviceUnit).hasNonPersistentTopic() : false; + if (isIsolationPoliciesPresent) { + if (LOG.isDebugEnabled()) { + LOG.debug("Isolation Policies Present for namespace - [{}]", namespace.toString()); + } + } + for (final String broker : availableBrokers) { + final String brokerUrlString = String.format("http://%s", broker); + URL brokerUrl; + try { + brokerUrl = new URL(brokerUrlString); + } catch (MalformedURLException e) { + LOG.error("Unable to parse brokerUrl from ResourceUnitId", e); + continue; + } + // todo: in future check if the resource unit has resources to take the namespace + if (isIsolationPoliciesPresent) { + // note: serviceUnitID is namespace name and ResourceID is brokerName + if (policies.isPrimaryBroker(namespace, brokerUrl.getHost())) { + primariesCache.add(broker); + if (LOG.isDebugEnabled()) { + LOG.debug("Added Primary Broker - [{}] as possible Candidates for" + + " namespace - [{}] with policies", brokerUrl.getHost(), namespace.toString()); + } + } else if (policies.isSecondaryBroker(namespace, brokerUrl.getHost())) { + secondaryCache.add(broker); + if (LOG.isDebugEnabled()) { + LOG.debug( + "Added Shared Broker - [{}] as possible " + + "Candidates for namespace - [{}] with policies", + brokerUrl.getHost(), namespace.toString()); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping Broker - [{}] not primary broker and not shared" + + " for namespace - [{}] ", brokerUrl.getHost(), namespace.toString()); + } + + } + } else { + // non-persistent topic can be assigned to only those brokers that enabled for non-persistent topic + if (isNonPersistentTopic + && !brokerTopicLoadingPredicate.isEnableNonPersistentTopics(brokerUrlString)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Filter broker- [{}] because it doesn't support non-persistent namespace - [{}]", + brokerUrl.getHost(), namespace.toString()); + } + } else if (!isNonPersistentTopic + && !brokerTopicLoadingPredicate.isEnablePersistentTopics(brokerUrlString)) { + // persistent topic can be assigned to only brokers that enabled for persistent-topic + if (LOG.isDebugEnabled()) { + LOG.debug("Filter broker- [{}] because broker only supports non-persistent " + + "namespace - [{}]", brokerUrl.getHost(), namespace.toString()); + } + } else if (policies.isSharedBroker(brokerUrl.getHost())) { + secondaryCache.add(broker); + if (LOG.isDebugEnabled()) { + LOG.debug("Added Shared Broker - [{}] as possible Candidates for namespace - [{}]", + brokerUrl.getHost(), namespace.toString()); + } + } + } + } + if (isIsolationPoliciesPresent) { + brokerCandidateCache.addAll(primariesCache); + if (policies.shouldFailoverToSecondaries(namespace, primariesCache.size())) { + if (LOG.isDebugEnabled()) { + LOG.debug( + "Not enough of primaries [{}] available for namespace - [{}], " + + "adding shared [{}] as possible candidate owners", + primariesCache.size(), namespace.toString(), secondaryCache.size()); + } + brokerCandidateCache.addAll(secondaryCache); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug( + "Policies not present for namespace - [{}] so only " + + "considering shared [{}] brokers for possible owner", + namespace.toString(), secondaryCache.size()); + } + brokerCandidateCache.addAll(secondaryCache); + } + return brokerCandidateCache; + }); + } /** * Using the given bundles, populate the namespace to bundle range map. * @@ -408,6 +506,22 @@ public static void filterAntiAffinityGroupOwnedBrokers( } } + public static CompletableFuture filterAntiAffinityGroupOwnedBrokersAsync( + final PulsarService pulsar, final String assignedBundleName, + final Set candidates, + Set> bundleOwnershipData, + Map brokerToDomainMap + ) { + if (candidates.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + final String namespaceName = getNamespaceNameFromBundleName(assignedBundleName); + return getAntiAffinityNamespaceOwnedBrokers(pulsar, namespaceName, bundleOwnershipData) + .thenAccept(brokerToAntiAffinityNamespaceCount -> + filterAntiAffinityGroupOwnedBrokers(pulsar, candidates, + brokerToDomainMap, brokerToAntiAffinityNamespaceCount)); + } + /** * It computes least number of namespace owned by any of the domain and then it filters out all the domains that own * namespaces more than this count. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java index 4a1577a4e28b6..25fbc152b1163 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.LoadReport; import org.apache.pulsar.broker.loadbalance.ResourceUnit; @@ -53,6 +54,20 @@ private Optional getIsolationPolicies(String cluster } } + private CompletableFuture> getIsolationPoliciesAsync(String clusterName) { + return this.pulsar.getPulsarResources().getNamespaceResources() + .getIsolationPolicies().getIsolationDataPoliciesAsync(clusterName); + } + + public CompletableFuture areIsolationPoliciesPresentAsync(NamespaceName namespace) { + return getIsolationPoliciesAsync(pulsar.getConfiguration().getClusterName()) + .thenApply(policies -> { + return policies.filter(isolationPolicies -> + isolationPolicies.getPolicyByNamespace(namespace) != null) + .isPresent(); + }); + } + public boolean areIsolationPoliciesPresent(NamespaceName namespace) { try { Optional policies = diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index 437a6527e8538..50f05f80e3f8b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -171,7 +171,7 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i entry.release(); continue; } - if (!isReplayRead && msgMetadata != null && msgMetadata.hasTxnidMostBits() + if (msgMetadata != null && msgMetadata.hasTxnidMostBits() && msgMetadata.hasTxnidLeastBits()) { if (Markers.isTxnMarker(msgMetadata)) { // because consumer can receive message is smaller than maxReadPosition, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 98c0e5b497998..0be78ca4ecfe1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1191,7 +1191,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { "Consumer that failed is already present on the connection"); } else { Consumer consumer = existingConsumerFuture.getNow(null); - log.info("[{}] Consumer with the same id is already created:" + log.warn("[{}] Consumer with the same id is already created:" + " consumerId={}, consumer={}", remoteAddress, consumerId, consumer); commandSender.sendSuccessResponse(requestId); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/GenerateDocsCommand.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/GenerateDocsCommand.java index 1819a56baa12a..b020b4bfd8bd1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/GenerateDocsCommand.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/tools/GenerateDocsCommand.java @@ -26,7 +26,7 @@ import org.apache.bookkeeper.tools.framework.CliCommand; import org.apache.bookkeeper.tools.framework.CliFlags; import org.apache.bookkeeper.tools.framework.CliSpec; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; /** * The command to generate documents of broker-tool. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java index e2d3de9c19a29..1f117338ea074 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java @@ -60,7 +60,7 @@ public class CompactedTopicImpl implements CompactedTopic { private final BookKeeper bk; - private PositionImpl compactionHorizon = null; + private volatile PositionImpl compactionHorizon = null; private volatile CompletableFuture compactedTopicContext = null; public CompactedTopicImpl(BookKeeper bk) { @@ -91,7 +91,6 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, int numberOfEntriesToRead, boolean isFirstRead, ReadEntriesCallback callback, Consumer consumer) { - synchronized (this) { PositionImpl cursorPosition; if (isFirstRead && MessageId.earliest.equals(consumer.getStartMessageId())){ cursorPosition = PositionImpl.EARLIEST; @@ -101,8 +100,11 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, // TODO: redeliver epoch link https://github.com/apache/pulsar/issues/13690 ReadEntriesCtx readEntriesCtx = ReadEntriesCtx.create(consumer, DEFAULT_CONSUMER_EPOCH); - if (compactionHorizon == null - || compactionHorizon.compareTo(cursorPosition) < 0) { + + final PositionImpl currentCompactionHorizon = compactionHorizon; + + if (currentCompactionHorizon == null + || currentCompactionHorizon.compareTo(cursorPosition) < 0) { cursor.asyncReadEntriesOrWait(numberOfEntriesToRead, callback, readEntriesCtx, PositionImpl.LATEST); } else { compactedTopicContext.thenCompose( @@ -110,23 +112,13 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, .thenCompose((startPoint) -> { // do not need to read the compaction ledger if it is empty. // the cursor just needs to be set to the compaction horizon - if (startPoint == COMPACT_LEDGER_EMPTY) { - cursor.seek(compactionHorizon.getNext()); + if (startPoint == COMPACT_LEDGER_EMPTY || startPoint == NEWER_THAN_COMPACTED) { + cursor.seek(currentCompactionHorizon.getNext()); callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx); return CompletableFuture.completedFuture(null); - } - if (startPoint == NEWER_THAN_COMPACTED && compactionHorizon.compareTo(cursorPosition) < 0) { - cursor.asyncReadEntriesOrWait(numberOfEntriesToRead, callback, readEntriesCtx, - PositionImpl.LATEST); - return CompletableFuture.completedFuture(null); } else { long endPoint = Math.min(context.ledger.getLastAddConfirmed(), startPoint + numberOfEntriesToRead); - if (startPoint == NEWER_THAN_COMPACTED) { - cursor.seek(compactionHorizon.getNext()); - callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx); - return CompletableFuture.completedFuture(null); - } return readEntries(context.ledger, startPoint, endPoint) .thenAccept((entries) -> { Entry lastEntry = entries.get(entries.size() - 1); @@ -142,7 +134,7 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, })) .exceptionally((exception) -> { if (exception.getCause() instanceof NoSuchElementException) { - cursor.seek(compactionHorizon.getNext()); + cursor.seek(currentCompactionHorizon.getNext()); callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx); } else { callback.readEntriesFailed(new ManagedLedgerException(exception), readEntriesCtx); @@ -150,7 +142,6 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, return null; }); } - } } static CompletableFuture findStartPoint(PositionImpl p, @@ -318,7 +309,7 @@ private static int comparePositionAndMessageId(PositionImpl p, MessageIdData m) .compare(p.getEntryId(), m.getEntryId()).result(); } - public synchronized Optional getCompactionHorizon() { + public Optional getCompactionHorizon() { return Optional.ofNullable(this.compactionHorizon); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java index 2539c306500a2..83ff790228108 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java @@ -41,8 +41,8 @@ import org.apache.pulsar.client.api.SizeUnit; import org.apache.pulsar.client.internal.PropertiesUtils; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.netty.EventLoopUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java deleted file mode 100644 index 4a8639c060603..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdGenerateDocumentation.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.utils; - -import com.beust.jcommander.Parameters; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.apache.pulsar.common.util.BaseGenerateDocumentation; -import org.apache.pulsar.websocket.service.WebSocketProxyConfiguration; - -@Data -@Parameters(commandDescription = "Generate documentation automatically.") -@Slf4j -public class CmdGenerateDocumentation extends BaseGenerateDocumentation { - - @Override - public String generateDocumentByClassName(String className) throws Exception { - StringBuilder sb = new StringBuilder(); - if (ServiceConfiguration.class.getName().equals(className)) { - return generateDocByFieldContext(className, "Broker", sb); - } else if (ClientConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Client", sb); - } else if (WebSocketProxyConfiguration.class.getName().equals(className)) { - return generateDocByFieldContext(className, "WebSocket", sb); - } - - return "Class [" + className + "] not found"; - } - - public static void main(String[] args) throws Exception { - CmdGenerateDocumentation generateDocumentation = new CmdGenerateDocumentation(); - generateDocumentation.run(args); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdUtility.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdUtility.java deleted file mode 100644 index f0f88562953e5..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/utils/CmdUtility.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.utils; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Writer; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class CmdUtility { - private static final Logger LOG = LoggerFactory.getLogger(CmdUtility.class); - private static final Charset UTF_8 = StandardCharsets.UTF_8; - - /** - * Executes the specified string command in a separate process. STDOUT and STDERR output will be buffered to the - * given writer ( {@link Writer}) argument. - * - * @param writer - * stdout and stderr output - * @param command - * a specified system command - * @return exitValue 0: success, Non-zero: failure - * @throws IOException - */ - public static int exec(Writer writer, String... command) throws IOException { - if (LOG.isDebugEnabled()) { - StringBuilder sb = new StringBuilder(); - for (String str : command) { - sb.append(str).append(' '); - } - LOG.debug("command={}", sb); - } - - ProcessBuilder pb = new ProcessBuilder(command); - pb.redirectErrorStream(true); - Process proc = null; - BufferedReader reader = null; - try { - proc = pb.start(); - reader = new BufferedReader(new InputStreamReader(proc.getInputStream(), UTF_8)); - String line = null; - while ((line = reader.readLine()) != null) { - if (writer != null) { - writer.write(line); - writer.write('\n'); - } - } - LOG.debug("sending the command to the host"); - int exitValue = proc.waitFor(); - if (LOG.isDebugEnabled()) { - LOG.debug("command exit value={}", exitValue); - } - return exitValue; - } catch (InterruptedException e) { - throw new RuntimeException(e); - } finally { - if (reader != null) { - reader.close(); - } - if (proc != null) { - proc.destroy(); - } - } - } - -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java index 2477e1ad2bb1b..fa91ef79d4067 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java @@ -45,8 +45,8 @@ import javax.crypto.SecretKey; import lombok.Cleanup; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.RelativeTimeUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; public class TokensCliUtils { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java index c6945169512b5..1bc3bd26f1294 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java @@ -38,7 +38,7 @@ import java.lang.reflect.Method; import java.util.Arrays; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.testng.annotations.Test; @Test(groups = "broker") diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index a5de86b08df2b..83ed63bf0d95d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -930,19 +930,6 @@ public void testGrantPartitionedTopic() { verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Map> permissions = (Map>) responseCaptor.getValue(); Assert.assertEquals(permissions.get(role), expectActions); - TopicName topicName = TopicName.get(TopicDomain.persistent.value(), testTenant, testNamespace, - partitionedTopicName); - for (int i = 0; i < numPartitions; i++) { - TopicName partition = topicName.getPartition(i); - response = mock(AsyncResponse.class); - responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.getPermissionsOnTopic(response, testTenant, testNamespace, - partition.getEncodedLocalName()); - verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); - Map> partitionPermissions = - (Map>) responseCaptor.getValue(); - Assert.assertEquals(partitionPermissions.get(role), expectActions); - } } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java index c612104f8bf1b..d1cf91635f992 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java @@ -20,12 +20,9 @@ import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; - import java.io.IOException; import java.util.HashMap; import java.util.HashSet; @@ -39,7 +36,6 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; -import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -311,25 +307,4 @@ public void requestInterceptorFailedTest() { } } - @Test - public void testLoadWhenDisableBrokerInterceptorsIsTrue() throws IOException { - ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class); - serviceConfiguration.setDisableBrokerInterceptors(true); - BrokerInterceptor brokerInterceptor = BrokerInterceptors.load(serviceConfiguration); - assertNull(brokerInterceptor); - - verify(serviceConfiguration, times(1)).isDisableBrokerInterceptors(); - verify(serviceConfiguration, times(0)).getBrokerInterceptorsDirectory(); - } - - @Test - public void testLoadWhenDisableBrokerInterceptorsIsFalse() throws IOException { - ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class); - serviceConfiguration.setDisableBrokerInterceptors(false); - BrokerInterceptor brokerInterceptor = BrokerInterceptors.load(serviceConfiguration); - assertNull(brokerInterceptor); - - verify(serviceConfiguration, times(1)).isDisableBrokerInterceptors(); - verify(serviceConfiguration, times(1)).getBrokerInterceptorsDirectory(); - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java index 014e6fa3c7a04..b766c7de5a5f9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java @@ -131,7 +131,7 @@ public void testAntiAffinityGroupPolicyFilter() var expected = new HashMap<>(brokers); var actual = antiAffinityGroupPolicyFilter.filter( - brokers, namespaceBundle, context); + brokers, namespaceBundle, context).get(); assertEquals(actual, expected); doReturn(antiAffinityEnabledNameSpace + "/" + bundle).when(namespaceBundle).toString(); @@ -142,7 +142,7 @@ public void testAntiAffinityGroupPolicyFilter() .get(5, TimeUnit.SECONDS).get(); expected.remove(srcBroker); actual = antiAffinityGroupPolicyFilter.filter( - brokers, namespaceBundle, context); + brokers, namespaceBundle, context).get(); assertEquals(actual, expected); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index d348347515f03..4d15a5a5e2725 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -106,6 +106,7 @@ import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.stats.Metrics; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; import org.awaitility.Awaitility; @@ -267,11 +268,11 @@ public String name() { } @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { brokers.remove(pulsar1.getLookupServiceAddress()); - return brokers; + return CompletableFuture.completedFuture(brokers); } })).when(primaryLoadManager).getBrokerFilterPipeline(); @@ -288,11 +289,11 @@ public void testFilterHasException() throws Exception { doReturn(List.of(new MockBrokerFilter() { @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { - brokers.clear(); - throw new BrokerFilterException("Test"); + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + brokers.remove(brokers.keySet().iterator().next()); + return FutureUtil.failedFuture(new BrokerFilterException("Test")); } })).when(primaryLoadManager).getBrokerFilterPipeline(); @@ -531,19 +532,18 @@ public void testMoreThenOneFilter() throws Exception { String lookupServiceAddress1 = pulsar1.getLookupServiceAddress(); doReturn(List.of(new MockBrokerFilter() { @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { brokers.remove(lookupServiceAddress1); - return brokers; + return CompletableFuture.completedFuture(brokers); } },new MockBrokerFilter() { @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { - brokers.clear(); - throw new BrokerFilterException("Test"); + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + return FutureUtil.failedFuture(new BrokerFilterException("Test")); } })).when(primaryLoadManager).getBrokerFilterPipeline(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java index c2c534f72e9db..b8d5d98fd4c08 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java @@ -31,6 +31,9 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; @@ -67,7 +70,7 @@ public class BrokerIsolationPoliciesFilterTest { */ @Test public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() - throws IllegalAccessException, BrokerFilterException { + throws IllegalAccessException, BrokerFilterException, ExecutionException, InterruptedException { var namespace = "my-tenant/my-ns"; NamespaceName namespaceName = NamespaceName.get(namespace); @@ -83,18 +86,18 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker Map result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()); + "broker3", getLookupData())), namespaceName, getContext()).get(); assertEquals(result.keySet(), Set.of("broker1")); // b. available-brokers: broker2, broker3 => result: broker2 result = filter.filter(new HashMap<>(Map.of( "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()); + "broker3", getLookupData())), namespaceName, getContext()).get(); assertEquals(result.keySet(), Set.of("broker2")); // c. available-brokers: broker3 => result: NULL result = filter.filter(new HashMap<>(Map.of( - "broker3", getLookupData())), namespaceName, getContext()); + "broker3", getLookupData())), namespaceName, getContext()).get(); assertTrue(result.isEmpty()); // 2. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 2 @@ -104,24 +107,24 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()); + "broker3", getLookupData())), namespaceName, getContext()).get(); assertEquals(result.keySet(), Set.of("broker1", "broker2")); // b. available-brokers: broker2, broker3 => result: broker2 result = filter.filter(new HashMap<>(Map.of( "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()); + "broker3", getLookupData())), namespaceName, getContext()).get(); assertEquals(result.keySet(), Set.of("broker2")); // c. available-brokers: broker3 => result: NULL result = filter.filter(new HashMap<>(Map.of( - "broker3", getLookupData())), namespaceName, getContext()); + "broker3", getLookupData())), namespaceName, getContext()).get(); assertTrue(result.isEmpty()); } @Test public void testFilterWithPersistentOrNonPersistentDisabled() - throws IllegalAccessException, BrokerFilterException { + throws IllegalAccessException, BrokerFilterException, ExecutionException, InterruptedException { var namespace = "my-tenant/my-ns"; NamespaceName namespaceName = NamespaceName.get(namespace); NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); @@ -129,7 +132,8 @@ public void testFilterWithPersistentOrNonPersistentDisabled() doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); var policies = mock(SimpleResourceAllocationPolicies.class); - doReturn(false).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(CompletableFuture.completedFuture(false)) + .when(policies).areIsolationPoliciesPresentAsync(eq(namespaceName)); doReturn(true).when(policies).isSharedBroker(any()); IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(policies); @@ -140,14 +144,14 @@ public void testFilterWithPersistentOrNonPersistentDisabled() Map result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()); + "broker3", getLookupData())), namespaceBundle, getContext()).get(); assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(true, false), "broker2", getLookupData(true, false), - "broker3", getLookupData())), namespaceBundle, getContext()); + "broker3", getLookupData())), namespaceBundle, getContext()).get(); assertEquals(result.keySet(), Set.of("broker3")); doReturn(false).when(namespaceBundle).hasNonPersistentTopic(); @@ -155,13 +159,13 @@ public void testFilterWithPersistentOrNonPersistentDisabled() result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()); + "broker3", getLookupData())), namespaceBundle, getContext()).get(); assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(false, true), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()); + "broker3", getLookupData())), namespaceBundle, getContext()).get(); assertEquals(result.keySet(), Set.of("broker2", "broker3")); } @@ -172,7 +176,8 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, Set shared, int min_limit) { reset(policies); - doReturn(true).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(CompletableFuture.completedFuture(true)) + .when(policies).areIsolationPoliciesPresentAsync(eq(namespaceName)); doReturn(false).when(policies).isPrimaryBroker(eq(namespaceName), any()); doReturn(false).when(policies).isSecondaryBroker(eq(namespaceName), any()); doReturn(false).when(policies).isSharedBroker(any()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java index 4aef87cf63aa8..6b35bf71a0317 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java @@ -27,6 +27,7 @@ import org.testng.annotations.Test; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; /** @@ -35,7 +36,7 @@ public class BrokerLoadManagerClassFilterTest extends BrokerFilterTestBase { @Test - public void test() throws BrokerFilterException { + public void test() throws BrokerFilterException, ExecutionException, InterruptedException { LoadManagerContext context = getContext(); context.brokerConfiguration().setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); BrokerLoadManagerClassFilter filter = new BrokerLoadManagerClassFilter(); @@ -48,14 +49,14 @@ public void test() throws BrokerFilterException { "broker5", getLookupData("3.0.0", null) ); - Map result = filter.filter(new HashMap<>(originalBrokers), null, context); + Map result = filter.filter(new HashMap<>(originalBrokers), null, context).get(); assertEquals(result, Map.of( "broker1", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()), "broker2", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()) )); context.brokerConfiguration().setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); - result = filter.filter(new HashMap<>(originalBrokers), null, context); + result = filter.filter(new HashMap<>(originalBrokers), null, context).get(); assertEquals(result, Map.of( "broker3", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java index da13a9526a881..a6214304bd201 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; import static org.testng.Assert.assertEquals; @@ -38,7 +39,7 @@ public class BrokerMaxTopicCountFilterTest extends BrokerFilterTestBase { @Test - public void test() throws IllegalAccessException, BrokerFilterException { + public void test() throws IllegalAccessException, BrokerFilterException, ExecutionException, InterruptedException { LoadManagerContext context = getContext(); LoadDataStore store = context.brokerLoadDataStore(); BrokerLoadData maxTopicLoadData = new BrokerLoadData(); @@ -58,7 +59,8 @@ public void test() throws IllegalAccessException, BrokerFilterException { "broker3", getLookupData(), "broker4", getLookupData() ); - Map result = filter.filter(new HashMap<>(originalBrokers), null, context); + Map result = + filter.filter(new HashMap<>(originalBrokers), null, context).get(); assertEquals(result, Map.of( "broker2", getLookupData(), "broker4", getLookupData() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java index cafd8f0ea7a4c..4a1a45439384e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java @@ -21,9 +21,11 @@ import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterBadVersionException; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; @@ -39,14 +41,14 @@ public class BrokerVersionFilterTest extends BrokerFilterTestBase { @Test - public void testFilterEmptyBrokerList() throws BrokerFilterException { + public void testFilterEmptyBrokerList() throws BrokerFilterException, ExecutionException, InterruptedException { BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(new HashMap<>(), null, getContext()); + Map result = brokerVersionFilter.filter(new HashMap<>(), null, getContext()).get(); assertTrue(result.isEmpty()); } @Test - public void testDisabledFilter() throws BrokerFilterException { + public void testDisabledFilter() throws BrokerFilterException, ExecutionException, InterruptedException { LoadManagerContext context = getContext(); ServiceConfiguration configuration = new ServiceConfiguration(); configuration.setPreferLaterVersions(false); @@ -58,12 +60,12 @@ public void testDisabledFilter() throws BrokerFilterException { ); Map brokers = new HashMap<>(originalBrokers); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(brokers, null, context); + Map result = brokerVersionFilter.filter(brokers, null, context).get(); assertEquals(result, originalBrokers); } @Test - public void testFilter() throws BrokerFilterException { + public void testFilter() throws BrokerFilterException, ExecutionException, InterruptedException { Map originalBrokers = Map.of( "localhost:6650", getLookupData("2.10.0"), "localhost:6651", getLookupData("2.10.1"), @@ -72,7 +74,7 @@ public void testFilter() throws BrokerFilterException { ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); Map result = brokerVersionFilter.filter( - new HashMap<>(originalBrokers), null, getContext()); + new HashMap<>(originalBrokers), null, getContext()).get(); assertEquals(result, Map.of( "localhost:6651", getLookupData("2.10.1"), "localhost:6652", getLookupData("2.10.1"), @@ -85,7 +87,7 @@ public void testFilter() throws BrokerFilterException { "localhost:6652", getLookupData("2.10.1"), "localhost:6653", getLookupData("2.10.1") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); + result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()).get(); assertEquals(result, Map.of( "localhost:6652", getLookupData("2.10.1"), @@ -99,19 +101,24 @@ public void testFilter() throws BrokerFilterException { "localhost:6653", getLookupData("2.10.2-SNAPSHOT") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); + result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()).get(); assertEquals(result, Map.of( "localhost:6653", getLookupData("2.10.2-SNAPSHOT") )); } - @Test(expectedExceptions = BrokerFilterBadVersionException.class) - public void testInvalidVersionString() throws BrokerFilterException { + @Test + public void testInvalidVersionString() { Map originalBrokers = Map.of( "localhost:6650", getLookupData("xxx") ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); + try { + brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()).get(); + fail(); + } catch (Exception ex) { + assertEquals(ex.getCause().getClass(), BrokerFilterBadVersionException.class); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index af5890fcacbb2..3dd41a92e682c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -634,7 +634,8 @@ public void testBundlesWithIsolationPolicies() { // test setLoadBalancerSheddingBundlesWithPoliciesEnabled=false; - doReturn(true).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any()); + doReturn(CompletableFuture.completedFuture(true)) + .when(allocationPoliciesSpy).areIsolationPoliciesPresentAsync(any()); ctx.brokerConfiguration().setLoadBalancerTransferEnabled(true); ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(false); res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -679,12 +680,14 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); doReturn(true).when(namespaceBundle).hasNonPersistentTopic(); doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); - doReturn(false).when(policies).areIsolationPoliciesPresent(any()); + doReturn(CompletableFuture.completedFuture(false)) + .when(policies).areIsolationPoliciesPresentAsync(any()); doReturn(false).when(policies).isPrimaryBroker(any(), any()); doReturn(false).when(policies).isSecondaryBroker(any(), any()); doReturn(true).when(policies).isSharedBroker(any()); - doReturn(true).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(CompletableFuture.completedFuture(true)) + .when(policies).areIsolationPoliciesPresentAsync(eq(namespaceName)); primary.forEach(broker -> { doReturn(true).when(policies).isPrimaryBroker(eq(namespaceName), eq(broker)); @@ -723,8 +726,8 @@ public void testBundlesWithAntiAffinityGroup() throws MetadataStoreException { doAnswer(invocationOnMock -> { Map brokers = invocationOnMock.getArgument(0); brokers.clear(); - return brokers; - }).when(antiAffinityGroupPolicyHelper).filter(any(), any()); + return CompletableFuture.completedFuture(brokers); + }).when(antiAffinityGroupPolicyHelper).filterAsync(any(), any()); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); assertTrue(res.isEmpty()); @@ -737,11 +740,11 @@ public void testBundlesWithAntiAffinityGroup() throws MetadataStoreException { String bundle = invocationOnMock.getArgument(1, String.class); if (bundle.equalsIgnoreCase(bundleE1)) { - return brokers; + return CompletableFuture.completedFuture(brokers); } brokers.clear(); - return brokers; - }).when(antiAffinityGroupPolicyHelper).filter(any(), any()); + return CompletableFuture.completedFuture(brokers); + }).when(antiAffinityGroupPolicyHelper).filterAsync(any(), any()); var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected2 = new HashSet<>(); expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), @@ -761,10 +764,10 @@ public String name() { } @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { - throw new BrokerFilterException("test"); + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + return FutureUtil.failedFuture(new BrokerFilterException("test")); } }; filters.add(filter); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java index b934ced08c5db..f3cb25e789f08 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java @@ -21,6 +21,7 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; import org.apache.pulsar.common.util.RateLimiter; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -51,6 +52,14 @@ public void setup() throws Exception { publishRateLimiter = new PublishRateLimiterImpl(policies, CLUSTER_NAME); } + @AfterMethod + public void cleanup() throws Exception { + policies.publishMaxMessageRate.clear(); + policies.publishMaxMessageRate = null; + precisePublishLimiter.close(); + publishRateLimiter.close(); + } + @Test public void testPublishRateLimiterImplExceed() throws Exception { // increment not exceed diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java index 78846fb75922a..0e17719aca7e7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java @@ -22,6 +22,7 @@ import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -32,6 +33,9 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.MessageRedeliveryController; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -224,6 +228,62 @@ public void sortedTest() throws Exception { log.info("TransactionConsumeTest sortedTest finish."); } + @Test + public void testMessageRedelivery() throws Exception { + int transactionMessageCnt = 10; + String subName = "shared-test"; + + @Cleanup + Consumer sharedConsumer = pulsarClient.newConsumer() + .topic(CONSUME_TOPIC) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + Awaitility.await().until(sharedConsumer::isConnected); + + long mostSigBits = 2L; + long leastSigBits = 5L; + TxnID txnID = new TxnID(mostSigBits, leastSigBits); + + // produce batch message with txn and then abort + PersistentTopic persistentTopic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic(CONSUME_TOPIC, false).get().get(); + + List sendMessageList = new ArrayList<>(); + List messageIdDataList = appendTransactionMessages(txnID, persistentTopic, transactionMessageCnt, sendMessageList); + + persistentTopic.endTxn(txnID, TxnAction.ABORT_VALUE, 0L).get(); + log.info("Abort txn."); + + // redeliver transaction messages to shared consumer + PersistentSubscription subRef = persistentTopic.getSubscription(subName); + PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef + .getDispatcher(); + Field redeliveryMessagesField = PersistentDispatcherMultipleConsumers.class + .getDeclaredField("redeliveryMessages"); + redeliveryMessagesField.setAccessible(true); + MessageRedeliveryController redeliveryMessages = new MessageRedeliveryController(true); + + final Field totalAvailablePermitsField = PersistentDispatcherMultipleConsumers.class + .getDeclaredField("totalAvailablePermits"); + totalAvailablePermitsField.setAccessible(true); + totalAvailablePermitsField.set(dispatcher, 1000); + + for (MessageIdData messageIdData : messageIdDataList) { + redeliveryMessages.add(messageIdData.getLedgerId(), messageIdData.getEntryId()); + } + + redeliveryMessagesField.set(dispatcher, redeliveryMessages); + dispatcher.readMoreEntries(); + + // shared consumer should not receive the redelivered aborted transaction messages + Message message = sharedConsumer.receive(5, TimeUnit.SECONDS); + Assert.assertNull(message); + + log.info("TransactionConsumeTest testMessageRedelivery finish."); + } + private void sendNormalMessages(Producer producer, int startMsgCnt, int messageCnt, List sendMessageList) throws PulsarClientException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java index 75ae91f18a305..8189f8e86b5b3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticatedProducerConsumerTest.java @@ -396,11 +396,6 @@ public void testDeleteAuthenticationPoliciesOfTopic() throws Exception { Awaitility.await().untilAsserted(() -> { assertTrue(pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get("p1/ns1")) .get().auth_policies.getTopicAuthentication().containsKey(partitionedTopic)); - for (int i = 0; i < numPartitions; i++) { - assertTrue(pulsar.getPulsarResources().getNamespaceResources().getPolicies(NamespaceName.get("p1/ns1")) - .get().auth_policies.getTopicAuthentication() - .containsKey(TopicName.get(partitionedTopic).getPartition(i).toString())); - } }); admin.topics().deletePartitionedTopic("persistent://p1/ns1/partitioned-topic"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java index b1f653fae012a..fb8d6566d9a0d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java @@ -35,7 +35,7 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.common.util.CmdGenerateDocs; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.testng.annotations.Test; /** diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index f7a48b9b97736..257d2c87550e4 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -111,6 +111,7 @@ io.swagger swagger-core + provided diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index bf302fcffec00..e19ab654a3cc8 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -2288,6 +2288,20 @@ public boolean isS3Driver(String driver) { return driver.equalsIgnoreCase(driverNames.get(0)) || driver.equalsIgnoreCase(driverNames.get(1)); } + public boolean positiveCheck(String paramName, long value) { + if (value <= 0) { + throw new ParameterException(paramName + " cannot be less than or equal to 0!"); + } + return true; + } + + public boolean maxValueCheck(String paramName, long value, long maxValue) { + if (value > maxValue) { + throw new ParameterException(paramName + " cannot be greater than " + maxValue + "!"); + } + return true; + } + @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index b07ad0056617e..31d9648babcff 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -1004,7 +1004,7 @@ private class CreateSubscription extends CliCommand { private java.util.List params; @Parameter(names = { "-s", - "--subscription" }, description = "Subscription to reset position on", required = true) + "--subscription" }, description = "Name of subscription to be created", required = true) private String subscriptionName; @Parameter(names = { "-m" , "--messageId" }, description = "messageId where to create the subscription. " @@ -2198,14 +2198,14 @@ public boolean isS3Driver(String driver) { public boolean positiveCheck(String paramName, long value) { if (value <= 0) { - throw new ParameterException(paramName + " is not be negative or 0!"); + throw new ParameterException(paramName + " cannot be less than or equal to 0!"); } return true; } public boolean maxValueCheck(String paramName, long value, long maxValue) { if (value > maxValue) { - throw new ParameterException(paramName + " is not bigger than " + maxValue + "!"); + throw new ParameterException(paramName + " cannot be greater than " + maxValue + "!"); } return true; } diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index db2297100eeb2..0e45e73636612 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -76,6 +76,7 @@ io.swagger swagger-annotations + provided diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java index 365abce3b90d2..263507dac1dc6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java @@ -64,6 +64,12 @@ protected ConnectionHandler(HandlerState state, Backoff backoff, Connection conn } protected void grabCnx() { + if (!duringConnect.compareAndSet(false, true)) { + log.info("[{}] [{}] Skip grabbing the connection since there is a pending connection", + state.topic, state.getHandlerName()); + return; + } + if (CLIENT_CNX_UPDATER.get(this) != null) { log.warn("[{}] [{}] Client cnx already set, ignoring reconnection request", state.topic, state.getHandlerName()); @@ -76,11 +82,6 @@ protected void grabCnx() { state.topic, state.getHandlerName(), state.getState()); return; } - if (!duringConnect.compareAndSet(false, true)) { - log.info("[{}] [{}] Skip grabbing the connection since there is a pending connection", - state.topic, state.getHandlerName()); - return; - } try { CompletableFuture cnxFuture; @@ -123,8 +124,8 @@ private Void handleConnectionError(Throwable exception) { } void reconnectLater(Throwable exception) { - duringConnect.set(false); CLIENT_CNX_UPDATER.set(this, null); + duringConnect.set(false); if (!isValidStateForReconnection()) { log.info("[{}] [{}] Ignoring reconnection request (state: {})", state.topic, state.getHandlerName(), state.getState()); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/CmdGenerateDocumentation.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/CmdGenerateDocumentation.java deleted file mode 100644 index ae6210b978edc..0000000000000 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/CmdGenerateDocumentation.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.client.impl.conf; - -import com.beust.jcommander.Parameters; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.util.BaseGenerateDocumentation; - -@Data -@Parameters(commandDescription = "Generate documentation automatically.") -@Slf4j -public class CmdGenerateDocumentation extends BaseGenerateDocumentation { - - @Override - public String generateDocumentByClassName(String className) throws Exception { - StringBuilder sb = new StringBuilder(); - if (ClientConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Client", sb); - } else if (ProducerConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Producer", sb); - } else if (ConsumerConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Consumer", sb); - } else if (ReaderConfigurationData.class.getName().equals(className)) { - return generateDocByApiModelProperty(className, "Reader", sb); - } - - return "Class [" + className + "] not found"; - } - - public static void main(String[] args) throws Exception { - CmdGenerateDocumentation generateDocumentation = new CmdGenerateDocumentation(); - generateDocumentation.run(args); - } -} diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index cfc9cba7ede7f..8c99638878f53 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -50,6 +50,7 @@ io.swagger swagger-annotations + provided @@ -226,16 +227,12 @@ snappy-java test + com.google.code.gson gson - - com.beust - jcommander - - org.awaitility awaitility diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java index f0023ce5a42dd..8dfd478cc8fbe 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java @@ -130,7 +130,8 @@ private static Provider loadConscryptProvider() { log.warn("Conscrypt isn't available for {} {}. Using JDK default security provider.", System.getProperty("os.name"), System.getProperty("os.arch")); } else { - log.warn("Conscrypt isn't available. Using JDK default security provider.", e); + log.warn("Conscrypt isn't available. Using JDK default security provider." + + " Cause : {}, Reason : {}", e.getCause(), e.getMessage()); } return null; } diff --git a/pulsar-docs-tools/pom.xml b/pulsar-docs-tools/pom.xml new file mode 100644 index 0000000000000..9074ad0d6c591 --- /dev/null +++ b/pulsar-docs-tools/pom.xml @@ -0,0 +1,51 @@ + + + + 4.0.0 + + + org.apache.pulsar + pulsar + 3.1.0-SNAPSHOT + + + pulsar-docs-tools + Pulsar Documentation Generators + + + + io.swagger + swagger-annotations + + + io.swagger + swagger-core + + + com.beust + jcommander + + + + diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/BaseGenerateDocumentation.java b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/BaseGenerateDocumentation.java similarity index 97% rename from pulsar-common/src/main/java/org/apache/pulsar/common/util/BaseGenerateDocumentation.java rename to pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/BaseGenerateDocumentation.java index b88ed197e8f2e..db6178a7fda4d 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/BaseGenerateDocumentation.java +++ b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/BaseGenerateDocumentation.java @@ -16,11 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.common.util; +package org.apache.pulsar.docs.tools; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; import io.swagger.annotations.ApiModelProperty; import java.io.Serializable; import java.lang.annotation.Annotation; @@ -31,15 +30,12 @@ import java.util.List; import java.util.function.Predicate; import java.util.stream.Collectors; -import lombok.Data; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.MethodUtils; import org.apache.commons.lang3.tuple.Pair; -@Data -@Parameters(commandDescription = "Generate documentation automatically.") @Slf4j public abstract class BaseGenerateDocumentation { @@ -186,7 +182,9 @@ public int compare(Pair o1, Pair clazz = Class.forName(className); Object obj = clazz.getDeclaredConstructor().newInstance(); Field[] fields = clazz.getDeclaredFields(); @@ -218,7 +216,9 @@ protected String generateDocByFieldContext(String className, String type, String return sb.toString(); } - protected String generateDocByApiModelProperty(String className, String type, StringBuilder sb) throws Exception { + protected String generateDocByApiModelProperty(String className, String type) throws Exception { + final StringBuilder sb = new StringBuilder(); + Class clazz = Class.forName(className); Object obj = clazz.getDeclaredConstructor().newInstance(); Field[] fields = clazz.getDeclaredFields(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/CmdGenerateDocs.java similarity index 90% rename from pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java rename to pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/CmdGenerateDocs.java index f41e53a80e7f3..8f784c1eca1fa 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/CmdGenerateDocs.java +++ b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/CmdGenerateDocs.java @@ -16,13 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.common.util; +package org.apache.pulsar.docs.tools; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterDescription; import com.beust.jcommander.Parameters; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; import lombok.Getter; @@ -145,11 +146,15 @@ private String generateDocument(String module, JCommander commander) { sb.append("|Flag|Description|Default|\n"); sb.append("|---|---|---|\n"); List options = cmd.getParameters(); - options.forEach((option) -> - sb.append("| `").append(option.getNames()) - .append("` | ").append(option.getDescription().replace("\n", " ")) - .append("|").append(option.getDefault()).append("|\n") - ); + options.stream().sorted(Comparator.comparing(ParameterDescription::getLongestName)) + .forEach((option) -> + sb.append("| `") + .append(option.getNames()) + .append("` | ") + .append(option.getDescription().replace("\n", " ")) + .append("|") + .append(option.getDefault()).append("|\n") + ); } return sb.toString(); } diff --git a/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/package-info.java b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/package-info.java new file mode 100644 index 0000000000000..d971fc8ee6b11 --- /dev/null +++ b/pulsar-docs-tools/src/main/java/org/apache/pulsar/docs/tools/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.docs.tools; diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/CmdGenerateDocsTest.java b/pulsar-docs-tools/src/test/java/org/apache/pulsar/docs/tools/CmdGenerateDocsTest.java similarity index 98% rename from pulsar-common/src/test/java/org/apache/pulsar/common/util/CmdGenerateDocsTest.java rename to pulsar-docs-tools/src/test/java/org/apache/pulsar/docs/tools/CmdGenerateDocsTest.java index 68d8a02f48673..3f96ddaef591a 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/CmdGenerateDocsTest.java +++ b/pulsar-docs-tools/src/test/java/org/apache/pulsar/docs/tools/CmdGenerateDocsTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.common.util; +package org.apache.pulsar.docs.tools; import static org.testng.Assert.assertEquals; import com.beust.jcommander.Parameter; @@ -86,8 +86,8 @@ public void testGenerateDocs() { + "\n" + "|Flag|Description|Default|\n" + "|---|---|---|\n" - + "| `-n, --name` | Name|null|\n" + "| `-h, --help` | Show this help message|false|\n" + + "| `-n, --name` | Name|null|\n" + System.lineSeparator(); assertEquals(rightMsg, message); } finally { diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index da7c83689c1c9..c6db572f57d7e 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -129,9 +129,22 @@ commons-io + + ${project.groupId} + pulsar-docs-tools + ${project.version} + + + io.swagger + * + + + + io.swagger swagger-core + provided com.fasterxml.jackson.core diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionWorkerStarter.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionWorkerStarter.java index fa988a0927bd6..679ce1db70d97 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionWorkerStarter.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionWorkerStarter.java @@ -22,8 +22,8 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.ShutdownUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; /** * A starter to start function worker. diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index a607b9e835f8e..de1eb9a16c05b 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -49,6 +49,12 @@ ${project.version} + + ${project.groupId} + pulsar-docs-tools + ${project.version} + + ${project.groupId} pulsar-websocket @@ -60,6 +66,12 @@ commons-lang3 + + io.swagger + swagger-annotations + provided + + org.eclipse.jetty jetty-server diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index beee9f1a4f763..d8bb048710770 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -46,9 +46,9 @@ import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.configuration.VipStatus; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.DirectMemoryUtils; import org.apache.pulsar.common.util.ShutdownUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.proxy.stats.ProxyStats; import org.apache.pulsar.websocket.WebSocketConsumerServlet; import org.apache.pulsar.websocket.WebSocketPingPongServlet; @@ -61,10 +61,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * Starts an instance of the Pulsar ProxyService. - * */ public class ProxyServiceStarter { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java index 27e61c90e9e26..afa2e12dabb4f 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java @@ -38,8 +38,6 @@ import javax.ws.rs.core.Response.Status; import org.apache.pulsar.proxy.server.ProxyService; - - @Path("/") @Api(value = "/proxy-stats", description = "Stats for proxy", tags = "proxy-stats", hidden = true) @Produces(MediaType.APPLICATION_JSON) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentation.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentation.java index a1c7e30a25ed6..a9164b6d95393 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentation.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentation.java @@ -16,26 +16,41 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.proxy.util; -import com.beust.jcommander.Parameters; -import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.util.BaseGenerateDocumentation; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; +import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; +import org.apache.pulsar.client.impl.conf.ReaderConfigurationData; +import org.apache.pulsar.docs.tools.BaseGenerateDocumentation; import org.apache.pulsar.proxy.server.ProxyConfiguration; +import org.apache.pulsar.websocket.service.WebSocketProxyConfiguration; -@Data -@Parameters(commandDescription = "Generate documentation automatically.") @Slf4j public class CmdGenerateDocumentation extends BaseGenerateDocumentation { @Override public String generateDocumentByClassName(String className) throws Exception { - StringBuilder sb = new StringBuilder(); if (ProxyConfiguration.class.getName().equals(className)) { - return generateDocByFieldContext(className, "Pulsar proxy", sb); + return generateDocByFieldContext(className, "Pulsar proxy"); + } else if (ServiceConfiguration.class.getName().equals(className)) { + return generateDocByFieldContext(className, "Broker"); + } else if (ClientConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Client"); + } else if (WebSocketProxyConfiguration.class.getName().equals(className)) { + return generateDocByFieldContext(className, "WebSocket"); + } else if (ProducerConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Producer"); + } else if (ConsumerConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Consumer"); + } else if (ReaderConfigurationData.class.getName().equals(className)) { + return generateDocByApiModelProperty(className, "Reader"); + } else { + return "Class [" + className + "] not found"; } - return "Class [" + className + "] not found"; } public static void main(String[] args) throws Exception { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/package-info.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/package-info.java index 1501a16f55813..85284c0dd229a 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/package-info.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/util/package-info.java @@ -16,4 +16,5 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.proxy.util; \ No newline at end of file + +package org.apache.pulsar.proxy.util; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/utils/CmdTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentationTest.java similarity index 83% rename from pulsar-broker/src/test/java/org/apache/pulsar/utils/CmdTest.java rename to pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentationTest.java index a571a1cd0c92d..b76463e862f50 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/utils/CmdTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdGenerateDocumentationTest.java @@ -16,17 +16,22 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.utils; -import static org.testng.Assert.assertTrue; +package org.apache.pulsar.proxy.util; + import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.lang.reflect.Field; import org.apache.pulsar.common.configuration.FieldContext; import org.testng.annotations.Test; +import static org.testng.Assert.assertTrue; -@Test(groups = "utils") -public class CmdTest { +public class CmdGenerateDocumentationTest { + @Test + public void cmdParserProxyConfigurationTest() throws Exception { + String value = generateDoc("org.apache.pulsar.proxy.server.ProxyConfiguration"); + assertTrue(value.contains("Pulsar proxy")); + } @Test public void cmdParserTest() throws Exception { @@ -43,14 +48,14 @@ public void cmdParserClientTest() throws Exception { generateDoc("org.apache.pulsar.client.impl.conf.ClientConfigurationData"); } - private void generateDoc(String clazz) throws Exception { + private String generateDoc(String clazz) throws Exception { PrintStream oldStream = System.out; try (ByteArrayOutputStream baoStream = new ByteArrayOutputStream(2048); PrintStream cacheStream = new PrintStream(baoStream);) { System.setOut(cacheStream); CmdGenerateDocumentation.main(("-c " + clazz).split(" ")); String message = baoStream.toString(); - Class cls = Class.forName(clazz); + Class cls = Class.forName(clazz); Field[] fields = cls.getDeclaredFields(); for (Field field : fields) { field.setAccessible(true); @@ -60,6 +65,7 @@ private void generateDoc(String clazz) throws Exception { } assertTrue(message.indexOf(field.getName()) > 0); } + return message; } finally { System.setOut(oldStream); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdTest.java deleted file mode 100644 index 59e1a43310d67..0000000000000 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/util/CmdTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.proxy.util; - -import static org.testng.Assert.assertTrue; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.lang.reflect.Field; -import org.apache.pulsar.common.configuration.FieldContext; -import org.testng.annotations.Test; - -public class CmdTest { - - @Test - public void cmdParserProxyConfigurationTest() throws Exception { - String value = generateDoc("org.apache.pulsar.proxy.server.ProxyConfiguration"); - assertTrue(value.contains("Pulsar proxy")); - } - - private String generateDoc(String clazz) throws Exception { - PrintStream oldStream = System.out; - try (ByteArrayOutputStream baoStream = new ByteArrayOutputStream(2048); - PrintStream cacheStream = new PrintStream(baoStream);) { - System.setOut(cacheStream); - CmdGenerateDocumentation.main(("-c " + clazz).split(" ")); - String message = baoStream.toString(); - Class cls = Class.forName(clazz); - Field[] fields = cls.getDeclaredFields(); - for (Field field : fields) { - field.setAccessible(true); - FieldContext fieldContext = field.getAnnotation(FieldContext.class); - if (fieldContext == null) { - continue; - } - assertTrue(message.indexOf(field.getName()) > 0); - } - return message; - } finally { - System.setOut(oldStream); - } - } -} diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 15f20ae7e4ad6..9afd6c98bc2ff 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -221,7 +221,7 @@ The Apache Software License, Version 2.0 - jackson-module-jaxb-annotations-2.14.2.jar - jackson-module-jsonSchema-2.14.2.jar * Guava - - guava-32.0.0-jre.jar + - guava-32.1.1-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - failureaccess-1.0.1.jar * Google Guice @@ -430,21 +430,21 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.16.1.jar - - bookkeeper-common-allocator-4.16.1.jar - - bookkeeper-proto-4.16.1.jar - - bookkeeper-server-4.16.1.jar - - bookkeeper-stats-api-4.16.1.jar - - bookkeeper-tools-framework-4.16.1.jar - - circe-checksum-4.16.1.jar - - codahale-metrics-provider-4.16.1.jar - - cpu-affinity-4.16.1.jar - - http-server-4.16.1.jar - - prometheus-metrics-provider-4.16.1.jar - - codahale-metrics-provider-4.16.1.jar - - bookkeeper-slogger-api-4.16.1.jar - - bookkeeper-slogger-slf4j-4.16.1.jar - - native-io-4.16.1.jar + - bookkeeper-common-4.16.2.jar + - bookkeeper-common-allocator-4.16.2.jar + - bookkeeper-proto-4.16.2.jar + - bookkeeper-server-4.16.2.jar + - bookkeeper-stats-api-4.16.2.jar + - bookkeeper-tools-framework-4.16.2.jar + - circe-checksum-4.16.2.jar + - codahale-metrics-provider-4.16.2.jar + - cpu-affinity-4.16.2.jar + - http-server-4.16.2.jar + - prometheus-metrics-provider-4.16.2.jar + - codahale-metrics-provider-4.16.2.jar + - bookkeeper-slogger-api-4.16.2.jar + - bookkeeper-slogger-slf4j-4.16.2.jar + - native-io-4.16.2.jar * Apache Commons - commons-cli-1.5.0.jar - commons-codec-1.15.jar diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 8335aa3603f63..1ac764a3c64d9 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -37,7 +37,7 @@ 2.6 0.0.12 3.0.5 - 32.0.0-jre + 32.1.1-jre 2.12.1 2.5.1 4.0.1 diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index f742c286f0b68..bf8f38f2da1af 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -52,6 +52,18 @@ test + + ${project.groupId} + pulsar-docs-tools + ${project.version} + + + io.swagger + * + + + + org.apache.commons commons-lang3 @@ -80,6 +92,7 @@ io.swagger swagger-core + provided diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java index fbcecc0642e34..530f0796d488d 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java @@ -26,8 +26,8 @@ import com.beust.jcommander.Parameter; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.configuration.VipStatus; -import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.ShutdownUtil; +import org.apache.pulsar.docs.tools.CmdGenerateDocs; import org.apache.pulsar.websocket.WebSocketConsumerServlet; import org.apache.pulsar.websocket.WebSocketPingPongServlet; import org.apache.pulsar.websocket.WebSocketProducerServlet; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java index 86228d25a793f..51dc9d7dd72da 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -261,8 +261,7 @@ public void testStopBroker() throws Exception { assertNotEquals(broker1, broker); } - // TODO: This test is very flaky and it's disabled for now to unblock CI - @Test(timeOut = 40 * 1000, enabled = false) + @Test(timeOut = 40 * 1000) public void testAntiaffinityPolicy() throws PulsarAdminException { final String namespaceAntiAffinityGroup = "my-anti-affinity-filter"; final String antiAffinityEnabledNameSpace = DEFAULT_TENANT + "/my-ns-filter" + nsSuffix; From e9f3630dc885f4def5db7bf55f5ee77cbad71b5b Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Thu, 6 Jul 2023 07:59:39 +0900 Subject: [PATCH 11/14] Revert "Temp change" This reverts commit 26d33482235e283c80b041186b66f856e9258fd9. --- pulsar-client-tools/pom.xml | 9 --------- .../java/org/apache/pulsar/admin/cli/CmdNamespaces.java | 3 +-- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 257d2c87550e4..16a30f8d63040 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -38,10 +38,6 @@ jcommander compile - - com.google.guava - guava - ${project.groupId} pulsar-client-admin-api @@ -141,11 +137,6 @@ org.apache.commons commons-text - - com.google.guava - guava - 32.0.0-jre - diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index e19ab654a3cc8..ef23b944eddaf 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -27,7 +27,6 @@ import com.google.common.collect.Sets; import java.io.IOException; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -2275,7 +2274,7 @@ private class SetOffloadPolicies extends CliCommand { ) private String offloadReadPriorityStr; - public final List driverNames = Collections.singletonList(OffloadPoliciesImpl.DRIVER_NAMES); + public final List driverNames = OffloadPoliciesImpl.DRIVER_NAMES; public boolean driverSupported(String driver) { return driverNames.stream().anyMatch(d -> d.equalsIgnoreCase(driver)); From df3279ff571f28781720ecafda3db3bac9dd51d8 Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Thu, 6 Jul 2023 08:27:40 +0900 Subject: [PATCH 12/14] Fixing changes via PulsarAdminToolConverterTest --- .../cli/PulsarAdminToolConverterTest.java | 569 ++++++++++++++++++ .../pulsar/admin/cli/CmdNamespaces.java | 13 +- .../admin/cli/utils/ValueValidationUtils.java | 6 + .../PositiveIntegerValueValidator.java | 31 + ...r.java => PositiveLongValueValidator.java} | 2 +- 5 files changed, 613 insertions(+), 8 deletions(-) create mode 100644 pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolConverterTest.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveIntegerValueValidator.java rename pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/{PositiveValueValidator.java => PositiveLongValueValidator.java} (94%) diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolConverterTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolConverterTest.java new file mode 100644 index 0000000000000..54612fbd6d9e2 --- /dev/null +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolConverterTest.java @@ -0,0 +1,569 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import com.google.common.collect.Sets; +import java.util.EnumSet; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.admin.ListNamespaceTopicsOptions; +import org.apache.pulsar.client.admin.Lookup; +import org.apache.pulsar.client.admin.Namespaces; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride; +import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.BacklogQuota; +import org.apache.pulsar.common.policies.data.BacklogQuota.RetentionPolicy; +import org.apache.pulsar.common.policies.data.BookieAffinityGroupData; +import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; +import org.apache.pulsar.common.policies.data.DispatchRate; +import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; +import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; +import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; +import org.apache.pulsar.common.policies.data.OffloadedReadPriority; +import org.apache.pulsar.common.policies.data.PersistencePolicies; +import org.apache.pulsar.common.policies.data.PublishRate; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.SubscribeRate; +import org.apache.pulsar.common.policies.data.TopicType; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +@Slf4j +public class PulsarAdminToolConverterTest { + + @Test + public void namespacesSimpl() throws Exception { + PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); + Namespaces mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + Lookup mockLookup = mock(Lookup.class); + when(admin.lookups()).thenReturn(mockLookup); + + CmdNamespaces namespaces = new CmdNamespaces(() -> admin); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split( + "set-offload-policies myprop/clust/ns1 -r test-region -d aws-s3 -b test-bucket -e http://test.endpoint -mbs 32M -rbs 5M -oat 10M -oats 100 -oae 10s -orp tiered-storage-first")); + verify(mockNamespaces).setOffloadPolicies("myprop/clust/ns1", + OffloadPoliciesImpl.create("aws-s3", "test-region", "test-bucket", + "http://test.endpoint", null, null, null, null, 32 * 1024 * 1024, 5 * 1024 * 1024, + 10 * 1024 * 1024L, 100L, 10000L, OffloadedReadPriority.TIERED_STORAGE_FIRST)); + + namespaces.run(split("remove-offload-policies myprop/clust/ns1")); + verify(mockNamespaces).removeOffloadPolicies("myprop/clust/ns1"); + + namespaces.run(split("get-offload-policies myprop/clust/ns1")); + verify(mockNamespaces).getOffloadPolicies("myprop/clust/ns1"); + + namespaces.run(split("remove-message-ttl myprop/clust/ns1")); + verify(mockNamespaces).removeNamespaceMessageTTL("myprop/clust/ns1"); + + namespaces.run(split("set-deduplication-snapshot-interval myprop/clust/ns1 -i 1000")); + verify(mockNamespaces).setDeduplicationSnapshotInterval("myprop/clust/ns1", 1000); + namespaces.run(split("get-deduplication-snapshot-interval myprop/clust/ns1")); + verify(mockNamespaces).getDeduplicationSnapshotInterval("myprop/clust/ns1"); + namespaces.run(split("remove-deduplication-snapshot-interval myprop/clust/ns1")); + verify(mockNamespaces).removeDeduplicationSnapshotInterval("myprop/clust/ns1"); + + } + + + @Test + public void namespaces() throws Exception { + PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); + Namespaces mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + Lookup mockLookup = mock(Lookup.class); + when(admin.lookups()).thenReturn(mockLookup); + + CmdNamespaces namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("list myprop")); + verify(mockNamespaces).getNamespaces("myprop"); + + namespaces.run(split("list-cluster myprop/clust")); + verify(mockNamespaces).getNamespaces("myprop", "clust"); + + namespaces.run(split("topics myprop/clust/ns1")); + verify(mockNamespaces).getTopics("myprop/clust/ns1", ListNamespaceTopicsOptions.builder().build()); + + namespaces.run(split("policies myprop/clust/ns1")); + verify(mockNamespaces).getPolicies("myprop/clust/ns1"); + + namespaces.run(split("create myprop/clust/ns1")); + verify(mockNamespaces).createNamespace("myprop/clust/ns1"); + + namespaces.run(split("delete myprop/clust/ns1")); + verify(mockNamespaces).deleteNamespace("myprop/clust/ns1", false); + + namespaces.run(split("permissions myprop/clust/ns1")); + verify(mockNamespaces).getPermissions("myprop/clust/ns1"); + + namespaces.run(split("grant-permission myprop/clust/ns1 --role role1 --actions produce,consume")); + verify(mockNamespaces).grantPermissionOnNamespace("myprop/clust/ns1", "role1", + EnumSet.of(AuthAction.produce, AuthAction.consume)); + + namespaces.run(split("revoke-permission myprop/clust/ns1 --role role1")); + verify(mockNamespaces).revokePermissionsOnNamespace("myprop/clust/ns1", "role1"); + + namespaces.run(split("set-clusters myprop/clust/ns1 -c use,usw,usc")); + verify(mockNamespaces).setNamespaceReplicationClusters("myprop/clust/ns1", + Sets.newHashSet("use", "usw", "usc")); + + namespaces.run(split("get-clusters myprop/clust/ns1")); + verify(mockNamespaces).getNamespaceReplicationClusters("myprop/clust/ns1"); + + namespaces.run(split("set-subscription-types-enabled myprop/clust/ns1 -t Shared,Failover")); + verify(mockNamespaces).setSubscriptionTypesEnabled("myprop/clust/ns1", + Sets.newHashSet(SubscriptionType.Shared, SubscriptionType.Failover)); + + namespaces.run(split("get-subscription-types-enabled myprop/clust/ns1")); + verify(mockNamespaces).getSubscriptionTypesEnabled("myprop/clust/ns1"); + + namespaces.run(split("remove-subscription-types-enabled myprop/clust/ns1")); + verify(mockNamespaces).removeSubscriptionTypesEnabled("myprop/clust/ns1"); + + namespaces.run(split("get-schema-validation-enforce myprop/clust/ns1 -ap")); + verify(mockNamespaces).getSchemaValidationEnforced("myprop/clust/ns1", true); + + namespaces + .run(split("set-bookie-affinity-group myprop/clust/ns1 --primary-group test1 --secondary-group test2")); + verify(mockNamespaces).setBookieAffinityGroup("myprop/clust/ns1", + BookieAffinityGroupData.builder() + .bookkeeperAffinityGroupPrimary("test1") + .bookkeeperAffinityGroupSecondary("test2") + .build()); + + namespaces.run(split("get-bookie-affinity-group myprop/clust/ns1")); + verify(mockNamespaces).getBookieAffinityGroup("myprop/clust/ns1"); + + namespaces.run(split("delete-bookie-affinity-group myprop/clust/ns1")); + verify(mockNamespaces).deleteBookieAffinityGroup("myprop/clust/ns1"); + + namespaces.run(split("set-replicator-dispatch-rate myprop/clust/ns1 -md 10 -bd 11 -dt 12")); + verify(mockNamespaces).setReplicatorDispatchRate("myprop/clust/ns1", DispatchRate.builder() + .dispatchThrottlingRateInMsg(10) + .dispatchThrottlingRateInByte(11) + .ratePeriodInSecond(12) + .build()); + + namespaces.run(split("get-replicator-dispatch-rate myprop/clust/ns1")); + verify(mockNamespaces).getReplicatorDispatchRate("myprop/clust/ns1"); + + namespaces.run(split("remove-replicator-dispatch-rate myprop/clust/ns1")); + verify(mockNamespaces).removeReplicatorDispatchRate("myprop/clust/ns1"); + + namespaces.run(split("unload myprop/clust/ns1")); + verify(mockNamespaces).unload("myprop/clust/ns1"); + + // message_age must have time limit, destination_storage must have size limit + Assert.assertFalse(namespaces.run( + split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10G -t message_age"))); + Assert.assertFalse(namespaces.run( + split("set-backlog-quota myprop/clust/ns1 -p producer_exception -lt 10h -t destination_storage"))); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("unload myprop/clust/ns1 -b 0x80000000_0xffffffff")); + verify(mockNamespaces).unloadNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff", null); + + namespaces.run(split("split-bundle myprop/clust/ns1 -b 0x00000000_0xffffffff")); + verify(mockNamespaces).splitNamespaceBundle("myprop/clust/ns1", "0x00000000_0xffffffff", false, null); + + namespaces.run(split("get-backlog-quotas myprop/clust/ns1")); + verify(mockNamespaces).getBacklogQuotaMap("myprop/clust/ns1"); + + namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_request_hold -l 10")); + verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", + BacklogQuota.builder() + .limitSize(10) + .retentionPolicy(RetentionPolicy.producer_request_hold) + .build(), + BacklogQuota.BacklogQuotaType.destination_storage); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10K")); + verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", + BacklogQuota.builder() + .limitSize(10 * 1024) + .retentionPolicy(RetentionPolicy.producer_exception) + .build(), + BacklogQuota.BacklogQuotaType.destination_storage); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10M")); + verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", + BacklogQuota.builder() + .limitSize(10 * 1024 * 1024) + .retentionPolicy(RetentionPolicy.producer_exception) + .build(), + BacklogQuota.BacklogQuotaType.destination_storage); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10G")); + verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", + BacklogQuota.builder() + .limitSize(10L * 1024 * 1024 * 1024) + .retentionPolicy(RetentionPolicy.producer_exception) + .build(), + BacklogQuota.BacklogQuotaType.destination_storage); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p consumer_backlog_eviction -lt 10m -t message_age")); + verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", + BacklogQuota.builder() + .limitTime(10 * 60) + .retentionPolicy(RetentionPolicy.consumer_backlog_eviction) + .build(), + BacklogQuota.BacklogQuotaType.message_age); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("set-message-ttl myprop/clust/ns1 -ttl 300")); + verify(mockNamespaces).setNamespaceMessageTTL("myprop/clust/ns1", 300); + + namespaces.run(split("set-subscription-expiration-time myprop/clust/ns1 -t 60")); + verify(mockNamespaces).setSubscriptionExpirationTime("myprop/clust/ns1", 60); + + namespaces.run(split("get-deduplication myprop/clust/ns1")); + verify(mockNamespaces).getDeduplicationStatus("myprop/clust/ns1"); + namespaces.run(split("set-deduplication myprop/clust/ns1 --enable")); + verify(mockNamespaces).setDeduplicationStatus("myprop/clust/ns1", true); + namespaces.run(split("remove-deduplication myprop/clust/ns1")); + verify(mockNamespaces).removeDeduplicationStatus("myprop/clust/ns1"); + + namespaces.run(split("set-auto-topic-creation myprop/clust/ns1 -e -t non-partitioned")); + verify(mockNamespaces).setAutoTopicCreation("myprop/clust/ns1", + AutoTopicCreationOverride.builder() + .allowAutoTopicCreation(true) + .topicType(TopicType.NON_PARTITIONED.toString()) + .build()); + + namespaces.run(split("get-auto-topic-creation myprop/clust/ns1")); + verify(mockNamespaces).getAutoTopicCreation("myprop/clust/ns1"); + + namespaces.run(split("remove-auto-topic-creation myprop/clust/ns1")); + verify(mockNamespaces).removeAutoTopicCreation("myprop/clust/ns1"); + + namespaces.run(split("set-auto-subscription-creation myprop/clust/ns1 -e")); + verify(mockNamespaces).setAutoSubscriptionCreation("myprop/clust/ns1", + AutoSubscriptionCreationOverride.builder().allowAutoSubscriptionCreation(true).build()); + + namespaces.run(split("get-auto-subscription-creation myprop/clust/ns1")); + verify(mockNamespaces).getAutoSubscriptionCreation("myprop/clust/ns1"); + + namespaces.run(split("remove-auto-subscription-creation myprop/clust/ns1")); + verify(mockNamespaces).removeAutoSubscriptionCreation("myprop/clust/ns1"); + + namespaces.run(split("get-message-ttl myprop/clust/ns1")); + verify(mockNamespaces).getNamespaceMessageTTL("myprop/clust/ns1"); + + namespaces.run(split("get-subscription-expiration-time myprop/clust/ns1")); + verify(mockNamespaces).getSubscriptionExpirationTime("myprop/clust/ns1"); + + namespaces.run(split("remove-subscription-expiration-time myprop/clust/ns1")); + verify(mockNamespaces).removeSubscriptionExpirationTime("myprop/clust/ns1"); + + namespaces.run(split("set-anti-affinity-group myprop/clust/ns1 -g group")); + verify(mockNamespaces).setNamespaceAntiAffinityGroup("myprop/clust/ns1", "group"); + + namespaces.run(split("get-anti-affinity-group myprop/clust/ns1")); + verify(mockNamespaces).getNamespaceAntiAffinityGroup("myprop/clust/ns1"); + + namespaces.run(split("get-anti-affinity-namespaces -p dummy -c cluster -g group")); + verify(mockNamespaces).getAntiAffinityNamespaces("dummy", "cluster", "group"); + + namespaces.run(split("delete-anti-affinity-group myprop/clust/ns1 ")); + verify(mockNamespaces).deleteNamespaceAntiAffinityGroup("myprop/clust/ns1"); + + + namespaces.run(split("set-retention myprop/clust/ns1 -t 1h -s 1M")); + verify(mockNamespaces).setRetention("myprop/clust/ns1", + new RetentionPolicies(60, 1)); + + // Test with default time unit (seconds) + namespaces = new CmdNamespaces(() -> admin); + reset(mockNamespaces); + namespaces.run(split("set-retention myprop/clust/ns1 -t 120 -s 20M")); + verify(mockNamespaces).setRetention("myprop/clust/ns1", + new RetentionPolicies(2, 20)); + + // Test with explicit time unit (seconds) + namespaces = new CmdNamespaces(() -> admin); + reset(mockNamespaces); + namespaces.run(split("set-retention myprop/clust/ns1 -t 120s -s 20M")); + verify(mockNamespaces).setRetention("myprop/clust/ns1", + new RetentionPolicies(2, 20)); + + // Test size with default size less than 1 mb + namespaces = new CmdNamespaces(() -> admin); + reset(mockNamespaces); + namespaces.run(split("set-retention myprop/clust/ns1 -t 120s -s 4096")); + verify(mockNamespaces).setRetention("myprop/clust/ns1", + new RetentionPolicies(2, 0)); + + // Test size with default size greater than 1mb + namespaces = new CmdNamespaces(() -> admin); + reset(mockNamespaces); + namespaces.run(split("set-retention myprop/clust/ns1 -t 180 -s " + (2 * 1024 * 1024))); + verify(mockNamespaces).setRetention("myprop/clust/ns1", + new RetentionPolicies(3, 2)); + + namespaces.run(split("get-retention myprop/clust/ns1")); + verify(mockNamespaces).getRetention("myprop/clust/ns1"); + + namespaces.run(split("remove-retention myprop/clust/ns1")); + verify(mockNamespaces).removeRetention("myprop/clust/ns1"); + + namespaces.run(split("set-delayed-delivery myprop/clust/ns1 -e -t 1s")); + verify(mockNamespaces).setDelayedDeliveryMessages("myprop/clust/ns1", + DelayedDeliveryPolicies.builder().tickTime(1000).active(true).build()); + + namespaces.run(split("get-delayed-delivery myprop/clust/ns1")); + verify(mockNamespaces).getDelayedDelivery("myprop/clust/ns1"); + + namespaces.run(split("remove-delayed-delivery myprop/clust/ns1")); + verify(mockNamespaces).removeDelayedDeliveryMessages("myprop/clust/ns1"); + + namespaces.run(split("set-inactive-topic-policies myprop/clust/ns1 -e -t 1s -m delete_when_no_subscriptions")); + verify(mockNamespaces).setInactiveTopicPolicies("myprop/clust/ns1", + new InactiveTopicPolicies(InactiveTopicDeleteMode.delete_when_no_subscriptions, 1, + true)); + + namespaces.run(split("get-inactive-topic-policies myprop/clust/ns1")); + verify(mockNamespaces).getInactiveTopicPolicies("myprop/clust/ns1"); + + namespaces.run(split("remove-inactive-topic-policies myprop/clust/ns1")); + verify(mockNamespaces).removeInactiveTopicPolicies("myprop/clust/ns1"); + + namespaces.run(split("clear-backlog myprop/clust/ns1 -force")); + verify(mockNamespaces).clearNamespaceBacklog("myprop/clust/ns1"); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("set-message-ttl myprop/clust/ns1 -ttl 6m")); + verify(mockNamespaces).setNamespaceMessageTTL("myprop/clust/ns1", 6 * 60); + + namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff myprop/clust/ns1 -force")); + verify(mockNamespaces).clearNamespaceBundleBacklog("myprop/clust/ns1", "0x80000000_0xffffffff"); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("clear-backlog -s my-sub myprop/clust/ns1 -force")); + verify(mockNamespaces).clearNamespaceBacklogForSubscription("myprop/clust/ns1", "my-sub"); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1 -force")); + verify(mockNamespaces).clearNamespaceBundleBacklogForSubscription("myprop/clust/ns1", "0x80000000_0xffffffff", + "my-sub"); + + namespaces.run(split("unsubscribe -s my-sub myprop/clust/ns1")); + verify(mockNamespaces).unsubscribeNamespace("myprop/clust/ns1", "my-sub"); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("unsubscribe -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1")); + verify(mockNamespaces).unsubscribeNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub"); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("get-max-producers-per-topic myprop/clust/ns1")); + verify(mockNamespaces).getMaxProducersPerTopic("myprop/clust/ns1"); + + namespaces.run(split("set-max-producers-per-topic myprop/clust/ns1 -p 1")); + verify(mockNamespaces).setMaxProducersPerTopic("myprop/clust/ns1", 1); + + namespaces.run(split("remove-max-producers-per-topic myprop/clust/ns1")); + verify(mockNamespaces).removeMaxProducersPerTopic("myprop/clust/ns1"); + + namespaces.run(split("get-max-consumers-per-topic myprop/clust/ns1")); + verify(mockNamespaces).getMaxConsumersPerTopic("myprop/clust/ns1"); + + namespaces.run(split("set-max-consumers-per-topic myprop/clust/ns1 -c 2")); + verify(mockNamespaces).setMaxConsumersPerTopic("myprop/clust/ns1", 2); + + namespaces.run(split("remove-max-consumers-per-topic myprop/clust/ns1")); + verify(mockNamespaces).removeMaxConsumersPerTopic("myprop/clust/ns1"); + + namespaces.run(split("get-max-consumers-per-subscription myprop/clust/ns1")); + verify(mockNamespaces).getMaxConsumersPerSubscription("myprop/clust/ns1"); + + namespaces.run(split("remove-max-consumers-per-subscription myprop/clust/ns1")); + verify(mockNamespaces).removeMaxConsumersPerSubscription("myprop/clust/ns1"); + + namespaces.run(split("set-max-consumers-per-subscription myprop/clust/ns1 -c 3")); + verify(mockNamespaces).setMaxConsumersPerSubscription("myprop/clust/ns1", 3); + + namespaces.run(split("get-max-unacked-messages-per-subscription myprop/clust/ns1")); + verify(mockNamespaces).getMaxUnackedMessagesPerSubscription("myprop/clust/ns1"); + + namespaces.run(split("set-max-unacked-messages-per-subscription myprop/clust/ns1 -c 3")); + verify(mockNamespaces).setMaxUnackedMessagesPerSubscription("myprop/clust/ns1", 3); + + namespaces.run(split("remove-max-unacked-messages-per-subscription myprop/clust/ns1")); + verify(mockNamespaces).removeMaxUnackedMessagesPerSubscription("myprop/clust/ns1"); + + namespaces.run(split("get-max-unacked-messages-per-consumer myprop/clust/ns1")); + verify(mockNamespaces).getMaxUnackedMessagesPerConsumer("myprop/clust/ns1"); + + namespaces.run(split("set-max-unacked-messages-per-consumer myprop/clust/ns1 -c 3")); + verify(mockNamespaces).setMaxUnackedMessagesPerConsumer("myprop/clust/ns1", 3); + + namespaces.run(split("remove-max-unacked-messages-per-consumer myprop/clust/ns1")); + verify(mockNamespaces).removeMaxUnackedMessagesPerConsumer("myprop/clust/ns1"); + + mockNamespaces = mock(Namespaces.class); + when(admin.namespaces()).thenReturn(mockNamespaces); + namespaces = new CmdNamespaces(() -> admin); + + namespaces.run(split("set-dispatch-rate myprop/clust/ns1 -md -1 -bd -1 -dt 2")); + verify(mockNamespaces).setDispatchRate("myprop/clust/ns1", DispatchRate.builder() + .dispatchThrottlingRateInMsg(-1) + .dispatchThrottlingRateInByte(-1) + .ratePeriodInSecond(2) + .build()); + + namespaces.run(split("get-dispatch-rate myprop/clust/ns1")); + verify(mockNamespaces).getDispatchRate("myprop/clust/ns1"); + + namespaces.run(split("remove-dispatch-rate myprop/clust/ns1")); + verify(mockNamespaces).removeDispatchRate("myprop/clust/ns1"); + + namespaces.run(split("set-publish-rate myprop/clust/ns1 -m 10 -b 20")); + verify(mockNamespaces).setPublishRate("myprop/clust/ns1", new PublishRate(10, 20)); + + namespaces.run(split("get-publish-rate myprop/clust/ns1")); + verify(mockNamespaces).getPublishRate("myprop/clust/ns1"); + + namespaces.run(split("remove-publish-rate myprop/clust/ns1")); + verify(mockNamespaces).removePublishRate("myprop/clust/ns1"); + + namespaces.run(split("set-subscribe-rate myprop/clust/ns1 -sr 2 -st 60")); + verify(mockNamespaces).setSubscribeRate("myprop/clust/ns1", new SubscribeRate(2, 60)); + + namespaces.run(split("get-subscribe-rate myprop/clust/ns1")); + verify(mockNamespaces).getSubscribeRate("myprop/clust/ns1"); + + namespaces.run(split("remove-subscribe-rate myprop/clust/ns1")); + verify(mockNamespaces).removeSubscribeRate("myprop/clust/ns1"); + + namespaces.run(split("set-subscription-dispatch-rate myprop/clust/ns1 -md -1 -bd -1 -dt 2")); + verify(mockNamespaces).setSubscriptionDispatchRate("myprop/clust/ns1", DispatchRate.builder() + .dispatchThrottlingRateInMsg(-1) + .dispatchThrottlingRateInByte(-1) + .ratePeriodInSecond(2) + .build()); + + namespaces.run(split("get-subscription-dispatch-rate myprop/clust/ns1")); + verify(mockNamespaces).getSubscriptionDispatchRate("myprop/clust/ns1"); + + namespaces.run(split("remove-subscription-dispatch-rate myprop/clust/ns1")); + verify(mockNamespaces).removeSubscriptionDispatchRate("myprop/clust/ns1"); + + namespaces.run(split("get-compaction-threshold myprop/clust/ns1")); + verify(mockNamespaces).getCompactionThreshold("myprop/clust/ns1"); + + namespaces.run(split("remove-compaction-threshold myprop/clust/ns1")); + verify(mockNamespaces).removeCompactionThreshold("myprop/clust/ns1"); + + namespaces.run(split("set-compaction-threshold myprop/clust/ns1 -t 1G")); + verify(mockNamespaces).setCompactionThreshold("myprop/clust/ns1", 1024 * 1024 * 1024); + + namespaces.run(split("get-offload-threshold myprop/clust/ns1")); + verify(mockNamespaces).getOffloadThreshold("myprop/clust/ns1"); + + namespaces.run(split("set-offload-threshold myprop/clust/ns1 -s 1G")); + verify(mockNamespaces).setOffloadThreshold("myprop/clust/ns1", 1024 * 1024 * 1024); + + namespaces.run(split("get-offload-deletion-lag myprop/clust/ns1")); + verify(mockNamespaces).getOffloadDeleteLagMs("myprop/clust/ns1"); + + namespaces.run(split("set-offload-deletion-lag myprop/clust/ns1 -l 1d")); + verify(mockNamespaces).setOffloadDeleteLag("myprop/clust/ns1", 24 * 60 * 60, TimeUnit.SECONDS); + + namespaces.run(split("clear-offload-deletion-lag myprop/clust/ns1")); + verify(mockNamespaces).clearOffloadDeleteLag("myprop/clust/ns1"); + + namespaces.run(split( + "set-offload-policies myprop/clust/ns1 -r test-region -d aws-s3 -b test-bucket -e http://test.endpoint -mbs 32M -rbs 5M -oat 10M -oats 100 -oae 10s -orp tiered-storage-first")); + verify(mockNamespaces).setOffloadPolicies("myprop/clust/ns1", + OffloadPoliciesImpl.create("aws-s3", "test-region", "test-bucket", + "http://test.endpoint", null, null, null, null, 32 * 1024 * 1024, 5 * 1024 * 1024, + 10 * 1024 * 1024L, 100L, 10000L, OffloadedReadPriority.TIERED_STORAGE_FIRST)); + + namespaces.run(split("remove-offload-policies myprop/clust/ns1")); + verify(mockNamespaces).removeOffloadPolicies("myprop/clust/ns1"); + + namespaces.run(split("get-offload-policies myprop/clust/ns1")); + verify(mockNamespaces).getOffloadPolicies("myprop/clust/ns1"); + + namespaces.run(split("remove-message-ttl myprop/clust/ns1")); + verify(mockNamespaces).removeNamespaceMessageTTL("myprop/clust/ns1"); + + namespaces.run(split("set-deduplication-snapshot-interval myprop/clust/ns1 -i 1000")); + verify(mockNamespaces).setDeduplicationSnapshotInterval("myprop/clust/ns1", 1000); + namespaces.run(split("get-deduplication-snapshot-interval myprop/clust/ns1")); + verify(mockNamespaces).getDeduplicationSnapshotInterval("myprop/clust/ns1"); + namespaces.run(split("remove-deduplication-snapshot-interval myprop/clust/ns1")); + verify(mockNamespaces).removeDeduplicationSnapshotInterval("myprop/clust/ns1"); + + } + + String[] split(String s) { + return s.split(" "); + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index ef23b944eddaf..1ecc8968a866f 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -40,10 +40,9 @@ import org.apache.pulsar.admin.cli.utils.converters.ByteUnitToLongConverter; import org.apache.pulsar.admin.cli.utils.converters.TimeUnitToMillisConverter; import org.apache.pulsar.admin.cli.utils.converters.TimeUnitToSecondsConverter; -import org.apache.pulsar.admin.cli.utils.validators.IntegerMaxValueLongValidator; import org.apache.pulsar.admin.cli.utils.validators.MinNegativeOneValidator; import org.apache.pulsar.admin.cli.utils.validators.NonNegativeValueValidator; -import org.apache.pulsar.admin.cli.utils.validators.PositiveValueValidator; +import org.apache.pulsar.admin.cli.utils.validators.PositiveIntegerValueValidator; import org.apache.pulsar.client.admin.ListNamespaceTopicsOptions; import org.apache.pulsar.client.admin.Mode; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -411,7 +410,7 @@ private class SetMessageTTL extends CliCommand { description = "Message TTL in seconds (or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w). " + "When the value is set to `0`, TTL is disabled.", required = true, converter = TimeUnitToSecondsConverter.class, - validateValueWith = {IntegerMaxValueLongValidator.class, NonNegativeValueValidator.class}) + validateValueWith = {NonNegativeValueValidator.class}) private Long messageTTLInSecond; @Override @@ -1252,7 +1251,7 @@ private class SetBacklogQuota extends CliCommand { @Parameter(names = { "-l", "--limit" }, description = "Size limit (eg: 10M, 16G)", converter = ByteUnitToLongConverter.class) - private Long limit; + private Long limit = 0L; @Parameter(names = { "-lt", "--limitTime" }, description = "Time limit in second (or minutes, hours, days, weeks eg: 100m, 3h, 2d, 5w), " @@ -2229,7 +2228,7 @@ private class SetOffloadPolicies extends CliCommand { + "s3 and google-cloud-storage requires this parameter", required = false, converter = ByteUnitIntegerConverter.class, - validateValueWith = {PositiveValueValidator.class, IntegerMaxValueLongValidator.class}) + validateValueWith = {PositiveIntegerValueValidator.class}) private Integer maxBlockSizeInBytes = OffloadPoliciesImpl.DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; @Parameter( @@ -2237,7 +2236,7 @@ private class SetOffloadPolicies extends CliCommand { description = "Read buffer size (eg: 1M, 5M), default is 1MB", required = false, converter = ByteUnitIntegerConverter.class, - validateValueWith = {PositiveValueValidator.class, IntegerMaxValueLongValidator.class}) + validateValueWith = {PositiveIntegerValueValidator.class}) private Integer readBufferSizeInBytes = OffloadPoliciesImpl.DEFAULT_READ_BUFFER_SIZE_IN_BYTES; @Parameter( @@ -2246,7 +2245,7 @@ private class SetOffloadPolicies extends CliCommand { + "(or seconds,minutes,hours,days,weeks eg: 10s, 100m, 3h, 2d, 5w).", required = false, converter = TimeUnitToMillisConverter.class, - validateValueWith = PositiveValueValidator.class) + validateValueWith = PositiveIntegerValueValidator.class) private Long offloadAfterElapsedInMillis = OffloadPoliciesImpl.DEFAULT_OFFLOAD_DELETION_LAG_IN_MILLIS; @Parameter( diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/ValueValidationUtils.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/ValueValidationUtils.java index 280ec774096f9..9c5c6f9b99ce2 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/ValueValidationUtils.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/ValueValidationUtils.java @@ -37,6 +37,12 @@ public static void positiveCheck(String paramName, long value) { } } + public static void positiveCheck(String paramName, int value) { + if (value <= 0) { + throw new ParameterException(paramName + " cannot be less than or equal to <0>!"); + } + } + public static void emptyCheck(String paramName, String value) { if (StringUtils.isEmpty(value)) { throw new ParameterException("The value of " + paramName + " can't be empty"); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveIntegerValueValidator.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveIntegerValueValidator.java new file mode 100644 index 0000000000000..d3d44d60cb889 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveIntegerValueValidator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils.validators; + +import com.beust.jcommander.IValueValidator; +import com.beust.jcommander.ParameterException; +import org.apache.pulsar.admin.cli.utils.ValueValidationUtils; + +public class PositiveIntegerValueValidator implements IValueValidator { + + @Override + public void validate(String name, Integer value) throws ParameterException { + ValueValidationUtils.positiveCheck(name, value); + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveValueValidator.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveLongValueValidator.java similarity index 94% rename from pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveValueValidator.java rename to pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveLongValueValidator.java index 901903f74b7b1..d95d41b2c0762 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveValueValidator.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/validators/PositiveLongValueValidator.java @@ -22,7 +22,7 @@ import com.beust.jcommander.ParameterException; import org.apache.pulsar.admin.cli.utils.ValueValidationUtils; -public class PositiveValueValidator implements IValueValidator { +public class PositiveLongValueValidator implements IValueValidator { @Override public void validate(String name, Long value) throws ParameterException { From 9d3a9c1f39b4a129ee424194a310dd357c37cd66 Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Thu, 6 Jul 2023 23:20:42 +0900 Subject: [PATCH 13/14] Fix human error wrt long/integer converter --- .../main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 1ecc8968a866f..d93433c786444 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -43,6 +43,7 @@ import org.apache.pulsar.admin.cli.utils.validators.MinNegativeOneValidator; import org.apache.pulsar.admin.cli.utils.validators.NonNegativeValueValidator; import org.apache.pulsar.admin.cli.utils.validators.PositiveIntegerValueValidator; +import org.apache.pulsar.admin.cli.utils.validators.PositiveLongValueValidator; import org.apache.pulsar.client.admin.ListNamespaceTopicsOptions; import org.apache.pulsar.client.admin.Mode; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -2245,7 +2246,7 @@ private class SetOffloadPolicies extends CliCommand { + "(or seconds,minutes,hours,days,weeks eg: 10s, 100m, 3h, 2d, 5w).", required = false, converter = TimeUnitToMillisConverter.class, - validateValueWith = PositiveIntegerValueValidator.class) + validateValueWith = PositiveLongValueValidator.class) private Long offloadAfterElapsedInMillis = OffloadPoliciesImpl.DEFAULT_OFFLOAD_DELETION_LAG_IN_MILLIS; @Parameter( From 8aee04f7dcd3ec5cf61889d7d4a12cd3df9e4819 Mon Sep 17 00:00:00 2001 From: joohyukkim Date: Fri, 7 Jul 2023 08:43:27 +0900 Subject: [PATCH 14/14] Update PulsarAdminToolConverterTest.java --- .../cli/PulsarAdminToolConverterTest.java | 64 ++++++++----------- 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolConverterTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolConverterTest.java index 54612fbd6d9e2..fc56c12029fe8 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolConverterTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolConverterTest.java @@ -55,46 +55,7 @@ @Slf4j public class PulsarAdminToolConverterTest { - @Test - public void namespacesSimpl() throws Exception { - PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); - Namespaces mockNamespaces = mock(Namespaces.class); - when(admin.namespaces()).thenReturn(mockNamespaces); - Lookup mockLookup = mock(Lookup.class); - when(admin.lookups()).thenReturn(mockLookup); - - CmdNamespaces namespaces = new CmdNamespaces(() -> admin); - - mockNamespaces = mock(Namespaces.class); - when(admin.namespaces()).thenReturn(mockNamespaces); - namespaces = new CmdNamespaces(() -> admin); - namespaces.run(split( - "set-offload-policies myprop/clust/ns1 -r test-region -d aws-s3 -b test-bucket -e http://test.endpoint -mbs 32M -rbs 5M -oat 10M -oats 100 -oae 10s -orp tiered-storage-first")); - verify(mockNamespaces).setOffloadPolicies("myprop/clust/ns1", - OffloadPoliciesImpl.create("aws-s3", "test-region", "test-bucket", - "http://test.endpoint", null, null, null, null, 32 * 1024 * 1024, 5 * 1024 * 1024, - 10 * 1024 * 1024L, 100L, 10000L, OffloadedReadPriority.TIERED_STORAGE_FIRST)); - - namespaces.run(split("remove-offload-policies myprop/clust/ns1")); - verify(mockNamespaces).removeOffloadPolicies("myprop/clust/ns1"); - - namespaces.run(split("get-offload-policies myprop/clust/ns1")); - verify(mockNamespaces).getOffloadPolicies("myprop/clust/ns1"); - - namespaces.run(split("remove-message-ttl myprop/clust/ns1")); - verify(mockNamespaces).removeNamespaceMessageTTL("myprop/clust/ns1"); - - namespaces.run(split("set-deduplication-snapshot-interval myprop/clust/ns1 -i 1000")); - verify(mockNamespaces).setDeduplicationSnapshotInterval("myprop/clust/ns1", 1000); - namespaces.run(split("get-deduplication-snapshot-interval myprop/clust/ns1")); - verify(mockNamespaces).getDeduplicationSnapshotInterval("myprop/clust/ns1"); - namespaces.run(split("remove-deduplication-snapshot-interval myprop/clust/ns1")); - verify(mockNamespaces).removeDeduplicationSnapshotInterval("myprop/clust/ns1"); - - } - - @Test public void namespaces() throws Exception { PulsarAdmin admin = Mockito.mock(PulsarAdmin.class); @@ -262,6 +223,31 @@ public void namespaces() throws Exception { when(admin.namespaces()).thenReturn(mockNamespaces); namespaces = new CmdNamespaces(() -> admin); + namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -lt 10000 -t message_age")); + verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", + BacklogQuota.builder() + .limitTime(10000) + .retentionPolicy(RetentionPolicy.producer_exception) + .build(), + BacklogQuota.BacklogQuotaType.message_age); + + namespaces.run(split("set-persistence myprop/clust/ns1 -e 2 -w 1 -a 1 -r 100.0")); + verify(mockNamespaces).setPersistence("myprop/clust/ns1", + new PersistencePolicies(2, 1, 1, 100.0d)); + + namespaces.run(split("get-persistence myprop/clust/ns1")); + verify(mockNamespaces).getPersistence("myprop/clust/ns1"); + + namespaces.run(split("remove-persistence myprop/clust/ns1")); + verify(mockNamespaces).removePersistence("myprop/clust/ns1"); + + namespaces.run(split("get-max-subscriptions-per-topic myprop/clust/ns1")); + verify(mockNamespaces).getMaxSubscriptionsPerTopic("myprop/clust/ns1"); + namespaces.run(split("set-max-subscriptions-per-topic myprop/clust/ns1 -m 300")); + verify(mockNamespaces).setMaxSubscriptionsPerTopic("myprop/clust/ns1", 300); + namespaces.run(split("remove-max-subscriptions-per-topic myprop/clust/ns1")); + verify(mockNamespaces).removeMaxSubscriptionsPerTopic("myprop/clust/ns1"); + namespaces.run(split("set-message-ttl myprop/clust/ns1 -ttl 300")); verify(mockNamespaces).setNamespaceMessageTTL("myprop/clust/ns1", 300);