diff --git a/docs/api-reference/dynamic-configuration-api.md b/docs/api-reference/dynamic-configuration-api.md index 85c7dcf5e492..90b7028f2478 100644 --- a/docs/api-reference/dynamic-configuration-api.md +++ b/docs/api-reference/dynamic-configuration-api.md @@ -91,8 +91,6 @@ Host: http://ROUTER_IP:ROUTER_PORT ```json { "millisToWaitBeforeDeleting": 900000, - "mergeBytesLimit": 524288000, - "mergeSegmentsLimit": 100, "maxSegmentsToMove": 100, "replicantLifetime": 15, "replicationThrottleLimit": 500, @@ -159,8 +157,6 @@ curl "http://ROUTER_IP:ROUTER_PORT/druid/coordinator/v1/config" \ --header 'Content-Type: application/json' \ --data '{ "millisToWaitBeforeDeleting": 900000, - "mergeBytesLimit": 524288000, - "mergeSegmentsLimit": 100, "maxSegmentsToMove": 5, "percentOfSegmentsToConsiderPerMove": 100, "useBatchedSegmentSampler": true, @@ -192,8 +188,6 @@ Content-Length: 683 { "millisToWaitBeforeDeleting": 900000, - "mergeBytesLimit": 524288000, - "mergeSegmentsLimit": 100, "maxSegmentsToMove": 5, "percentOfSegmentsToConsiderPerMove": 100, "useBatchedSegmentSampler": true, @@ -295,7 +289,7 @@ Host: http://ROUTER_IP:ROUTER_PORT "comment": "", "ip": "127.0.0.1" }, - "payload": "{\"millisToWaitBeforeDeleting\":900000,\"mergeBytesLimit\":524288000,\"mergeSegmentsLimit\":100,\"maxSegmentsToMove\":5,\"replicantLifetime\":15,\"replicationThrottleLimit\":10,\"balancerComputeThreads\":1,\"killDataSourceWhitelist\":[],\"killPendingSegmentsSkipList\":[],\"maxSegmentsInNodeLoadingQueue\":100,\"decommissioningNodes\":[],\"decommissioningMaxPercentOfMaxSegmentsToMove\":70,\"pauseCoordination\":false,\"replicateAfterLoadTimeout\":false,\"maxNonPrimaryReplicantsToLoad\":2147483647,\"useRoundRobinSegmentAssignment\":true,\"smartSegmentLoading\":true,\"debugDimensions\":null}", + "payload": "{\"millisToWaitBeforeDeleting\":900000,\"maxSegmentsToMove\":5,\"replicantLifetime\":15,\"replicationThrottleLimit\":10,\"balancerComputeThreads\":1,\"killDataSourceWhitelist\":[],\"killPendingSegmentsSkipList\":[],\"maxSegmentsInNodeLoadingQueue\":100,\"decommissioningNodes\":[],\"decommissioningMaxPercentOfMaxSegmentsToMove\":70,\"pauseCoordination\":false,\"replicateAfterLoadTimeout\":false,\"maxNonPrimaryReplicantsToLoad\":2147483647,\"useRoundRobinSegmentAssignment\":true,\"smartSegmentLoading\":true,\"debugDimensions\":null}", "auditTime": "2023-10-03T20:59:51.622Z" } ] diff --git a/docs/configuration/index.md b/docs/configuration/index.md index a8dd77efc01a..8f4f8065f1d9 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -951,8 +951,6 @@ The following table shows the dynamic configuration properties for the Coordinat |Property|Description|Default| |--------|-----------|-------| |`millisToWaitBeforeDeleting`|How long does the Coordinator need to be a leader before it can start marking overshadowed segments as unused in metadata storage.| 900000 (15 mins)| -|`mergeBytesLimit`|The maximum total uncompressed size in bytes of segments to merge.|524288000L| -|`mergeSegmentsLimit`|The maximum number of segments that can be in a single [append task](../ingestion/tasks.md).|100| |`smartSegmentLoading`|Enables ["smart" segment loading mode](#smart-segment-loading) which dynamically computes the optimal values of several properties that maximize Coordinator performance.|true| |`maxSegmentsToMove`|The maximum number of segments that can be moved in a Historical tier at any given time.|100| |`replicantLifetime`|The maximum number of Coordinator runs for which a segment can wait in the load queue of a Historical before Druid raises an alert.|15| diff --git a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java index e834c7b1fd33..d805bad5e01c 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java @@ -51,8 +51,6 @@ public class CoordinatorDynamicConfig public static final String CONFIG_KEY = "coordinator.config"; private final long markSegmentAsUnusedDelayMillis; - private final long mergeBytesLimit; - private final int mergeSegmentsLimit; private final int maxSegmentsToMove; private final int replicantLifetime; private final int replicationThrottleLimit; @@ -98,8 +96,6 @@ public CoordinatorDynamicConfig( // updated to Jackson 2.9 it could be changed, see https://github.com/apache/druid/issues/7152 @JsonProperty("millisToWaitBeforeDeleting") long markSegmentAsUnusedDelayMillis, - @JsonProperty("mergeBytesLimit") long mergeBytesLimit, - @JsonProperty("mergeSegmentsLimit") int mergeSegmentsLimit, @JsonProperty("maxSegmentsToMove") int maxSegmentsToMove, @JsonProperty("replicantLifetime") int replicantLifetime, @JsonProperty("replicationThrottleLimit") int replicationThrottleLimit, @@ -127,8 +123,6 @@ public CoordinatorDynamicConfig( { this.markSegmentAsUnusedDelayMillis = markSegmentAsUnusedDelayMillis; - this.mergeBytesLimit = mergeBytesLimit; - this.mergeSegmentsLimit = mergeSegmentsLimit; this.maxSegmentsToMove = maxSegmentsToMove; this.smartSegmentLoading = Builder.valueOrDefault(smartSegmentLoading, Defaults.SMART_SEGMENT_LOADING); @@ -212,18 +206,6 @@ public long getMarkSegmentAsUnusedDelayMillis() return markSegmentAsUnusedDelayMillis; } - @JsonProperty - public long getMergeBytesLimit() - { - return mergeBytesLimit; - } - - @JsonProperty - public int getMergeSegmentsLimit() - { - return mergeSegmentsLimit; - } - @JsonProperty public int getMaxSegmentsToMove() { @@ -294,7 +276,7 @@ public boolean isSmartSegmentLoading() * List of historical servers to 'decommission'. Coordinator will not assign new segments to 'decommissioning' * servers, and segments will be moved away from them to be placed on non-decommissioning servers. * - * @return list of host:port entries + * @return Set of host:port entries */ @JsonProperty public Set getDecommissioningNodes() @@ -332,8 +314,6 @@ public String toString() return "CoordinatorDynamicConfig{" + "leadingTimeMillisBeforeCanMarkAsUnusedOvershadowedSegments=" + markSegmentAsUnusedDelayMillis + - ", mergeBytesLimit=" + mergeBytesLimit + - ", mergeSegmentsLimit=" + mergeSegmentsLimit + ", maxSegmentsToMove=" + maxSegmentsToMove + ", replicantLifetime=" + replicantLifetime + ", replicationThrottleLimit=" + replicationThrottleLimit + @@ -362,8 +342,6 @@ public boolean equals(Object o) CoordinatorDynamicConfig that = (CoordinatorDynamicConfig) o; return markSegmentAsUnusedDelayMillis == that.markSegmentAsUnusedDelayMillis - && mergeBytesLimit == that.mergeBytesLimit - && mergeSegmentsLimit == that.mergeSegmentsLimit && maxSegmentsToMove == that.maxSegmentsToMove && balancerComputeThreads == that.balancerComputeThreads && replicantLifetime == that.replicantLifetime @@ -389,8 +367,6 @@ public int hashCode() { return Objects.hash( markSegmentAsUnusedDelayMillis, - mergeBytesLimit, - mergeSegmentsLimit, maxSegmentsToMove, replicantLifetime, replicationThrottleLimit, @@ -423,8 +399,6 @@ public static int getDefaultBalancerComputeThreads() private static class Defaults { static final long LEADING_MILLIS_BEFORE_MARK_UNUSED = TimeUnit.MINUTES.toMillis(15); - static final long MERGE_BYTES_LIMIT = 524_288_000L; - static final int MERGE_SEGMENTS_LIMIT = 100; static final int MAX_SEGMENTS_TO_MOVE = 100; static final int REPLICANT_LIFETIME = 15; static final int REPLICATION_THROTTLE_LIMIT = 500; @@ -441,8 +415,6 @@ private static class Defaults public static class Builder { private Long markSegmentAsUnusedDelayMillis; - private Long mergeBytesLimit; - private Integer mergeSegmentsLimit; private Integer maxSegmentsToMove; private Integer replicantLifetime; private Integer replicationThrottleLimit; @@ -466,8 +438,6 @@ public Builder() @JsonCreator public Builder( @JsonProperty("millisToWaitBeforeDeleting") @Nullable Long markSegmentAsUnusedDelayMillis, - @JsonProperty("mergeBytesLimit") @Nullable Long mergeBytesLimit, - @JsonProperty("mergeSegmentsLimit") @Nullable Integer mergeSegmentsLimit, @JsonProperty("maxSegmentsToMove") @Nullable Integer maxSegmentsToMove, @JsonProperty("replicantLifetime") @Nullable Integer replicantLifetime, @JsonProperty("replicationThrottleLimit") @Nullable Integer replicationThrottleLimit, @@ -486,8 +456,6 @@ public Builder( ) { this.markSegmentAsUnusedDelayMillis = markSegmentAsUnusedDelayMillis; - this.mergeBytesLimit = mergeBytesLimit; - this.mergeSegmentsLimit = mergeSegmentsLimit; this.maxSegmentsToMove = maxSegmentsToMove; this.replicantLifetime = replicantLifetime; this.replicationThrottleLimit = replicationThrottleLimit; @@ -600,8 +568,6 @@ public CoordinatorDynamicConfig build() markSegmentAsUnusedDelayMillis, Defaults.LEADING_MILLIS_BEFORE_MARK_UNUSED ), - valueOrDefault(mergeBytesLimit, Defaults.MERGE_BYTES_LIMIT), - valueOrDefault(mergeSegmentsLimit, Defaults.MERGE_SEGMENTS_LIMIT), valueOrDefault(maxSegmentsToMove, Defaults.MAX_SEGMENTS_TO_MOVE), valueOrDefault(replicantLifetime, Defaults.REPLICANT_LIFETIME), valueOrDefault(replicationThrottleLimit, Defaults.REPLICATION_THROTTLE_LIMIT), @@ -632,8 +598,6 @@ public CoordinatorDynamicConfig build(CoordinatorDynamicConfig defaults) markSegmentAsUnusedDelayMillis, defaults.getMarkSegmentAsUnusedDelayMillis() ), - valueOrDefault(mergeBytesLimit, defaults.getMergeBytesLimit()), - valueOrDefault(mergeSegmentsLimit, defaults.getMergeSegmentsLimit()), valueOrDefault(maxSegmentsToMove, defaults.getMaxSegmentsToMove()), valueOrDefault(replicantLifetime, defaults.getReplicantLifetime()), valueOrDefault(replicationThrottleLimit, defaults.getReplicationThrottleLimit()), diff --git a/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java b/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java index a97160ed27be..eb7fb1992873 100644 --- a/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java @@ -41,8 +41,6 @@ public void testSerde() throws Exception { String jsonStr = "{\n" + " \"millisToWaitBeforeDeleting\": 1,\n" - + " \"mergeBytesLimit\": 1,\n" - + " \"mergeSegmentsLimit\" : 1,\n" + " \"maxSegmentsToMove\": 1,\n" + " \"replicantLifetime\": 1,\n" + " \"replicationThrottleLimit\": 1,\n" @@ -73,13 +71,10 @@ public void testSerde() throws Exception 1, 1, 1, - 1, - 1, 2, whitelist, 0.15, 2, - false, 1, decommissioning, false, @@ -93,13 +88,10 @@ public void testSerde() throws Exception 1, 1, 1, - 1, - 1, 2, whitelist, 0.15, 2, - false, 1, ImmutableSet.of("host1"), false, @@ -113,13 +105,10 @@ public void testSerde() throws Exception 1, 1, 1, - 1, - 1, 2, whitelist, 0.15, 2, - false, 1, ImmutableSet.of("host1"), false, @@ -133,13 +122,10 @@ public void testSerde() throws Exception 1, 1, 1, - 1, - 1, 2, whitelist, 0.15, 2, - false, 1, ImmutableSet.of("host1"), true, @@ -153,13 +139,10 @@ public void testSerde() throws Exception 1, 1, 1, - 1, - 1, 2, whitelist, 0.15, 2, - false, 1, ImmutableSet.of("host1"), true, @@ -173,13 +156,10 @@ public void testSerde() throws Exception 1, 1, 1, - 1, - 1, 2, whitelist, 0.15, 2, - false, 1, ImmutableSet.of("host1"), true, @@ -193,13 +173,10 @@ public void testSerde() throws Exception 1, 1, 1, - 1, - 1, 2, whitelist, 0.1, 2, - false, 1, ImmutableSet.of("host1"), true, @@ -213,13 +190,10 @@ public void testSerde() throws Exception 1, 1, 1, - 1, - 1, 2, whitelist, 0.1, 5, - false, 1, ImmutableSet.of("host1"), true, @@ -244,8 +218,6 @@ public void testDeserializationWithUnknownProperties() throws Exception public void testConstructorWithNullsShouldKillUnusedSegmentsInAllDataSources() { CoordinatorDynamicConfig config = new CoordinatorDynamicConfig( - 1, - 1, 1, 1, 1, @@ -270,8 +242,6 @@ public void testConstructorWithNullsShouldKillUnusedSegmentsInAllDataSources() public void testConstructorWithSpecificDataSourcesToKillShouldNotKillUnusedSegmentsInAllDatasources() { CoordinatorDynamicConfig config = new CoordinatorDynamicConfig( - 1, - 1, 1, 1, 1, @@ -297,8 +267,6 @@ public void testDecommissioningParametersBackwardCompatibility() throws Exceptio { String jsonStr = "{\n" + " \"millisToWaitBeforeDeleting\": 1,\n" - + " \"mergeBytesLimit\": 1,\n" - + " \"mergeSegmentsLimit\" : 1,\n" + " \"maxSegmentsToMove\": 1,\n" + " \"replicantLifetime\": 1,\n" + " \"replicationThrottleLimit\": 1,\n" @@ -324,13 +292,10 @@ public void testDecommissioningParametersBackwardCompatibility() throws Exceptio 1, 1, 1, - 1, - 1, 2, whitelist, 0.1, Integer.MAX_VALUE, - false, 1, decommissioning, false, @@ -344,13 +309,10 @@ public void testDecommissioningParametersBackwardCompatibility() throws Exceptio 1, 1, 1, - 1, - 1, 2, whitelist, 0.1, Integer.MAX_VALUE, - false, 1, ImmutableSet.of("host1"), false, @@ -364,13 +326,10 @@ public void testDecommissioningParametersBackwardCompatibility() throws Exceptio 1, 1, 1, - 1, - 1, 2, whitelist, 0.1, Integer.MAX_VALUE, - false, 1, ImmutableSet.of("host1"), false, @@ -383,8 +342,6 @@ public void testSerdeWithStringInKillDataSourceWhitelist() throws Exception { String jsonStr = "{\n" + " \"millisToWaitBeforeDeleting\": 1,\n" - + " \"mergeBytesLimit\": 1,\n" - + " \"mergeSegmentsLimit\" : 1,\n" + " \"maxSegmentsToMove\": 1,\n" + " \"replicantLifetime\": 1,\n" + " \"replicationThrottleLimit\": 1,\n" @@ -408,13 +365,10 @@ public void testSerdeWithStringInKillDataSourceWhitelist() throws Exception 1, 1, 1, - 1, - 1, 2, ImmutableSet.of("test1", "test2"), 0.1, Integer.MAX_VALUE, - false, 1, ImmutableSet.of(), false, @@ -427,8 +381,6 @@ public void testHandleMissingPercentOfSegmentsToConsiderPerMove() throws Excepti { String jsonStr = "{\n" + " \"millisToWaitBeforeDeleting\": 1,\n" - + " \"mergeBytesLimit\": 1,\n" - + " \"mergeSegmentsLimit\" : 1,\n" + " \"maxSegmentsToMove\": 1,\n" + " \"replicantLifetime\": 1,\n" + " \"replicationThrottleLimit\": 1,\n" @@ -452,13 +404,10 @@ public void testHandleMissingPercentOfSegmentsToConsiderPerMove() throws Excepti 1, 1, 1, - 1, - 1, 2, whitelist, 0.1, Integer.MAX_VALUE, - false, 1, decommissioning, false, @@ -471,8 +420,6 @@ public void testDeserializeWithoutMaxSegmentsInNodeLoadingQueue() throws Excepti { String jsonStr = "{\n" + " \"millisToWaitBeforeDeleting\": 1,\n" - + " \"mergeBytesLimit\": 1,\n" - + " \"mergeSegmentsLimit\" : 1,\n" + " \"maxSegmentsToMove\": 1,\n" + " \"replicantLifetime\": 1,\n" + " \"replicationThrottleLimit\": 1,\n" @@ -492,13 +439,10 @@ public void testDeserializeWithoutMaxSegmentsInNodeLoadingQueue() throws Excepti 1, 1, 1, - 1, - 1, 2, ImmutableSet.of(), 0.1, Integer.MAX_VALUE, - true, EXPECTED_DEFAULT_MAX_SEGMENTS_IN_NODE_LOADING_QUEUE, ImmutableSet.of(), false, @@ -514,8 +458,6 @@ public void testBuilderDefaults() assertConfig( defaultConfig, 900000, - 524288000, - 100, 100, 15, 500, @@ -523,7 +465,6 @@ public void testBuilderDefaults() emptyList, 0.1, Integer.MAX_VALUE, - true, EXPECTED_DEFAULT_MAX_SEGMENTS_IN_NODE_LOADING_QUEUE, emptyList, false, @@ -542,8 +483,6 @@ public void testBuilderWithDefaultSpecificDataSourcesToKillUnusedSegmentsInSpeci assertConfig( config, 900000, - 524288000, - 100, 100, 15, 500, @@ -551,7 +490,6 @@ public void testBuilderWithDefaultSpecificDataSourcesToKillUnusedSegmentsInSpeci ImmutableSet.of("DATASOURCE"), 0.1, Integer.MAX_VALUE, - false, EXPECTED_DEFAULT_MAX_SEGMENTS_IN_NODE_LOADING_QUEUE, ImmutableSet.of(), false, @@ -569,26 +507,7 @@ public void testUpdate() Assert.assertEquals( current, - new CoordinatorDynamicConfig.Builder( - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null - ).build(current) + CoordinatorDynamicConfig.builder().build(current) ); } @@ -604,8 +523,6 @@ public void testEqualsAndHashCode() private void assertConfig( CoordinatorDynamicConfig config, long expectedLeadingTimeMillisBeforeCanMarkAsUnusedOvershadowedSegments, - long expectedMergeBytesLimit, - int expectedMergeSegmentsLimit, int expectedMaxSegmentsToMove, int expectedReplicantLifetime, int expectedReplicationThrottleLimit, @@ -613,7 +530,6 @@ private void assertConfig( Set expectedSpecificDataSourcesToKillUnusedSegmentsIn, Double expectedKillTaskSlotRatio, @Nullable Integer expectedMaxKillTaskSlots, - boolean expectedKillUnusedSegmentsInAllDataSources, int expectedMaxSegmentsInNodeLoadingQueue, Set decommissioningNodes, boolean pauseCoordination, @@ -624,8 +540,6 @@ private void assertConfig( expectedLeadingTimeMillisBeforeCanMarkAsUnusedOvershadowedSegments, config.getMarkSegmentAsUnusedDelayMillis() ); - Assert.assertEquals(expectedMergeBytesLimit, config.getMergeBytesLimit()); - Assert.assertEquals(expectedMergeSegmentsLimit, config.getMergeSegmentsLimit()); Assert.assertEquals(expectedMaxSegmentsToMove, config.getMaxSegmentsToMove()); Assert.assertEquals(expectedReplicantLifetime, config.getReplicantLifetime()); Assert.assertEquals(expectedReplicationThrottleLimit, config.getReplicationThrottleLimit()); diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts index e46d1f27c964..60ab353f8ec0 100644 --- a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts +++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts @@ -20,8 +20,6 @@ import type { CoordinatorDynamicConfig } from './coordinator-dynamic-config'; export const DEFAULT_COORDINATOR_DYNAMIC_CONFIG: CoordinatorDynamicConfig = { millisToWaitBeforeDeleting: 900000, - mergeBytesLimit: 524288000, - mergeSegmentsLimit: 100, maxSegmentsToMove: 100, replicantLifetime: 15, replicationThrottleLimit: 500, diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx index b60bcfc2f43c..bdd47a317225 100644 --- a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx +++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx @@ -30,8 +30,6 @@ export interface CoordinatorDynamicConfig { maxKillTaskSlots?: number; killPendingSegmentsSkipList?: string[]; maxSegmentsInNodeLoadingQueue?: number; - mergeBytesLimit?: number; - mergeSegmentsLimit?: number; millisToWaitBeforeDeleting?: number; replicantLifetime?: number; replicationThrottleLimit?: number; @@ -220,18 +218,6 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[ ), }, - { - name: 'mergeBytesLimit', - type: 'size-bytes', - defaultValue: 524288000, - info: <>The maximum total uncompressed size in bytes of segments to merge., - }, - { - name: 'mergeSegmentsLimit', - type: 'number', - defaultValue: 100, - info: <>The maximum number of segments that can be in a single append task., - }, { name: 'millisToWaitBeforeDeleting', type: 'number',