From e291fa03322bdc21e4a8d63124cac2ad748098ae Mon Sep 17 00:00:00 2001 From: "qiumingming.2018" Date: Wed, 13 Feb 2019 21:26:57 +0800 Subject: [PATCH 1/8] Support assign tasks to run on different tiers of MiddleManagers --- docs/content/configuration/index.md | 58 ++++++ ...ngTaskBasedWorkerProvisioningStrategy.java | 3 +- .../overlord/hrtr/HttpRemoteTaskRunner.java | 3 +- .../indexing/overlord/hrtr/WorkerHolder.java | 3 +- ...utionWithTierSpecWorkerSelectStrategy.java | 93 +++++++++ ...EqualDistributionWorkerSelectStrategy.java | 2 +- ...acityWithTierSpecWorkerSelectStrategy.java | 93 +++++++++ .../FillCapacityWorkerSelectStrategy.java | 2 +- .../overlord/setup/WorkerSelectStrategy.java | 4 +- .../overlord/setup/WorkerSelectUtils.java | 73 ++++++++ .../overlord/setup/WorkerTierSpec.java | 142 ++++++++++++++ .../apache/druid/indexing/worker/Worker.java | 18 +- .../indexing/worker/config/WorkerConfig.java | 11 ++ .../indexing/worker/http/WorkerResource.java | 3 +- .../overlord/ImmutableWorkerInfoTest.java | 50 +++-- .../overlord/RemoteTaskRunnerTestUtils.java | 13 +- .../overlord/TaskRunnerUtilsTest.java | 3 +- ...dingTaskBasedProvisioningStrategyTest.java | 3 +- .../SimpleProvisioningStrategyTest.java | 3 +- .../hrtr/HttpRemoteTaskRunnerTest.java | 35 ++-- .../overlord/hrtr/WorkerHolderTest.java | 3 +- ...nWithAffinityWorkerSelectStrategyTest.java | 15 +- ...nWithTierSpecWorkerSelectStrategyTest.java | 176 ++++++++++++++++++ ...lDistributionWorkerSelectStrategyTest.java | 25 +-- ...yWithAffinityWorkerSelectStrategyTest.java | 11 +- ...yWithTierSpecWorkerSelectStrategyTest.java | 176 ++++++++++++++++++ .../overlord/setup/WorkerTierSpecTest.java | 64 +++++++ .../worker/WorkerTaskMonitorTest.java | 4 +- .../worker/http/WorkerResourceTest.java | 4 +- .../druid/discovery/WorkerNodeService.java | 17 +- .../discovery/WorkerNodeServiceTest.java | 3 +- .../apache/druid/cli/CliMiddleManager.java | 6 +- 32 files changed, 1040 insertions(+), 79 deletions(-) create mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategy.java create mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategy.java create mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java create mode 100644 indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java create mode 100644 indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java create mode 100644 indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpecTest.java diff --git a/docs/content/configuration/index.md b/docs/content/configuration/index.md index 639bb45cce27..121bf1e2d24a 100644 --- a/docs/content/configuration/index.md +++ b/docs/content/configuration/index.md @@ -1028,6 +1028,37 @@ useful if you want work evenly distributed across your middleManagers. |`type`|`equalDistribution`.|required; must be `equalDistribution`| |`affinityConfig`|[Affinity config](#affinity) object|null (no affinity)| +###### Equal Distribution With Tier Spec + +This strategy is a variant of `Equal Distribution`, which support `workerTierSpec` field rather than `affinityConfig`. By specifying `workerTierSpec`, you can assign tasks to run on different tiers of MiddleManagers based on the tasks' **taskType** and **dataSource name**. + +|Property|Description|Default| +|--------|-----------|-------| +|`type`|`equalDistributionWithTierSpec`.|required; must be `equalDistributionWithTierSpec`| +|`workerTierSpec`|[Worker Tier Spec](#workerTierSpec) object|null (no worker tier spec)| + +Example: specify tasks default to run on _tier1_ whose task +type is "index_kafka", while dataSource "ds1" run on _tier2_. + +```json +{ + "selectStrategy": { + "type": "equalDistributionWithTierSpec", + "workerTierSpec": { + "strong": false, + "tierMap": { + "index_kafka": { + "defaultTier": "tier1", + "tiers": { + "ds1": "tier2" + } + } + } + } + } +} +``` + ###### Fill Capacity Tasks are assigned to the worker with the most currently-running tasks at the time the task begins running. This is @@ -1042,6 +1073,15 @@ middleManagers up to capacity simultaneously, rather than a single middleManager |`type`|`fillCapacity`.|required; must be `fillCapacity`| |`affinityConfig`|[Affinity config](#affinity) object|null (no affinity)| +###### Fill Capacity With Tier Spec + +This strategy is a variant of `Fill Capacity`, which support `workerTierSpec` field rather than `affinityConfig`. The usage is the same with _equalDistributionWithTierSpec_ strategy. + +|Property|Description|Default| +|--------|-----------|-------| +|`type`|`fillCapacityWithTierSpec`.|required; must be `fillCapacityWithTierSpec`| +|`workerTierSpec`|[Worker Tier Spec](#workerTierSpec) object|null (no worker tier spec)| + ###### Javascript Allows defining arbitrary logic for selecting workers to run task using a JavaScript function. @@ -1078,6 +1118,23 @@ field. If not provided, the default is to not use affinity at all. |`affinity`|JSON object mapping a datasource String name to a list of indexing service middleManager host:port String values. Druid doesn't perform DNS resolution, so the 'host' value must match what is configured on the middleManager and what the middleManager announces itself as (examine the Overlord logs to see what your middleManager announces itself as).|{}| |`strong`|With weak affinity (the default), tasks for a dataSource may be assigned to other middleManagers if their affinity-mapped middleManagers are not able to run all pending tasks in the queue for that dataSource. With strong affinity, tasks for a dataSource will only ever be assigned to their affinity-mapped middleManagers, and will wait in the pending queue if necessary.|false| +###### WorkerTierSpec + +WorkerTierSpec can be provided to the _equalDistributionWithTierSpec_ and _fillCapacityWithTierSpec_ strategies using the "workerTierSpec" +field. If not provided, the default is to not use it at all. + +|Property|Description|Default| +|--------|-----------|-------| +|`tierMap`|A JSON map object mapping a task type String name to a [TierConfig](tierConfig) object, by which you can specify tier config for different task type.|{}| +|`strong`|With weak workerTierSpec (the default), tasks for a dataSource may be assigned to other middleManagers if the middleManagers specified in `tierMap` are not able to run all pending tasks in the queue for that dataSource. With strong workerTierSpec, tasks for a dataSource will only ever be assigned to their specified middleManagers, and will wait in the pending queue if necessary.|false| + +###### TierConfig + +|Property|Description|Default| +|--------|-----------|-------| +|`defaultTier`|Specify default tier for a task type.|null| +|`tiers`|A JSON map object mapping a datasource String name to a tier String name of the MiddleManager. If tier isn't specified for a datasource, then using the `defaultTier`. If no specified tier and the `defaultTier` is also null, then tasks can run on any available MiddleManagers.|null| + ##### Autoscaler Amazon's EC2 is currently the only supported autoscaler. @@ -1127,6 +1184,7 @@ Middle managers pass their configurations down to their child peons. The MiddleM |`druid.worker.ip`|The IP of the worker.|localhost| |`druid.worker.version`|Version identifier for the MiddleManager.|0| |`druid.worker.capacity`|Maximum number of tasks the MiddleManager can accept.|Number of available processors - 1| +|`druid.worker.tier`|A string to name the tier that the MiddleManager node belongs to.|`_default_worker_tier`| #### Peon Processing diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java index c908d282a30c..f58275a663de 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java @@ -37,6 +37,7 @@ import org.apache.druid.indexing.overlord.setup.WorkerBehaviorConfig; import org.apache.druid.indexing.overlord.setup.WorkerSelectStrategy; import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.concurrent.ScheduledExecutors; import org.apache.druid.java.util.emitter.EmittingLogger; @@ -477,7 +478,7 @@ private static ImmutableWorkerInfo workerWithTask(ImmutableWorkerInfo immutableW private static ImmutableWorkerInfo createDummyWorker(String scheme, String host, int capacity, String version) { return new ImmutableWorkerInfo( - new Worker(scheme, host, "-2", capacity, version), + new Worker(scheme, host, "-2", capacity, version, WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunner.java index 5cb99ba569a1..e0138024284e 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunner.java @@ -489,7 +489,8 @@ private Worker toWorker(DiscoveryDruidNode node) node.getDruidNode().getHostAndPortToUse(), ((WorkerNodeService) node.getServices().get(WorkerNodeService.DISCOVERY_SERVICE_KEY)).getIp(), ((WorkerNodeService) node.getServices().get(WorkerNodeService.DISCOVERY_SERVICE_KEY)).getCapacity(), - ((WorkerNodeService) node.getServices().get(WorkerNodeService.DISCOVERY_SERVICE_KEY)).getVersion() + ((WorkerNodeService) node.getServices().get(WorkerNodeService.DISCOVERY_SERVICE_KEY)).getVersion(), + ((WorkerNodeService) node.getServices().get(WorkerNodeService.DISCOVERY_SERVICE_KEY)).getTier() ); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolder.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolder.java index caee05714c69..a1d18601f8f4 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolder.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolder.java @@ -178,7 +178,8 @@ public ImmutableWorkerInfo toImmutable() worker.getHost(), worker.getIp(), worker.getCapacity(), - "" + "", + worker.getTier() ); } w = disabledWorker; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategy.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategy.java new file mode 100644 index 000000000000..d04d1eaabed7 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategy.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.setup; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.ImmutableWorkerInfo; +import org.apache.druid.indexing.overlord.config.WorkerTaskRunnerConfig; + +import javax.annotation.Nullable; +import java.util.Objects; + +public class EqualDistributionWithTierSpecWorkerSelectStrategy implements WorkerSelectStrategy +{ + private final WorkerTierSpec workerTierSpec; + + @JsonCreator + public EqualDistributionWithTierSpecWorkerSelectStrategy( + @JsonProperty("workerTierSpec") WorkerTierSpec workerTierSpec + ) + { + this.workerTierSpec = workerTierSpec; + } + + @JsonProperty + public WorkerTierSpec getWorkerTierSpec() + { + return workerTierSpec; + } + + @Nullable + @Override + public ImmutableWorkerInfo findWorkerForTask( + final WorkerTaskRunnerConfig config, + final ImmutableMap zkWorkers, + final Task task + ) + { + return WorkerSelectUtils.selectWorker( + task, + zkWorkers, + config, + workerTierSpec, + EqualDistributionWorkerSelectStrategy::selectFromEligibleWorkers + ); + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final EqualDistributionWithTierSpecWorkerSelectStrategy that = (EqualDistributionWithTierSpecWorkerSelectStrategy) o; + return Objects.equals(workerTierSpec, that.workerTierSpec); + } + + @Override + public int hashCode() + { + return Objects.hash(workerTierSpec); + } + + @Override + public String toString() + { + return "EqualDistributionWithTierSpecWorkerSelectStrategy{" + + "workerTierSpec=" + workerTierSpec + + '}'; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategy.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategy.java index eb9a3c57909c..afb020e9f7ab 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategy.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategy.java @@ -66,7 +66,7 @@ public ImmutableWorkerInfo findWorkerForTask( ); } - private static ImmutableWorkerInfo selectFromEligibleWorkers(final Map eligibleWorkers) + static ImmutableWorkerInfo selectFromEligibleWorkers(final Map eligibleWorkers) { return eligibleWorkers.values().stream().max( Comparator.comparing(ImmutableWorkerInfo::getAvailableCapacity) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategy.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategy.java new file mode 100644 index 000000000000..d35452929498 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategy.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.setup; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.ImmutableWorkerInfo; +import org.apache.druid.indexing.overlord.config.WorkerTaskRunnerConfig; + +import javax.annotation.Nullable; +import java.util.Objects; + +public class FillCapacityWithTierSpecWorkerSelectStrategy implements WorkerSelectStrategy +{ + private final WorkerTierSpec workerTierSpec; + + @JsonCreator + public FillCapacityWithTierSpecWorkerSelectStrategy( + @JsonProperty("workerTierSpec") WorkerTierSpec workerTierSpec + ) + { + this.workerTierSpec = workerTierSpec; + } + + @JsonProperty + public WorkerTierSpec getWorkerTierSpec() + { + return workerTierSpec; + } + + @Nullable + @Override + public ImmutableWorkerInfo findWorkerForTask( + final WorkerTaskRunnerConfig config, + final ImmutableMap zkWorkers, + final Task task + ) + { + return WorkerSelectUtils.selectWorker( + task, + zkWorkers, + config, + workerTierSpec, + FillCapacityWorkerSelectStrategy::selectFromEligibleWorkers + ); + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final FillCapacityWithTierSpecWorkerSelectStrategy that = (FillCapacityWithTierSpecWorkerSelectStrategy) o; + return Objects.equals(workerTierSpec, that.workerTierSpec); + } + + @Override + public int hashCode() + { + return Objects.hash(workerTierSpec); + } + + @Override + public String toString() + { + return "FillCapacityWithTierSpecWorkerSelectStrategy{" + + "workerTierSpec=" + workerTierSpec + + '}'; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWorkerSelectStrategy.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWorkerSelectStrategy.java index 2c82215abff0..47ac65b8271d 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWorkerSelectStrategy.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWorkerSelectStrategy.java @@ -64,7 +64,7 @@ public ImmutableWorkerInfo findWorkerForTask( ); } - private static ImmutableWorkerInfo selectFromEligibleWorkers(final Map eligibleWorkers) + static ImmutableWorkerInfo selectFromEligibleWorkers(final Map eligibleWorkers) { return eligibleWorkers.values().stream().max( Comparator.comparing(ImmutableWorkerInfo::getCurrCapacityUsed) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectStrategy.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectStrategy.java index fbb8638907d6..d63bc569cf2d 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectStrategy.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectStrategy.java @@ -38,7 +38,9 @@ @JsonSubTypes.Type(name = "fillCapacityWithAffinity", value = FillCapacityWithAffinityWorkerSelectStrategy.class), @JsonSubTypes.Type(name = "equalDistribution", value = EqualDistributionWorkerSelectStrategy.class), @JsonSubTypes.Type(name = "equalDistributionWithAffinity", value = EqualDistributionWithAffinityWorkerSelectStrategy.class), - @JsonSubTypes.Type(name = "javascript", value = JavaScriptWorkerSelectStrategy.class) + @JsonSubTypes.Type(name = "javascript", value = JavaScriptWorkerSelectStrategy.class), + @JsonSubTypes.Type(name = "fillCapacityWithTierSpec", value = FillCapacityWithTierSpecWorkerSelectStrategy.class), + @JsonSubTypes.Type(name = "equalDistributionWithTierSpec", value = EqualDistributionWithTierSpecWorkerSelectStrategy.class) }) @PublicApi public interface WorkerSelectStrategy diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java index c5ccd15bebe7..726779bd1ee3 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java @@ -95,6 +95,79 @@ public static ImmutableWorkerInfo selectWorker( } } + /** + * Helper for {@link WorkerSelectStrategy} implementations. + * + * @param allWorkers map of all workers, in the style provided to {@link WorkerSelectStrategy} + * @param workerTierSpec worker tier spec, or null + * @param workerSelector function that receives a list of eligible workers: version is high enough, worker can run + * the task, and worker satisfies the worker tier spec. may return null. + * + * @return selected worker from "allWorkers", or null. + */ + @Nullable + public static ImmutableWorkerInfo selectWorker( + final Task task, + final Map allWorkers, + final WorkerTaskRunnerConfig workerTaskRunnerConfig, + @Nullable final WorkerTierSpec workerTierSpec, + final Function, ImmutableWorkerInfo> workerSelector + ) + { + // Workers that could potentially run this task, ignoring workerTierSpec. + final Map runnableWorkers = allWorkers + .values() + .stream() + .filter(worker -> worker.canRunTask(task) + && worker.isValidVersion(workerTaskRunnerConfig.getMinWorkerVersion())) + .collect(Collectors.toMap(w -> w.getWorker().getHost(), Function.identity())); + + // select worker according to worker tier spec + if (workerTierSpec != null && workerTierSpec.getTierMap() != null) { + final WorkerTierSpec.TierConfig tierConfig = workerTierSpec.getTierMap() + .getOrDefault( + task.getType(), + new WorkerTierSpec.TierConfig(null, null) + ); + final String defaultTier = tierConfig.getDefaultTier(); + final Map tiers = tierConfig.getTiers(); + + // select worker from preferred tier + final String preferredTier = tiers != null ? tiers.get(task.getDataSource()) : defaultTier; + if (preferredTier != null) { + final ImmutableMap tierWorkers = getTierWorkers(preferredTier, runnableWorkers); + final ImmutableWorkerInfo selected = workerSelector.apply(tierWorkers); + + if (selected != null) { + return selected; + } else if (workerTierSpec.isStrong()) { + return null; + } + } + } + + // select worker from all runnable workers by default + return workerSelector.apply(ImmutableMap.copyOf(runnableWorkers)); + } + + /** + * Return workers belong to this tier. + * + * @param tier worker tier name + * @param workerMap map of worker hostname to worker info + * + * @return map of worker hostname to worker info + */ + private static ImmutableMap getTierWorkers( + final String tier, + final Map workerMap + ) + { + return ImmutableMap.copyOf( + Maps.filterValues(workerMap, workerInfo -> workerInfo.getWorker().getTier().equals(tier)) + ); + } + /** * Return workers not assigned to any affinity pool at all. * diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java new file mode 100644 index 000000000000..88d46131f4b0 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.setup; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; +import java.util.Objects; + +public class WorkerTierSpec +{ + // key: taskType, value: tierConfig + private final Map tierMap; + private final boolean strong; + + @JsonCreator + public WorkerTierSpec( + @JsonProperty("tierMap") Map tierMap, + @JsonProperty("strong") boolean strong + ) + { + this.tierMap = tierMap; + this.strong = strong; + } + + @JsonProperty + public Map getTierMap() + { + return tierMap; + } + + @JsonProperty + public boolean isStrong() + { + return strong; + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final WorkerTierSpec that = (WorkerTierSpec) o; + return strong == that.strong && + Objects.equals(tierMap, that.tierMap); + } + + @Override + public int hashCode() + { + return Objects.hash(tierMap, strong); + } + + @Override + public String toString() + { + return "WorkerTierSpec{" + + "tierMap=" + tierMap + + ", strong=" + strong + + '}'; + } + + public static class TierConfig + { + private final String defaultTier; + // key: datasource, value: tier + private final Map tiers; + + @JsonCreator + public TierConfig( + @JsonProperty("defaultTier") String defaultTier, + @JsonProperty("tiers") Map tiers + ) + { + this.defaultTier = defaultTier; + this.tiers = tiers; + } + + @JsonProperty + public String getDefaultTier() + { + return defaultTier; + } + + @JsonProperty + public Map getTiers() + { + return tiers; + } + + @Override + public boolean equals(final Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final TierConfig that = (TierConfig) o; + return Objects.equals(defaultTier, that.defaultTier) && + Objects.equals(tiers, that.tiers); + } + + @Override + public int hashCode() + { + return Objects.hash(defaultTier, tiers); + } + + @Override + public String toString() + { + return "TierConfig{" + + "defaultTier=" + defaultTier + + ", tiers=" + tiers + + '}'; + } + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/worker/Worker.java b/indexing-service/src/main/java/org/apache/druid/indexing/worker/Worker.java index fd141daf2668..ae31a3711e90 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/worker/Worker.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/worker/Worker.java @@ -34,6 +34,7 @@ public class Worker private final String ip; private final int capacity; private final String version; + private final String tier; @JsonCreator public Worker( @@ -41,7 +42,8 @@ public Worker( @JsonProperty("host") String host, @JsonProperty("ip") String ip, @JsonProperty("capacity") int capacity, - @JsonProperty("version") String version + @JsonProperty("version") String version, + @JsonProperty("tier") String tier ) { this.scheme = scheme == null ? "http" : scheme; // needed for backwards compatibility with older workers (pre-#4270) @@ -49,6 +51,7 @@ public Worker( this.ip = ip; this.capacity = capacity; this.version = version; + this.tier = tier; } @JsonProperty @@ -81,6 +84,12 @@ public String getVersion() return version; } + @JsonProperty + public String getTier() + { + return tier; + } + @Override public boolean equals(Object o) { @@ -105,7 +114,10 @@ public boolean equals(Object o) if (!ip.equals(worker.ip)) { return false; } - return version.equals(worker.version); + if (!version.equals(worker.version)) { + return false; + } + return tier.equals(worker.tier); } @Override @@ -116,6 +128,7 @@ public int hashCode() result = 31 * result + ip.hashCode(); result = 31 * result + capacity; result = 31 * result + version.hashCode(); + result = 31 * result + tier.hashCode(); return result; } @@ -128,6 +141,7 @@ public String toString() ", ip='" + ip + '\'' + ", capacity=" + capacity + ", version='" + version + '\'' + + ", tier='" + tier + '\'' + '}'; } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/worker/config/WorkerConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/worker/config/WorkerConfig.java index e574a0d151cf..e38bf168a14e 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/worker/config/WorkerConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/worker/config/WorkerConfig.java @@ -30,6 +30,8 @@ */ public class WorkerConfig { + public static final String DEFAULT_TIER = "_default_worker_tier"; + @JsonProperty @NotNull private String ip = DruidNode.getDefaultHost(); @@ -42,6 +44,10 @@ public class WorkerConfig @Min(1) private int capacity = Math.max(1, JvmUtils.getRuntimeInfo().getAvailableProcessors() - 1); + @JsonProperty + @NotNull + private String tier = DEFAULT_TIER; + public String getIp() { return ip; @@ -56,4 +62,9 @@ public int getCapacity() { return capacity; } + + public String getTier() + { + return tier; + } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/worker/http/WorkerResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/worker/http/WorkerResource.java index 52513215aba8..53034228192e 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/worker/http/WorkerResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/worker/http/WorkerResource.java @@ -91,7 +91,8 @@ public Response doDisable() enabledWorker.getHost(), enabledWorker.getIp(), enabledWorker.getCapacity(), - DISABLED_VERSION + DISABLED_VERSION, + enabledWorker.getTier() ); curatorCoordinator.updateWorkerAnnouncement(disabledWorker); workerTaskManager.workerDisabled(); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfoTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfoTest.java index 3c1530d3f6d0..8155fc663884 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfoTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfoTest.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableSet; import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; import org.junit.Assert; @@ -35,7 +36,7 @@ public void testSerde() throws Exception { ImmutableWorkerInfo workerInfo = new ImmutableWorkerInfo( new Worker( - "http", "testWorker", "192.0.0.1", 10, "v1" + "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp1", "grp2"), @@ -56,7 +57,7 @@ public void testEqualsAndSerde() // Everything equal assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker", "192.0.0.1", 10, "v1" + "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp1", "grp2"), @@ -64,7 +65,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker", "192.0.0.1", 10, "v1" + "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp1", "grp2"), @@ -72,10 +73,29 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), true); + // same worker different tier + assertEqualsAndHashCode(new ImmutableWorkerInfo( + new Worker( + "http", "testWorker", "192.0.0.1", 10, "v1", "t1" + ), + 2, + ImmutableSet.of("grp1", "grp2"), + ImmutableSet.of("task1", "task2"), + DateTimes.of("2015-01-01T01:01:01Z") + ), new ImmutableWorkerInfo( + new Worker( + "http", "testWorker", "192.0.0.1", 10, "v1", "t2" + ), + 2, + ImmutableSet.of("grp1", "grp2"), + ImmutableSet.of("task1", "task2"), + DateTimes.of("2015-01-01T01:01:01Z") + ), false); + // different worker same tasks assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker1", "192.0.0.1", 10, "v1" + "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp1", "grp2"), @@ -83,7 +103,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker2", "192.0.0.1", 10, "v1" + "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp1", "grp2"), @@ -94,7 +114,7 @@ public void testEqualsAndSerde() // same worker different task groups assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker", "192.0.0.1", 10, "v1" + "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp3", "grp2"), @@ -102,7 +122,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker", "192.0.0.1", 10, "v1" + "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp1", "grp2"), @@ -113,7 +133,7 @@ public void testEqualsAndSerde() // same worker different tasks assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker1", "192.0.0.1", 10, "v1" + "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp1", "grp2"), @@ -121,7 +141,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker2", "192.0.0.1", 10, "v1" + "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp1", "grp2"), @@ -132,7 +152,7 @@ public void testEqualsAndSerde() // same worker different capacity assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker1", "192.0.0.1", 10, "v1" + "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 3, ImmutableSet.of("grp1", "grp2"), @@ -140,7 +160,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker2", "192.0.0.1", 10, "v1" + "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp1", "grp2"), @@ -151,7 +171,7 @@ public void testEqualsAndSerde() // same worker different lastCompletedTaskTime assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker1", "192.0.0.1", 10, "v1" + "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 3, ImmutableSet.of("grp1", "grp2"), @@ -159,7 +179,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker2", "192.0.0.1", 10, "v1" + "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp1", "grp2"), @@ -170,7 +190,7 @@ public void testEqualsAndSerde() // same worker different blacklistedUntil assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker1", "192.0.0.1", 10, "v1" + "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 3, ImmutableSet.of("grp1", "grp2"), @@ -179,7 +199,7 @@ public void testEqualsAndSerde() DateTimes.of("2017-07-30") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker2", "192.0.0.1", 10, "v1" + "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp1", "grp2"), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTestUtils.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTestUtils.java index ffd5d685a437..3ec1af3e1eec 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTestUtils.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTestUtils.java @@ -43,6 +43,7 @@ import org.apache.druid.indexing.overlord.setup.WorkerBehaviorConfig; import org.apache.druid.indexing.worker.TaskAnnouncement; import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.server.initialization.IndexerZkConfig; @@ -147,7 +148,8 @@ Worker makeWorker(final String workerId, final int capacity) throws Exception workerId, workerId, capacity, - "0" + "0", + WorkerConfig.DEFAULT_TIER ); cf.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath( @@ -163,7 +165,14 @@ void disableWorker(Worker worker) throws Exception { cf.setData().forPath( joiner.join(announcementsPath, worker.getHost()), - jsonMapper.writeValueAsBytes(new Worker(worker.getScheme(), worker.getHost(), worker.getIp(), worker.getCapacity(), "")) + jsonMapper.writeValueAsBytes(new Worker( + worker.getScheme(), + worker.getHost(), + worker.getIp(), + worker.getCapacity(), + "", + worker.getTier() + )) ); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskRunnerUtilsTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskRunnerUtilsTest.java index b21a98fda4ce..ea09d9fa03c1 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskRunnerUtilsTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskRunnerUtilsTest.java @@ -20,6 +20,7 @@ package org.apache.druid.indexing.overlord; import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.junit.Assert; import org.junit.Test; @@ -31,7 +32,7 @@ public class TaskRunnerUtilsTest public void testMakeWorkerURL() { final URL url = TaskRunnerUtils.makeWorkerURL( - new Worker("https", "1.2.3.4:8290", "1.2.3.4", 1, "0"), + new Worker("https", "1.2.3.4:8290", "1.2.3.4", 1, "0", WorkerConfig.DEFAULT_TIER), "/druid/worker/v1/task/%s/log", "foo bar&" ); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedProvisioningStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedProvisioningStrategyTest.java index 2fb0949c96fc..f2be7889bd8a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedProvisioningStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedProvisioningStrategyTest.java @@ -36,6 +36,7 @@ import org.apache.druid.indexing.overlord.setup.WorkerBehaviorConfig; import org.apache.druid.indexing.worker.TaskAnnouncement; import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.concurrent.Execs; @@ -577,7 +578,7 @@ public TestZkWorker( int capacity ) { - super(new Worker(scheme, host, ip, capacity, version), null, new DefaultObjectMapper()); + super(new Worker(scheme, host, ip, capacity, version, WorkerConfig.DEFAULT_TIER), null, new DefaultObjectMapper()); this.testTask = testTask; } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/SimpleProvisioningStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/SimpleProvisioningStrategyTest.java index 7796dfcb4178..3e0777e88940 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/SimpleProvisioningStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/SimpleProvisioningStrategyTest.java @@ -34,6 +34,7 @@ import org.apache.druid.indexing.overlord.setup.WorkerBehaviorConfig; import org.apache.druid.indexing.worker.TaskAnnouncement; import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.concurrent.Execs; @@ -507,7 +508,7 @@ public TestZkWorker( String version ) { - super(new Worker(scheme, host, ip, 3, version), null, new DefaultObjectMapper()); + super(new Worker(scheme, host, ip, 3, version, WorkerConfig.DEFAULT_TIER), null, new DefaultObjectMapper()); this.testTask = testTask; } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunnerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunnerTest.java index 0adcf266014f..9290b8a9fb8a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunnerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunnerTest.java @@ -47,6 +47,7 @@ import org.apache.druid.indexing.overlord.setup.DefaultWorkerBehaviorConfig; import org.apache.druid.indexing.worker.TaskAnnouncement; import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.http.client.HttpClient; @@ -141,7 +142,7 @@ protected WorkerHolder createWorkerHolder( new DruidNode("service", "host1", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0") + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_TIER) ) ); @@ -149,7 +150,7 @@ WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0") new DruidNode("service", "host2", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0") + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0", WorkerConfig.DEFAULT_TIER) ) ); @@ -236,7 +237,7 @@ protected WorkerHolder createWorkerHolder( new DruidNode("service", "host1", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0") + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_TIER) ) ); @@ -244,7 +245,7 @@ WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0") new DruidNode("service", "host2", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0") + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0", WorkerConfig.DEFAULT_TIER) ) ); @@ -338,7 +339,7 @@ protected WorkerHolder createWorkerHolder( new DruidNode("service", "host", false, 1234, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0") + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_TIER) ) ); @@ -480,7 +481,7 @@ protected WorkerHolder createWorkerHolder( new DruidNode("service", "host", false, 1234, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0") + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_TIER) ) ); @@ -655,7 +656,7 @@ protected WorkerHolder createWorkerHolder( DiscoveryDruidNode druidNode = new DiscoveryDruidNode( new DruidNode("service", "host", false, 1234, null, true, false), NodeType.MIDDLE_MANAGER, - ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0")) + ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_TIER)) ); workerHolders.put( @@ -832,7 +833,7 @@ protected WorkerHolder createWorkerHolder( new DruidNode("service", "host1", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 1, "0") + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 1, "0", WorkerConfig.DEFAULT_TIER) ) ); @@ -876,7 +877,7 @@ WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 1, "0") DiscoveryDruidNode druidNode2 = new DiscoveryDruidNode( new DruidNode("service", "host2", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, - ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 1, "0")) + ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 1, "0", WorkerConfig.DEFAULT_TIER)) ); workerHolders.put( @@ -906,7 +907,7 @@ WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 1, "0") DiscoveryDruidNode druidNode3 = new DiscoveryDruidNode( new DruidNode("service", "host3", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, - ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 1, "0")) + ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 1, "0", WorkerConfig.DEFAULT_TIER)) ); workerHolders.put( @@ -951,7 +952,7 @@ public void testTaskAddedOrUpdated1() throws Exception ); WorkerHolder workerHolder = EasyMock.createMock(WorkerHolder.class); - EasyMock.expect(workerHolder.getWorker()).andReturn(new Worker("http", "worker", "127.0.0.1", 1, "v1")).anyTimes(); + EasyMock.expect(workerHolder.getWorker()).andReturn(new Worker("http", "worker", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_TIER)).anyTimes(); workerHolder.setLastCompletedTaskTime(EasyMock.anyObject()); workerHolder.resetContinuouslyFailedTasksCount(); EasyMock.expect(workerHolder.getContinuouslyFailedTasksCount()).andReturn(0); @@ -987,7 +988,7 @@ public void testTaskAddedOrUpdated1() throws Exception // Another "rogue-worker" reports running it, and gets asked to shutdown the task WorkerHolder rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class); EasyMock.expect(rogueWorkerHolder.getWorker()) - .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1")) + .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_TIER)) .anyTimes(); rogueWorkerHolder.shutdownTask(task.getId()); EasyMock.replay(rogueWorkerHolder); @@ -1002,7 +1003,7 @@ public void testTaskAddedOrUpdated1() throws Exception // "rogue-worker" reports FAILURE for the task, ignored rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class); EasyMock.expect(rogueWorkerHolder.getWorker()) - .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1")) + .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_TIER)) .anyTimes(); EasyMock.replay(rogueWorkerHolder); taskRunner.taskAddedOrUpdated(TaskAnnouncement.create( @@ -1025,7 +1026,7 @@ public void testTaskAddedOrUpdated1() throws Exception // "rogue-worker" reports running it, and gets asked to shutdown the task rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class); EasyMock.expect(rogueWorkerHolder.getWorker()) - .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1")) + .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_TIER)) .anyTimes(); rogueWorkerHolder.shutdownTask(task.getId()); EasyMock.replay(rogueWorkerHolder); @@ -1040,7 +1041,7 @@ public void testTaskAddedOrUpdated1() throws Exception // "rogue-worker" reports FAILURE for the tasks, ignored rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class); EasyMock.expect(rogueWorkerHolder.getWorker()) - .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1")) + .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_TIER)) .anyTimes(); EasyMock.replay(rogueWorkerHolder); taskRunner.taskAddedOrUpdated(TaskAnnouncement.create( @@ -1079,7 +1080,7 @@ public void testTaskAddedOrUpdated2() throws Exception listenerNotificationsAccumulator ); - Worker worker = new Worker("http", "localhost", "127.0.0.1", 1, "v1"); + Worker worker = new Worker("http", "localhost", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_TIER); WorkerHolder workerHolder = EasyMock.createMock(WorkerHolder.class); EasyMock.expect(workerHolder.getWorker()).andReturn(worker).anyTimes(); @@ -1138,7 +1139,7 @@ public void testTaskAddedOrUpdated3() HttpRemoteTaskRunner taskRunner = createTaskRunnerForTestTaskAddedOrUpdated(taskStorage, listenerNotificationsAccumulator); - Worker worker = new Worker("http", "localhost", "127.0.0.1", 1, "v1"); + Worker worker = new Worker("http", "localhost", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_TIER); WorkerHolder workerHolder = EasyMock.createMock(WorkerHolder.class); EasyMock.expect(workerHolder.getWorker()).andReturn(worker).anyTimes(); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolderTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolderTest.java index cc26b66859b2..84ef23a6d015 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolderTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolderTest.java @@ -28,6 +28,7 @@ import org.apache.druid.indexing.worker.TaskAnnouncement; import org.apache.druid.indexing.worker.Worker; import org.apache.druid.indexing.worker.WorkerHistoryItem; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.segment.TestHelper; import org.apache.druid.server.coordination.ChangeRequestHttpSyncer; @@ -54,7 +55,7 @@ public void testSyncListener() new HttpRemoteTaskRunnerConfig(), EasyMock.createNiceMock(ScheduledExecutorService.class), (taskAnnouncement, holder) -> updates.add(taskAnnouncement), - new Worker("http", "localhost", "127.0.0.1", 5, "v0") + new Worker("http", "localhost", "127.0.0.1", 5, "v0", WorkerConfig.DEFAULT_TIER) ); ChangeRequestHttpSyncer.Listener syncListener = workerHolder.createSyncListener(); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java index 29294b8ca20c..c0d99b803afb 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java @@ -26,6 +26,7 @@ import org.apache.druid.indexing.overlord.ImmutableWorkerInfo; import org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig; import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.segment.TestHelper; import org.junit.Assert; @@ -55,28 +56,28 @@ public String getDataSource() ImmutableMap.of( "localhost0", new ImmutableWorkerInfo( - new Worker("http", "localhost0", "localhost0", 2, "v1"), 0, + new Worker("http", "localhost0", "localhost0", 2, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost1", new ImmutableWorkerInfo( - new Worker("http", "localhost1", "localhost1", 2, "v1"), 0, + new Worker("http", "localhost1", "localhost1", 2, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost2", new ImmutableWorkerInfo( - new Worker("http", "localhost2", "localhost2", 2, "v1"), 1, + new Worker("http", "localhost2", "localhost2", 2, "v1", WorkerConfig.DEFAULT_TIER), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost3", new ImmutableWorkerInfo( - new Worker("http", "localhost3", "localhost3", 2, "v1"), 1, + new Worker("http", "localhost3", "localhost3", 2, "v1", WorkerConfig.DEFAULT_TIER), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -99,14 +100,14 @@ public void testFindWorkerForTaskWithNulls() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "lhost", "lhost", 1, "v1"), 0, + new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 1, "v1"), 0, + new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -129,7 +130,7 @@ public void testIsolation() ImmutableMap.of( "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 1, "v1"), 0, + new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java new file mode 100644 index 000000000000..eb86e4969144 --- /dev/null +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.setup; + +import com.google.common.collect.ImmutableMap; +import org.apache.druid.indexing.common.task.NoopTask; +import org.apache.druid.indexing.overlord.ImmutableWorkerInfo; +import org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig; +import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.java.util.common.DateTimes; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashSet; + +public class EqualDistributionWithTierSpecWorkerSelectStrategyTest +{ + private static final ImmutableMap WORKERS_FOR_TIER_TESTS = + ImmutableMap.of( + "localhost0", + new ImmutableWorkerInfo( + new Worker("http", "localhost0", "localhost0", 1, "v1", "t1"), 0, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() + ), + "localhost1", + new ImmutableWorkerInfo( + new Worker("http", "localhost1", "localhost1", 2, "v1", "t1"), 0, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() + ), + "localhost2", + new ImmutableWorkerInfo( + new Worker("http", "localhost2", "localhost2", 3, "v1", "t2"), 0, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() + ), + "localhost3", + new ImmutableWorkerInfo( + new Worker("http", "localhost3", "localhost3", 4, "v1", "t2"), 0, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() + ) + ); + + @Test + public void testFindWorkerForTask() + { + final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + ImmutableMap.of( + "noop", + new WorkerTierSpec.TierConfig( + "t1", + ImmutableMap.of("ds1", "t1") + ) + ), + false + ); + final EqualDistributionWithTierSpecWorkerSelectStrategy strategy = new EqualDistributionWithTierSpecWorkerSelectStrategy( + workerTierSpec); + + ImmutableWorkerInfo worker = strategy.findWorkerForTask( + new RemoteTaskRunnerConfig(), + WORKERS_FOR_TIER_TESTS, + new NoopTask(null, "ds1", 1, 0, null, null, null) + ); + Assert.assertEquals("localhost1", worker.getWorker().getHost()); + } + + @Test + public void testFindWorkerForTaskWithNullWorkerTierSpec() + { + final EqualDistributionWithTierSpecWorkerSelectStrategy strategy = new EqualDistributionWithTierSpecWorkerSelectStrategy( + null); + + ImmutableWorkerInfo worker = strategy.findWorkerForTask( + new RemoteTaskRunnerConfig(), + WORKERS_FOR_TIER_TESTS, + new NoopTask(null, "ds1", 1, 0, null, null, null) + ); + Assert.assertEquals("localhost3", worker.getWorker().getHost()); + } + + @Test + public void testFindWorkerForTaskWithNullPreferredTier() + { + final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + ImmutableMap.of( + "noop", + new WorkerTierSpec.TierConfig( + null, + null + ) + ), + false + ); + final EqualDistributionWithTierSpecWorkerSelectStrategy strategy = new EqualDistributionWithTierSpecWorkerSelectStrategy( + workerTierSpec); + + ImmutableWorkerInfo worker = strategy.findWorkerForTask( + new RemoteTaskRunnerConfig(), + WORKERS_FOR_TIER_TESTS, + new NoopTask(null, "ds1", 1, 0, null, null, null) + ); + Assert.assertEquals("localhost3", worker.getWorker().getHost()); + } + + @Test + public void testWeakTierSpec() + { + final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + ImmutableMap.of( + "noop", + new WorkerTierSpec.TierConfig( + "t1", + ImmutableMap.of("ds1", "t3") + ) + ), + false + ); + final EqualDistributionWithTierSpecWorkerSelectStrategy strategy = new EqualDistributionWithTierSpecWorkerSelectStrategy( + workerTierSpec); + + ImmutableWorkerInfo worker = strategy.findWorkerForTask( + new RemoteTaskRunnerConfig(), + WORKERS_FOR_TIER_TESTS, + new NoopTask(null, "ds1", 1, 0, null, null, null) + ); + Assert.assertEquals("localhost3", worker.getWorker().getHost()); + } + + @Test + public void testStrongTierSpec() + { + final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + ImmutableMap.of( + "noop", + new WorkerTierSpec.TierConfig( + "t1", + ImmutableMap.of("ds1", "t3") + ) + ), + true + ); + final EqualDistributionWithTierSpecWorkerSelectStrategy strategy = new EqualDistributionWithTierSpecWorkerSelectStrategy( + workerTierSpec); + + ImmutableWorkerInfo worker = strategy.findWorkerForTask( + new RemoteTaskRunnerConfig(), + WORKERS_FOR_TIER_TESTS, + new NoopTask(null, "ds1", 1, 0, null, null, null) + ); + Assert.assertNull(worker); + } +} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java index e68d0a9312a6..60c4bd066369 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java @@ -25,6 +25,7 @@ import org.apache.druid.indexing.overlord.ImmutableWorkerInfo; import org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig; import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.java.util.common.DateTimes; import org.junit.Assert; import org.junit.Test; @@ -37,28 +38,28 @@ public class EqualDistributionWorkerSelectStrategyTest ImmutableMap.of( "localhost0", new ImmutableWorkerInfo( - new Worker("http", "localhost0", "localhost0", 2, "v1"), 0, + new Worker("http", "localhost0", "localhost0", 2, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost1", new ImmutableWorkerInfo( - new Worker("http", "localhost1", "localhost1", 2, "v1"), 0, + new Worker("http", "localhost1", "localhost1", 2, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost2", new ImmutableWorkerInfo( - new Worker("http", "localhost2", "localhost2", 2, "v1"), 1, + new Worker("http", "localhost2", "localhost2", 2, "v1", WorkerConfig.DEFAULT_TIER), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost3", new ImmutableWorkerInfo( - new Worker("http", "localhost3", "localhost3", 2, "v1"), 1, + new Worker("http", "localhost3", "localhost3", 2, "v1", WorkerConfig.DEFAULT_TIER), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -75,14 +76,14 @@ public void testFindWorkerForTask() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "lhost", "lhost", 1, "v1"), 0, + new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 1, "v1"), 1, + new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -110,14 +111,14 @@ public void testFindWorkerForTaskWhenSameCurrCapacityUsed() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "lhost", "lhost", 5, "v1"), 5, + new Worker("http", "lhost", "lhost", 5, "v1", WorkerConfig.DEFAULT_TIER), 5, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 10, "v1"), 5, + new Worker("http", "localhost", "localhost", 10, "v1", WorkerConfig.DEFAULT_TIER), 5, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -146,14 +147,14 @@ public void testOneDisableWorkerDifferentUsedCapacity() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "disableHost", "disableHost", 10, DISABLED_VERSION), 2, + new Worker("http", "disableHost", "disableHost", 10, DISABLED_VERSION, WorkerConfig.DEFAULT_TIER), 2, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "enableHost", "enableHost", 10, "v1"), 5, + new Worker("http", "enableHost", "enableHost", 10, "v1", WorkerConfig.DEFAULT_TIER), 5, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -182,14 +183,14 @@ public void testOneDisableWorkerSameUsedCapacity() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "disableHost", "disableHost", 10, DISABLED_VERSION), 5, + new Worker("http", "disableHost", "disableHost", 10, DISABLED_VERSION, WorkerConfig.DEFAULT_TIER), 5, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "enableHost", "enableHost", 10, "v1"), 5, + new Worker("http", "enableHost", "enableHost", 10, "v1", WorkerConfig.DEFAULT_TIER), 5, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java index 73599f502ed6..0134ea5b73ae 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java @@ -25,6 +25,7 @@ import org.apache.druid.indexing.overlord.ImmutableWorkerInfo; import org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig; import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.java.util.common.DateTimes; import org.junit.Assert; import org.junit.Test; @@ -45,14 +46,14 @@ public void testFindWorkerForTask() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "lhost", "lhost", 1, "v1"), 0, + new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 1, "v1"), 0, + new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -82,14 +83,14 @@ public void testFindWorkerForTaskWithNulls() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "lhost", "lhost", 1, "v1"), 0, + new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 1, "v1"), 0, + new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -112,7 +113,7 @@ public void testIsolation() ImmutableMap.of( "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 1, "v1"), 0, + new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java new file mode 100644 index 000000000000..62fc71352483 --- /dev/null +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.setup; + +import com.google.common.collect.ImmutableMap; +import org.apache.druid.indexing.common.task.NoopTask; +import org.apache.druid.indexing.overlord.ImmutableWorkerInfo; +import org.apache.druid.indexing.overlord.config.RemoteTaskRunnerConfig; +import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.java.util.common.DateTimes; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashSet; + +public class FillCapacityWithTierSpecWorkerSelectStrategyTest +{ + private static final ImmutableMap WORKERS_FOR_TIER_TESTS = + ImmutableMap.of( + "localhost0", + new ImmutableWorkerInfo( + new Worker("http", "localhost0", "localhost0", 5, "v1", "t1"), 1, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() + ), + "localhost1", + new ImmutableWorkerInfo( + new Worker("http", "localhost1", "localhost1", 5, "v1", "t1"), 2, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() + ), + "localhost2", + new ImmutableWorkerInfo( + new Worker("http", "localhost2", "localhost2", 5, "v1", "t2"), 3, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() + ), + "localhost3", + new ImmutableWorkerInfo( + new Worker("http", "localhost3", "localhost3", 5, "v1", "t2"), 4, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() + ) + ); + + @Test + public void testFindWorkerForTask() + { + final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + ImmutableMap.of( + "noop", + new WorkerTierSpec.TierConfig( + "t1", + ImmutableMap.of("ds1", "t1") + ) + ), + false + ); + final FillCapacityWithTierSpecWorkerSelectStrategy strategy = new FillCapacityWithTierSpecWorkerSelectStrategy( + workerTierSpec); + + ImmutableWorkerInfo worker = strategy.findWorkerForTask( + new RemoteTaskRunnerConfig(), + WORKERS_FOR_TIER_TESTS, + new NoopTask(null, "ds1", 1, 0, null, null, null) + ); + Assert.assertEquals("localhost1", worker.getWorker().getHost()); + } + + @Test + public void testFindWorkerForTaskWithNullWorkerTierSpec() + { + final FillCapacityWithTierSpecWorkerSelectStrategy strategy = new FillCapacityWithTierSpecWorkerSelectStrategy( + null); + + ImmutableWorkerInfo worker = strategy.findWorkerForTask( + new RemoteTaskRunnerConfig(), + WORKERS_FOR_TIER_TESTS, + new NoopTask(null, "ds1", 1, 0, null, null, null) + ); + Assert.assertEquals("localhost3", worker.getWorker().getHost()); + } + + @Test + public void testFindWorkerForTaskWithNullPreferredTier() + { + final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + ImmutableMap.of( + "noop", + new WorkerTierSpec.TierConfig( + null, + null + ) + ), + false + ); + final FillCapacityWithTierSpecWorkerSelectStrategy strategy = new FillCapacityWithTierSpecWorkerSelectStrategy( + workerTierSpec); + + ImmutableWorkerInfo worker = strategy.findWorkerForTask( + new RemoteTaskRunnerConfig(), + WORKERS_FOR_TIER_TESTS, + new NoopTask(null, "ds1", 1, 0, null, null, null) + ); + Assert.assertEquals("localhost3", worker.getWorker().getHost()); + } + + @Test + public void testWeakTierSpec() + { + final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + ImmutableMap.of( + "noop", + new WorkerTierSpec.TierConfig( + "t1", + ImmutableMap.of("ds1", "t3") + ) + ), + false + ); + final FillCapacityWithTierSpecWorkerSelectStrategy strategy = new FillCapacityWithTierSpecWorkerSelectStrategy( + workerTierSpec); + + ImmutableWorkerInfo worker = strategy.findWorkerForTask( + new RemoteTaskRunnerConfig(), + WORKERS_FOR_TIER_TESTS, + new NoopTask(null, "ds1", 1, 0, null, null, null) + ); + Assert.assertEquals("localhost3", worker.getWorker().getHost()); + } + + @Test + public void testStrongTierSpec() + { + final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + ImmutableMap.of( + "noop", + new WorkerTierSpec.TierConfig( + "t1", + ImmutableMap.of("ds1", "t3") + ) + ), + true + ); + final FillCapacityWithTierSpecWorkerSelectStrategy strategy = new FillCapacityWithTierSpecWorkerSelectStrategy( + workerTierSpec); + + ImmutableWorkerInfo worker = strategy.findWorkerForTask( + new RemoteTaskRunnerConfig(), + WORKERS_FOR_TIER_TESTS, + new NoopTask(null, "ds1", 1, 0, null, null, null) + ); + Assert.assertNull(worker); + } +} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpecTest.java new file mode 100644 index 000000000000..a6d9f8997666 --- /dev/null +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpecTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.setup; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class WorkerTierSpecTest +{ + private ObjectMapper mapper; + + @Before + public void setUp() + { + mapper = new DefaultObjectMapper(); + } + + @Test + public void testSerde() throws Exception + { + String jsonStr = "{\n" + + " \"strong\": true,\n" + + " \"tierMap\": {\n" + + " \"index_kafka\": {\"defaultTier\": \"t1\", \"tiers\": {\"ds1\": \"t2\"}}\n" + + " }\n" + + "}"; + + WorkerTierSpec workerTierSpec = mapper.readValue( + mapper.writeValueAsString( + mapper.readValue( + jsonStr, + WorkerTierSpec.class + ) + ), WorkerTierSpec.class + ); + + Assert.assertTrue(workerTierSpec.isStrong()); + Assert.assertEquals(ImmutableMap.of( + "index_kafka", + new WorkerTierSpec.TierConfig("t1", ImmutableMap.of("ds1", "t2")) + ), workerTierSpec.getTierMap()); + } +} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskMonitorTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskMonitorTest.java index 20eb5fc3d675..66e8adb36656 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskMonitorTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskMonitorTest.java @@ -43,6 +43,7 @@ import org.apache.druid.indexing.common.task.Task; import org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner; import org.apache.druid.indexing.overlord.TestRemoteTaskRunnerConfig; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.IndexMergerV9; @@ -115,7 +116,8 @@ public void setUp() throws Exception "worker", "localhost", 3, - "0" + "0", + WorkerConfig.DEFAULT_TIER ); workerCuratorCoordinator = new WorkerCuratorCoordinator( diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/worker/http/WorkerResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/worker/http/WorkerResourceTest.java index 78aead2bce5b..6d9fbd3f15db 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/worker/http/WorkerResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/worker/http/WorkerResourceTest.java @@ -29,6 +29,7 @@ import org.apache.druid.indexing.worker.Worker; import org.apache.druid.indexing.worker.WorkerCuratorCoordinator; import org.apache.druid.indexing.worker.WorkerTaskMonitor; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.server.initialization.IndexerZkConfig; @@ -77,7 +78,8 @@ public void setUp() throws Exception "host", "ip", 3, - "v1" + "v1", + WorkerConfig.DEFAULT_TIER ); curatorCoordinator = new WorkerCuratorCoordinator( diff --git a/server/src/main/java/org/apache/druid/discovery/WorkerNodeService.java b/server/src/main/java/org/apache/druid/discovery/WorkerNodeService.java index 5ccfec964693..682ae73519bf 100644 --- a/server/src/main/java/org/apache/druid/discovery/WorkerNodeService.java +++ b/server/src/main/java/org/apache/druid/discovery/WorkerNodeService.java @@ -33,16 +33,19 @@ public class WorkerNodeService extends DruidService private final String ip; private final int capacity; private final String version; + private final String tier; public WorkerNodeService( @JsonProperty("ip") String ip, @JsonProperty("capacity") int capacity, - @JsonProperty("version") String version + @JsonProperty("version") String version, + @JsonProperty("tier") String tier ) { this.ip = ip; this.capacity = capacity; this.version = version; + this.tier = tier; } @Override @@ -69,6 +72,12 @@ public String getVersion() return version; } + @JsonProperty + public String getTier() + { + return tier; + } + @Override public boolean equals(Object o) { @@ -81,13 +90,14 @@ public boolean equals(Object o) WorkerNodeService that = (WorkerNodeService) o; return capacity == that.capacity && Objects.equals(ip, that.ip) && - Objects.equals(version, that.version); + Objects.equals(version, that.version) && + Objects.equals(tier, that.tier); } @Override public int hashCode() { - return Objects.hash(ip, capacity, version); + return Objects.hash(ip, capacity, version, tier); } @Override @@ -97,6 +107,7 @@ public String toString() "ip='" + ip + '\'' + ", capacity=" + capacity + ", version='" + version + '\'' + + ", tier='" + tier + '\'' + '}'; } } diff --git a/server/src/test/java/org/apache/druid/discovery/WorkerNodeServiceTest.java b/server/src/test/java/org/apache/druid/discovery/WorkerNodeServiceTest.java index 873b189d0a9a..80ce1bdb2bfc 100644 --- a/server/src/test/java/org/apache/druid/discovery/WorkerNodeServiceTest.java +++ b/server/src/test/java/org/apache/druid/discovery/WorkerNodeServiceTest.java @@ -34,7 +34,8 @@ public void testSerde() throws Exception DruidService expected = new WorkerNodeService( "1.1.1.1", 100, - "v1" + "v1", + "t1" ); ObjectMapper mapper = TestHelper.makeJsonMapper(); diff --git a/services/src/main/java/org/apache/druid/cli/CliMiddleManager.java b/services/src/main/java/org/apache/druid/cli/CliMiddleManager.java index 25c4e13e32dc..87dc8168e2c0 100644 --- a/services/src/main/java/org/apache/druid/cli/CliMiddleManager.java +++ b/services/src/main/java/org/apache/druid/cli/CliMiddleManager.java @@ -147,7 +147,8 @@ public Worker getWorker(@Self DruidNode node, WorkerConfig config) node.getHostAndPortToUse(), config.getIp(), config.getCapacity(), - config.getVersion() + config.getVersion(), + config.getTier() ); } @@ -158,7 +159,8 @@ public WorkerNodeService getWorkerNodeService(WorkerConfig workerConfig) return new WorkerNodeService( workerConfig.getIp(), workerConfig.getCapacity(), - workerConfig.getVersion() + workerConfig.getVersion(), + workerConfig.getTier() ); } }, From f79b831b6526113c443d0abfc2dc97145e3589de Mon Sep 17 00:00:00 2001 From: "qiumingming.2018" Date: Fri, 17 May 2019 19:52:52 +0800 Subject: [PATCH 2/8] address comments --- docs/content/configuration/index.md | 8 +- .../overlord/setup/WorkerSelectUtils.java | 4 +- .../overlord/setup/WorkerTierSpec.java | 16 ++-- ...nWithTierSpecWorkerSelectStrategyTest.java | 88 +++++++++++-------- ...yWithTierSpecWorkerSelectStrategyTest.java | 84 ++++++++++-------- .../overlord/setup/WorkerTierSpecTest.java | 2 +- 6 files changed, 115 insertions(+), 87 deletions(-) diff --git a/docs/content/configuration/index.md b/docs/content/configuration/index.md index 121bf1e2d24a..7bbeb504869d 100644 --- a/docs/content/configuration/index.md +++ b/docs/content/configuration/index.md @@ -1030,7 +1030,7 @@ useful if you want work evenly distributed across your middleManagers. ###### Equal Distribution With Tier Spec -This strategy is a variant of `Equal Distribution`, which support `workerTierSpec` field rather than `affinityConfig`. By specifying `workerTierSpec`, you can assign tasks to run on different tiers of MiddleManagers based on the tasks' **taskType** and **dataSource name**. +This strategy is a variant of `Equal Distribution`, which support `workerTierSpec` field rather than `affinityConfig`. By specifying `workerTierSpec`, you can assign tasks to run on different tiers of MiddleManagers based on the tasks' **taskType** and **dataSource name**. This strategy can't work with `AutoScaler` since the behavior is undefined. |Property|Description|Default| |--------|-----------|-------| @@ -1049,7 +1049,7 @@ type is "index_kafka", while dataSource "ds1" run on _tier2_. "tierMap": { "index_kafka": { "defaultTier": "tier1", - "tiers": { + "tierAffinity": { "ds1": "tier2" } } @@ -1075,7 +1075,7 @@ middleManagers up to capacity simultaneously, rather than a single middleManager ###### Fill Capacity With Tier Spec -This strategy is a variant of `Fill Capacity`, which support `workerTierSpec` field rather than `affinityConfig`. The usage is the same with _equalDistributionWithTierSpec_ strategy. +This strategy is a variant of `Fill Capacity`, which support `workerTierSpec` field rather than `affinityConfig`. The usage is the same with _equalDistributionWithTierSpec_ strategy. This strategy can't work with `AutoScaler` since the behavior is undefined. |Property|Description|Default| |--------|-----------|-------| @@ -1133,7 +1133,7 @@ field. If not provided, the default is to not use it at all. |Property|Description|Default| |--------|-----------|-------| |`defaultTier`|Specify default tier for a task type.|null| -|`tiers`|A JSON map object mapping a datasource String name to a tier String name of the MiddleManager. If tier isn't specified for a datasource, then using the `defaultTier`. If no specified tier and the `defaultTier` is also null, then tasks can run on any available MiddleManagers.|null| +|`tierAffinity`|A JSON map object mapping a datasource String name to a tier String name of the MiddleManager. If tier isn't specified for a datasource, then using the `defaultTier`. If no specified tier and the `defaultTier` is also null, then tasks can run on any available MiddleManagers.|null| ##### Autoscaler diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java index 726779bd1ee3..6df581933659 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java @@ -130,10 +130,10 @@ public static ImmutableWorkerInfo selectWorker( new WorkerTierSpec.TierConfig(null, null) ); final String defaultTier = tierConfig.getDefaultTier(); - final Map tiers = tierConfig.getTiers(); + final Map tierAffinity = tierConfig.getTierAffinity(); // select worker from preferred tier - final String preferredTier = tiers != null ? tiers.get(task.getDataSource()) : defaultTier; + final String preferredTier = tierAffinity != null ? tierAffinity.get(task.getDataSource()) : defaultTier; if (preferredTier != null) { final ImmutableMap tierWorkers = getTierWorkers(preferredTier, runnableWorkers); final ImmutableWorkerInfo selected = workerSelector.apply(tierWorkers); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java index 88d46131f4b0..b178795d9e23 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java @@ -86,16 +86,16 @@ public static class TierConfig { private final String defaultTier; // key: datasource, value: tier - private final Map tiers; + private final Map tierAffinity; @JsonCreator public TierConfig( @JsonProperty("defaultTier") String defaultTier, - @JsonProperty("tiers") Map tiers + @JsonProperty("tierAffinity") Map tierAffinity ) { this.defaultTier = defaultTier; - this.tiers = tiers; + this.tierAffinity = tierAffinity; } @JsonProperty @@ -105,9 +105,9 @@ public String getDefaultTier() } @JsonProperty - public Map getTiers() + public Map getTierAffinity() { - return tiers; + return tierAffinity; } @Override @@ -121,13 +121,13 @@ public boolean equals(final Object o) } final TierConfig that = (TierConfig) o; return Objects.equals(defaultTier, that.defaultTier) && - Objects.equals(tiers, that.tiers); + Objects.equals(tierAffinity, that.tierAffinity); } @Override public int hashCode() { - return Objects.hash(defaultTier, tiers); + return Objects.hash(defaultTier, tierAffinity); } @Override @@ -135,7 +135,7 @@ public String toString() { return "TierConfig{" + "defaultTier=" + defaultTier + - ", tiers=" + tiers + + ", tierAffinity=" + tierAffinity + '}'; } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java index eb86e4969144..21c3d51e6670 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java @@ -65,41 +65,59 @@ public class EqualDistributionWithTierSpecWorkerSelectStrategyTest ); @Test - public void testFindWorkerForTask() + public void testFindWorkerForTaskWithNullWorkerTierSpec() { - final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + ImmutableWorkerInfo worker = selectWorker(null); + Assert.assertEquals("localhost3", worker.getWorker().getHost()); + } + + @Test + public void testFindWorkerForTaskWithPreferredTier() + { + // test defaultTier != null and tierAffinity != null + final WorkerTierSpec workerTierSpec1 = new WorkerTierSpec( ImmutableMap.of( "noop", new WorkerTierSpec.TierConfig( - "t1", - ImmutableMap.of("ds1", "t1") + "t2", + ImmutableMap.of("ds1", "t2") ) ), false ); - final EqualDistributionWithTierSpecWorkerSelectStrategy strategy = new EqualDistributionWithTierSpecWorkerSelectStrategy( - workerTierSpec); - ImmutableWorkerInfo worker = strategy.findWorkerForTask( - new RemoteTaskRunnerConfig(), - WORKERS_FOR_TIER_TESTS, - new NoopTask(null, "ds1", 1, 0, null, null, null) + ImmutableWorkerInfo worker1 = selectWorker(workerTierSpec1); + Assert.assertEquals("localhost3", worker1.getWorker().getHost()); + + // test defaultTier == null and tierAffinity != null + final WorkerTierSpec workerTierSpec2 = new WorkerTierSpec( + ImmutableMap.of( + "noop", + new WorkerTierSpec.TierConfig( + null, + ImmutableMap.of("ds1", "t2") + ) + ), + false ); - Assert.assertEquals("localhost1", worker.getWorker().getHost()); - } - @Test - public void testFindWorkerForTaskWithNullWorkerTierSpec() - { - final EqualDistributionWithTierSpecWorkerSelectStrategy strategy = new EqualDistributionWithTierSpecWorkerSelectStrategy( - null); + ImmutableWorkerInfo worker2 = selectWorker(workerTierSpec2); + Assert.assertEquals("localhost3", worker2.getWorker().getHost()); - ImmutableWorkerInfo worker = strategy.findWorkerForTask( - new RemoteTaskRunnerConfig(), - WORKERS_FOR_TIER_TESTS, - new NoopTask(null, "ds1", 1, 0, null, null, null) + // test defaultTier != null and tierAffinity == null + final WorkerTierSpec workerTierSpec3 = new WorkerTierSpec( + ImmutableMap.of( + "noop", + new WorkerTierSpec.TierConfig( + "t2", + null + ) + ), + false ); - Assert.assertEquals("localhost3", worker.getWorker().getHost()); + + ImmutableWorkerInfo worker3 = selectWorker(workerTierSpec3); + Assert.assertEquals("localhost3", worker3.getWorker().getHost()); } @Test @@ -115,14 +133,8 @@ public void testFindWorkerForTaskWithNullPreferredTier() ), false ); - final EqualDistributionWithTierSpecWorkerSelectStrategy strategy = new EqualDistributionWithTierSpecWorkerSelectStrategy( - workerTierSpec); - ImmutableWorkerInfo worker = strategy.findWorkerForTask( - new RemoteTaskRunnerConfig(), - WORKERS_FOR_TIER_TESTS, - new NoopTask(null, "ds1", 1, 0, null, null, null) - ); + ImmutableWorkerInfo worker = selectWorker(workerTierSpec); Assert.assertEquals("localhost3", worker.getWorker().getHost()); } @@ -139,14 +151,8 @@ public void testWeakTierSpec() ), false ); - final EqualDistributionWithTierSpecWorkerSelectStrategy strategy = new EqualDistributionWithTierSpecWorkerSelectStrategy( - workerTierSpec); - ImmutableWorkerInfo worker = strategy.findWorkerForTask( - new RemoteTaskRunnerConfig(), - WORKERS_FOR_TIER_TESTS, - new NoopTask(null, "ds1", 1, 0, null, null, null) - ); + ImmutableWorkerInfo worker = selectWorker(workerTierSpec); Assert.assertEquals("localhost3", worker.getWorker().getHost()); } @@ -163,6 +169,13 @@ public void testStrongTierSpec() ), true ); + + ImmutableWorkerInfo worker = selectWorker(workerTierSpec); + Assert.assertNull(worker); + } + + private ImmutableWorkerInfo selectWorker(WorkerTierSpec workerTierSpec) + { final EqualDistributionWithTierSpecWorkerSelectStrategy strategy = new EqualDistributionWithTierSpecWorkerSelectStrategy( workerTierSpec); @@ -171,6 +184,7 @@ public void testStrongTierSpec() WORKERS_FOR_TIER_TESTS, new NoopTask(null, "ds1", 1, 0, null, null, null) ); - Assert.assertNull(worker); + + return worker; } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java index 62fc71352483..e9656ed1d3a5 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java @@ -65,9 +65,17 @@ public class FillCapacityWithTierSpecWorkerSelectStrategyTest ); @Test - public void testFindWorkerForTask() + public void testFindWorkerForTaskWithNullWorkerTierSpec() { - final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + ImmutableWorkerInfo worker = selectWorker(null); + Assert.assertEquals("localhost3", worker.getWorker().getHost()); + } + + @Test + public void testFindWorkerForTaskWithPreferredTier() + { + // test defaultTier != null and tierAffinity != null + final WorkerTierSpec workerTierSpec1 = new WorkerTierSpec( ImmutableMap.of( "noop", new WorkerTierSpec.TierConfig( @@ -77,29 +85,39 @@ public void testFindWorkerForTask() ), false ); - final FillCapacityWithTierSpecWorkerSelectStrategy strategy = new FillCapacityWithTierSpecWorkerSelectStrategy( - workerTierSpec); - ImmutableWorkerInfo worker = strategy.findWorkerForTask( - new RemoteTaskRunnerConfig(), - WORKERS_FOR_TIER_TESTS, - new NoopTask(null, "ds1", 1, 0, null, null, null) + ImmutableWorkerInfo worker1 = selectWorker(workerTierSpec1); + Assert.assertEquals("localhost1", worker1.getWorker().getHost()); + + // test defaultTier == null and tierAffinity != null + final WorkerTierSpec workerTierSpec2 = new WorkerTierSpec( + ImmutableMap.of( + "noop", + new WorkerTierSpec.TierConfig( + null, + ImmutableMap.of("ds1", "t1") + ) + ), + false ); - Assert.assertEquals("localhost1", worker.getWorker().getHost()); - } - @Test - public void testFindWorkerForTaskWithNullWorkerTierSpec() - { - final FillCapacityWithTierSpecWorkerSelectStrategy strategy = new FillCapacityWithTierSpecWorkerSelectStrategy( - null); + ImmutableWorkerInfo worker2 = selectWorker(workerTierSpec2); + Assert.assertEquals("localhost1", worker2.getWorker().getHost()); - ImmutableWorkerInfo worker = strategy.findWorkerForTask( - new RemoteTaskRunnerConfig(), - WORKERS_FOR_TIER_TESTS, - new NoopTask(null, "ds1", 1, 0, null, null, null) + // test defaultTier != null and tierAffinity == null + final WorkerTierSpec workerTierSpec3 = new WorkerTierSpec( + ImmutableMap.of( + "noop", + new WorkerTierSpec.TierConfig( + "t1", + null + ) + ), + false ); - Assert.assertEquals("localhost3", worker.getWorker().getHost()); + + ImmutableWorkerInfo worker3 = selectWorker(workerTierSpec3); + Assert.assertEquals("localhost1", worker3.getWorker().getHost()); } @Test @@ -115,14 +133,8 @@ public void testFindWorkerForTaskWithNullPreferredTier() ), false ); - final FillCapacityWithTierSpecWorkerSelectStrategy strategy = new FillCapacityWithTierSpecWorkerSelectStrategy( - workerTierSpec); - ImmutableWorkerInfo worker = strategy.findWorkerForTask( - new RemoteTaskRunnerConfig(), - WORKERS_FOR_TIER_TESTS, - new NoopTask(null, "ds1", 1, 0, null, null, null) - ); + ImmutableWorkerInfo worker = selectWorker(workerTierSpec); Assert.assertEquals("localhost3", worker.getWorker().getHost()); } @@ -139,14 +151,8 @@ public void testWeakTierSpec() ), false ); - final FillCapacityWithTierSpecWorkerSelectStrategy strategy = new FillCapacityWithTierSpecWorkerSelectStrategy( - workerTierSpec); - ImmutableWorkerInfo worker = strategy.findWorkerForTask( - new RemoteTaskRunnerConfig(), - WORKERS_FOR_TIER_TESTS, - new NoopTask(null, "ds1", 1, 0, null, null, null) - ); + ImmutableWorkerInfo worker = selectWorker(workerTierSpec); Assert.assertEquals("localhost3", worker.getWorker().getHost()); } @@ -163,6 +169,13 @@ public void testStrongTierSpec() ), true ); + + ImmutableWorkerInfo worker = selectWorker(workerTierSpec); + Assert.assertNull(worker); + } + + private ImmutableWorkerInfo selectWorker(WorkerTierSpec workerTierSpec) + { final FillCapacityWithTierSpecWorkerSelectStrategy strategy = new FillCapacityWithTierSpecWorkerSelectStrategy( workerTierSpec); @@ -171,6 +184,7 @@ public void testStrongTierSpec() WORKERS_FOR_TIER_TESTS, new NoopTask(null, "ds1", 1, 0, null, null, null) ); - Assert.assertNull(worker); + + return worker; } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpecTest.java index a6d9f8997666..7dcfc74ae199 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpecTest.java @@ -42,7 +42,7 @@ public void testSerde() throws Exception String jsonStr = "{\n" + " \"strong\": true,\n" + " \"tierMap\": {\n" - + " \"index_kafka\": {\"defaultTier\": \"t1\", \"tiers\": {\"ds1\": \"t2\"}}\n" + + " \"index_kafka\": {\"defaultTier\": \"t1\", \"tierAffinity\": {\"ds1\": \"t2\"}}\n" + " }\n" + "}"; From b171c78ef32feff41d8ab61e587f2f0f5691e11c Mon Sep 17 00:00:00 2001 From: "qiumingming.2018" Date: Mon, 7 Oct 2019 22:57:44 +0800 Subject: [PATCH 3/8] address comments --- docs/configuration/index.md | 2 + .../overlord/setup/WorkerSelectUtils.java | 71 ++++++++++--------- .../overlord/setup/WorkerTierSpec.java | 5 +- .../WorkerTaskRunnerQueryAdpaterTest.java | 5 +- ...nWithTierSpecWorkerSelectStrategyTest.java | 8 +-- ...yWithTierSpecWorkerSelectStrategyTest.java | 8 +-- .../java/org/apache/druid/cli/CliIndexer.java | 6 +- 7 files changed, 58 insertions(+), 47 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 91b2e412ba12..54616b604457 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1037,6 +1037,8 @@ This strategy is a variant of `Fill Capacity`, which support `workerTierSpec` fi |`type`|`fillCapacityWithTierSpec`.|required; must be `fillCapacityWithTierSpec`| |`workerTierSpec`|[Worker Tier Spec](#workerTierSpec) object|null (no worker tier spec)| +> Before using the _equalDistributionWithTierSpec_ and _fillCapacityWithTierSpec_ strategies, you must upgrade overlord and all MiddleManagers to the version that support this feature. + ###### Javascript Allows defining arbitrary logic for selecting workers to run task using a JavaScript function. diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java index 6df581933659..f9c9baf329a0 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java @@ -57,13 +57,7 @@ public static ImmutableWorkerInfo selectWorker( final Function, ImmutableWorkerInfo> workerSelector ) { - // Workers that could potentially run this task, ignoring affinityConfig. - final Map runnableWorkers = allWorkers - .values() - .stream() - .filter(worker -> worker.canRunTask(task) - && worker.isValidVersion(workerTaskRunnerConfig.getMinWorkerVersion())) - .collect(Collectors.toMap(w -> w.getWorker().getHost(), Function.identity())); + final Map runnableWorkers = getRunnableWorkers(task, allWorkers, workerTaskRunnerConfig); if (affinityConfig == null) { // All runnable workers are valid. @@ -114,34 +108,31 @@ public static ImmutableWorkerInfo selectWorker( final Function, ImmutableWorkerInfo> workerSelector ) { - // Workers that could potentially run this task, ignoring workerTierSpec. - final Map runnableWorkers = allWorkers - .values() - .stream() - .filter(worker -> worker.canRunTask(task) - && worker.isValidVersion(workerTaskRunnerConfig.getMinWorkerVersion())) - .collect(Collectors.toMap(w -> w.getWorker().getHost(), Function.identity())); + final Map runnableWorkers = getRunnableWorkers(task, allWorkers, workerTaskRunnerConfig); // select worker according to worker tier spec - if (workerTierSpec != null && workerTierSpec.getTierMap() != null) { - final WorkerTierSpec.TierConfig tierConfig = workerTierSpec.getTierMap() - .getOrDefault( - task.getType(), - new WorkerTierSpec.TierConfig(null, null) - ); - final String defaultTier = tierConfig.getDefaultTier(); - final Map tierAffinity = tierConfig.getTierAffinity(); - - // select worker from preferred tier - final String preferredTier = tierAffinity != null ? tierAffinity.get(task.getDataSource()) : defaultTier; - if (preferredTier != null) { - final ImmutableMap tierWorkers = getTierWorkers(preferredTier, runnableWorkers); - final ImmutableWorkerInfo selected = workerSelector.apply(tierWorkers); - - if (selected != null) { - return selected; - } else if (workerTierSpec.isStrong()) { - return null; + if (workerTierSpec != null) { + final WorkerTierSpec.TierConfig tierConfig = workerTierSpec.getTierMap().get(task.getType()); + + if (tierConfig != null) { + final String defaultTier = tierConfig.getDefaultTier(); + final Map tierAffinity = tierConfig.getTierAffinity(); + + String preferredTier = tierAffinity.get(task.getDataSource()); + // If there is no preferred tier for the datasource, then using the defaultTier. However, the defaultTier + // may be null too, so we need to do one more null check (see below). + preferredTier = preferredTier == null ? defaultTier : preferredTier; + + if (preferredTier != null) { + // select worker from preferred tier + final ImmutableMap tierWorkers = getTierWorkers(preferredTier, runnableWorkers); + final ImmutableWorkerInfo selected = workerSelector.apply(tierWorkers); + + if (selected != null) { + return selected; + } else if (workerTierSpec.isStrong()) { + return null; + } } } } @@ -150,6 +141,20 @@ public static ImmutableWorkerInfo selectWorker( return workerSelector.apply(ImmutableMap.copyOf(runnableWorkers)); } + // Get workers that could potentially run this task, ignoring affinityConfig/workerTierSpec. + private static Map getRunnableWorkers( + final Task task, + final Map allWorkers, + final WorkerTaskRunnerConfig workerTaskRunnerConfig + ) + { + return allWorkers.values() + .stream() + .filter(worker -> worker.canRunTask(task) + && worker.isValidVersion(workerTaskRunnerConfig.getMinWorkerVersion())) + .collect(Collectors.toMap(w -> w.getWorker().getHost(), Function.identity())); + } + /** * Return workers belong to this tier. * diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java index b178795d9e23..1c9f64b79618 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -37,7 +38,7 @@ public WorkerTierSpec( @JsonProperty("strong") boolean strong ) { - this.tierMap = tierMap; + this.tierMap = tierMap == null ? Collections.EMPTY_MAP : tierMap; this.strong = strong; } @@ -95,7 +96,7 @@ public TierConfig( ) { this.defaultTier = defaultTier; - this.tierAffinity = tierAffinity; + this.tierAffinity = tierAffinity == null ? Collections.EMPTY_MAP : tierAffinity; } @JsonProperty diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdpaterTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdpaterTest.java index 330ff2c8b7e8..7effd5d00b6c 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdpaterTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdpaterTest.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.SettableFuture; import org.apache.druid.indexing.worker.Worker; +import org.apache.druid.indexing.worker.config.WorkerConfig; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.http.client.HttpClient; @@ -66,7 +67,7 @@ public void setup() ImmutableList.of( new ImmutableWorkerInfo( new Worker( - "http", "worker-host1", "192.0.0.1", 10, "v1" + "http", "worker-host1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER ), 2, ImmutableSet.of("grp1", "grp2"), @@ -75,7 +76,7 @@ public void setup() ), new ImmutableWorkerInfo( new Worker( - "https", "worker-host2", "192.0.0.2", 4, "v1" + "https", "worker-host2", "192.0.0.2", 4, "v1", WorkerConfig.DEFAULT_TIER ), 1, ImmutableSet.of("grp1"), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java index 21c3d51e6670..a26139b48f46 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java @@ -74,7 +74,7 @@ public void testFindWorkerForTaskWithNullWorkerTierSpec() @Test public void testFindWorkerForTaskWithPreferredTier() { - // test defaultTier != null and tierAffinity != null + // test defaultTier != null and tierAffinity is not empty final WorkerTierSpec workerTierSpec1 = new WorkerTierSpec( ImmutableMap.of( "noop", @@ -89,7 +89,7 @@ public void testFindWorkerForTaskWithPreferredTier() ImmutableWorkerInfo worker1 = selectWorker(workerTierSpec1); Assert.assertEquals("localhost3", worker1.getWorker().getHost()); - // test defaultTier == null and tierAffinity != null + // test defaultTier == null and tierAffinity is not empty final WorkerTierSpec workerTierSpec2 = new WorkerTierSpec( ImmutableMap.of( "noop", @@ -104,7 +104,7 @@ public void testFindWorkerForTaskWithPreferredTier() ImmutableWorkerInfo worker2 = selectWorker(workerTierSpec2); Assert.assertEquals("localhost3", worker2.getWorker().getHost()); - // test defaultTier != null and tierAffinity == null + // test defaultTier != null and tierAffinity is empty final WorkerTierSpec workerTierSpec3 = new WorkerTierSpec( ImmutableMap.of( "noop", @@ -182,7 +182,7 @@ private ImmutableWorkerInfo selectWorker(WorkerTierSpec workerTierSpec) ImmutableWorkerInfo worker = strategy.findWorkerForTask( new RemoteTaskRunnerConfig(), WORKERS_FOR_TIER_TESTS, - new NoopTask(null, "ds1", 1, 0, null, null, null) + new NoopTask(null, null, "ds1", 1, 0, null, null, null) ); return worker; diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java index e9656ed1d3a5..088078ddc6dc 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java @@ -74,7 +74,7 @@ public void testFindWorkerForTaskWithNullWorkerTierSpec() @Test public void testFindWorkerForTaskWithPreferredTier() { - // test defaultTier != null and tierAffinity != null + // test defaultTier != null and tierAffinity is not empty final WorkerTierSpec workerTierSpec1 = new WorkerTierSpec( ImmutableMap.of( "noop", @@ -89,7 +89,7 @@ public void testFindWorkerForTaskWithPreferredTier() ImmutableWorkerInfo worker1 = selectWorker(workerTierSpec1); Assert.assertEquals("localhost1", worker1.getWorker().getHost()); - // test defaultTier == null and tierAffinity != null + // test defaultTier == null and tierAffinity is not empty final WorkerTierSpec workerTierSpec2 = new WorkerTierSpec( ImmutableMap.of( "noop", @@ -104,7 +104,7 @@ public void testFindWorkerForTaskWithPreferredTier() ImmutableWorkerInfo worker2 = selectWorker(workerTierSpec2); Assert.assertEquals("localhost1", worker2.getWorker().getHost()); - // test defaultTier != null and tierAffinity == null + // test defaultTier != null and tierAffinity is empty final WorkerTierSpec workerTierSpec3 = new WorkerTierSpec( ImmutableMap.of( "noop", @@ -182,7 +182,7 @@ private ImmutableWorkerInfo selectWorker(WorkerTierSpec workerTierSpec) ImmutableWorkerInfo worker = strategy.findWorkerForTask( new RemoteTaskRunnerConfig(), WORKERS_FOR_TIER_TESTS, - new NoopTask(null, "ds1", 1, 0, null, null, null) + new NoopTask(null, null, "ds1", 1, 0, null, null, null) ); return worker; diff --git a/services/src/main/java/org/apache/druid/cli/CliIndexer.java b/services/src/main/java/org/apache/druid/cli/CliIndexer.java index 97afa5b413bd..41581a2111d2 100644 --- a/services/src/main/java/org/apache/druid/cli/CliIndexer.java +++ b/services/src/main/java/org/apache/druid/cli/CliIndexer.java @@ -161,7 +161,8 @@ public Worker getWorker(@Self DruidNode node, WorkerConfig config) node.getHostAndPortToUse(), config.getIp(), config.getCapacity(), - config.getVersion() + config.getVersion(), + WorkerConfig.DEFAULT_TIER ); } @@ -172,7 +173,8 @@ public WorkerNodeService getWorkerNodeService(WorkerConfig workerConfig) return new WorkerNodeService( workerConfig.getIp(), workerConfig.getCapacity(), - workerConfig.getVersion() + workerConfig.getVersion(), + WorkerConfig.DEFAULT_TIER ); } }, From b8fa638abcd98fd9e7f04689ac48d75f04280dc4 Mon Sep 17 00:00:00 2001 From: "qiumingming.2018" Date: Tue, 8 Oct 2019 01:31:37 +0800 Subject: [PATCH 4/8] rename tier to category and docs --- docs/configuration/index.md | 50 +++++++------- ...ngTaskBasedWorkerProvisioningStrategy.java | 2 +- .../overlord/hrtr/HttpRemoteTaskRunner.java | 2 +- .../indexing/overlord/hrtr/WorkerHolder.java | 2 +- ...WithCategorySpecWorkerSelectStrategy.java} | 26 +++---- ...WithCategorySpecWorkerSelectStrategy.java} | 26 +++---- ...rTierSpec.java => WorkerCategorySpec.java} | 66 +++++++++--------- .../overlord/setup/WorkerSelectStrategy.java | 4 +- .../overlord/setup/WorkerSelectUtils.java | 46 ++++++------- .../apache/druid/indexing/worker/Worker.java | 16 ++--- .../indexing/worker/http/WorkerResource.java | 2 +- .../overlord/ImmutableWorkerInfoTest.java | 36 +++++----- .../overlord/RemoteTaskRunnerTestUtils.java | 4 +- .../overlord/TaskRunnerUtilsTest.java | 2 +- .../WorkerTaskRunnerQueryAdpaterTest.java | 4 +- ...dingTaskBasedProvisioningStrategyTest.java | 2 +- .../SimpleProvisioningStrategyTest.java | 2 +- .../hrtr/HttpRemoteTaskRunnerTest.java | 34 +++++----- .../overlord/hrtr/WorkerHolderTest.java | 2 +- ...nWithAffinityWorkerSelectStrategyTest.java | 50 +++++++------- ...CategorySpecWorkerSelectStrategyTest.java} | 68 +++++++++---------- ...lDistributionWorkerSelectStrategyTest.java | 24 +++---- ...yWithAffinityWorkerSelectStrategyTest.java | 10 +-- ...CategorySpecWorkerSelectStrategyTest.java} | 68 +++++++++---------- ...cTest.java => WorkerCategorySpecTest.java} | 18 ++--- .../worker/WorkerTaskMonitorTest.java | 2 +- .../worker/http/WorkerResourceTest.java | 2 +- .../druid/discovery/WorkerNodeService.java | 16 ++--- .../indexing/worker/config/WorkerConfig.java | 8 +-- .../discovery/WorkerNodeServiceTest.java | 2 +- .../java/org/apache/druid/cli/CliIndexer.java | 4 +- .../apache/druid/cli/CliMiddleManager.java | 4 +- 32 files changed, 302 insertions(+), 302 deletions(-) rename indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/{EqualDistributionWithTierSpecWorkerSelectStrategy.java => EqualDistributionWithCategorySpecWorkerSelectStrategy.java} (70%) rename indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/{FillCapacityWithTierSpecWorkerSelectStrategy.java => FillCapacityWithCategorySpecWorkerSelectStrategy.java} (71%) rename indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/{WorkerTierSpec.java => WorkerCategorySpec.java} (55%) rename indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/{EqualDistributionWithTierSpecWorkerSelectStrategyTest.java => EqualDistributionWithCategorySpecWorkerSelectStrategyTest.java} (69%) rename indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/{FillCapacityWithTierSpecWorkerSelectStrategyTest.java => FillCapacityWithCategorySpecWorkerSelectStrategyTest.java} (70%) rename indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/{WorkerTierSpecTest.java => WorkerCategorySpecTest.java} (75%) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 54616b604457..59b74298a916 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -983,29 +983,29 @@ useful if you want work evenly distributed across your MiddleManagers. |`type`|`equalDistribution`.|required; must be `equalDistribution`| |`affinityConfig`|[Affinity config](#affinity) object|null (no affinity)| -###### Equal Distribution With Tier Spec +###### Equal Distribution With Category Spec -This strategy is a variant of `Equal Distribution`, which support `workerTierSpec` field rather than `affinityConfig`. By specifying `workerTierSpec`, you can assign tasks to run on different tiers of MiddleManagers based on the tasks' **taskType** and **dataSource name**. This strategy can't work with `AutoScaler` since the behavior is undefined. +This strategy is a variant of `Equal Distribution`, which support `workerCategorySpec` field rather than `affinityConfig`. By specifying `workerCategorySpec`, you can assign tasks to run on different categories of MiddleManagers based on the tasks' **taskType** and **dataSource name**. This strategy can't work with `AutoScaler` since the behavior is undefined. |Property|Description|Default| |--------|-----------|-------| -|`type`|`equalDistributionWithTierSpec`.|required; must be `equalDistributionWithTierSpec`| -|`workerTierSpec`|[Worker Tier Spec](#workerTierSpec) object|null (no worker tier spec)| +|`type`|`equalDistributionWithCategorySpec`.|required; must be `equalDistributionWithCategorySpec`| +|`workerCategorySpec`|[Worker Category Spec](#workerCategorySpec) object|null (no worker category spec)| -Example: specify tasks default to run on _tier1_ whose task -type is "index_kafka", while dataSource "ds1" run on _tier2_. +Example: specify tasks default to run on _c1_ whose task +type is "index_kafka", while dataSource "ds1" run on _c2_. ```json { "selectStrategy": { - "type": "equalDistributionWithTierSpec", - "workerTierSpec": { + "type": "equalDistributionWithCategorySpec", + "workerCategorySpec": { "strong": false, - "tierMap": { + "categoryMap": { "index_kafka": { - "defaultTier": "tier1", - "tierAffinity": { - "ds1": "tier2" + "defaultCategory": "c1", + "categoryAffinity": { + "ds1": "c2" } } } @@ -1028,16 +1028,16 @@ MiddleManagers up to capacity simultaneously, rather than a single MiddleManager |`type`|`fillCapacity`.|required; must be `fillCapacity`| |`affinityConfig`|[Affinity config](#affinity) object|null (no affinity)| -###### Fill Capacity With Tier Spec +###### Fill Capacity With Category Spec -This strategy is a variant of `Fill Capacity`, which support `workerTierSpec` field rather than `affinityConfig`. The usage is the same with _equalDistributionWithTierSpec_ strategy. This strategy can't work with `AutoScaler` since the behavior is undefined. +This strategy is a variant of `Fill Capacity`, which support `workerCategorySpec` field rather than `affinityConfig`. The usage is the same with _equalDistributionWithCategorySpec_ strategy. This strategy can't work with `AutoScaler` since the behavior is undefined. |Property|Description|Default| |--------|-----------|-------| -|`type`|`fillCapacityWithTierSpec`.|required; must be `fillCapacityWithTierSpec`| -|`workerTierSpec`|[Worker Tier Spec](#workerTierSpec) object|null (no worker tier spec)| +|`type`|`fillCapacityWithCategorySpec`.|required; must be `fillCapacityWithCategorySpec`| +|`workerCategorySpec`|[Worker Category Spec](#workerCategorySpec) object|null (no worker category spec)| -> Before using the _equalDistributionWithTierSpec_ and _fillCapacityWithTierSpec_ strategies, you must upgrade overlord and all MiddleManagers to the version that support this feature. +> Before using the _equalDistributionWithCategorySpec_ and _fillCapacityWithCategorySpec_ strategies, you must upgrade overlord and all MiddleManagers to the version that support this feature. ###### Javascript @@ -1073,22 +1073,22 @@ field. If not provided, the default is to not use affinity at all. |`affinity`|JSON object mapping a datasource String name to a list of indexing service MiddleManager host:port String values. Druid doesn't perform DNS resolution, so the 'host' value must match what is configured on the MiddleManager and what the MiddleManager announces itself as (examine the Overlord logs to see what your MiddleManager announces itself as).|{}| |`strong`|With weak affinity (the default), tasks for a dataSource may be assigned to other MiddleManagers if their affinity-mapped MiddleManagers are not able to run all pending tasks in the queue for that dataSource. With strong affinity, tasks for a dataSource will only ever be assigned to their affinity-mapped MiddleManagers, and will wait in the pending queue if necessary.|false| -###### WorkerTierSpec +###### WorkerCategorySpec -WorkerTierSpec can be provided to the _equalDistributionWithTierSpec_ and _fillCapacityWithTierSpec_ strategies using the "workerTierSpec" +WorkerCategorySpec can be provided to the _equalDistributionWithCategorySpec_ and _fillCapacityWithCategorySpec_ strategies using the "workerCategorySpec" field. If not provided, the default is to not use it at all. |Property|Description|Default| |--------|-----------|-------| -|`tierMap`|A JSON map object mapping a task type String name to a [TierConfig](tierConfig) object, by which you can specify tier config for different task type.|{}| -|`strong`|With weak workerTierSpec (the default), tasks for a dataSource may be assigned to other middleManagers if the middleManagers specified in `tierMap` are not able to run all pending tasks in the queue for that dataSource. With strong workerTierSpec, tasks for a dataSource will only ever be assigned to their specified middleManagers, and will wait in the pending queue if necessary.|false| +|`categoryMap`|A JSON map object mapping a task type String name to a [CategoryConfig](categoryConfig) object, by which you can specify category config for different task type.|{}| +|`strong`|With weak workerCategorySpec (the default), tasks for a dataSource may be assigned to other middleManagers if the middleManagers specified in `categoryMap` are not able to run all pending tasks in the queue for that dataSource. With strong workerCategorySpec, tasks for a dataSource will only ever be assigned to their specified middleManagers, and will wait in the pending queue if necessary.|false| -###### TierConfig +###### CategoryConfig |Property|Description|Default| |--------|-----------|-------| -|`defaultTier`|Specify default tier for a task type.|null| -|`tierAffinity`|A JSON map object mapping a datasource String name to a tier String name of the MiddleManager. If tier isn't specified for a datasource, then using the `defaultTier`. If no specified tier and the `defaultTier` is also null, then tasks can run on any available MiddleManagers.|null| +|`defaultCategory`|Specify default category for a task type.|null| +|`categoryAffinity`|A JSON map object mapping a datasource String name to a category String name of the MiddleManager. If category isn't specified for a datasource, then using the `defaultCategory`. If no specified category and the `defaultCategory` is also null, then tasks can run on any available MiddleManagers.|null| ##### Autoscaler @@ -1141,7 +1141,7 @@ Middle managers pass their configurations down to their child peons. The MiddleM |`druid.worker.ip`|The IP of the worker.|localhost| |`druid.worker.version`|Version identifier for the MiddleManager.|0| |`druid.worker.capacity`|Maximum number of tasks the MiddleManager can accept.|Number of available processors - 1| -|`druid.worker.tier`|A string to name the tier that the MiddleManager node belongs to.|`_default_worker_tier`| +|`druid.worker.category`|A string to name the category that the MiddleManager node belongs to.|`__default_worker_category`| #### Peon Processing diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java index f58275a663de..52624f41e4e0 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java @@ -478,7 +478,7 @@ private static ImmutableWorkerInfo workerWithTask(ImmutableWorkerInfo immutableW private static ImmutableWorkerInfo createDummyWorker(String scheme, String host, int capacity, String version) { return new ImmutableWorkerInfo( - new Worker(scheme, host, "-2", capacity, version, WorkerConfig.DEFAULT_TIER), + new Worker(scheme, host, "-2", capacity, version, WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunner.java index 54760b268766..afbdbd75b410 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunner.java @@ -486,7 +486,7 @@ private Worker toWorker(DiscoveryDruidNode node) ((WorkerNodeService) node.getServices().get(WorkerNodeService.DISCOVERY_SERVICE_KEY)).getIp(), ((WorkerNodeService) node.getServices().get(WorkerNodeService.DISCOVERY_SERVICE_KEY)).getCapacity(), ((WorkerNodeService) node.getServices().get(WorkerNodeService.DISCOVERY_SERVICE_KEY)).getVersion(), - ((WorkerNodeService) node.getServices().get(WorkerNodeService.DISCOVERY_SERVICE_KEY)).getTier() + ((WorkerNodeService) node.getServices().get(WorkerNodeService.DISCOVERY_SERVICE_KEY)).getCategory() ); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolder.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolder.java index 445bbac622b5..cfb618066bb0 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolder.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolder.java @@ -184,7 +184,7 @@ public ImmutableWorkerInfo toImmutable() worker.getIp(), worker.getCapacity(), "", - worker.getTier() + worker.getCategory() ); } w = disabledWorker; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategy.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithCategorySpecWorkerSelectStrategy.java similarity index 70% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategy.java rename to indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithCategorySpecWorkerSelectStrategy.java index d04d1eaabed7..ec65693ac3dd 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategy.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithCategorySpecWorkerSelectStrategy.java @@ -29,22 +29,22 @@ import javax.annotation.Nullable; import java.util.Objects; -public class EqualDistributionWithTierSpecWorkerSelectStrategy implements WorkerSelectStrategy +public class EqualDistributionWithCategorySpecWorkerSelectStrategy implements WorkerSelectStrategy { - private final WorkerTierSpec workerTierSpec; + private final WorkerCategorySpec workerCategorySpec; @JsonCreator - public EqualDistributionWithTierSpecWorkerSelectStrategy( - @JsonProperty("workerTierSpec") WorkerTierSpec workerTierSpec + public EqualDistributionWithCategorySpecWorkerSelectStrategy( + @JsonProperty("workerCategorySpec") WorkerCategorySpec workerCategorySpec ) { - this.workerTierSpec = workerTierSpec; + this.workerCategorySpec = workerCategorySpec; } @JsonProperty - public WorkerTierSpec getWorkerTierSpec() + public WorkerCategorySpec getWorkerCategorySpec() { - return workerTierSpec; + return workerCategorySpec; } @Nullable @@ -59,7 +59,7 @@ public ImmutableWorkerInfo findWorkerForTask( task, zkWorkers, config, - workerTierSpec, + workerCategorySpec, EqualDistributionWorkerSelectStrategy::selectFromEligibleWorkers ); } @@ -73,21 +73,21 @@ public boolean equals(final Object o) if (o == null || getClass() != o.getClass()) { return false; } - final EqualDistributionWithTierSpecWorkerSelectStrategy that = (EqualDistributionWithTierSpecWorkerSelectStrategy) o; - return Objects.equals(workerTierSpec, that.workerTierSpec); + final EqualDistributionWithCategorySpecWorkerSelectStrategy that = (EqualDistributionWithCategorySpecWorkerSelectStrategy) o; + return Objects.equals(workerCategorySpec, that.workerCategorySpec); } @Override public int hashCode() { - return Objects.hash(workerTierSpec); + return Objects.hash(workerCategorySpec); } @Override public String toString() { - return "EqualDistributionWithTierSpecWorkerSelectStrategy{" + - "workerTierSpec=" + workerTierSpec + + return "EqualDistributionWithCategorySpecWorkerSelectStrategy{" + + "workerCategorySpec=" + workerCategorySpec + '}'; } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategy.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithCategorySpecWorkerSelectStrategy.java similarity index 71% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategy.java rename to indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithCategorySpecWorkerSelectStrategy.java index d35452929498..3dcdfe9a5401 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategy.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithCategorySpecWorkerSelectStrategy.java @@ -29,22 +29,22 @@ import javax.annotation.Nullable; import java.util.Objects; -public class FillCapacityWithTierSpecWorkerSelectStrategy implements WorkerSelectStrategy +public class FillCapacityWithCategorySpecWorkerSelectStrategy implements WorkerSelectStrategy { - private final WorkerTierSpec workerTierSpec; + private final WorkerCategorySpec workerCategorySpec; @JsonCreator - public FillCapacityWithTierSpecWorkerSelectStrategy( - @JsonProperty("workerTierSpec") WorkerTierSpec workerTierSpec + public FillCapacityWithCategorySpecWorkerSelectStrategy( + @JsonProperty("workerCategorySpec") WorkerCategorySpec workerCategorySpec ) { - this.workerTierSpec = workerTierSpec; + this.workerCategorySpec = workerCategorySpec; } @JsonProperty - public WorkerTierSpec getWorkerTierSpec() + public WorkerCategorySpec getWorkerCategorySpec() { - return workerTierSpec; + return workerCategorySpec; } @Nullable @@ -59,7 +59,7 @@ public ImmutableWorkerInfo findWorkerForTask( task, zkWorkers, config, - workerTierSpec, + workerCategorySpec, FillCapacityWorkerSelectStrategy::selectFromEligibleWorkers ); } @@ -73,21 +73,21 @@ public boolean equals(final Object o) if (o == null || getClass() != o.getClass()) { return false; } - final FillCapacityWithTierSpecWorkerSelectStrategy that = (FillCapacityWithTierSpecWorkerSelectStrategy) o; - return Objects.equals(workerTierSpec, that.workerTierSpec); + final FillCapacityWithCategorySpecWorkerSelectStrategy that = (FillCapacityWithCategorySpecWorkerSelectStrategy) o; + return Objects.equals(workerCategorySpec, that.workerCategorySpec); } @Override public int hashCode() { - return Objects.hash(workerTierSpec); + return Objects.hash(workerCategorySpec); } @Override public String toString() { - return "FillCapacityWithTierSpecWorkerSelectStrategy{" + - "workerTierSpec=" + workerTierSpec + + return "FillCapacityWithCategorySpecWorkerSelectStrategy{" + + "workerCategorySpec=" + workerCategorySpec + '}'; } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerCategorySpec.java similarity index 55% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java rename to indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerCategorySpec.java index 1c9f64b79618..57340bc8f48b 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerCategorySpec.java @@ -26,26 +26,26 @@ import java.util.Map; import java.util.Objects; -public class WorkerTierSpec +public class WorkerCategorySpec { - // key: taskType, value: tierConfig - private final Map tierMap; + // key: taskType, value: categoryConfig + private final Map categoryMap; private final boolean strong; @JsonCreator - public WorkerTierSpec( - @JsonProperty("tierMap") Map tierMap, + public WorkerCategorySpec( + @JsonProperty("categoryMap") Map categoryMap, @JsonProperty("strong") boolean strong ) { - this.tierMap = tierMap == null ? Collections.EMPTY_MAP : tierMap; + this.categoryMap = categoryMap == null ? Collections.EMPTY_MAP : categoryMap; this.strong = strong; } @JsonProperty - public Map getTierMap() + public Map getCategoryMap() { - return tierMap; + return categoryMap; } @JsonProperty @@ -63,52 +63,52 @@ public boolean equals(final Object o) if (o == null || getClass() != o.getClass()) { return false; } - final WorkerTierSpec that = (WorkerTierSpec) o; + final WorkerCategorySpec that = (WorkerCategorySpec) o; return strong == that.strong && - Objects.equals(tierMap, that.tierMap); + Objects.equals(categoryMap, that.categoryMap); } @Override public int hashCode() { - return Objects.hash(tierMap, strong); + return Objects.hash(categoryMap, strong); } @Override public String toString() { - return "WorkerTierSpec{" + - "tierMap=" + tierMap + + return "WorkerCategorySpec{" + + "categoryMap=" + categoryMap + ", strong=" + strong + '}'; } - public static class TierConfig + public static class CategoryConfig { - private final String defaultTier; - // key: datasource, value: tier - private final Map tierAffinity; + private final String defaultCategory; + // key: datasource, value: category + private final Map categoryAffinity; @JsonCreator - public TierConfig( - @JsonProperty("defaultTier") String defaultTier, - @JsonProperty("tierAffinity") Map tierAffinity + public CategoryConfig( + @JsonProperty("defaultCategory") String defaultCategory, + @JsonProperty("categoryAffinity") Map categoryAffinity ) { - this.defaultTier = defaultTier; - this.tierAffinity = tierAffinity == null ? Collections.EMPTY_MAP : tierAffinity; + this.defaultCategory = defaultCategory; + this.categoryAffinity = categoryAffinity == null ? Collections.EMPTY_MAP : categoryAffinity; } @JsonProperty - public String getDefaultTier() + public String getDefaultCategory() { - return defaultTier; + return defaultCategory; } @JsonProperty - public Map getTierAffinity() + public Map getCategoryAffinity() { - return tierAffinity; + return categoryAffinity; } @Override @@ -120,23 +120,23 @@ public boolean equals(final Object o) if (o == null || getClass() != o.getClass()) { return false; } - final TierConfig that = (TierConfig) o; - return Objects.equals(defaultTier, that.defaultTier) && - Objects.equals(tierAffinity, that.tierAffinity); + final CategoryConfig that = (CategoryConfig) o; + return Objects.equals(defaultCategory, that.defaultCategory) && + Objects.equals(categoryAffinity, that.categoryAffinity); } @Override public int hashCode() { - return Objects.hash(defaultTier, tierAffinity); + return Objects.hash(defaultCategory, categoryAffinity); } @Override public String toString() { - return "TierConfig{" + - "defaultTier=" + defaultTier + - ", tierAffinity=" + tierAffinity + + return "CategoryConfig{" + + "defaultCategory=" + defaultCategory + + ", categoryAffinity=" + categoryAffinity + '}'; } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectStrategy.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectStrategy.java index d63bc569cf2d..a3443ee73583 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectStrategy.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectStrategy.java @@ -39,8 +39,8 @@ @JsonSubTypes.Type(name = "equalDistribution", value = EqualDistributionWorkerSelectStrategy.class), @JsonSubTypes.Type(name = "equalDistributionWithAffinity", value = EqualDistributionWithAffinityWorkerSelectStrategy.class), @JsonSubTypes.Type(name = "javascript", value = JavaScriptWorkerSelectStrategy.class), - @JsonSubTypes.Type(name = "fillCapacityWithTierSpec", value = FillCapacityWithTierSpecWorkerSelectStrategy.class), - @JsonSubTypes.Type(name = "equalDistributionWithTierSpec", value = EqualDistributionWithTierSpecWorkerSelectStrategy.class) + @JsonSubTypes.Type(name = "fillCapacityWithCategorySpec", value = FillCapacityWithCategorySpecWorkerSelectStrategy.class), + @JsonSubTypes.Type(name = "equalDistributionWithCategorySpec", value = EqualDistributionWithCategorySpecWorkerSelectStrategy.class) }) @PublicApi public interface WorkerSelectStrategy diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java index f9c9baf329a0..24721e85d068 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/setup/WorkerSelectUtils.java @@ -93,9 +93,9 @@ public static ImmutableWorkerInfo selectWorker( * Helper for {@link WorkerSelectStrategy} implementations. * * @param allWorkers map of all workers, in the style provided to {@link WorkerSelectStrategy} - * @param workerTierSpec worker tier spec, or null + * @param workerCategorySpec worker category spec, or null * @param workerSelector function that receives a list of eligible workers: version is high enough, worker can run - * the task, and worker satisfies the worker tier spec. may return null. + * the task, and worker satisfies the worker category spec. may return null. * * @return selected worker from "allWorkers", or null. */ @@ -104,33 +104,33 @@ public static ImmutableWorkerInfo selectWorker( final Task task, final Map allWorkers, final WorkerTaskRunnerConfig workerTaskRunnerConfig, - @Nullable final WorkerTierSpec workerTierSpec, + @Nullable final WorkerCategorySpec workerCategorySpec, final Function, ImmutableWorkerInfo> workerSelector ) { final Map runnableWorkers = getRunnableWorkers(task, allWorkers, workerTaskRunnerConfig); - // select worker according to worker tier spec - if (workerTierSpec != null) { - final WorkerTierSpec.TierConfig tierConfig = workerTierSpec.getTierMap().get(task.getType()); + // select worker according to worker category spec + if (workerCategorySpec != null) { + final WorkerCategorySpec.CategoryConfig categoryConfig = workerCategorySpec.getCategoryMap().get(task.getType()); - if (tierConfig != null) { - final String defaultTier = tierConfig.getDefaultTier(); - final Map tierAffinity = tierConfig.getTierAffinity(); + if (categoryConfig != null) { + final String defaultCategory = categoryConfig.getDefaultCategory(); + final Map categoryAffinity = categoryConfig.getCategoryAffinity(); - String preferredTier = tierAffinity.get(task.getDataSource()); - // If there is no preferred tier for the datasource, then using the defaultTier. However, the defaultTier + String preferredCategory = categoryAffinity.get(task.getDataSource()); + // If there is no preferred category for the datasource, then using the defaultCategory. However, the defaultCategory // may be null too, so we need to do one more null check (see below). - preferredTier = preferredTier == null ? defaultTier : preferredTier; + preferredCategory = preferredCategory == null ? defaultCategory : preferredCategory; - if (preferredTier != null) { - // select worker from preferred tier - final ImmutableMap tierWorkers = getTierWorkers(preferredTier, runnableWorkers); - final ImmutableWorkerInfo selected = workerSelector.apply(tierWorkers); + if (preferredCategory != null) { + // select worker from preferred category + final ImmutableMap categoryWorkers = getCategoryWorkers(preferredCategory, runnableWorkers); + final ImmutableWorkerInfo selected = workerSelector.apply(categoryWorkers); if (selected != null) { return selected; - } else if (workerTierSpec.isStrong()) { + } else if (workerCategorySpec.isStrong()) { return null; } } @@ -141,7 +141,7 @@ public static ImmutableWorkerInfo selectWorker( return workerSelector.apply(ImmutableMap.copyOf(runnableWorkers)); } - // Get workers that could potentially run this task, ignoring affinityConfig/workerTierSpec. + // Get workers that could potentially run this task, ignoring affinityConfig/workerCategorySpec. private static Map getRunnableWorkers( final Task task, final Map allWorkers, @@ -156,20 +156,20 @@ private static Map getRunnableWorkers( } /** - * Return workers belong to this tier. + * Return workers belong to this category. * - * @param tier worker tier name + * @param category worker category name * @param workerMap map of worker hostname to worker info * * @return map of worker hostname to worker info */ - private static ImmutableMap getTierWorkers( - final String tier, + private static ImmutableMap getCategoryWorkers( + final String category, final Map workerMap ) { return ImmutableMap.copyOf( - Maps.filterValues(workerMap, workerInfo -> workerInfo.getWorker().getTier().equals(tier)) + Maps.filterValues(workerMap, workerInfo -> workerInfo.getWorker().getCategory().equals(category)) ); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/worker/Worker.java b/indexing-service/src/main/java/org/apache/druid/indexing/worker/Worker.java index ae31a3711e90..2394a464e5c2 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/worker/Worker.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/worker/Worker.java @@ -34,7 +34,7 @@ public class Worker private final String ip; private final int capacity; private final String version; - private final String tier; + private final String category; @JsonCreator public Worker( @@ -43,7 +43,7 @@ public Worker( @JsonProperty("ip") String ip, @JsonProperty("capacity") int capacity, @JsonProperty("version") String version, - @JsonProperty("tier") String tier + @JsonProperty("category") String category ) { this.scheme = scheme == null ? "http" : scheme; // needed for backwards compatibility with older workers (pre-#4270) @@ -51,7 +51,7 @@ public Worker( this.ip = ip; this.capacity = capacity; this.version = version; - this.tier = tier; + this.category = category; } @JsonProperty @@ -85,9 +85,9 @@ public String getVersion() } @JsonProperty - public String getTier() + public String getCategory() { - return tier; + return category; } @Override @@ -117,7 +117,7 @@ public boolean equals(Object o) if (!version.equals(worker.version)) { return false; } - return tier.equals(worker.tier); + return category.equals(worker.category); } @Override @@ -128,7 +128,7 @@ public int hashCode() result = 31 * result + ip.hashCode(); result = 31 * result + capacity; result = 31 * result + version.hashCode(); - result = 31 * result + tier.hashCode(); + result = 31 * result + category.hashCode(); return result; } @@ -141,7 +141,7 @@ public String toString() ", ip='" + ip + '\'' + ", capacity=" + capacity + ", version='" + version + '\'' + - ", tier='" + tier + '\'' + + ", category='" + category + '\'' + '}'; } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/worker/http/WorkerResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/worker/http/WorkerResource.java index f86ee4815d82..cafabc9e525f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/worker/http/WorkerResource.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/worker/http/WorkerResource.java @@ -92,7 +92,7 @@ public Response doDisable() enabledWorker.getIp(), enabledWorker.getCapacity(), DISABLED_VERSION, - enabledWorker.getTier() + enabledWorker.getCategory() ); curatorCoordinator.updateWorkerAnnouncement(disabledWorker); workerTaskManager.workerDisabled(); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfoTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfoTest.java index 8155fc663884..785698e68f0e 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfoTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/ImmutableWorkerInfoTest.java @@ -36,7 +36,7 @@ public void testSerde() throws Exception { ImmutableWorkerInfo workerInfo = new ImmutableWorkerInfo( new Worker( - "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp1", "grp2"), @@ -57,7 +57,7 @@ public void testEqualsAndSerde() // Everything equal assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp1", "grp2"), @@ -65,7 +65,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp1", "grp2"), @@ -73,10 +73,10 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), true); - // same worker different tier + // same worker different category assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker", "192.0.0.1", 10, "v1", "t1" + "http", "testWorker", "192.0.0.1", 10, "v1", "c1" ), 2, ImmutableSet.of("grp1", "grp2"), @@ -84,7 +84,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker", "192.0.0.1", 10, "v1", "t2" + "http", "testWorker", "192.0.0.1", 10, "v1", "c2" ), 2, ImmutableSet.of("grp1", "grp2"), @@ -95,7 +95,7 @@ public void testEqualsAndSerde() // different worker same tasks assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp1", "grp2"), @@ -103,7 +103,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp1", "grp2"), @@ -114,7 +114,7 @@ public void testEqualsAndSerde() // same worker different task groups assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp3", "grp2"), @@ -122,7 +122,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp1", "grp2"), @@ -133,7 +133,7 @@ public void testEqualsAndSerde() // same worker different tasks assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp1", "grp2"), @@ -141,7 +141,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp1", "grp2"), @@ -152,7 +152,7 @@ public void testEqualsAndSerde() // same worker different capacity assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 3, ImmutableSet.of("grp1", "grp2"), @@ -160,7 +160,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp1", "grp2"), @@ -171,7 +171,7 @@ public void testEqualsAndSerde() // same worker different lastCompletedTaskTime assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 3, ImmutableSet.of("grp1", "grp2"), @@ -179,7 +179,7 @@ public void testEqualsAndSerde() DateTimes.of("2015-01-01T01:01:01Z") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp1", "grp2"), @@ -190,7 +190,7 @@ public void testEqualsAndSerde() // same worker different blacklistedUntil assertEqualsAndHashCode(new ImmutableWorkerInfo( new Worker( - "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 3, ImmutableSet.of("grp1", "grp2"), @@ -199,7 +199,7 @@ public void testEqualsAndSerde() DateTimes.of("2017-07-30") ), new ImmutableWorkerInfo( new Worker( - "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "testWorker2", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp1", "grp2"), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTestUtils.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTestUtils.java index 4a38ce103a9f..46239e3d76fc 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTestUtils.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/RemoteTaskRunnerTestUtils.java @@ -148,7 +148,7 @@ Worker makeWorker(final String workerId, final int capacity) throws Exception workerId, capacity, "0", - WorkerConfig.DEFAULT_TIER + WorkerConfig.DEFAULT_CATEGORY ); cf.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath( @@ -170,7 +170,7 @@ void disableWorker(Worker worker) throws Exception worker.getIp(), worker.getCapacity(), "", - worker.getTier() + worker.getCategory() )) ); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskRunnerUtilsTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskRunnerUtilsTest.java index ea09d9fa03c1..194fb9c86514 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskRunnerUtilsTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskRunnerUtilsTest.java @@ -32,7 +32,7 @@ public class TaskRunnerUtilsTest public void testMakeWorkerURL() { final URL url = TaskRunnerUtils.makeWorkerURL( - new Worker("https", "1.2.3.4:8290", "1.2.3.4", 1, "0", WorkerConfig.DEFAULT_TIER), + new Worker("https", "1.2.3.4:8290", "1.2.3.4", 1, "0", WorkerConfig.DEFAULT_CATEGORY), "/druid/worker/v1/task/%s/log", "foo bar&" ); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdpaterTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdpaterTest.java index 7effd5d00b6c..4bb1c9b24d8c 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdpaterTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/WorkerTaskRunnerQueryAdpaterTest.java @@ -67,7 +67,7 @@ public void setup() ImmutableList.of( new ImmutableWorkerInfo( new Worker( - "http", "worker-host1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_TIER + "http", "worker-host1", "192.0.0.1", 10, "v1", WorkerConfig.DEFAULT_CATEGORY ), 2, ImmutableSet.of("grp1", "grp2"), @@ -76,7 +76,7 @@ public void setup() ), new ImmutableWorkerInfo( new Worker( - "https", "worker-host2", "192.0.0.2", 4, "v1", WorkerConfig.DEFAULT_TIER + "https", "worker-host2", "192.0.0.2", 4, "v1", WorkerConfig.DEFAULT_CATEGORY ), 1, ImmutableSet.of("grp1"), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedProvisioningStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedProvisioningStrategyTest.java index f2be7889bd8a..45207940668e 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedProvisioningStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/PendingTaskBasedProvisioningStrategyTest.java @@ -578,7 +578,7 @@ public TestZkWorker( int capacity ) { - super(new Worker(scheme, host, ip, capacity, version, WorkerConfig.DEFAULT_TIER), null, new DefaultObjectMapper()); + super(new Worker(scheme, host, ip, capacity, version, WorkerConfig.DEFAULT_CATEGORY), null, new DefaultObjectMapper()); this.testTask = testTask; } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/SimpleProvisioningStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/SimpleProvisioningStrategyTest.java index 3e0777e88940..7f9739ad55ff 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/SimpleProvisioningStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/SimpleProvisioningStrategyTest.java @@ -508,7 +508,7 @@ public TestZkWorker( String version ) { - super(new Worker(scheme, host, ip, 3, version, WorkerConfig.DEFAULT_TIER), null, new DefaultObjectMapper()); + super(new Worker(scheme, host, ip, 3, version, WorkerConfig.DEFAULT_CATEGORY), null, new DefaultObjectMapper()); this.testTask = testTask; } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunnerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunnerTest.java index a943567fb239..dc87de699bdb 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunnerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/HttpRemoteTaskRunnerTest.java @@ -144,7 +144,7 @@ protected WorkerHolder createWorkerHolder( new DruidNode("service", "host1", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_TIER) + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY) ) ); @@ -152,7 +152,7 @@ WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", Wo new DruidNode("service", "host2", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0", WorkerConfig.DEFAULT_TIER) + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0", WorkerConfig.DEFAULT_CATEGORY) ) ); @@ -241,7 +241,7 @@ protected WorkerHolder createWorkerHolder( new DruidNode("service", "host1", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_TIER) + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY) ) ); @@ -249,7 +249,7 @@ WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", Wo new DruidNode("service", "host2", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0", WorkerConfig.DEFAULT_TIER) + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 2, "0", WorkerConfig.DEFAULT_CATEGORY) ) ); @@ -345,7 +345,7 @@ protected WorkerHolder createWorkerHolder( new DruidNode("service", "host", false, 1234, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_TIER) + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY) ) ); @@ -490,7 +490,7 @@ protected WorkerHolder createWorkerHolder( new DruidNode("service", "host", false, 1234, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_TIER) + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY) ) ); @@ -664,7 +664,7 @@ protected WorkerHolder createWorkerHolder( DiscoveryDruidNode druidNode = new DiscoveryDruidNode( new DruidNode("service", "host", false, 1234, null, true, false), NodeType.MIDDLE_MANAGER, - ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_TIER)) + ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 2, "0", WorkerConfig.DEFAULT_CATEGORY)) ); workerHolders.put( @@ -845,7 +845,7 @@ protected WorkerHolder createWorkerHolder( new DruidNode("service", "host1", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, ImmutableMap.of( - WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 1, "0", WorkerConfig.DEFAULT_TIER) + WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 1, "0", WorkerConfig.DEFAULT_CATEGORY) ) ); @@ -890,7 +890,7 @@ WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 1, "0", Wo DiscoveryDruidNode druidNode2 = new DiscoveryDruidNode( new DruidNode("service", "host2", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, - ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 1, "0", WorkerConfig.DEFAULT_TIER)) + ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 1, "0", WorkerConfig.DEFAULT_CATEGORY)) ); workerHolders.put( @@ -921,7 +921,7 @@ WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip1", 1, "0", Wo DiscoveryDruidNode druidNode3 = new DiscoveryDruidNode( new DruidNode("service", "host3", false, 8080, null, true, false), NodeType.MIDDLE_MANAGER, - ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 1, "0", WorkerConfig.DEFAULT_TIER)) + ImmutableMap.of(WorkerNodeService.DISCOVERY_SERVICE_KEY, new WorkerNodeService("ip2", 1, "0", WorkerConfig.DEFAULT_CATEGORY)) ); workerHolders.put( @@ -967,7 +967,7 @@ public void testTaskAddedOrUpdated1() throws Exception ); WorkerHolder workerHolder = EasyMock.createMock(WorkerHolder.class); - EasyMock.expect(workerHolder.getWorker()).andReturn(new Worker("http", "worker", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_TIER)).anyTimes(); + EasyMock.expect(workerHolder.getWorker()).andReturn(new Worker("http", "worker", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_CATEGORY)).anyTimes(); workerHolder.setLastCompletedTaskTime(EasyMock.anyObject()); workerHolder.resetContinuouslyFailedTasksCount(); EasyMock.expect(workerHolder.getContinuouslyFailedTasksCount()).andReturn(0); @@ -1003,7 +1003,7 @@ public void testTaskAddedOrUpdated1() throws Exception // Another "rogue-worker" reports running it, and gets asked to shutdown the task WorkerHolder rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class); EasyMock.expect(rogueWorkerHolder.getWorker()) - .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_TIER)) + .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_CATEGORY)) .anyTimes(); rogueWorkerHolder.shutdownTask(task.getId()); EasyMock.replay(rogueWorkerHolder); @@ -1018,7 +1018,7 @@ public void testTaskAddedOrUpdated1() throws Exception // "rogue-worker" reports FAILURE for the task, ignored rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class); EasyMock.expect(rogueWorkerHolder.getWorker()) - .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_TIER)) + .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_CATEGORY)) .anyTimes(); EasyMock.replay(rogueWorkerHolder); taskRunner.taskAddedOrUpdated(TaskAnnouncement.create( @@ -1041,7 +1041,7 @@ public void testTaskAddedOrUpdated1() throws Exception // "rogue-worker" reports running it, and gets asked to shutdown the task rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class); EasyMock.expect(rogueWorkerHolder.getWorker()) - .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_TIER)) + .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_CATEGORY)) .anyTimes(); rogueWorkerHolder.shutdownTask(task.getId()); EasyMock.replay(rogueWorkerHolder); @@ -1056,7 +1056,7 @@ public void testTaskAddedOrUpdated1() throws Exception // "rogue-worker" reports FAILURE for the tasks, ignored rogueWorkerHolder = EasyMock.createMock(WorkerHolder.class); EasyMock.expect(rogueWorkerHolder.getWorker()) - .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_TIER)) + .andReturn(new Worker("http", "rogue-worker", "127.0.0.1", 5, "v1", WorkerConfig.DEFAULT_CATEGORY)) .anyTimes(); EasyMock.replay(rogueWorkerHolder); taskRunner.taskAddedOrUpdated(TaskAnnouncement.create( @@ -1095,7 +1095,7 @@ public void testTaskAddedOrUpdated2() throws Exception listenerNotificationsAccumulator ); - Worker worker = new Worker("http", "localhost", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_TIER); + Worker worker = new Worker("http", "localhost", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_CATEGORY); WorkerHolder workerHolder = EasyMock.createMock(WorkerHolder.class); EasyMock.expect(workerHolder.getWorker()).andReturn(worker).anyTimes(); @@ -1154,7 +1154,7 @@ public void testTaskAddedOrUpdated3() HttpRemoteTaskRunner taskRunner = createTaskRunnerForTestTaskAddedOrUpdated(taskStorage, listenerNotificationsAccumulator); - Worker worker = new Worker("http", "localhost", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_TIER); + Worker worker = new Worker("http", "localhost", "127.0.0.1", 1, "v1", WorkerConfig.DEFAULT_CATEGORY); WorkerHolder workerHolder = EasyMock.createMock(WorkerHolder.class); EasyMock.expect(workerHolder.getWorker()).andReturn(worker).anyTimes(); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolderTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolderTest.java index be4836c41089..3319d8125713 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolderTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/hrtr/WorkerHolderTest.java @@ -60,7 +60,7 @@ public void testSyncListener() new HttpRemoteTaskRunnerConfig(), EasyMock.createNiceMock(ScheduledExecutorService.class), (taskAnnouncement, holder) -> updates.add(taskAnnouncement), - new Worker("http", "localhost", "127.0.0.1", 5, "v0", WorkerConfig.DEFAULT_TIER), + new Worker("http", "localhost", "127.0.0.1", 5, "v0", WorkerConfig.DEFAULT_CATEGORY), ImmutableList.of( TaskAnnouncement.create( task0, diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java index 92279747e68a..05d1bd0ed76f 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithAffinityWorkerSelectStrategyTest.java @@ -56,31 +56,31 @@ public String getDataSource() ImmutableMap.of( "localhost0", new ImmutableWorkerInfo( - new Worker("http", "localhost0", "localhost0", 2, "v1", WorkerConfig.DEFAULT_TIER), 0, + new Worker("http", "localhost0", "localhost0", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost1", new ImmutableWorkerInfo( - new Worker("http", "localhost1", "localhost1", 2, "v1", WorkerConfig.DEFAULT_TIER), 0, - new HashSet<>(), - new HashSet<>(), - DateTimes.nowUtc() + new Worker("http", "localhost1", "localhost1", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() ), "localhost2", new ImmutableWorkerInfo( - new Worker("http", "localhost2", "localhost2", 2, "v1", WorkerConfig.DEFAULT_TIER), 1, - new HashSet<>(), - new HashSet<>(), - DateTimes.nowUtc() + new Worker("http", "localhost2", "localhost2", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 1, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() ), "localhost3", new ImmutableWorkerInfo( - new Worker("http", "localhost3", "localhost3", 2, "v1", WorkerConfig.DEFAULT_TIER), 1, - new HashSet<>(), - new HashSet<>(), - DateTimes.nowUtc() + new Worker("http", "localhost3", "localhost3", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 1, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() ) ), noopTask @@ -100,17 +100,17 @@ public void testFindWorkerForTaskWithNulls() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, - new HashSet<>(), - new HashSet<>(), - DateTimes.nowUtc() + new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, - new HashSet<>(), - new HashSet<>(), - DateTimes.nowUtc() + new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() ) ), new NoopTask(null, null, null, 1, 0, null, null, null) @@ -130,10 +130,10 @@ public void testIsolation() ImmutableMap.of( "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, - new HashSet<>(), - new HashSet<>(), - DateTimes.nowUtc() + new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, + new HashSet<>(), + new HashSet<>(), + DateTimes.nowUtc() ) ), new NoopTask(null, null, null, 1, 0, null, null, null) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithCategorySpecWorkerSelectStrategyTest.java similarity index 69% rename from indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java rename to indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithCategorySpecWorkerSelectStrategyTest.java index a26139b48f46..80366ea03ad6 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithTierSpecWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWithCategorySpecWorkerSelectStrategyTest.java @@ -30,34 +30,34 @@ import java.util.HashSet; -public class EqualDistributionWithTierSpecWorkerSelectStrategyTest +public class EqualDistributionWithCategorySpecWorkerSelectStrategyTest { private static final ImmutableMap WORKERS_FOR_TIER_TESTS = ImmutableMap.of( "localhost0", new ImmutableWorkerInfo( - new Worker("http", "localhost0", "localhost0", 1, "v1", "t1"), 0, + new Worker("http", "localhost0", "localhost0", 1, "v1", "c1"), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost1", new ImmutableWorkerInfo( - new Worker("http", "localhost1", "localhost1", 2, "v1", "t1"), 0, + new Worker("http", "localhost1", "localhost1", 2, "v1", "c1"), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost2", new ImmutableWorkerInfo( - new Worker("http", "localhost2", "localhost2", 3, "v1", "t2"), 0, + new Worker("http", "localhost2", "localhost2", 3, "v1", "c2"), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost3", new ImmutableWorkerInfo( - new Worker("http", "localhost3", "localhost3", 4, "v1", "t2"), 0, + new Worker("http", "localhost3", "localhost3", 4, "v1", "c2"), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -75,58 +75,58 @@ public void testFindWorkerForTaskWithNullWorkerTierSpec() public void testFindWorkerForTaskWithPreferredTier() { // test defaultTier != null and tierAffinity is not empty - final WorkerTierSpec workerTierSpec1 = new WorkerTierSpec( + final WorkerCategorySpec workerCategorySpec1 = new WorkerCategorySpec( ImmutableMap.of( "noop", - new WorkerTierSpec.TierConfig( - "t2", - ImmutableMap.of("ds1", "t2") + new WorkerCategorySpec.CategoryConfig( + "c2", + ImmutableMap.of("ds1", "c2") ) ), false ); - ImmutableWorkerInfo worker1 = selectWorker(workerTierSpec1); + ImmutableWorkerInfo worker1 = selectWorker(workerCategorySpec1); Assert.assertEquals("localhost3", worker1.getWorker().getHost()); // test defaultTier == null and tierAffinity is not empty - final WorkerTierSpec workerTierSpec2 = new WorkerTierSpec( + final WorkerCategorySpec workerCategorySpec2 = new WorkerCategorySpec( ImmutableMap.of( "noop", - new WorkerTierSpec.TierConfig( + new WorkerCategorySpec.CategoryConfig( null, - ImmutableMap.of("ds1", "t2") + ImmutableMap.of("ds1", "c2") ) ), false ); - ImmutableWorkerInfo worker2 = selectWorker(workerTierSpec2); + ImmutableWorkerInfo worker2 = selectWorker(workerCategorySpec2); Assert.assertEquals("localhost3", worker2.getWorker().getHost()); // test defaultTier != null and tierAffinity is empty - final WorkerTierSpec workerTierSpec3 = new WorkerTierSpec( + final WorkerCategorySpec workerCategorySpec3 = new WorkerCategorySpec( ImmutableMap.of( "noop", - new WorkerTierSpec.TierConfig( - "t2", + new WorkerCategorySpec.CategoryConfig( + "c2", null ) ), false ); - ImmutableWorkerInfo worker3 = selectWorker(workerTierSpec3); + ImmutableWorkerInfo worker3 = selectWorker(workerCategorySpec3); Assert.assertEquals("localhost3", worker3.getWorker().getHost()); } @Test public void testFindWorkerForTaskWithNullPreferredTier() { - final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + final WorkerCategorySpec workerCategorySpec = new WorkerCategorySpec( ImmutableMap.of( "noop", - new WorkerTierSpec.TierConfig( + new WorkerCategorySpec.CategoryConfig( null, null ) @@ -134,50 +134,50 @@ public void testFindWorkerForTaskWithNullPreferredTier() false ); - ImmutableWorkerInfo worker = selectWorker(workerTierSpec); + ImmutableWorkerInfo worker = selectWorker(workerCategorySpec); Assert.assertEquals("localhost3", worker.getWorker().getHost()); } @Test public void testWeakTierSpec() { - final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + final WorkerCategorySpec workerCategorySpec = new WorkerCategorySpec( ImmutableMap.of( "noop", - new WorkerTierSpec.TierConfig( - "t1", - ImmutableMap.of("ds1", "t3") + new WorkerCategorySpec.CategoryConfig( + "c1", + ImmutableMap.of("ds1", "c3") ) ), false ); - ImmutableWorkerInfo worker = selectWorker(workerTierSpec); + ImmutableWorkerInfo worker = selectWorker(workerCategorySpec); Assert.assertEquals("localhost3", worker.getWorker().getHost()); } @Test public void testStrongTierSpec() { - final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + final WorkerCategorySpec workerCategorySpec = new WorkerCategorySpec( ImmutableMap.of( "noop", - new WorkerTierSpec.TierConfig( - "t1", - ImmutableMap.of("ds1", "t3") + new WorkerCategorySpec.CategoryConfig( + "c1", + ImmutableMap.of("ds1", "c3") ) ), true ); - ImmutableWorkerInfo worker = selectWorker(workerTierSpec); + ImmutableWorkerInfo worker = selectWorker(workerCategorySpec); Assert.assertNull(worker); } - private ImmutableWorkerInfo selectWorker(WorkerTierSpec workerTierSpec) + private ImmutableWorkerInfo selectWorker(WorkerCategorySpec workerCategorySpec) { - final EqualDistributionWithTierSpecWorkerSelectStrategy strategy = new EqualDistributionWithTierSpecWorkerSelectStrategy( - workerTierSpec); + final EqualDistributionWithCategorySpecWorkerSelectStrategy strategy = new EqualDistributionWithCategorySpecWorkerSelectStrategy( + workerCategorySpec); ImmutableWorkerInfo worker = strategy.findWorkerForTask( new RemoteTaskRunnerConfig(), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java index 01d38af89dc2..20259c3a52ed 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java @@ -38,28 +38,28 @@ public class EqualDistributionWorkerSelectStrategyTest ImmutableMap.of( "localhost0", new ImmutableWorkerInfo( - new Worker("http", "localhost0", "localhost0", 2, "v1", WorkerConfig.DEFAULT_TIER), 0, + new Worker("http", "localhost0", "localhost0", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost1", new ImmutableWorkerInfo( - new Worker("http", "localhost1", "localhost1", 2, "v1", WorkerConfig.DEFAULT_TIER), 0, + new Worker("http", "localhost1", "localhost1", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost2", new ImmutableWorkerInfo( - new Worker("http", "localhost2", "localhost2", 2, "v1", WorkerConfig.DEFAULT_TIER), 1, + new Worker("http", "localhost2", "localhost2", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost3", new ImmutableWorkerInfo( - new Worker("http", "localhost3", "localhost3", 2, "v1", WorkerConfig.DEFAULT_TIER), 1, + new Worker("http", "localhost3", "localhost3", 2, "v1", WorkerConfig.DEFAULT_CATEGORY), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -76,14 +76,14 @@ public void testFindWorkerForTask() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, + new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 1, + new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -111,14 +111,14 @@ public void testFindWorkerForTaskWhenSameCurrCapacityUsed() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "lhost", "lhost", 5, "v1", WorkerConfig.DEFAULT_TIER), 5, + new Worker("http", "lhost", "lhost", 5, "v1", WorkerConfig.DEFAULT_CATEGORY), 5, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 10, "v1", WorkerConfig.DEFAULT_TIER), 5, + new Worker("http", "localhost", "localhost", 10, "v1", WorkerConfig.DEFAULT_CATEGORY), 5, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -147,14 +147,14 @@ public void testOneDisableWorkerDifferentUsedCapacity() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "disableHost", "disableHost", 10, disabledVersion, WorkerConfig.DEFAULT_TIER), 2, + new Worker("http", "disableHost", "disableHost", 10, disabledVersion, WorkerConfig.DEFAULT_CATEGORY), 2, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "enableHost", "enableHost", 10, "v1", WorkerConfig.DEFAULT_TIER), 5, + new Worker("http", "enableHost", "enableHost", 10, "v1", WorkerConfig.DEFAULT_CATEGORY), 5, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -183,14 +183,14 @@ public void testOneDisableWorkerSameUsedCapacity() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "disableHost", "disableHost", 10, disabledVersion, WorkerConfig.DEFAULT_TIER), 5, + new Worker("http", "disableHost", "disableHost", 10, disabledVersion, WorkerConfig.DEFAULT_CATEGORY), 5, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "enableHost", "enableHost", 10, "v1", WorkerConfig.DEFAULT_TIER), 5, + new Worker("http", "enableHost", "enableHost", 10, "v1", WorkerConfig.DEFAULT_CATEGORY), 5, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java index 5bd55f5f48f4..efdcb8235565 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithAffinityWorkerSelectStrategyTest.java @@ -46,14 +46,14 @@ public void testFindWorkerForTask() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, + new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, + new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -83,14 +83,14 @@ public void testFindWorkerForTaskWithNulls() ImmutableMap.of( "lhost", new ImmutableWorkerInfo( - new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, + new Worker("http", "lhost", "lhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, + new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -113,7 +113,7 @@ public void testIsolation() ImmutableMap.of( "localhost", new ImmutableWorkerInfo( - new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_TIER), 0, + new Worker("http", "localhost", "localhost", 1, "v1", WorkerConfig.DEFAULT_CATEGORY), 0, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithCategorySpecWorkerSelectStrategyTest.java similarity index 70% rename from indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java rename to indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithCategorySpecWorkerSelectStrategyTest.java index 088078ddc6dc..8968a7538555 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithTierSpecWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/FillCapacityWithCategorySpecWorkerSelectStrategyTest.java @@ -30,34 +30,34 @@ import java.util.HashSet; -public class FillCapacityWithTierSpecWorkerSelectStrategyTest +public class FillCapacityWithCategorySpecWorkerSelectStrategyTest { private static final ImmutableMap WORKERS_FOR_TIER_TESTS = ImmutableMap.of( "localhost0", new ImmutableWorkerInfo( - new Worker("http", "localhost0", "localhost0", 5, "v1", "t1"), 1, + new Worker("http", "localhost0", "localhost0", 5, "v1", "c1"), 1, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost1", new ImmutableWorkerInfo( - new Worker("http", "localhost1", "localhost1", 5, "v1", "t1"), 2, + new Worker("http", "localhost1", "localhost1", 5, "v1", "c1"), 2, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost2", new ImmutableWorkerInfo( - new Worker("http", "localhost2", "localhost2", 5, "v1", "t2"), 3, + new Worker("http", "localhost2", "localhost2", 5, "v1", "c2"), 3, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() ), "localhost3", new ImmutableWorkerInfo( - new Worker("http", "localhost3", "localhost3", 5, "v1", "t2"), 4, + new Worker("http", "localhost3", "localhost3", 5, "v1", "c2"), 4, new HashSet<>(), new HashSet<>(), DateTimes.nowUtc() @@ -75,58 +75,58 @@ public void testFindWorkerForTaskWithNullWorkerTierSpec() public void testFindWorkerForTaskWithPreferredTier() { // test defaultTier != null and tierAffinity is not empty - final WorkerTierSpec workerTierSpec1 = new WorkerTierSpec( + final WorkerCategorySpec workerCategorySpec1 = new WorkerCategorySpec( ImmutableMap.of( "noop", - new WorkerTierSpec.TierConfig( - "t1", - ImmutableMap.of("ds1", "t1") + new WorkerCategorySpec.CategoryConfig( + "c1", + ImmutableMap.of("ds1", "c1") ) ), false ); - ImmutableWorkerInfo worker1 = selectWorker(workerTierSpec1); + ImmutableWorkerInfo worker1 = selectWorker(workerCategorySpec1); Assert.assertEquals("localhost1", worker1.getWorker().getHost()); // test defaultTier == null and tierAffinity is not empty - final WorkerTierSpec workerTierSpec2 = new WorkerTierSpec( + final WorkerCategorySpec workerCategorySpec2 = new WorkerCategorySpec( ImmutableMap.of( "noop", - new WorkerTierSpec.TierConfig( + new WorkerCategorySpec.CategoryConfig( null, - ImmutableMap.of("ds1", "t1") + ImmutableMap.of("ds1", "c1") ) ), false ); - ImmutableWorkerInfo worker2 = selectWorker(workerTierSpec2); + ImmutableWorkerInfo worker2 = selectWorker(workerCategorySpec2); Assert.assertEquals("localhost1", worker2.getWorker().getHost()); // test defaultTier != null and tierAffinity is empty - final WorkerTierSpec workerTierSpec3 = new WorkerTierSpec( + final WorkerCategorySpec workerCategorySpec3 = new WorkerCategorySpec( ImmutableMap.of( "noop", - new WorkerTierSpec.TierConfig( - "t1", + new WorkerCategorySpec.CategoryConfig( + "c1", null ) ), false ); - ImmutableWorkerInfo worker3 = selectWorker(workerTierSpec3); + ImmutableWorkerInfo worker3 = selectWorker(workerCategorySpec3); Assert.assertEquals("localhost1", worker3.getWorker().getHost()); } @Test public void testFindWorkerForTaskWithNullPreferredTier() { - final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + final WorkerCategorySpec workerCategorySpec = new WorkerCategorySpec( ImmutableMap.of( "noop", - new WorkerTierSpec.TierConfig( + new WorkerCategorySpec.CategoryConfig( null, null ) @@ -134,50 +134,50 @@ public void testFindWorkerForTaskWithNullPreferredTier() false ); - ImmutableWorkerInfo worker = selectWorker(workerTierSpec); + ImmutableWorkerInfo worker = selectWorker(workerCategorySpec); Assert.assertEquals("localhost3", worker.getWorker().getHost()); } @Test public void testWeakTierSpec() { - final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + final WorkerCategorySpec workerCategorySpec = new WorkerCategorySpec( ImmutableMap.of( "noop", - new WorkerTierSpec.TierConfig( - "t1", - ImmutableMap.of("ds1", "t3") + new WorkerCategorySpec.CategoryConfig( + "c1", + ImmutableMap.of("ds1", "c3") ) ), false ); - ImmutableWorkerInfo worker = selectWorker(workerTierSpec); + ImmutableWorkerInfo worker = selectWorker(workerCategorySpec); Assert.assertEquals("localhost3", worker.getWorker().getHost()); } @Test public void testStrongTierSpec() { - final WorkerTierSpec workerTierSpec = new WorkerTierSpec( + final WorkerCategorySpec workerCategorySpec = new WorkerCategorySpec( ImmutableMap.of( "noop", - new WorkerTierSpec.TierConfig( - "t1", - ImmutableMap.of("ds1", "t3") + new WorkerCategorySpec.CategoryConfig( + "c1", + ImmutableMap.of("ds1", "c3") ) ), true ); - ImmutableWorkerInfo worker = selectWorker(workerTierSpec); + ImmutableWorkerInfo worker = selectWorker(workerCategorySpec); Assert.assertNull(worker); } - private ImmutableWorkerInfo selectWorker(WorkerTierSpec workerTierSpec) + private ImmutableWorkerInfo selectWorker(WorkerCategorySpec workerCategorySpec) { - final FillCapacityWithTierSpecWorkerSelectStrategy strategy = new FillCapacityWithTierSpecWorkerSelectStrategy( - workerTierSpec); + final FillCapacityWithCategorySpecWorkerSelectStrategy strategy = new FillCapacityWithCategorySpecWorkerSelectStrategy( + workerCategorySpec); ImmutableWorkerInfo worker = strategy.findWorkerForTask( new RemoteTaskRunnerConfig(), diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerCategorySpecTest.java similarity index 75% rename from indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpecTest.java rename to indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerCategorySpecTest.java index 7dcfc74ae199..4277984fc10a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerTierSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerCategorySpecTest.java @@ -26,7 +26,7 @@ import org.junit.Before; import org.junit.Test; -public class WorkerTierSpecTest +public class WorkerCategorySpecTest { private ObjectMapper mapper; @@ -41,24 +41,24 @@ public void testSerde() throws Exception { String jsonStr = "{\n" + " \"strong\": true,\n" - + " \"tierMap\": {\n" - + " \"index_kafka\": {\"defaultTier\": \"t1\", \"tierAffinity\": {\"ds1\": \"t2\"}}\n" + + " \"categoryMap\": {\n" + + " \"index_kafka\": {\"defaultCategory\": \"c1\", \"categoryAffinity\": {\"ds1\": \"c2\"}}\n" + " }\n" + "}"; - WorkerTierSpec workerTierSpec = mapper.readValue( + WorkerCategorySpec workerCategorySpec = mapper.readValue( mapper.writeValueAsString( mapper.readValue( jsonStr, - WorkerTierSpec.class + WorkerCategorySpec.class ) - ), WorkerTierSpec.class + ), WorkerCategorySpec.class ); - Assert.assertTrue(workerTierSpec.isStrong()); + Assert.assertTrue(workerCategorySpec.isStrong()); Assert.assertEquals(ImmutableMap.of( "index_kafka", - new WorkerTierSpec.TierConfig("t1", ImmutableMap.of("ds1", "t2")) - ), workerTierSpec.getTierMap()); + new WorkerCategorySpec.CategoryConfig("c1", ImmutableMap.of("ds1", "c2")) + ), workerCategorySpec.getCategoryMap()); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskMonitorTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskMonitorTest.java index 7ec7549158fb..93379f843326 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskMonitorTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/worker/WorkerTaskMonitorTest.java @@ -114,7 +114,7 @@ public void setUp() throws Exception "localhost", 3, "0", - WorkerConfig.DEFAULT_TIER + WorkerConfig.DEFAULT_CATEGORY ); workerCuratorCoordinator = new WorkerCuratorCoordinator( diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/worker/http/WorkerResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/worker/http/WorkerResourceTest.java index ca9af8412a52..3667ef74e27a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/worker/http/WorkerResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/worker/http/WorkerResourceTest.java @@ -79,7 +79,7 @@ public void setUp() throws Exception "ip", 3, "v1", - WorkerConfig.DEFAULT_TIER + WorkerConfig.DEFAULT_CATEGORY ); curatorCoordinator = new WorkerCuratorCoordinator( diff --git a/server/src/main/java/org/apache/druid/discovery/WorkerNodeService.java b/server/src/main/java/org/apache/druid/discovery/WorkerNodeService.java index 682ae73519bf..71ad41aacbea 100644 --- a/server/src/main/java/org/apache/druid/discovery/WorkerNodeService.java +++ b/server/src/main/java/org/apache/druid/discovery/WorkerNodeService.java @@ -33,19 +33,19 @@ public class WorkerNodeService extends DruidService private final String ip; private final int capacity; private final String version; - private final String tier; + private final String category; public WorkerNodeService( @JsonProperty("ip") String ip, @JsonProperty("capacity") int capacity, @JsonProperty("version") String version, - @JsonProperty("tier") String tier + @JsonProperty("category") String category ) { this.ip = ip; this.capacity = capacity; this.version = version; - this.tier = tier; + this.category = category; } @Override @@ -73,9 +73,9 @@ public String getVersion() } @JsonProperty - public String getTier() + public String getCategory() { - return tier; + return category; } @Override @@ -91,13 +91,13 @@ public boolean equals(Object o) return capacity == that.capacity && Objects.equals(ip, that.ip) && Objects.equals(version, that.version) && - Objects.equals(tier, that.tier); + Objects.equals(category, that.category); } @Override public int hashCode() { - return Objects.hash(ip, capacity, version, tier); + return Objects.hash(ip, capacity, version, category); } @Override @@ -107,7 +107,7 @@ public String toString() "ip='" + ip + '\'' + ", capacity=" + capacity + ", version='" + version + '\'' + - ", tier='" + tier + '\'' + + ", category='" + category + '\'' + '}'; } } diff --git a/server/src/main/java/org/apache/druid/indexing/worker/config/WorkerConfig.java b/server/src/main/java/org/apache/druid/indexing/worker/config/WorkerConfig.java index e62bfdc63b5c..49e3c2a7b9b7 100644 --- a/server/src/main/java/org/apache/druid/indexing/worker/config/WorkerConfig.java +++ b/server/src/main/java/org/apache/druid/indexing/worker/config/WorkerConfig.java @@ -31,7 +31,7 @@ */ public class WorkerConfig { - public static final String DEFAULT_TIER = "_default_worker_tier"; + public static final String DEFAULT_CATEGORY = "_default_worker_category"; @JsonProperty private String ip = DruidNode.getDefaultHost(); @@ -45,7 +45,7 @@ public class WorkerConfig @JsonProperty @NotNull - private String tier = DEFAULT_TIER; + private String category = DEFAULT_CATEGORY; private long intermediaryPartitionDiscoveryPeriodSec = 60L; @@ -76,9 +76,9 @@ public int getCapacity() return capacity; } - public String getTier() + public String getCategory() { - return tier; + return category; } public long getIntermediaryPartitionDiscoveryPeriodSec() diff --git a/server/src/test/java/org/apache/druid/discovery/WorkerNodeServiceTest.java b/server/src/test/java/org/apache/druid/discovery/WorkerNodeServiceTest.java index 80ce1bdb2bfc..b4a08e942b99 100644 --- a/server/src/test/java/org/apache/druid/discovery/WorkerNodeServiceTest.java +++ b/server/src/test/java/org/apache/druid/discovery/WorkerNodeServiceTest.java @@ -35,7 +35,7 @@ public void testSerde() throws Exception "1.1.1.1", 100, "v1", - "t1" + "c1" ); ObjectMapper mapper = TestHelper.makeJsonMapper(); diff --git a/services/src/main/java/org/apache/druid/cli/CliIndexer.java b/services/src/main/java/org/apache/druid/cli/CliIndexer.java index 41581a2111d2..42d24ac2661b 100644 --- a/services/src/main/java/org/apache/druid/cli/CliIndexer.java +++ b/services/src/main/java/org/apache/druid/cli/CliIndexer.java @@ -162,7 +162,7 @@ public Worker getWorker(@Self DruidNode node, WorkerConfig config) config.getIp(), config.getCapacity(), config.getVersion(), - WorkerConfig.DEFAULT_TIER + WorkerConfig.DEFAULT_CATEGORY ); } @@ -174,7 +174,7 @@ public WorkerNodeService getWorkerNodeService(WorkerConfig workerConfig) workerConfig.getIp(), workerConfig.getCapacity(), workerConfig.getVersion(), - WorkerConfig.DEFAULT_TIER + WorkerConfig.DEFAULT_CATEGORY ); } }, diff --git a/services/src/main/java/org/apache/druid/cli/CliMiddleManager.java b/services/src/main/java/org/apache/druid/cli/CliMiddleManager.java index 526813aef0f7..8efeb3cecb67 100644 --- a/services/src/main/java/org/apache/druid/cli/CliMiddleManager.java +++ b/services/src/main/java/org/apache/druid/cli/CliMiddleManager.java @@ -155,7 +155,7 @@ public Worker getWorker(@Self DruidNode node, WorkerConfig config) config.getIp(), config.getCapacity(), config.getVersion(), - config.getTier() + config.getCategory() ); } @@ -167,7 +167,7 @@ public WorkerNodeService getWorkerNodeService(WorkerConfig workerConfig) workerConfig.getIp(), workerConfig.getCapacity(), workerConfig.getVersion(), - workerConfig.getTier() + workerConfig.getCategory() ); } }, From 73bc542a9656200921e1e4f598506d917704c8cb Mon Sep 17 00:00:00 2001 From: "qiumingming.2018" Date: Wed, 9 Oct 2019 00:30:26 +0800 Subject: [PATCH 5/8] doc --- docs/configuration/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 59b74298a916..8cc3338fd988 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -990,7 +990,7 @@ This strategy is a variant of `Equal Distribution`, which support `workerCategor |Property|Description|Default| |--------|-----------|-------| |`type`|`equalDistributionWithCategorySpec`.|required; must be `equalDistributionWithCategorySpec`| -|`workerCategorySpec`|[Worker Category Spec](#workerCategorySpec) object|null (no worker category spec)| +|`workerCategorySpec`|[Worker Category Spec](#WorkerCategorySpec) object|null (no worker category spec)| Example: specify tasks default to run on _c1_ whose task type is "index_kafka", while dataSource "ds1" run on _c2_. @@ -1035,7 +1035,7 @@ This strategy is a variant of `Fill Capacity`, which support `workerCategorySpec |Property|Description|Default| |--------|-----------|-------| |`type`|`fillCapacityWithCategorySpec`.|required; must be `fillCapacityWithCategorySpec`| -|`workerCategorySpec`|[Worker Category Spec](#workerCategorySpec) object|null (no worker category spec)| +|`workerCategorySpec`|[Worker Category Spec](#WorkerCategorySpec) object|null (no worker category spec)| > Before using the _equalDistributionWithCategorySpec_ and _fillCapacityWithCategorySpec_ strategies, you must upgrade overlord and all MiddleManagers to the version that support this feature. @@ -1080,7 +1080,7 @@ field. If not provided, the default is to not use it at all. |Property|Description|Default| |--------|-----------|-------| -|`categoryMap`|A JSON map object mapping a task type String name to a [CategoryConfig](categoryConfig) object, by which you can specify category config for different task type.|{}| +|`categoryMap`|A JSON map object mapping a task type String name to a [CategoryConfig](#CategoryConfig) object, by which you can specify category config for different task type.|{}| |`strong`|With weak workerCategorySpec (the default), tasks for a dataSource may be assigned to other middleManagers if the middleManagers specified in `categoryMap` are not able to run all pending tasks in the queue for that dataSource. With strong workerCategorySpec, tasks for a dataSource will only ever be assigned to their specified middleManagers, and will wait in the pending queue if necessary.|false| ###### CategoryConfig From 01f047f921489a4169265e746cec1044f8ec2c82 Mon Sep 17 00:00:00 2001 From: "qiumingming.2018" Date: Fri, 11 Oct 2019 11:45:04 +0800 Subject: [PATCH 6/8] fix doc --- docs/configuration/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 8cc3338fd988..1290ad269cba 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -990,7 +990,7 @@ This strategy is a variant of `Equal Distribution`, which support `workerCategor |Property|Description|Default| |--------|-----------|-------| |`type`|`equalDistributionWithCategorySpec`.|required; must be `equalDistributionWithCategorySpec`| -|`workerCategorySpec`|[Worker Category Spec](#WorkerCategorySpec) object|null (no worker category spec)| +|`workerCategorySpec`|[Worker Category Spec](#workercategoryspec) object|null (no worker category spec)| Example: specify tasks default to run on _c1_ whose task type is "index_kafka", while dataSource "ds1" run on _c2_. @@ -1035,7 +1035,7 @@ This strategy is a variant of `Fill Capacity`, which support `workerCategorySpec |Property|Description|Default| |--------|-----------|-------| |`type`|`fillCapacityWithCategorySpec`.|required; must be `fillCapacityWithCategorySpec`| -|`workerCategorySpec`|[Worker Category Spec](#WorkerCategorySpec) object|null (no worker category spec)| +|`workerCategorySpec`|[Worker Category Spec](#workercategoryspec) object|null (no worker category spec)| > Before using the _equalDistributionWithCategorySpec_ and _fillCapacityWithCategorySpec_ strategies, you must upgrade overlord and all MiddleManagers to the version that support this feature. @@ -1080,7 +1080,7 @@ field. If not provided, the default is to not use it at all. |Property|Description|Default| |--------|-----------|-------| -|`categoryMap`|A JSON map object mapping a task type String name to a [CategoryConfig](#CategoryConfig) object, by which you can specify category config for different task type.|{}| +|`categoryMap`|A JSON map object mapping a task type String name to a [CategoryConfig](#categoryconfig) object, by which you can specify category config for different task type.|{}| |`strong`|With weak workerCategorySpec (the default), tasks for a dataSource may be assigned to other middleManagers if the middleManagers specified in `categoryMap` are not able to run all pending tasks in the queue for that dataSource. With strong workerCategorySpec, tasks for a dataSource will only ever be assigned to their specified middleManagers, and will wait in the pending queue if necessary.|false| ###### CategoryConfig From 367a342b46099cc033c2c67282312bbef2abab57 Mon Sep 17 00:00:00 2001 From: "qiumingming.2018" Date: Mon, 14 Oct 2019 00:43:23 +0800 Subject: [PATCH 7/8] fix spelling errors --- docs/configuration/index.md | 10 ++++++---- website/.spelling | 8 ++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 1290ad269cba..08d58e82d7fa 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -992,8 +992,8 @@ This strategy is a variant of `Equal Distribution`, which support `workerCategor |`type`|`equalDistributionWithCategorySpec`.|required; must be `equalDistributionWithCategorySpec`| |`workerCategorySpec`|[Worker Category Spec](#workercategoryspec) object|null (no worker category spec)| -Example: specify tasks default to run on _c1_ whose task -type is "index_kafka", while dataSource "ds1" run on _c2_. +Example: specify tasks default to run on **c1** whose task +type is "index_kafka", while dataSource "ds1" run on **c2**. ```json { @@ -1039,7 +1039,9 @@ This strategy is a variant of `Fill Capacity`, which support `workerCategorySpec > Before using the _equalDistributionWithCategorySpec_ and _fillCapacityWithCategorySpec_ strategies, you must upgrade overlord and all MiddleManagers to the version that support this feature. -###### Javascript + + +###### JavaScript Allows defining arbitrary logic for selecting workers to run task using a JavaScript function. The function is passed remoteTaskRunnerConfig, map of workerId to available workers and task to be executed and returns the workerId on which the task should be run or null if the task cannot be run. @@ -1081,7 +1083,7 @@ field. If not provided, the default is to not use it at all. |Property|Description|Default| |--------|-----------|-------| |`categoryMap`|A JSON map object mapping a task type String name to a [CategoryConfig](#categoryconfig) object, by which you can specify category config for different task type.|{}| -|`strong`|With weak workerCategorySpec (the default), tasks for a dataSource may be assigned to other middleManagers if the middleManagers specified in `categoryMap` are not able to run all pending tasks in the queue for that dataSource. With strong workerCategorySpec, tasks for a dataSource will only ever be assigned to their specified middleManagers, and will wait in the pending queue if necessary.|false| +|`strong`|With weak workerCategorySpec (the default), tasks for a dataSource may be assigned to other MiddleManagers if the MiddleManagers specified in `categoryMap` are not able to run all pending tasks in the queue for that dataSource. With strong workerCategorySpec, tasks for a dataSource will only ever be assigned to their specified MiddleManagers, and will wait in the pending queue if necessary.|false| ###### CategoryConfig diff --git a/website/.spelling b/website/.spelling index 3167f85d1f3f..bc553f79f7bb 100644 --- a/website/.spelling +++ b/website/.spelling @@ -1628,3 +1628,11 @@ isUnpatrolled metroCode regionIsoCode regionName +taskType +index_kafka +ds1 +equalDistributionWithCategorySpec +fillCapacityWithCategorySpec +WorkerCategorySpec +workerCategorySpec +CategoryConfig From 0e0bcee080e505aa1ec01eb5e89e33464e45aa01 Mon Sep 17 00:00:00 2001 From: "qiumingming.2018" Date: Thu, 17 Oct 2019 15:35:17 +0800 Subject: [PATCH 8/8] docs --- website/.spelling | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/website/.spelling b/website/.spelling index bc553f79f7bb..b4efcdfc23a9 100644 --- a/website/.spelling +++ b/website/.spelling @@ -1600,6 +1600,16 @@ v0.12.0 versionReplacementString workerId yyyy-MM-dd +taskType +index_kafka +c1 +c2 +ds1 +equalDistributionWithCategorySpec +fillCapacityWithCategorySpec +WorkerCategorySpec +workerCategorySpec +CategoryConfig - ../docs/design/index.md logsearch - ../docs/ingestion/index.md @@ -1628,11 +1638,3 @@ isUnpatrolled metroCode regionIsoCode regionName -taskType -index_kafka -ds1 -equalDistributionWithCategorySpec -fillCapacityWithCategorySpec -WorkerCategorySpec -workerCategorySpec -CategoryConfig