From c1ea353da73a5567e5e10b5bac51a4b9590332fb Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Tue, 1 Apr 2025 14:29:37 -0700 Subject: [PATCH 1/7] add turboLoadingNodes --- .../coordinator-dynamic-config.tsx | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx index bdd47a317225..290accc75ea2 100644 --- a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx +++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx @@ -38,6 +38,7 @@ export interface CoordinatorDynamicConfig { replicateAfterLoadTimeout?: boolean; useRoundRobinSegmentAssignment?: boolean; smartSegmentLoading?: boolean; + turboLoadingNodes?: string[]; // Undocumented debugDimensions?: any; @@ -156,6 +157,19 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[ ), }, + { + name: 'turboLoadingNodes', + type: 'string-array', + info: ( + <> + List of Historical servers to place in turbo loading mode. These servers use a larger + thread-pool to load segments faster but at the cost of query performance. For servers + specified in turboLoadingNodes,{' '} + druid.coordinator.loadqueuepeon.http.batchSize is ignored and the coordinator + uses the value of the respective numLoadingThreads instead. + + ), + }, { name: 'killDataSourceWhitelist', label: 'Kill datasource whitelist', From 5d8c1a4ef6a947929d1d59573d23e9d7ffb2fbb5 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 2 Apr 2025 08:10:38 -0700 Subject: [PATCH 2/7] Update web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx Co-authored-by: Karan Kumar --- .../coordinator-dynamic-config/coordinator-dynamic-config.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx index 290accc75ea2..7cae154b8d65 100644 --- a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx +++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx @@ -162,7 +162,7 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[ type: 'string-array', info: ( <> - List of Historical servers to place in turbo loading mode. These servers use a larger + [Experimental] List of Historical servers to place in turbo loading mode. These servers use a larger thread-pool to load segments faster but at the cost of query performance. For servers specified in turboLoadingNodes,{' '} druid.coordinator.loadqueuepeon.http.batchSize is ignored and the coordinator From 094e04f742f9ea5c03291e4eb163daff1e133a36 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 2 Apr 2025 08:10:45 -0700 Subject: [PATCH 3/7] Update web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx Co-authored-by: Karan Kumar --- .../coordinator-dynamic-config/coordinator-dynamic-config.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx index 7cae154b8d65..ce7ab193f90b 100644 --- a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx +++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx @@ -166,7 +166,7 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[ thread-pool to load segments faster but at the cost of query performance. For servers specified in turboLoadingNodes,{' '} druid.coordinator.loadqueuepeon.http.batchSize is ignored and the coordinator - uses the value of the respective numLoadingThreads instead. + uses the value of the respective numLoadingThreads instead. Please use this config with caution. All servers should eventually be removed from this list once the segment loading on the respective historicals is finished. ), }, From beabb64044836510b97a458f69160ecfc5393e4f Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 2 Apr 2025 11:21:45 -0700 Subject: [PATCH 4/7] added Experimental icon --- .../src/components/auto-form/auto-form.tsx | 15 +++++++++++++-- .../src/components/header-bar/header-bar.tsx | 3 ++- .../coordinator-dynamic-config.tsx | 17 ++++++++++++----- web-console/src/utils/general.tsx | 6 +++++- .../schema-step/schema-step.tsx | 4 +--- 5 files changed, 33 insertions(+), 12 deletions(-) diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx index 7026ca5d5dc6..746e7dc4ce84 100644 --- a/web-console/src/components/auto-form/auto-form.tsx +++ b/web-console/src/components/auto-form/auto-form.tsx @@ -28,7 +28,7 @@ import { IconNames } from '@blueprintjs/icons'; import type { JSX } from 'react'; import React from 'react'; -import { deepDelete, deepGet, deepSet, durationSanitizer } from '../../utils'; +import { deepDelete, deepGet, deepSet, durationSanitizer, EXPERIMENTAL_ICON } from '../../utils'; import { ArrayInput } from '../array-input/array-input'; import { FancyNumericInput } from '../fancy-numeric-input/fancy-numeric-input'; import { FormGroupWithInfo } from '../form-group-with-info/form-group-with-info'; @@ -65,6 +65,7 @@ export interface Field { max?: number; zeroMeansUndefined?: boolean; height?: string; + experimental?: Functor; disabled?: Functor; defined?: Functor; required?: Functor; @@ -509,10 +510,20 @@ export class AutoForm> extends React.PureComponent if (!model) return; const label = field.label || AutoForm.makeLabelName(field.name); + const experimental = AutoForm.evaluateFunctor(field.experimental, model, false); return ( + {`${label} `} + {EXPERIMENTAL_ICON} + + ) : ( + label + ) + } info={field.info ? {field.info} : undefined} > {this.renderFieldInput(field)} diff --git a/web-console/src/components/header-bar/header-bar.tsx b/web-console/src/components/header-bar/header-bar.tsx index 46c1e9340fde..140edc051c1b 100644 --- a/web-console/src/components/header-bar/header-bar.tsx +++ b/web-console/src/components/header-bar/header-bar.tsx @@ -44,6 +44,7 @@ import { import { Capabilities } from '../../helpers'; import { getLink } from '../../links'; import { + EXPERIMENTAL_ICON, localStorageGetJson, LocalStorageKeys, localStorageRemove, @@ -153,7 +154,7 @@ export const HeaderBar = React.memo(function HeaderBar(props: HeaderBarProps) { [ { name: 'turboLoadingNodes', type: 'string-array', + experimental: true, info: ( <> - [Experimental] List of Historical servers to place in turbo loading mode. These servers use a larger - thread-pool to load segments faster but at the cost of query performance. For servers - specified in turboLoadingNodes,{' '} - druid.coordinator.loadqueuepeon.http.batchSize is ignored and the coordinator - uses the value of the respective numLoadingThreads instead. Please use this config with caution. All servers should eventually be removed from this list once the segment loading on the respective historicals is finished. +

+ List of Historical servers to place in turbo loading mode. These servers use a larger + thread-pool to load segments faster but at the cost of query performance. For servers + specified in turboLoadingNodes,{' '} + druid.coordinator.loadqueuepeon.http.batchSize is ignored and the coordinator + uses the value of the respective numLoadingThreads instead. +

+

+ Please use this config with caution. All servers should eventually be removed from this + list once the segment loading on the respective historicals is finished. +

), }, diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index d204c7a89e07..1cb6765b86f8 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -16,7 +16,7 @@ * limitations under the License. */ -import { Classes, Intent } from '@blueprintjs/core'; +import { Classes, Icon, Intent } from '@blueprintjs/core'; import type { IconName } from '@blueprintjs/icons'; import { IconNames } from '@blueprintjs/icons'; import copy from 'copy-to-clipboard'; @@ -708,3 +708,7 @@ export function toggle(xs: readonly T[], x: T, eq?: (a: T, b: T) => boolean): const e = eq || ((a, b) => a === b); return xs.find(_ => e(_, x)) ? xs.filter(d => !e(d, x)) : xs.concat([x]); } + +export const EXPERIMENTAL_ICON = ( + +); diff --git a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx index 232c6635361f..f3ee9fa2adc4 100644 --- a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx +++ b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx @@ -22,7 +22,6 @@ import { ButtonGroup, Callout, FormGroup, - Icon, Intent, Menu, MenuDivider, @@ -83,6 +82,7 @@ import { dataTypeToIcon, deepSet, DruidError, + EXPERIMENTAL_ICON, filterMap, oneOf, queryDruidSql, @@ -107,8 +107,6 @@ import { RollupAnalysisPane } from './rollup-analysis-pane/rollup-analysis-pane' import './schema-step.scss'; -const EXPERIMENTAL_ICON = ; - const queryRunner = new QueryRunner(); function digestQueryString(queryString: string): { From 679d17ac3a0849afaf8dd5a0ec907d53bdfd9672 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 2 Apr 2025 11:24:45 -0700 Subject: [PATCH 5/7] added feedback to docs also --- docs/configuration/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index b3f4f38d5ca4..903f824097ee 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -953,7 +953,7 @@ The following table shows the dynamic configuration properties for the Coordinat |`decommissioningNodes`|List of Historical servers to decommission. Coordinator will not assign new segments to decommissioning servers, and segments will be moved away from them to be placed on non-decommissioning servers at the maximum rate specified by `maxSegmentsToMove`.|none| |`pauseCoordination`|Boolean flag for whether or not the Coordinator should execute its various duties of coordinating the cluster. Setting this to true essentially pauses all coordination work while allowing the API to remain up. Duties that are paused include all classes that implement the `CoordinatorDuty` interface. Such duties include: segment balancing, segment compaction, submitting kill tasks for unused segments (if enabled), logging of used segments in the cluster, marking of newly unused or overshadowed segments, matching and execution of load/drop rules for used segments, unloading segments that are no longer marked as used from Historical servers. An example of when an admin may want to pause coordination would be if they are doing deep storage maintenance on HDFS name nodes with downtime and don't want the Coordinator to be directing Historical nodes to hit the name node with API requests until maintenance is done and the deep store is declared healthy for use again.|false| |`replicateAfterLoadTimeout`|Boolean flag for whether or not additional replication is needed for segments that have failed to load due to the expiry of `druid.coordinator.load.timeout`. If this is set to true, the Coordinator will attempt to replicate the failed segment on a different historical server. This helps improve the segment availability if there are a few slow Historicals in the cluster. However, the slow Historical may still load the segment later and the Coordinator may issue drop requests if the segment is over-replicated.|false| -|`turboLoadingNodes`| List of Historical servers to place in turbo loading mode. These servers use a larger thread-pool to load segments faster but at the cost of query performance. For servers specified in `turboLoadingNodes`, `druid.coordinator.loadqueuepeon.http.batchSize` is ignored and the coordinator uses the value of the respective `numLoadingThreads` instead. |none| +|`turboLoadingNodes`| Experimental. List of Historical servers to place in turbo loading mode. These servers use a larger thread-pool to load segments faster but at the cost of query performance. For servers specified in `turboLoadingNodes`, `druid.coordinator.loadqueuepeon.http.batchSize` is ignored and the coordinator uses the value of the respective `numLoadingThreads` instead.
Please use this config with caution. All servers should eventually be removed from this list once the segment loading on the respective historicals is finished. |none| ##### Smart segment loading From 622facb3fb837f382e7ceb577f813a2669fe36b0 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 2 Apr 2025 11:27:02 -0700 Subject: [PATCH 6/7] update tests --- .../header-bar/__snapshots__/header-bar.spec.tsx.snap | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap index d8e883e50e4c..d713f2fa86f1 100644 --- a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap +++ b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap @@ -168,7 +168,15 @@ exports[`HeaderBar matches snapshot 1`] = ` disabled={false} href="#explore" icon="map" - label="(experimental)" + labelElement={ + + } multiline={false} popoverProps={{}} selected={false} From f6cf8377c4e441c7560de19bfffb4d3aa3df210c Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 2 Apr 2025 13:10:32 -0700 Subject: [PATCH 7/7] add slash --- docs/configuration/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 9885fe5d81e9..91168d9471a1 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -953,7 +953,7 @@ The following table shows the dynamic configuration properties for the Coordinat |`decommissioningNodes`|List of Historical servers to decommission. Coordinator will not assign new segments to decommissioning servers, and segments will be moved away from them to be placed on non-decommissioning servers at the maximum rate specified by `maxSegmentsToMove`.|none| |`pauseCoordination`|Boolean flag for whether or not the Coordinator should execute its various duties of coordinating the cluster. Setting this to true essentially pauses all coordination work while allowing the API to remain up. Duties that are paused include all classes that implement the `CoordinatorDuty` interface. Such duties include: segment balancing, segment compaction, submitting kill tasks for unused segments (if enabled), logging of used segments in the cluster, marking of newly unused or overshadowed segments, matching and execution of load/drop rules for used segments, unloading segments that are no longer marked as used from Historical servers. An example of when an admin may want to pause coordination would be if they are doing deep storage maintenance on HDFS name nodes with downtime and don't want the Coordinator to be directing Historical nodes to hit the name node with API requests until maintenance is done and the deep store is declared healthy for use again.|false| |`replicateAfterLoadTimeout`|Boolean flag for whether or not additional replication is needed for segments that have failed to load due to the expiry of `druid.coordinator.load.timeout`. If this is set to true, the Coordinator will attempt to replicate the failed segment on a different historical server. This helps improve the segment availability if there are a few slow Historicals in the cluster. However, the slow Historical may still load the segment later and the Coordinator may issue drop requests if the segment is over-replicated.|false| -|`turboLoadingNodes`| Experimental. List of Historical servers to place in turbo loading mode. These servers use a larger thread-pool to load segments faster but at the cost of query performance. For servers specified in `turboLoadingNodes`, `druid.coordinator.loadqueuepeon.http.batchSize` is ignored and the coordinator uses the value of the respective `numLoadingThreads` instead.
Please use this config with caution. All servers should eventually be removed from this list once the segment loading on the respective historicals is finished. |none| +|`turboLoadingNodes`| Experimental. List of Historical servers to place in turbo loading mode. These servers use a larger thread-pool to load segments faster but at the cost of query performance. For servers specified in `turboLoadingNodes`, `druid.coordinator.loadqueuepeon.http.batchSize` is ignored and the coordinator uses the value of the respective `numLoadingThreads` instead.
Please use this config with caution. All servers should eventually be removed from this list once the segment loading on the respective historicals is finished. |none| ##### Smart segment loading