From 7a12afe986ad97963a29d48e72c5697e492e882b Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 21 Jun 2023 13:13:38 -0700 Subject: [PATCH 01/18] sys.segments view and segments card --- .../home-view/segments-card/segments-card.tsx | 18 ++++++++++++----- .../src/views/segments-view/segments-view.tsx | 20 ++++++++++++++++--- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/web-console/src/views/home-view/segments-card/segments-card.tsx b/web-console/src/views/home-view/segments-card/segments-card.tsx index 9872f27cc1f9..479b1d844ea0 100644 --- a/web-console/src/views/home-view/segments-card/segments-card.tsx +++ b/web-console/src/views/home-view/segments-card/segments-card.tsx @@ -28,6 +28,7 @@ import { HomeViewCard } from '../home-view-card/home-view-card'; export interface SegmentCounts { total: number; + cached_on_historical: number; unavailable: number; } @@ -37,12 +38,14 @@ export interface SegmentsCardProps { export const SegmentsCard = React.memo(function SegmentsCard(props: SegmentsCardProps) { const [segmentCountState] = useQueryManager({ + initQuery: props.capabilities, processQuery: async capabilities => { if (capabilities.hasSql()) { const segments = await queryDruidSql({ query: `SELECT COUNT(*) as "total", - COUNT(*) FILTER (WHERE is_active = 1 AND is_available = 0) as "unavailable" + COUNT(*) FILTER (WHERE is_active = 1 AND is_available = 1) as "cached_on_historical", + COUNT(*) FILTER (WHERE is_active = 1 AND is_available = 0 AND replication_factor > 0) + 1 as "unavailable" FROM sys.segments`, }); return segments.length === 1 ? segments[0] : null; @@ -61,16 +64,20 @@ FROM sys.segments`, return { total: availableSegmentNum + unavailableSegmentNum, + cached_on_historical: availableSegmentNum, // This is not correct unavailable: unavailableSegmentNum, }; } else { throw new Error(`must have SQL or coordinator access`); } }, - initQuery: props.capabilities, }); - const segmentCount = segmentCountState.data || { total: 0, unavailable: 0 }; + const segmentCount = segmentCountState.data || { + total: 0, + cached_on_historical: 0, + unavailable: 0, + }; return ( -

{pluralIfNeeded(segmentCount.total, 'segment')}

+

{pluralIfNeeded(segmentCount.total, 'segment')} total

+

{pluralIfNeeded(segmentCount.cached_on_historical, 'segment')} cached on historicals

{Boolean(segmentCount.unavailable) && ( -

{pluralIfNeeded(segmentCount.unavailable, 'unavailable segment')}

+

{pluralIfNeeded(segmentCount.unavailable, 'segment')} waiting to be cached

)} ); diff --git a/web-console/src/views/segments-view/segments-view.tsx b/web-console/src/views/segments-view/segments-view.tsx index 6f5496162c9c..cd4034963050 100644 --- a/web-console/src/views/segments-view/segments-view.tsx +++ b/web-console/src/views/segments-view/segments-view.tsx @@ -88,6 +88,7 @@ const tableColumns: Record = { 'Num rows', 'Avg. row size', 'Replicas', + 'Replication factor', 'Is available', 'Is active', 'Is realtime', @@ -118,6 +119,7 @@ const tableColumns: Record = { 'Num rows', 'Avg. row size', 'Replicas', + 'Replication factor', 'Is available', 'Is active', 'Is realtime', @@ -162,6 +164,7 @@ interface SegmentQueryResultRow { num_rows: NumberLike; avg_row_size: NumberLike; num_replicas: number; + replication_factor: number; is_available: number; is_active: number; is_realtime: number; @@ -214,6 +217,7 @@ END AS "time_span"`, visibleColumns.shown('Avg. row size') && `CASE WHEN "num_rows" <> 0 THEN ("size" / "num_rows") ELSE 0 END AS "avg_row_size"`, visibleColumns.shown('Replicas') && `"num_replicas"`, + visibleColumns.shown('Replication factor') && `"replication_factor"`, visibleColumns.shown('Is available') && `"is_available"`, visibleColumns.shown('Is active') && `"is_active"`, visibleColumns.shown('Is realtime') && `"is_realtime"`, @@ -413,6 +417,7 @@ END AS "time_span"`, num_rows: -1, avg_row_size: -1, num_replicas: -1, + replication_factor: -1, is_available: -1, is_active: -1, is_realtime: -1, @@ -781,7 +786,7 @@ END AS "time_span"`, ), }, { - Header: twoLines('Avg. row size', '(bytes)'), + Header: twoLines('Avg. row size', (bytes)), show: capabilities.hasSql() && visibleColumns.shown('Avg. row size'), accessor: 'avg_row_size', filterable: false, @@ -799,10 +804,19 @@ END AS "time_span"`, }, }, { - Header: 'Replicas', + Header: twoLines('Replicas', (actual)), show: hasSql && visibleColumns.shown('Replicas'), accessor: 'num_replicas', - width: 60, + width: 80, + filterable: false, + defaultSortDesc: true, + className: 'padded', + }, + { + Header: twoLines('Replication factor', (desired)), + show: hasSql && visibleColumns.shown('Replication factor'), + accessor: 'replication_factor', + width: 80, filterable: false, defaultSortDesc: true, className: 'padded', From ac14228dbf45af6113aeb95e0bbb78668fb50d3b Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 21 Jun 2023 13:56:17 -0700 Subject: [PATCH 02/18] more card fixes --- .../home-view/segments-card/segments-card.tsx | 29 +++++++++++------ .../supervisors-view/supervisors-view.tsx | 31 +++++++++++-------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/web-console/src/views/home-view/segments-card/segments-card.tsx b/web-console/src/views/home-view/segments-card/segments-card.tsx index 479b1d844ea0..26ce7f833139 100644 --- a/web-console/src/views/home-view/segments-card/segments-card.tsx +++ b/web-console/src/views/home-view/segments-card/segments-card.tsx @@ -27,9 +27,10 @@ import { deepGet, pluralIfNeeded, queryDruidSql } from '../../../utils'; import { HomeViewCard } from '../home-view-card/home-view-card'; export interface SegmentCounts { - total: number; + active: number; cached_on_historical: number; unavailable: number; + realtime: number; } export interface SegmentsCardProps { @@ -43,10 +44,12 @@ export const SegmentsCard = React.memo(function SegmentsCard(props: SegmentsCard if (capabilities.hasSql()) { const segments = await queryDruidSql({ query: `SELECT - COUNT(*) as "total", - COUNT(*) FILTER (WHERE is_active = 1 AND is_available = 1) as "cached_on_historical", - COUNT(*) FILTER (WHERE is_active = 1 AND is_available = 0 AND replication_factor > 0) + 1 as "unavailable" -FROM sys.segments`, + COUNT(*) AS "active", + COUNT(*) FILTER (WHERE is_available = 1) AS "cached_on_historical", + COUNT(*) FILTER (WHERE is_available = 0 AND replication_factor > 0) AS "unavailable", + COUNT(*) FILTER (WHERE is_realtime = 1) AS "realtime" +FROM sys.segments +WHERE is_active = 1`, }); return segments.length === 1 ? segments[0] : null; } else if (capabilities.hasCoordinatorAccess()) { @@ -63,7 +66,7 @@ FROM sys.segments`, ); return { - total: availableSegmentNum + unavailableSegmentNum, + active: availableSegmentNum + unavailableSegmentNum, cached_on_historical: availableSegmentNum, // This is not correct unavailable: unavailableSegmentNum, }; @@ -73,10 +76,11 @@ FROM sys.segments`, }, }); - const segmentCount = segmentCountState.data || { - total: 0, + const segmentCount: SegmentCounts = segmentCountState.data || { + active: 0, cached_on_historical: 0, unavailable: 0, + realtime: 0, }; return ( -

{pluralIfNeeded(segmentCount.total, 'segment')} total

+

{pluralIfNeeded(segmentCount.active, 'active segment')}

{pluralIfNeeded(segmentCount.cached_on_historical, 'segment')} cached on historicals

{Boolean(segmentCount.unavailable) && ( -

{pluralIfNeeded(segmentCount.unavailable, 'segment')} waiting to be cached

+

+ {pluralIfNeeded(segmentCount.unavailable, 'segment')} waiting to be cached on historicals +

+ )} + {Boolean(segmentCount.realtime) && ( +

{pluralIfNeeded(segmentCount.realtime, 'realtime segment')}

)} ); diff --git a/web-console/src/views/supervisors-view/supervisors-view.tsx b/web-console/src/views/supervisors-view/supervisors-view.tsx index 4e505f50407b..17fbe6ddd7c8 100644 --- a/web-console/src/views/supervisors-view/supervisors-view.tsx +++ b/web-console/src/views/supervisors-view/supervisors-view.tsx @@ -81,7 +81,6 @@ interface SupervisorQueryResultRow { supervisor_id: string; type: string; source: string; - state: string; detailed_state: string; suspended: boolean; running_tasks?: number; @@ -124,8 +123,8 @@ export interface SupervisorsViewState { visibleColumns: LocalStorageBackedVisibility; } -function stateToColor(status: string): string { - switch (status) { +function detailedStateToColor(detailedState: string): string { + switch (detailedState) { case 'UNHEALTHY_SUPERVISOR': case 'UNHEALTHY_TASKS': return '#d5100a'; @@ -157,13 +156,17 @@ export class SupervisorsView extends React.PureComponent< >; static SUPERVISOR_SQL = `SELECT - "supervisor_id", "type", "source", "state", "detailed_state", "suspended" = 1 AS "suspended" -FROM sys.supervisors + "supervisor_id", + "type", + "source", + CASE WHEN "suspended" = 0 THEN "detailed_state" ELSE 'SUSPENDED' END AS "detailed_state", + "suspended" = 1 AS "suspended" +FROM "sys"."supervisors" ORDER BY "supervisor_id"`; static RUNNING_TASK_SQL = `SELECT "datasource", "type", COUNT(*) AS "num_running_tasks" -FROM sys.tasks WHERE "status" = 'RUNNING' AND "runner_status" = 'RUNNING' +FROM "sys"."tasks" WHERE "status" = 'RUNNING' AND "runner_status" = 'RUNNING' GROUP BY 1, 2`; constructor(props: SupervisorsViewProps) { @@ -549,16 +552,16 @@ GROUP BY 1, 2`; id: 'status', width: 250, accessor: 'detailed_state', - Cell: row => ( + Cell: ({ value }) => ( - ●  - {row.value} + ●  + {value} ), @@ -578,9 +581,11 @@ GROUP BY 1, 2`; > {typeof value === 'undefined' ? 'n/a' - : !value - ? `No running tasks` - : pluralIfNeeded(value, 'running task')} + : value > 0 + ? pluralIfNeeded(value, 'running task') + : original.suspended + ? '' + : `No running tasks`} ), show: visibleColumns.shown('Running tasks'), From 50071d7ae03ae0516ec84fdc7f7a6f925be153c7 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Thu, 22 Jun 2023 22:18:21 -0700 Subject: [PATCH 03/18] fix refresh --- .../src/components/auto-form/auto-form.tsx | 3 +- .../components/braced-text/braced-text.tsx | 1 + .../src/components/deferred/deferred.tsx | 1 + .../form-group-with-info.tsx | 1 + .../src/components/header-bar/header-bar.tsx | 1 + .../highlight-text/highlight-text.tsx | 31 ++++- .../query-error-pane/query-error-pane.tsx | 15 ++- .../refresh-button/refresh-button.tsx | 7 +- .../components/segment-timeline/bar-group.tsx | 2 +- .../segment-timeline/segment-timeline.tsx | 2 +- .../src/components/show-log/show-log.tsx | 2 +- .../src/components/table-cell/table-cell.tsx | 4 +- web-console/src/console-application.tsx | 2 +- .../dialogs/doctor-dialog/doctor-dialog.tsx | 2 +- .../dialogs/history-dialog/history-dialog.tsx | 1 + .../numeric-input-dialog.tsx | 1 + .../dialogs/snitch-dialog/snitch-dialog.tsx | 4 +- .../dialogs/status-dialog/status-dialog.tsx | 2 +- .../ingestion-spec/ingestion-spec.tsx | 5 +- .../workbench-query/workbench-query.ts | 2 +- web-console/src/utils/basic-action.tsx | 1 + web-console/src/utils/druid-query.spec.ts | 51 ++++---- web-console/src/utils/druid-query.ts | 123 ++++++++++-------- web-console/src/utils/general.tsx | 3 +- web-console/src/utils/query-cursor.ts | 2 - web-console/src/utils/query-manager.tsx | 9 +- .../datasources-view/datasources-view.tsx | 4 +- .../views/load-data-view/load-data-view.tsx | 3 +- .../src/views/lookups-view/lookups-view.tsx | 2 +- .../src/views/segments-view/segments-view.tsx | 2 +- .../src/views/services-view/services-view.tsx | 2 +- .../column-actions/column-actions.tsx | 1 + .../column-editor/column-editor.tsx | 1 + .../rollup-analysis-pane.tsx | 1 + .../schema-step/schema-step.tsx | 1 + .../sql-data-loader-view.tsx | 1 + .../supervisors-view/supervisors-view.tsx | 2 +- .../src/views/tasks-view/tasks-view.tsx | 2 +- .../number-menu-items/number-menu-items.tsx | 1 + .../string-menu-items/string-menu-items.tsx | 1 + .../time-menu-items/time-menu-items.tsx | 1 + .../column-tree/column-tree.tsx | 2 +- .../execution-summary-panel.tsx | 1 + .../explain-dialog/explain-dialog.tsx | 1 + .../flexible-query-input.tsx | 2 +- .../helper-query/helper-query.tsx | 1 + .../input-source-step/input-source-step.tsx | 1 + .../workbench-view/query-tab/query-tab.tsx | 1 + .../result-table-pane/result-table-pane.tsx | 1 + .../workbench-view/run-panel/run-panel.tsx | 1 + .../workbench-history-dialog.tsx | 1 + .../views/workbench-view/workbench-view.tsx | 2 +- 52 files changed, 188 insertions(+), 129 deletions(-) diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx index a1e19174ffcd..1a342d567eed 100644 --- a/web-console/src/components/auto-form/auto-form.tsx +++ b/web-console/src/components/auto-form/auto-form.tsx @@ -25,6 +25,7 @@ import { NumericInput, } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; +import type { JSX } from 'react'; import React from 'react'; import { deepDelete, deepGet, deepSet, durationSanitizer } from '../../utils'; @@ -510,7 +511,7 @@ export class AutoForm> extends React.PureComponent ); } - render(): JSX.Element { + render() { const { fields, model, showCustom } = this.props; const { showMore, customDialog } = this.state; diff --git a/web-console/src/components/braced-text/braced-text.tsx b/web-console/src/components/braced-text/braced-text.tsx index 59840215ce35..69aafe08b02b 100644 --- a/web-console/src/components/braced-text/braced-text.tsx +++ b/web-console/src/components/braced-text/braced-text.tsx @@ -18,6 +18,7 @@ import classNames from 'classnames'; import { max } from 'd3-array'; +import type { JSX } from 'react'; import React, { Fragment } from 'react'; import './braced-text.scss'; diff --git a/web-console/src/components/deferred/deferred.tsx b/web-console/src/components/deferred/deferred.tsx index f4a9deb68c42..8e3979a2583c 100644 --- a/web-console/src/components/deferred/deferred.tsx +++ b/web-console/src/components/deferred/deferred.tsx @@ -16,6 +16,7 @@ * limitations under the License. */ +import type { JSX } from 'react'; import React from 'react'; export interface DeferredProps { diff --git a/web-console/src/components/form-group-with-info/form-group-with-info.tsx b/web-console/src/components/form-group-with-info/form-group-with-info.tsx index a42c4ca61d16..b3fa0deebc19 100644 --- a/web-console/src/components/form-group-with-info/form-group-with-info.tsx +++ b/web-console/src/components/form-group-with-info/form-group-with-info.tsx @@ -19,6 +19,7 @@ import { FormGroup, Icon } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import { Popover2 } from '@blueprintjs/popover2'; +import type { JSX } from 'react'; import React from 'react'; import './form-group-with-info.scss'; diff --git a/web-console/src/components/header-bar/header-bar.tsx b/web-console/src/components/header-bar/header-bar.tsx index 462aa60a22ae..928a9d92f4ee 100644 --- a/web-console/src/components/header-bar/header-bar.tsx +++ b/web-console/src/components/header-bar/header-bar.tsx @@ -32,6 +32,7 @@ import { } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import { Popover2 } from '@blueprintjs/popover2'; +import type { JSX } from 'react'; import React, { useState } from 'react'; import { diff --git a/web-console/src/components/highlight-text/highlight-text.tsx b/web-console/src/components/highlight-text/highlight-text.tsx index 85f55c44da72..4a50d103cbfa 100644 --- a/web-console/src/components/highlight-text/highlight-text.tsx +++ b/web-console/src/components/highlight-text/highlight-text.tsx @@ -16,29 +16,50 @@ * limitations under the License. */ +import type { JSX } from 'react'; import React from 'react'; import './highlight-text.scss'; export interface HighlightTextProps { text: string; - find: string; - replace: string | JSX.Element; + find: string | RegExp; + replace: string | JSX.Element | ((found: string) => string | JSX.Element); } export const HighlightText = React.memo(function HighlightText(props: HighlightTextProps) { const { text, find, replace } = props; - const startIndex = text.indexOf(find); + let startIndex = -1; + let found = ''; + + if (typeof find === 'string') { + startIndex = text.indexOf(find); + if (startIndex !== -1) { + found = find; + } + } else { + const m = find.exec(text); + if (m) { + startIndex = m.index; + found = m[0]; + } + } + if (startIndex === -1) return text; - const endIndex = startIndex + find.length; + const endIndex = startIndex + found.length; const pre = text.substring(0, startIndex); const post = text.substring(endIndex); + const replaceValue = typeof replace === 'function' ? replace(found) : replace; return ( {Boolean(pre) && {text.substring(0, startIndex)}} - {typeof replace === 'string' ? {replace} : replace} + {typeof replaceValue === 'string' ? ( + {replaceValue} + ) : ( + replaceValue + )} {Boolean(post) && {text.substring(endIndex)}} ); diff --git a/web-console/src/components/query-error-pane/query-error-pane.tsx b/web-console/src/components/query-error-pane/query-error-pane.tsx index 3ddb722c34a9..f8e0d3a622c1 100644 --- a/web-console/src/components/query-error-pane/query-error-pane.tsx +++ b/web-console/src/components/query-error-pane/query-error-pane.tsx @@ -16,6 +16,7 @@ * limitations under the License. */ +import type { JSX } from 'react'; import React, { useState } from 'react'; import type { DruidError, RowColumn } from '../../utils'; @@ -61,22 +62,26 @@ export const QueryErrorPane = React.memo(function QueryErrorPane(props: QueryErr return (
{suggestionElement} - {error.error &&

{`Error: ${error.error}`}

} + {error.error && ( +

{`Error: ${error.category}${ + error.persona && error.persona !== 'USER' ? ` (${error.persona})` : '' + }`}

+ )} {error.errorMessageWithoutExpectation && (

{position ? ( ( { moveCursorTo(position); }} > - {position.match} + {found} - } + )} /> ) : ( error.errorMessageWithoutExpectation diff --git a/web-console/src/components/refresh-button/refresh-button.tsx b/web-console/src/components/refresh-button/refresh-button.tsx index d2c38ac506e0..bbf7ed2f0714 100644 --- a/web-console/src/components/refresh-button/refresh-button.tsx +++ b/web-console/src/components/refresh-button/refresh-button.tsx @@ -33,13 +33,11 @@ const DELAYS: DelayLabel[] = [ ]; export interface RefreshButtonProps { - onRefresh: (auto: boolean) => void; + onRefresh(auto: boolean): void; localStorageKey?: LocalStorageKeys; } export const RefreshButton = React.memo(function RefreshButton(props: RefreshButtonProps) { - const { onRefresh, localStorageKey } = props; - return ( ); }); diff --git a/web-console/src/components/segment-timeline/bar-group.tsx b/web-console/src/components/segment-timeline/bar-group.tsx index 6c335d5c2892..d0cf867e2b2d 100644 --- a/web-console/src/components/segment-timeline/bar-group.tsx +++ b/web-console/src/components/segment-timeline/bar-group.tsx @@ -39,7 +39,7 @@ export class BarGroup extends React.Component { return nextProps.hoverOn === this.props.hoverOn; } - render(): JSX.Element[] | null { + render() { const { dataToRender, changeActiveDatasource, xScale, yScale, onHoverBar, barWidth } = this.props; if (dataToRender === undefined) return null; diff --git a/web-console/src/components/segment-timeline/segment-timeline.tsx b/web-console/src/components/segment-timeline/segment-timeline.tsx index 27fb9b9bfd65..150e94f2abe9 100644 --- a/web-console/src/components/segment-timeline/segment-timeline.tsx +++ b/web-console/src/components/segment-timeline/segment-timeline.tsx @@ -525,7 +525,7 @@ ORDER BY "start" DESC`; ); } - render(): JSX.Element { + render() { const { capabilities } = this.props; const { datasources, activeDataType, activeDatasource, startDate, endDate } = this.state; diff --git a/web-console/src/components/show-log/show-log.tsx b/web-console/src/components/show-log/show-log.tsx index 7134aa00568c..b072ac8e3039 100644 --- a/web-console/src/components/show-log/show-log.tsx +++ b/web-console/src/components/show-log/show-log.tsx @@ -137,7 +137,7 @@ export class ShowLog extends React.PureComponent { } }; - render(): JSX.Element { + render() { const { endpoint, downloadFilename, tail } = this.props; const { logState } = this.state; diff --git a/web-console/src/components/table-cell/table-cell.tsx b/web-console/src/components/table-cell/table-cell.tsx index 4e78b4f7fa73..3b31caa85f25 100644 --- a/web-console/src/components/table-cell/table-cell.tsx +++ b/web-console/src/components/table-cell/table-cell.tsx @@ -57,13 +57,13 @@ export const TableCell = React.memo(function TableCell(props: TableCellProps) { const { value, unlimited } = props; const [showValue, setShowValue] = useState(); - function renderShowValueDialog(): JSX.Element | undefined { + function renderShowValueDialog() { if (!showValue) return; return setShowValue(undefined)} str={showValue} />; } - function renderTruncated(str: string): JSX.Element { + function renderTruncated(str: string) { if (str.length <= MAX_CHARS_TO_SHOW) { return

{str}
; } diff --git a/web-console/src/console-application.tsx b/web-console/src/console-application.tsx index 8ea58d176015..9215d65b7c42 100644 --- a/web-console/src/console-application.tsx +++ b/web-console/src/console-application.tsx @@ -414,7 +414,7 @@ export class ConsoleApplication extends React.PureComponent< ); }; - render(): JSX.Element { + render() { const { capabilities, capabilitiesLoading } = this.state; if (capabilitiesLoading) { diff --git a/web-console/src/dialogs/doctor-dialog/doctor-dialog.tsx b/web-console/src/dialogs/doctor-dialog/doctor-dialog.tsx index 2e916697531d..560e22725c0e 100644 --- a/web-console/src/dialogs/doctor-dialog/doctor-dialog.tsx +++ b/web-console/src/dialogs/doctor-dialog/doctor-dialog.tsx @@ -175,7 +175,7 @@ export class DoctorDialog extends React.PureComponent; if (responseState.error) { diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx index 24d52373d0a0..136b4bb5217d 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx @@ -18,6 +18,7 @@ import { Code } from '@blueprintjs/core'; import { range } from 'd3-array'; +import type { JSX } from 'react'; import React from 'react'; import type { Field } from '../../components'; @@ -1921,7 +1922,7 @@ const TUNING_FORM_FIELDS: Field[] = [ defined: s => s.type === 'index_parallel' && deepGet(s, 'spec.ioConfig.inputSource.type') !== 'http', hideInMore: true, - adjustment: s => deepSet(s, 'splitHintSpec.type', 'maxSize'), + adjustment: s => deepSet(s, 'spec.tuningConfig.splitHintSpec.type', 'maxSize'), info: ( <> Maximum number of bytes of input files to process in a single subtask. If a single file is @@ -1937,7 +1938,7 @@ const TUNING_FORM_FIELDS: Field[] = [ min: 1, defined: typeIs('index_parallel'), hideInMore: true, - adjustment: s => deepSet(s, 'splitHintSpec.type', 'maxSize'), + adjustment: s => deepSet(s, 'spec.tuningConfig.splitHintSpec.type', 'maxSize'), info: ( <> Maximum number of input files to process in a single subtask. This limit is to avoid task diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index 68d2dcd71f82..53b974c87805 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -228,7 +228,7 @@ export class WorkbenchQuery { static getRowColumnFromIssue(issue: string): RowColumn | undefined { const m = issue.match(/at line (\d+),(\d+)/); if (!m) return; - return { match: '', row: Number(m[1]) - 1, column: Number(m[2]) - 1 }; + return { row: Number(m[1]) - 1, column: Number(m[2]) - 1 }; } public readonly queryParts: WorkbenchQueryPart[]; diff --git a/web-console/src/utils/basic-action.tsx b/web-console/src/utils/basic-action.tsx index 75461cc74538..5e0b270cc125 100644 --- a/web-console/src/utils/basic-action.tsx +++ b/web-console/src/utils/basic-action.tsx @@ -18,6 +18,7 @@ import type { IconName, Intent } from '@blueprintjs/core'; import { Menu, MenuItem } from '@blueprintjs/core'; +import type { JSX } from 'react'; import React from 'react'; export interface BasicAction { diff --git a/web-console/src/utils/druid-query.spec.ts b/web-console/src/utils/druid-query.spec.ts index e19a5c3df17c..150775dd8318 100644 --- a/web-console/src/utils/druid-query.spec.ts +++ b/web-console/src/utils/druid-query.spec.ts @@ -22,35 +22,32 @@ import { DruidError, getDruidErrorMessage } from './druid-query'; describe('DruidQuery', () => { describe('DruidError.parsePosition', () => { - it('works for single error 1', () => { - const message = `Encountered "COUNT" at line 2, column 12. Was expecting one of: "AS" ... "EXCEPT" ... "FETCH" ... "FROM" ... "INTERSECT" ... "LIMIT" ...`; - - expect(DruidError.parsePosition(message)).toEqual({ - match: 'at line 2, column 12', - row: 1, - column: 11, - }); - }); - - it('works for single error 2', () => { - const message = `org.apache.calcite.runtime.CalciteContextException: At line 2, column 20: Unknown identifier '*'`; - - expect(DruidError.parsePosition(message)).toEqual({ - match: 'At line 2, column 20', - row: 1, - column: 19, - }); - }); + // it('works for single error 1', () => { + // const message = `Encountered "COUNT" at line 2, column 12. Was expecting one of: "AS" ... "EXCEPT" ... "FETCH" ... "FROM" ... "INTERSECT" ... "LIMIT" ...`; + // + // expect(DruidError.extractPosition(message)).toEqual({ + // match: 'at line 2, column 12', + // row: 1, + // column: 11, + // }); + // }); it('works for range', () => { - const message = `org.apache.calcite.runtime.CalciteContextException: From line 2, column 13 to line 2, column 25: No match found for function signature SUMP()`; - - expect(DruidError.parsePosition(message)).toEqual({ - match: 'From line 2, column 13 to line 2, column 25', - row: 1, - column: 12, - endRow: 1, - endColumn: 25, + expect( + DruidError.extractPosition({ + sourceType: 'sql', + line: '1', + column: '16', + endLine: '1', + endColumn: '17', + token: "AS \\'l\\'", + expected: '...', + }), + ).toEqual({ + row: 0, + column: 15, + endRow: 0, + endColumn: 16, }); }); }); diff --git a/web-console/src/utils/druid-query.ts b/web-console/src/utils/druid-query.ts index 030a830c6732..482538e70abe 100644 --- a/web-console/src/utils/druid-query.ts +++ b/web-console/src/utils/druid-query.ts @@ -27,9 +27,26 @@ import type { RowColumn } from './query-cursor'; const CANCELED_MESSAGE = 'Query canceled by user.'; -export interface DruidErrorResponse { +export type ErrorResponsePersona = 'USER' | 'ADMIN' | 'OPERATOR' | 'DEVELOPER'; +export type ErrorResponseCategory = + | 'DEFENSIVE' + | 'INVALID_INPUT' + | 'UNAUTHORIZED' + | 'CAPACITY_EXCEEDED' + | 'CANCELED' + | 'RUNTIME_FAILURE' + | 'TIMEOUT' + | 'UNSUPPORTED' + | 'UNCATEGORIZED'; + +export interface ErrorResponse { + persona: ErrorResponsePersona; + category: ErrorResponseCategory; + errorMessage: string; // a message for the intended audience + context?: Record; // a map of extra context values that might be helpful + + // Deprecated as per https://github.com/apache/druid/blob/master/processing/src/main/java/org/apache/druid/error/ErrorResponse.java error?: string; - errorMessage?: string; errorClass?: string; host?: string; } @@ -51,7 +68,7 @@ export function parseHtmlError(htmlStr: string): string | undefined { .replace(/>/g, '>'); } -function getDruidErrorObject(e: any): DruidErrorResponse | string { +function errorResponseFromWhatever(e: any): ErrorResponse | string { if (e.response) { // This is a direct axios response error let data = e.response.data || {}; @@ -64,7 +81,7 @@ function getDruidErrorObject(e: any): DruidErrorResponse | string { } export function getDruidErrorMessage(e: any): string { - const data = getDruidErrorObject(e); + const data = errorResponseFromWhatever(e); switch (typeof data) { case 'object': return ( @@ -87,30 +104,20 @@ export function getDruidErrorMessage(e: any): string { } export class DruidError extends Error { - static parsePosition(errorMessage: string): RowColumn | undefined { - const range = /from line (\d+), column (\d+) to line (\d+), column (\d+)/i.exec( - String(errorMessage), - ); - if (range) { - return { - match: range[0], - row: Number(range[1]) - 1, - column: Number(range[2]) - 1, - endRow: Number(range[3]) - 1, - endColumn: Number(range[4]), // No -1 because we need to include the last char - }; - } + static extractPosition(context: Record | undefined): RowColumn | undefined { + if (context?.sourceType !== 'sql' || !context.line || !context.column) return; - const single = /at line (\d+), column (\d+)/i.exec(String(errorMessage)); - if (single) { - return { - match: single[0], - row: Number(single[1]) - 1, - column: Number(single[2]) - 1, - }; + const rowColumn: RowColumn = { + row: Number(context.line) - 1, + column: Number(context.column) - 1, + }; + + if (context.endLine && context.endColumn) { + rowColumn.endRow = Number(context.endLine) - 1; + rowColumn.endColumn = Number(context.endColumn) - 1; } - return; + return rowColumn; } static positionToIndex(str: string, line: number, column: number): number { @@ -123,8 +130,9 @@ export class DruidError extends Error { static getSuggestion(errorMessage: string): QuerySuggestion | undefined { // == is used instead of = // ex: SELECT * FROM wikipedia WHERE channel == '#en.wikipedia' - // ex: Encountered "= =" at line 3, column 15. Was expecting one of - const matchEquals = /Encountered "= =" at line (\d+), column (\d+)./.exec(errorMessage); + // er: Received an unexpected token [= =] (line [1], column [39]), acceptable options: + const matchEquals = + /Received an unexpected token \[= =] \(line \[(\d+)], column \[(\d+)]\),/.exec(errorMessage); if (matchEquals) { const line = Number(matchEquals[1]); const column = Number(matchEquals[2]); @@ -140,6 +148,7 @@ export class DruidError extends Error { // Mangled quotes from copy/paste // ex: SELECT * FROM wikipedia WHERE channel = ‘#en.wikipedia‛ + // er: Lexical error at line 1, column 41. Encountered: "\u2018" const matchLexical = /Lexical error at line (\d+), column (\d+).\s+Encountered: "\\u201\w"/.exec(errorMessage); if (matchLexical) { @@ -157,15 +166,15 @@ export class DruidError extends Error { // Incorrect quoting on table column // ex: SELECT * FROM wikipedia WHERE channel = "#en.wikipedia" - // ex: org.apache.calcite.runtime.CalciteContextException: From line 3, column 17 to line 3, column 31: Column '#ar.wikipedia' not found in any table + // er: Column '#en.wikipedia' not found in any table (line [1], column [41]) const matchQuotes = - /org.apache.calcite.runtime.CalciteContextException: From line (\d+), column (\d+) to line \d+, column \d+: Column '([^']+)' not found in any table/.exec( + /Column '([^']+)' not found in any table \(line \[(\d+)], column \[(\d+)]\)/.exec( errorMessage, ); if (matchQuotes) { - const line = Number(matchQuotes[1]); - const column = Number(matchQuotes[2]); - const literalString = matchQuotes[3]; + const literalString = matchQuotes[1]; + const line = Number(matchQuotes[2]); + const column = Number(matchQuotes[3]); return { label: `Replace "${literalString}" with '${literalString}'`, fn: str => { @@ -180,7 +189,10 @@ export class DruidError extends Error { // Single quotes on AS alias // ex: SELECT channel AS 'c' FROM wikipedia - const matchSingleQuotesAlias = /Encountered "AS \\'([\w-]+)\\'" at/i.exec(errorMessage); + // er: Received an unexpected token [AS \'c\'] (line [1], column [16]), acceptable options: + const matchSingleQuotesAlias = /Received an unexpected token \[AS \\'([\w-]+)\\']/i.exec( + errorMessage, + ); if (matchSingleQuotesAlias) { const alias = matchSingleQuotesAlias[1]; return { @@ -193,9 +205,12 @@ export class DruidError extends Error { }; } - // , before FROM, GROUP, ORDER, or LIMIT + // Comma (,) before FROM, GROUP, ORDER, or LIMIT // ex: SELECT channel, FROM wikipedia - const matchComma = /Encountered ", (FROM|GROUP|ORDER|LIMIT)" at/i.exec(errorMessage); + // er: Received an unexpected token [, FROM] (line [1], column [15]), acceptable options: + const matchComma = /Received an unexpected token \[, (FROM|GROUP|ORDER|LIMIT)]/i.exec( + errorMessage, + ); if (matchComma) { const keyword = matchComma[1]; return { @@ -208,10 +223,11 @@ export class DruidError extends Error { }; } - // ; at the end. https://bit.ly/1n1yfkJ + // Semicolon (;) at the end. https://bit.ly/1n1yfkJ // ex: SELECT 1; - // ex: Encountered ";" at line 6, column 16. - const matchSemicolon = /Encountered ";" at line (\d+), column (\d+)./i.exec(errorMessage); + // ex: Received an unexpected token [;] (line [1], column [9]), acceptable options: + const matchSemicolon = + /Received an unexpected token \[;] \(line \[(\d+)], column \[(\d+)]\),/i.exec(errorMessage); if (matchSemicolon) { const line = Number(matchSemicolon[1]); const column = Number(matchSemicolon[2]); @@ -229,49 +245,50 @@ export class DruidError extends Error { } public canceled?: boolean; - public error?: string; + public persona?: ErrorResponsePersona; + public category?: ErrorResponseCategory; + public context?: Record; public errorMessage?: string; public errorMessageWithoutExpectation?: string; public expectation?: string; public position?: RowColumn; + public suggestion?: QuerySuggestion; + + // Depricated + public error?: string; public errorClass?: string; public host?: string; - public suggestion?: QuerySuggestion; - constructor(e: any, removeLines?: number) { + constructor(e: any, skipLines = 0) { super(axios.isCancel(e) ? CANCELED_MESSAGE : getDruidErrorMessage(e)); if (axios.isCancel(e)) { this.canceled = true; } else { - const data = getDruidErrorObject(e); + const data = errorResponseFromWhatever(e); - let druidErrorResponse: DruidErrorResponse; + let druidErrorResponse: ErrorResponse; switch (typeof data) { case 'object': druidErrorResponse = data; break; - case 'string': + default: druidErrorResponse = { errorClass: 'HTML error', - }; - break; - - default: - druidErrorResponse = {}; + } as any; // ToDo break; } Object.assign(this, druidErrorResponse); if (this.errorMessage) { - if (removeLines) { + if (skipLines) { this.errorMessage = this.errorMessage.replace( - /line (\d+),/g, - (_, c) => `line ${Number(c) - removeLines},`, + /line \[(\d+)],/g, + (_, c) => `line [${Number(c) - skipLines}],`, ); } - this.position = DruidError.parsePosition(this.errorMessage); + this.position = DruidError.extractPosition(this.context); this.suggestion = DruidError.getSuggestion(this.errorMessage); const expectationIndex = this.errorMessage.indexOf('Was expecting one of'); diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index 6183bc4995f6..d1e8391898d6 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -23,6 +23,7 @@ import copy from 'copy-to-clipboard'; import hasOwnProp from 'has-own-prop'; import * as JSONBig from 'json-bigint-native'; import numeral from 'numeral'; +import type { JSX } from 'react'; import React from 'react'; import { AppToaster } from '../singletons'; @@ -306,7 +307,7 @@ export function compact(xs: (T | undefined | false | null | '')[]): T[] { } export function assemble(...xs: (T | undefined | false | null | '')[]): T[] { - return xs.filter(Boolean) as T[]; + return compact(xs); } export function moveToEnd( diff --git a/web-console/src/utils/query-cursor.ts b/web-console/src/utils/query-cursor.ts index f445406c031c..eb2ae8740dfc 100644 --- a/web-console/src/utils/query-cursor.ts +++ b/web-console/src/utils/query-cursor.ts @@ -37,7 +37,6 @@ export function prettyPrintSql(b: SqlBase): string { } export interface RowColumn { - match: string; row: number; column: number; endRow?: number; @@ -55,7 +54,6 @@ export function findEmptyLiteralPosition(query: SqlQuery): RowColumn | undefined const row = lines.length - 1; const lastLine = lines[row]; return { - match: '', row, column: lastLine.length, }; diff --git a/web-console/src/utils/query-manager.tsx b/web-console/src/utils/query-manager.tsx index 2cc9188eba9d..bc180c4b8820 100644 --- a/web-console/src/utils/query-manager.tsx +++ b/web-console/src/utils/query-manager.tsx @@ -217,14 +217,11 @@ export class QueryManager { this.trigger(); } - public rerunLastQuery(runInBackground = false): void { + public rerunLastQuery(onlyRunIfIdle = false): void { if (this.terminated) return; + if (onlyRunIfIdle && this.currentRunCancelFn) return; this.nextQuery = this.lastQuery; - if (runInBackground) { - void this.runWhenIdle(); - } else { - this.trigger(); - } + this.trigger(); } public cancelCurrent(): void { diff --git a/web-console/src/views/datasources-view/datasources-view.tsx b/web-console/src/views/datasources-view/datasources-view.tsx index 9a7950e3648c..fb778dfa2872 100644 --- a/web-console/src/views/datasources-view/datasources-view.tsx +++ b/web-console/src/views/datasources-view/datasources-view.tsx @@ -1031,7 +1031,7 @@ GROUP BY 1, 2`; } } - private renderRetentionDialog(): JSX.Element | undefined { + private renderRetentionDialog() { const { capabilities } = this.props; const { retentionDialogOpenOn, datasourcesAndDefaultRulesState } = this.state; const defaultRules = datasourcesAndDefaultRulesState.data?.defaultRules; @@ -1591,7 +1591,7 @@ GROUP BY 1, 2`; ); } - render(): JSX.Element { + render() { const { capabilities } = this.props; const { showUnused, diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index f046911d6abd..9dbf6e6eb750 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -41,6 +41,7 @@ import { Popover2 } from '@blueprintjs/popover2'; import classNames from 'classnames'; import * as JSONBig from 'json-bigint-native'; import memoize from 'memoize-one'; +import type { JSX } from 'react'; import React from 'react'; import { @@ -688,7 +689,7 @@ export class LoadDataView extends React.PureComponent Date: Fri, 23 Jun 2023 11:29:28 -0700 Subject: [PATCH 04/18] swap order --- web-console/src/views/home-view/segments-card/segments-card.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console/src/views/home-view/segments-card/segments-card.tsx b/web-console/src/views/home-view/segments-card/segments-card.tsx index 26ce7f833139..2f531715fb3e 100644 --- a/web-console/src/views/home-view/segments-card/segments-card.tsx +++ b/web-console/src/views/home-view/segments-card/segments-card.tsx @@ -92,12 +92,12 @@ WHERE is_active = 1`, error={segmentCountState.error} >

{pluralIfNeeded(segmentCount.active, 'active segment')}

-

{pluralIfNeeded(segmentCount.cached_on_historical, 'segment')} cached on historicals

{Boolean(segmentCount.unavailable) && (

{pluralIfNeeded(segmentCount.unavailable, 'segment')} waiting to be cached on historicals

)} +

{pluralIfNeeded(segmentCount.cached_on_historical, 'segment')} cached on historicals

{Boolean(segmentCount.realtime) && (

{pluralIfNeeded(segmentCount.realtime, 'realtime segment')}

)} From 52c6b9b0c705aa9fa32d5e121c6a86d786b9c5ab Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Sun, 25 Jun 2023 21:24:59 -0700 Subject: [PATCH 05/18] small cleanup --- .../src/views/segments-view/segments-view.tsx | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/web-console/src/views/segments-view/segments-view.tsx b/web-console/src/views/segments-view/segments-view.tsx index 80bb5dc4f8e2..86f74500fed7 100644 --- a/web-console/src/views/segments-view/segments-view.tsx +++ b/web-console/src/views/segments-view/segments-view.tsx @@ -16,7 +16,7 @@ * limitations under the License. */ -import { Button, ButtonGroup, Intent, Label, MenuItem, Switch } from '@blueprintjs/core'; +import { Button, ButtonGroup, Code, Intent, Label, MenuItem, Switch } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import classNames from 'classnames'; import { C, L, SqlComparison, SqlExpression } from 'druid-query-toolkit'; @@ -312,6 +312,17 @@ END AS "time_span"`, whereClause = SqlExpression.and(...whereParts).toString(); } + let effectiveSorted = sorted; + if (!effectiveSorted.find(sort => sort.id === 'version') && effectiveSorted.length) { + // Ensure there is a sort on version as a tiebreaker + effectiveSorted = effectiveSorted.concat([ + { + id: 'version', + desc: effectiveSorted[0].desc, // Take the first direction if it exists + }, + ]); + } + if (groupByInterval) { const innerQuery = compact([ `SELECT "start" || '/' || "end" AS "interval"`, @@ -336,11 +347,11 @@ END AS "time_span"`, whereClause ? ` AND ${whereClause}` : '', ]); - if (sorted.length) { + if (effectiveSorted.length) { queryParts.push( 'ORDER BY ' + - sorted - .map((sort: any) => `${C(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`) + effectiveSorted + .map(sort => `${C(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`) .join(', '), ); } @@ -353,11 +364,11 @@ END AS "time_span"`, queryParts.push(`WHERE ${whereClause}`); } - if (sorted.length) { + if (effectiveSorted.length) { queryParts.push( 'ORDER BY ' + - sorted - .map((sort: any) => `${C(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`) + effectiveSorted + .map(sort => `${C(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`) .join(', '), ); } @@ -534,7 +545,11 @@ END AS "time_span"`, data={segments} pages={10000000} // Dummy, we are hiding the page selector loading={segmentsState.loading} - noDataText={segmentsState.isEmpty() ? 'No segments' : segmentsState.getErrorMessage() || ''} + noDataText={ + segmentsState.isEmpty() + ? `No segments${filters.length ? ' matching filter' : ''}` + : segmentsState.getErrorMessage() || '' + } manual filterable filtered={filters} @@ -919,7 +934,9 @@ END AS "time_span"`, this.segmentsQueryManager.rerunLastQuery(); }} > -

{`Are you sure you want to drop segment '${terminateSegmentId}'?`}

+

+ Are you sure you want to drop segment {terminateSegmentId}? +

This action is not reversible.

); From cf23b5e75e1d6205602b39999df749119358eccd Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Tue, 27 Jun 2023 23:55:02 -0700 Subject: [PATCH 06/18] make better --- .../supervisors-view/supervisors-view.tsx | 26 +++++++++--- .../execution-stages-pane.tsx | 41 ++++++++++++++++--- 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/web-console/src/views/supervisors-view/supervisors-view.tsx b/web-console/src/views/supervisors-view/supervisors-view.tsx index 8d5a6c5ef861..c4bef912c3ad 100644 --- a/web-console/src/views/supervisors-view/supervisors-view.tsx +++ b/web-console/src/views/supervisors-view/supervisors-view.tsx @@ -16,7 +16,7 @@ * limitations under the License. */ -import { Intent, MenuItem } from '@blueprintjs/core'; +import { Code, Intent, MenuItem } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import React from 'react'; import type { Filter } from 'react-table'; @@ -127,14 +127,22 @@ function detailedStateToColor(detailedState: string): string { switch (detailedState) { case 'UNHEALTHY_SUPERVISOR': case 'UNHEALTHY_TASKS': + case 'UNABLE_TO_CONNECT_TO_STREAM': + case 'LOST_CONTACT_WITH_STREAM': return '#d5100a'; case 'PENDING': return '#00eaff'; + case 'DISCOVERING_INITIAL_TASKS': + case 'CREATING_TASKS': + case 'CONNECTING_TO_STREAM': case 'RUNNING': return '#2167d5'; + case 'IDLE': + return '#44659d'; + case 'STOPPING': return '#e75c06'; @@ -371,7 +379,9 @@ GROUP BY 1, 2`; this.supervisorQueryManager.rerunLastQuery(); }} > -

{`Are you sure you want to resume supervisor '${resumeSupervisorId}'?`}

+

+ Are you sure you want to resume supervisor {resumeSupervisorId}? +

); } @@ -400,7 +410,9 @@ GROUP BY 1, 2`; this.supervisorQueryManager.rerunLastQuery(); }} > -

{`Are you sure you want to suspend supervisor '${suspendSupervisorId}'?`}

+

+ Are you sure you want to suspend supervisor {suspendSupervisorId}? +

); } @@ -433,7 +445,9 @@ GROUP BY 1, 2`; 'I understand that this operation cannot be undone.', ]} > -

{`Are you sure you want to hard reset supervisor '${resetSupervisorId}'?`}

+

+ Are you sure you want to hard reset supervisor {resetSupervisorId}? +

Hard resetting a supervisor will lead to data loss or data duplication.

The reason for using this operation is to recover from a state in which the supervisor @@ -467,7 +481,9 @@ GROUP BY 1, 2`; this.supervisorQueryManager.rerunLastQuery(); }} > -

{`Are you sure you want to terminate supervisor '${terminateSupervisorId}'?`}

+

+ Are you sure you want to terminate supervisor {terminateSupervisorId}? +

This action is not reversible.

); diff --git a/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx b/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx index a59557fd4c4a..98035b3ed545 100644 --- a/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx +++ b/web-console/src/views/workbench-view/execution-stages-pane/execution-stages-pane.tsx @@ -104,6 +104,31 @@ function inputLabelContent(stage: StageDefinition, inputIndex: number) { ); } +function formatInputLabel(stage: StageDefinition, inputIndex: number) { + const { input, broadcast } = stage.definition; + const stageInput = input[inputIndex]; + let ret = 'Input '; + switch (stageInput.type) { + case 'stage': + ret += `Stage${stageInput.stage}`; + break; + + case 'table': + ret += stageInput.dataSource; + break; + + case 'external': + ret += `${stageInput.inputSource.type} external`; + break; + } + + if (broadcast?.includes(inputIndex)) { + ret += ` (broadcast)`; + } + + return ret; +} + export interface ExecutionStagesPaneProps { execution: Execution; onErrorClick?(): void; @@ -354,24 +379,30 @@ export const ExecutionStagesPane = React.memo(function ExecutionStagesPane( function dataProcessedInput(stage: StageDefinition, inputNumber: number) { const inputCounter: CounterName = `input${inputNumber}`; - if (!stages.hasCounterForStage(stage, inputCounter)) return; - const inputFileCount = stages.getTotalCounterForStage(stage, inputCounter, 'totalFiles'); - + const hasCounter = stages.hasCounterForStage(stage, inputCounter); const bytes = stages.getTotalCounterForStage(stage, inputCounter, 'bytes'); + const inputFileCount = stages.getTotalCounterForStage(stage, inputCounter, 'totalFiles'); return (
{inputFileCount ? ( From 0004a708979c043b69326b18381b2f566af019eb Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 5 Jul 2023 13:40:52 -0700 Subject: [PATCH 07/18] make updates --- .../__snapshots__/auto-form.spec.tsx.snap | 4 +- .../src/components/auto-form/auto-form.tsx | 58 ++++- .../fancy-numeric-input.spec.tsx.snap | 32 +++ .../fancy-numeric-input.spec.tsx} | 8 +- .../fancy-numeric-input.tsx | 223 ++++++++++++++++++ .../__snapshots__/header-bar.spec.tsx.snap | 11 + .../src/components/header-bar/header-bar.tsx | 13 + .../numeric-input-with-default.spec.tsx.snap | 20 -- .../numeric-input-with-default.tsx | 50 ---- .../compaction-dynamic-config-dialog.tsx | 150 ++++++++++++ web-console/src/dialogs/index.ts | 1 + .../dialogs/snitch-dialog/snitch-dialog.tsx | 4 +- .../workbench-query/workbench-query.spec.ts | 1 - web-console/src/utils/druid-query.spec.ts | 38 +-- web-console/src/utils/general.tsx | 2 +- .../__snapshots__/segments-view.spec.tsx.snap | 30 ++- .../max-tasks-button.spec.tsx.snap | 24 +- .../max-tasks-button/max-tasks-button.tsx | 9 +- 18 files changed, 563 insertions(+), 115 deletions(-) create mode 100644 web-console/src/components/fancy-numeric-input/__snapshots__/fancy-numeric-input.spec.tsx.snap rename web-console/src/components/{numeric-input-with-default/numeric-input-with-default.spec.tsx => fancy-numeric-input/fancy-numeric-input.spec.tsx} (81%) create mode 100644 web-console/src/components/fancy-numeric-input/fancy-numeric-input.tsx delete mode 100644 web-console/src/components/numeric-input-with-default/__snapshots__/numeric-input-with-default.spec.tsx.snap delete mode 100644 web-console/src/components/numeric-input-with-default/numeric-input-with-default.tsx create mode 100644 web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-dialog.tsx diff --git a/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap b/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap index 64c6e41420cd..30a25ee37e27 100644 --- a/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap +++ b/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap @@ -7,13 +7,15 @@ exports[`AutoForm matches snapshot 1`] = ` - { info?: React.ReactNode; type: | 'number' + | 'ratio' | 'size-bytes' | 'string' | 'duration' @@ -250,15 +251,14 @@ export class AutoForm> extends React.PureComponent const { required, defaultValue, modelValue } = AutoForm.computeFieldValues(model, field); return ( - { - let newValue: number | undefined; - if (valueAsString !== '' && !isNaN(valueAsNumber)) { - newValue = valueAsNumber === 0 && field.zeroMeansUndefined ? undefined : valueAsNumber; - } - this.fieldChange(field, newValue); + { + this.fieldChange( + field, + valueAsNumber === 0 && field.zeroMeansUndefined ? undefined : valueAsNumber, + ); }} onBlur={e => { if (e.target.value === '') { @@ -266,7 +266,7 @@ export class AutoForm> extends React.PureComponent } if (onFinalize) onFinalize(); }} - min={field.min || 0} + min={field.min ?? 0} max={field.max} fill large={large} @@ -277,6 +277,40 @@ export class AutoForm> extends React.PureComponent ); } + private renderRatioInput(field: Field): JSX.Element { + const { model, large, onFinalize } = this.props; + const { required, defaultValue, modelValue } = AutoForm.computeFieldValues(model, field); + + return ( + { + this.fieldChange( + field, + valueAsNumber === 0 && field.zeroMeansUndefined ? undefined : valueAsNumber, + ); + }} + onBlur={e => { + if (e.target.value === '') { + this.fieldChange(field, undefined); + } + if (onFinalize) onFinalize(); + }} + min={field.min ?? 0} + max={field.max ?? 1} + minorStepSize={0.001} + stepSize={0.01} + majorStepSize={0.05} + fill + large={large} + disabled={AutoForm.evaluateFunctor(field.disabled, model, false)} + placeholder={AutoForm.evaluateFunctor(field.placeholder, model, '')} + intent={required && modelValue == null ? AutoForm.REQUIRED_INTENT : undefined} + /> + ); + } + private renderSizeBytesInput(field: Field): JSX.Element { const { model, large, onFinalize } = this.props; const { required, defaultValue, modelValue } = AutoForm.computeFieldValues(model, field); @@ -446,6 +480,8 @@ export class AutoForm> extends React.PureComponent switch (field.type) { case 'number': return this.renderNumberInput(field); + case 'ratio': + return this.renderRatioInput(field); case 'size-bytes': return this.renderSizeBytesInput(field); case 'string': diff --git a/web-console/src/components/fancy-numeric-input/__snapshots__/fancy-numeric-input.spec.tsx.snap b/web-console/src/components/fancy-numeric-input/__snapshots__/fancy-numeric-input.spec.tsx.snap new file mode 100644 index 000000000000..c82556dbc6c4 --- /dev/null +++ b/web-console/src/components/fancy-numeric-input/__snapshots__/fancy-numeric-input.spec.tsx.snap @@ -0,0 +1,32 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`FancyNumericInput matches snapshot 1`] = ` + + + + + + + +`; diff --git a/web-console/src/components/numeric-input-with-default/numeric-input-with-default.spec.tsx b/web-console/src/components/fancy-numeric-input/fancy-numeric-input.spec.tsx similarity index 81% rename from web-console/src/components/numeric-input-with-default/numeric-input-with-default.spec.tsx rename to web-console/src/components/fancy-numeric-input/fancy-numeric-input.spec.tsx index dff5bd066502..2ede11e381f1 100644 --- a/web-console/src/components/numeric-input-with-default/numeric-input-with-default.spec.tsx +++ b/web-console/src/components/fancy-numeric-input/fancy-numeric-input.spec.tsx @@ -20,11 +20,13 @@ import React from 'react'; import { shallow } from '../../utils/shallow-renderer'; -import { NumericInputWithDefault } from './numeric-input-with-default'; +import { FancyNumericInput } from './fancy-numeric-input'; -describe('NumericInputWithDefault', () => { +describe('FancyNumericInput', () => { it('matches snapshot', () => { - const numericInputWithDefault = shallow(); + const numericInputWithDefault = shallow( + {}} />, + ); expect(numericInputWithDefault).toMatchSnapshot(); }); diff --git a/web-console/src/components/fancy-numeric-input/fancy-numeric-input.tsx b/web-console/src/components/fancy-numeric-input/fancy-numeric-input.tsx new file mode 100644 index 000000000000..8b70b4085c22 --- /dev/null +++ b/web-console/src/components/fancy-numeric-input/fancy-numeric-input.tsx @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { InputGroupProps2, Intent } from '@blueprintjs/core'; +import { Button, ButtonGroup, Classes, ControlGroup, InputGroup, Keys } from '@blueprintjs/core'; +import { IconNames } from '@blueprintjs/icons'; +import classNames from 'classnames'; +import { SqlExpression, SqlFunction, SqlLiteral, SqlMulti } from 'druid-query-toolkit'; +import React, { useEffect, useState } from 'react'; + +import { clamp } from '../../utils'; + +const MULTI_OP_TO_REDUCER: Record number> = { + '+': (a, b) => a + b, + '-': (a, b) => a - b, + '*': (a, b) => a * b, + '/': (a, b) => (b ? a / b : 0), +}; + +function evaluateSqlSimple(sql: SqlExpression): number | undefined { + if (sql instanceof SqlLiteral) { + return sql.getNumberValue(); + } else if (sql instanceof SqlMulti) { + const evaluatedArgs = sql.getArgArray().map(evaluateSqlSimple); + if (evaluatedArgs.some(x => typeof x === 'undefined')) return; + const reducer = MULTI_OP_TO_REDUCER[sql.op]; + if (!reducer) return; + return (evaluatedArgs as number[]).reduce(reducer); + } else if (sql instanceof SqlFunction && sql.getEffectiveFunctionName() === 'PI') { + return Math.PI; + } else { + return; + } +} + +function numberToShown(n: number): string { + return String(n); +} + +function shownToNumber(s: string): number | undefined { + const parsed = SqlExpression.maybeParse(s); + if (!parsed) return; + return evaluateSqlSimple(parsed); +} + +export interface FancyNumericInputProps { + className?: string; + intent?: Intent; + fill?: boolean; + large?: boolean; + small?: boolean; + disabled?: boolean; + readOnly?: boolean; + placeholder?: string; + onBlur?: InputGroupProps2['onBlur']; + + value: number; + defaultValue: number; + onValueChange(value: number): void; + + min?: number; + max?: number; + minorStepSize?: number; + stepSize?: number; + majorStepSize?: number; +} + +export const FancyNumericInput = React.memo(function FancyNumericInput( + props: FancyNumericInputProps, +) { + const { + className, + intent, + fill, + large, + small, + disabled, + readOnly, + placeholder, + onBlur, + + value, + defaultValue, + onValueChange, + + min, + max, + } = props; + + const stepSize = props.stepSize || 1; + const minorStepSize = props.minorStepSize || stepSize; + const majorStepSize = props.majorStepSize || stepSize * 10; + + function roundAndClamp(n: number): number { + const inv = 1 / minorStepSize; + return clamp(Math.floor(n * inv) / inv, min, max); + } + + const effectiveValue = value ?? defaultValue; + const [shownValue, setShownValue] = useState(numberToShown(effectiveValue)); + const shownNumberRaw = shownToNumber(shownValue); + const shownNumberClamped = shownNumberRaw ? roundAndClamp(shownNumberRaw) : undefined; + + useEffect(() => { + if (effectiveValue !== shownNumberClamped) { + setShownValue(numberToShown(effectiveValue)); + } + }, [effectiveValue]); + + const containerClasses = classNames( + 'fancy-numeric-input', + Classes.NUMERIC_INPUT, + { [Classes.LARGE]: large, [Classes.SMALL]: small }, + className, + ); + + const effectiveDisabled = disabled || readOnly; + const isIncrementDisabled = max !== undefined && +value >= max; + const isDecrementDisabled = min !== undefined && +value <= min; + + function changeValue(newValue: number): void { + onValueChange(roundAndClamp(newValue)); + } + + function increment(delta: number): void { + if (typeof shownNumberRaw !== 'number') return; + changeValue(shownNumberRaw + delta); + } + + function getIncrementSize(isShiftKeyPressed: boolean, isAltKeyPressed: boolean): number { + if (isShiftKeyPressed) { + return majorStepSize; + } + if (isAltKeyPressed) { + return minorStepSize; + } + return stepSize; + } + + return ( + + { + const valueAsString = (e.target as HTMLInputElement).value; + setShownValue(valueAsString); + + const shownNumber = shownToNumber(valueAsString); + if (typeof shownNumber === 'number') { + changeValue(shownNumber); + } + }} + onBlur={e => { + setShownValue(numberToShown(effectiveValue)); + onBlur?.(e); + }} + onKeyDown={e => { + const { keyCode } = e; + + if (keyCode === Keys.ENTER && typeof shownNumberClamped === 'number') { + setShownValue(numberToShown(shownNumberClamped)); + return; + } + + let direction = 0; + if (keyCode === Keys.ARROW_UP) { + direction = 1; + } else if (keyCode === Keys.ARROW_DOWN) { + direction = -1; + } + + if (direction) { + // when the input field has focus, some key combinations will modify + // the field's selection range. we'll actually want to select all + // text in the field after we modify the value on the following + // lines. preventing the default selection behavior lets us do that + // without interference. + e.preventDefault(); + + increment(direction * getIncrementSize(e.shiftKey, e.altKey)); + } + }} + /> + +
+
+ + ) : ( + + )} + + ); +}); diff --git a/web-console/src/dialogs/index.ts b/web-console/src/dialogs/index.ts index 37bf8d375229..468ccf8d6227 100644 --- a/web-console/src/dialogs/index.ts +++ b/web-console/src/dialogs/index.ts @@ -20,6 +20,7 @@ export * from './about-dialog/about-dialog'; export * from './alert-dialog/alert-dialog'; export * from './async-action-dialog/async-action-dialog'; export * from './compaction-config-dialog/compaction-config-dialog'; +export * from './compaction-dynamic-config-dialog/compaction-dynamic-config-dialog'; export * from './coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog'; export * from './diff-dialog/diff-dialog'; export * from './doctor-dialog/doctor-dialog'; diff --git a/web-console/src/dialogs/snitch-dialog/snitch-dialog.tsx b/web-console/src/dialogs/snitch-dialog/snitch-dialog.tsx index 924393136264..315841084907 100644 --- a/web-console/src/dialogs/snitch-dialog/snitch-dialog.tsx +++ b/web-console/src/dialogs/snitch-dialog/snitch-dialog.tsx @@ -155,7 +155,7 @@ export class SnitchDialog extends React.PureComponent ) : ( @@ -163,7 +163,7 @@ export class SnitchDialog extends React.PureComponent )} diff --git a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts index cef773abb0f8..b5dfff8c6b44 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts @@ -138,7 +138,6 @@ describe('WorkbenchQuery', () => { `End of input while parsing an object (missing '}') at line 40,2 >>>} ...`, ), ).toEqual({ - match: '', row: 39, column: 1, }); diff --git a/web-console/src/utils/druid-query.spec.ts b/web-console/src/utils/druid-query.spec.ts index 150775dd8318..8b460f0492ca 100644 --- a/web-console/src/utils/druid-query.spec.ts +++ b/web-console/src/utils/druid-query.spec.ts @@ -59,7 +59,9 @@ describe('DruidQuery', () => { FROM wikipedia -- test == WHERE channel == '#ar.wikipedia' `; - const suggestion = DruidError.getSuggestion(`Encountered "= =" at line 3, column 15.`); + const suggestion = DruidError.getSuggestion( + `Received an unexpected token [= =] (line [3], column [15]), acceptable options:`, + ); expect(suggestion!.label).toEqual(`Replace == with =`); expect(suggestion!.fn(sql)).toEqual(sane` SELECT * @@ -78,7 +80,7 @@ describe('DruidQuery', () => { ORDER BY 2 DESC `; const suggestion = DruidError.getSuggestion( - `Encountered "= =" at line 4, column 15. Was expecting one of: "EXCEPT" ... "FETCH" ... "GROUP" ...`, + `Received an unexpected token [= =] (line [4], column [15]), acceptable options:`, ); expect(suggestion!.label).toEqual(`Replace == with =`); expect(suggestion!.fn(sql)).toEqual(sane` @@ -137,7 +139,7 @@ describe('DruidQuery', () => { WHERE channel = "#ar.wikipedia" `; const suggestion = DruidError.getSuggestion( - `org.apache.calcite.runtime.CalciteContextException: From line 3, column 17 to line 3, column 31: Column '#ar.wikipedia' not found in any table`, + `Column '#ar.wikipedia' not found in any table (line [3], column [17])`, ); expect(suggestion!.label).toEqual(`Replace "#ar.wikipedia" with '#ar.wikipedia'`); expect(suggestion!.fn(sql)).toEqual(sane` @@ -148,41 +150,43 @@ describe('DruidQuery', () => { }); it('works for incorrectly quoted AS alias', () => { - const suggestion = DruidError.getSuggestion(`Encountered "AS \\'c\\'" at line 1, column 16.`); - expect(suggestion!.label).toEqual(`Replace 'c' with "c"`); - expect(suggestion!.fn(`SELECT channel AS 'c' FROM wikipedia`)).toEqual( - `SELECT channel AS "c" FROM wikipedia`, + const sql = `SELECT channel AS 'c' FROM wikipedia`; + const suggestion = DruidError.getSuggestion( + `Received an unexpected token [AS \\'c\\'] (line [1], column [16]), acceptable options:`, ); + expect(suggestion!.label).toEqual(`Replace 'c' with "c"`); + expect(suggestion!.fn(sql)).toEqual(`SELECT channel AS "c" FROM wikipedia`); }); it('removes comma (,) before FROM', () => { + const sql = `SELECT page, FROM wikipedia WHERE channel = '#ar.wikipedia'`; const suggestion = DruidError.getSuggestion( - `Encountered ", FROM" at line 1, column 12. Was expecting one of: "ABS" ...`, + `Received an unexpected token [, FROM] (line [1], column [12]), acceptable options:`, ); expect(suggestion!.label).toEqual(`Remove , before FROM`); - expect(suggestion!.fn(`SELECT page, FROM wikipedia WHERE channel = '#ar.wikipedia'`)).toEqual( + expect(suggestion!.fn(sql)).toEqual( `SELECT page FROM wikipedia WHERE channel = '#ar.wikipedia'`, ); }); it('removes comma (,) before ORDER', () => { + const sql = `SELECT page FROM wikipedia WHERE channel = '#ar.wikipedia' GROUP BY 1, ORDER BY 1`; const suggestion = DruidError.getSuggestion( - `Encountered ", ORDER" at line 1, column 14. Was expecting one of: "ABS" ...`, + `Received an unexpected token [, ORDER] (line [1], column [70]), acceptable options:`, ); expect(suggestion!.label).toEqual(`Remove , before ORDER`); - expect( - suggestion!.fn( - `SELECT page FROM wikipedia WHERE channel = '#ar.wikipedia' GROUP BY 1, ORDER BY 1`, - ), - ).toEqual(`SELECT page FROM wikipedia WHERE channel = '#ar.wikipedia' GROUP BY 1 ORDER BY 1`); + expect(suggestion!.fn(sql)).toEqual( + `SELECT page FROM wikipedia WHERE channel = '#ar.wikipedia' GROUP BY 1 ORDER BY 1`, + ); }); it('removes trailing semicolon (;)', () => { + const sql = `SELECT page FROM wikipedia WHERE channel = '#ar.wikipedia';`; const suggestion = DruidError.getSuggestion( - `Encountered ";" at line 1, column 59. Was expecting one of: "ABS" ...`, + `Received an unexpected token [;] (line [1], column [59]), acceptable options:`, ); expect(suggestion!.label).toEqual(`Remove trailing ;`); - expect(suggestion!.fn(`SELECT page FROM wikipedia WHERE channel = '#ar.wikipedia';`)).toEqual( + expect(suggestion!.fn(sql)).toEqual( `SELECT page FROM wikipedia WHERE channel = '#ar.wikipedia'`, ); }); diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index d1e8391898d6..acbca3cd4a87 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -62,7 +62,7 @@ export function wait(ms: number): Promise { }); } -export function clamp(n: number, min: number, max: number): number { +export function clamp(n: number, min = -Infinity, max = Infinity): number { return Math.min(Math.max(n, min), max); } diff --git a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap index 84718c5f09fe..aa62a576f54a 100755 --- a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap +++ b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap @@ -64,6 +64,7 @@ exports[`SegmentsView matches snapshot 1`] = ` "Num rows", "Avg. row size", "Replicas", + "Replication factor", "Is available", "Is active", "Is realtime", @@ -261,7 +262,9 @@ exports[`SegmentsView matches snapshot 1`] = ` "Header": Avg. row size
- (bytes) + + (bytes) + , "accessor": "avg_row_size", "className": "padded", @@ -270,13 +273,34 @@ exports[`SegmentsView matches snapshot 1`] = ` "width": 100, }, Object { - "Header": "Replicas", + "Header": + Replicas +
+ + (actual) + + , "accessor": "num_replicas", "className": "padded", "defaultSortDesc": true, "filterable": false, "show": true, - "width": 60, + "width": 80, + }, + Object { + "Header": + Replication factor +
+ + (desired) + + , + "accessor": "replication_factor", + "className": "padded", + "defaultSortDesc": true, + "filterable": false, + "show": true, + "width": 80, }, Object { "Filter": [Function], diff --git a/web-console/src/views/workbench-view/max-tasks-button/__snapshots__/max-tasks-button.spec.tsx.snap b/web-console/src/views/workbench-view/max-tasks-button/__snapshots__/max-tasks-button.spec.tsx.snap index d0275c90c8e9..9d621a02f7ee 100644 --- a/web-console/src/views/workbench-view/max-tasks-button/__snapshots__/max-tasks-button.spec.tsx.snap +++ b/web-console/src/views/workbench-view/max-tasks-button/__snapshots__/max-tasks-button.spec.tsx.snap @@ -97,23 +97,39 @@ exports[`MaxTasksButton matches snapshot 1`] = ` active={false} disabled={false} icon="tick" - multiline={false} + multiline={true} onClick={[Function]} popoverProps={Object {}} selected={false} shouldDismissPopover={false} - text="max - Use as many tasks as possible, up to the maximum." + text={ + + + max + + : + Use as many tasks as possible, up to the maximum. + + } /> + + auto + + : + Use as few tasks as possible without exceeding 512 MiB or 10,000 files per task, unless exceeding these limits is necessary to stay within 'maxNumTasks'. When calculating the size of files, the weighted size is used, which considers the file format and compression format used if any. When file sizes cannot be determined through directory listing (for example: http), behaves the same as 'max'. + + } /> diff --git a/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx b/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx index 0500dca3bae1..bf6bb165c85b 100644 --- a/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx +++ b/web-console/src/views/workbench-view/max-tasks-button/max-tasks-button.tsx @@ -37,7 +37,7 @@ const TASK_ASSIGNMENT_OPTIONS = ['max', 'auto']; const TASK_ASSIGNMENT_DESCRIPTION: Record = { max: 'Use as many tasks as possible, up to the maximum.', - auto: 'Use as few tasks as possible without exceeding 10 GiB or 10,000 files per task.', + auto: `Use as few tasks as possible without exceeding 512 MiB or 10,000 files per task, unless exceeding these limits is necessary to stay within 'maxNumTasks'. When calculating the size of files, the weighted size is used, which considers the file format and compression format used if any. When file sizes cannot be determined through directory listing (for example: http), behaves the same as 'max'.`, }; export interface MaxTasksButtonProps extends Omit { @@ -94,8 +94,13 @@ export const MaxTasksButton = function MaxTasksButton(props: MaxTasksButtonProps + {t}: {TASK_ASSIGNMENT_DESCRIPTION[t]} + + } shouldDismissPopover={false} + multiline onClick={() => changeQueryContext(changeTaskAssigment(queryContext, t))} /> ))} From 04e9b9f36528549f2562cce6470b26907c8e8744 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 5 Jul 2023 13:44:46 -0700 Subject: [PATCH 08/18] explicit --- web-console/src/utils/druid-query.spec.ts | 6 +++--- web-console/src/utils/druid-query.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/web-console/src/utils/druid-query.spec.ts b/web-console/src/utils/druid-query.spec.ts index 8b460f0492ca..2efccc1cd221 100644 --- a/web-console/src/utils/druid-query.spec.ts +++ b/web-console/src/utils/druid-query.spec.ts @@ -163,7 +163,7 @@ describe('DruidQuery', () => { const suggestion = DruidError.getSuggestion( `Received an unexpected token [, FROM] (line [1], column [12]), acceptable options:`, ); - expect(suggestion!.label).toEqual(`Remove , before FROM`); + expect(suggestion!.label).toEqual(`Remove comma (,) before FROM`); expect(suggestion!.fn(sql)).toEqual( `SELECT page FROM wikipedia WHERE channel = '#ar.wikipedia'`, ); @@ -174,7 +174,7 @@ describe('DruidQuery', () => { const suggestion = DruidError.getSuggestion( `Received an unexpected token [, ORDER] (line [1], column [70]), acceptable options:`, ); - expect(suggestion!.label).toEqual(`Remove , before ORDER`); + expect(suggestion!.label).toEqual(`Remove comma (,) before ORDER`); expect(suggestion!.fn(sql)).toEqual( `SELECT page FROM wikipedia WHERE channel = '#ar.wikipedia' GROUP BY 1 ORDER BY 1`, ); @@ -185,7 +185,7 @@ describe('DruidQuery', () => { const suggestion = DruidError.getSuggestion( `Received an unexpected token [;] (line [1], column [59]), acceptable options:`, ); - expect(suggestion!.label).toEqual(`Remove trailing ;`); + expect(suggestion!.label).toEqual(`Remove trailing semicolon (;)`); expect(suggestion!.fn(sql)).toEqual( `SELECT page FROM wikipedia WHERE channel = '#ar.wikipedia'`, ); diff --git a/web-console/src/utils/druid-query.ts b/web-console/src/utils/druid-query.ts index 482538e70abe..efdfd1bd6e77 100644 --- a/web-console/src/utils/druid-query.ts +++ b/web-console/src/utils/druid-query.ts @@ -214,7 +214,7 @@ export class DruidError extends Error { if (matchComma) { const keyword = matchComma[1]; return { - label: `Remove , before ${keyword}`, + label: `Remove comma (,) before ${keyword}`, fn: str => { const newQuery = str.replace(new RegExp(`,(\\s+${keyword})`, 'gim'), '$1'); if (newQuery === str) return; @@ -232,7 +232,7 @@ export class DruidError extends Error { const line = Number(matchSemicolon[1]); const column = Number(matchSemicolon[2]); return { - label: `Remove trailing ;`, + label: `Remove trailing semicolon (;)`, fn: str => { const index = DruidError.positionToIndex(str, line, column); if (str[index] !== ';') return; From 0c41232f5b52a864cc3170c547f87df14d6ef215 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Mon, 19 Jun 2023 11:05:45 -0700 Subject: [PATCH 09/18] New async API --- .../druid-models/async-query/async-query.ts | 41 ++ .../execution-ingest-complete.mock.ts | 584 ++++++++++------- .../execution/execution-ingest-error.mock.ts | 616 +++++++++++------- .../druid-models/execution/execution.spec.ts | 13 +- .../src/druid-models/execution/execution.ts | 145 +++-- web-console/src/druid-models/index.ts | 2 + web-console/src/druid-models/stages/stages.ts | 7 +- web-console/src/druid-models/task/task.ts | 102 +++ .../workbench-query/workbench-query.ts | 3 +- .../helpers/execution/sql-task-execution.ts | 117 ++-- .../execution-submit-dialog.tsx | 6 +- .../ingest-success-pane.tsx | 16 +- .../recent-query-task-panel.tsx | 20 +- 13 files changed, 1048 insertions(+), 624 deletions(-) create mode 100644 web-console/src/druid-models/async-query/async-query.ts create mode 100644 web-console/src/druid-models/task/task.ts diff --git a/web-console/src/druid-models/async-query/async-query.ts b/web-console/src/druid-models/async-query/async-query.ts new file mode 100644 index 000000000000..81469643fc95 --- /dev/null +++ b/web-console/src/druid-models/async-query/async-query.ts @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type AsyncState = 'ACCEPTED' | 'RUNNING' | 'FINISHED' | 'FAILED'; + +export interface AsyncStatusResponse { + queryId: string; + state: AsyncState; + createdAt: string; + durationInMs: number; + schema: { name: string; type: string; nativeType: string }[]; + resultSetInformation?: { + dataSource: string; + sampleRecords: any[][]; + numRows: number; + sizeInBytes: number; + }; + queryException?: AsyncError; +} + +export interface AsyncError { + error: string; + errorMessage: string; + errorClass: string; + host?: string; +} diff --git a/web-console/src/druid-models/execution/execution-ingest-complete.mock.ts b/web-console/src/druid-models/execution/execution-ingest-complete.mock.ts index aff948e8ee27..797fdd5f25c8 100644 --- a/web-console/src/druid-models/execution/execution-ingest-complete.mock.ts +++ b/web-console/src/druid-models/execution/execution-ingest-complete.mock.ts @@ -40,115 +40,53 @@ PARTITIONED BY ALL TIME } */ -export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport( - { - task: 'query-b55f3432-7810-4529-80ed-780a926a6f03', +export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskReport({ + multiStageQuery: { + type: 'multiStageQuery', + taskId: 'query-5aa683e2-a6ee-4655-a834-a643e91055b1', payload: { - type: 'query_controller', - id: 'query-b55f3432-7810-4529-80ed-780a926a6f03', - spec: { - query: { - queryType: 'scan', - dataSource: { - type: 'external', - inputSource: { - type: 'http', - uris: ['https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz'], - }, - inputFormat: { - type: 'json', - keepNullColumns: false, - assumeNewlineDelimited: false, - useJsonNodeReader: false, - }, - signature: [ - { name: 'timestamp', type: 'STRING' }, - { name: 'agent_type', type: 'STRING' }, - ], - }, - intervals: { - type: 'intervals', - intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], - }, - virtualColumns: [ - { - type: 'expression', - name: 'v0', - expression: 'timestamp_parse("timestamp",null,\'UTC\')', - outputType: 'LONG', - }, - ], - resultFormat: 'compactedList', - columns: ['agent_type', 'v0'], - legacy: false, - context: { - finalize: false, - finalizeAggregations: false, - groupByEnableMultiValueUnnesting: false, - maxNumTasks: 2, - queryId: 'b55f3432-7810-4529-80ed-780a926a6f03', - scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', - sqlInsertSegmentGranularity: '{"type":"all"}', - sqlQueryId: 'b55f3432-7810-4529-80ed-780a926a6f03', - sqlReplaceTimeChunks: 'all', - }, - granularity: { type: 'all' }, - }, - columnMappings: [ - { queryColumn: 'v0', outputColumn: '__time' }, - { queryColumn: 'agent_type', outputColumn: 'agent_type' }, - ], - destination: { - type: 'dataSource', - dataSource: 'kttm_simple', - segmentGranularity: { type: 'all' }, - replaceTimeChunks: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], - }, - assignmentStrategy: 'max', - tuningConfig: { maxNumWorkers: 1, maxRowsInMemory: 100000, rowsPerSegment: 3000000 }, + status: { + status: 'SUCCESS', + startTime: '2023-06-19T05:39:26.377Z', + durationMs: 23170, + pendingTasks: 0, + runningTasks: 2, }, - sqlQuery: - 'REPLACE INTO "kttm_simple" OVERWRITE ALL\nSELECT\n TIME_PARSE("timestamp") AS "__time",\n "agent_type"\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}\',\n \'{"type":"json"}\'\n )\n) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)\nPARTITIONED BY ALL TIME', - sqlQueryContext: { - finalizeAggregations: false, - maxParseExceptions: 0, - sqlQueryId: 'b55f3432-7810-4529-80ed-780a926a6f03', - groupByEnableMultiValueUnnesting: false, - sqlInsertSegmentGranularity: '{"type":"all"}', - maxNumTasks: 2, - sqlReplaceTimeChunks: 'all', - queryId: 'b55f3432-7810-4529-80ed-780a926a6f03', - }, - sqlTypeNames: ['TIMESTAMP', 'VARCHAR'], - context: { forceTimeChunkLock: true, useLineageBasedSegmentAllocation: true }, - groupId: 'query-b55f3432-7810-4529-80ed-780a926a6f03', - dataSource: 'kttm_simple', - resource: { - availabilityGroup: 'query-b55f3432-7810-4529-80ed-780a926a6f03', - requiredCapacity: 1, - }, - }, - }, - - { - multiStageQuery: { - type: 'multiStageQuery', - taskId: 'query-b55f3432-7810-4529-80ed-780a926a6f03', - payload: { - status: { - status: 'SUCCESS', - startTime: '2023-03-27T22:17:02.401Z', - durationMs: 28854, - pendingTasks: 0, - runningTasks: 2, - }, - stages: [ - { - stageNumber: 0, - definition: { - id: '8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_0', - input: [ - { + stages: [ + { + stageNumber: 0, + definition: { + id: '8af42220-2724-4a76-b39f-c2f98df2de69_0', + input: [ + { + type: 'external', + inputSource: { + type: 'http', + uris: ['https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz'], + }, + inputFormat: { + type: 'json', + keepNullColumns: false, + assumeNewlineDelimited: false, + useJsonNodeReader: false, + }, + signature: [ + { + name: 'timestamp', + type: 'STRING', + }, + { + name: 'agent_type', + type: 'STRING', + }, + ], + }, + ], + processor: { + type: 'scan', + query: { + queryType: 'scan', + dataSource: { type: 'external', inputSource: { type: 'http', @@ -163,157 +101,333 @@ export const EXECUTION_INGEST_COMPLETE = Execution.fromTaskPayloadAndReport( useJsonNodeReader: false, }, signature: [ - { name: 'timestamp', type: 'STRING' }, - { name: 'agent_type', type: 'STRING' }, - ], - }, - ], - processor: { - type: 'scan', - query: { - queryType: 'scan', - dataSource: { type: 'inputNumber', inputNumber: 0 }, - intervals: { - type: 'intervals', - intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], - }, - virtualColumns: [ { - type: 'expression', - name: 'v0', - expression: 'timestamp_parse("timestamp",null,\'UTC\')', - outputType: 'LONG', + name: 'timestamp', + type: 'STRING', + }, + { + name: 'agent_type', + type: 'STRING', }, ], - resultFormat: 'compactedList', - columns: ['agent_type', 'v0'], - legacy: false, - context: { - __timeColumn: 'v0', - finalize: false, - finalizeAggregations: false, - groupByEnableMultiValueUnnesting: false, - maxNumTasks: 2, - queryId: 'b55f3432-7810-4529-80ed-780a926a6f03', - scanSignature: - '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', - sqlInsertSegmentGranularity: '{"type":"all"}', - sqlQueryId: 'b55f3432-7810-4529-80ed-780a926a6f03', - sqlReplaceTimeChunks: 'all', + }, + intervals: { + type: 'intervals', + intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + }, + virtualColumns: [ + { + type: 'expression', + name: 'v0', + expression: 'timestamp_parse("timestamp",null,\'UTC\')', + outputType: 'LONG', }, - granularity: { type: 'all' }, + ], + resultFormat: 'compactedList', + columns: ['agent_type', 'v0'], + legacy: false, + context: { + __timeColumn: 'v0', + __user: 'allowAll', + finalize: false, + finalizeAggregations: false, + groupByEnableMultiValueUnnesting: false, + maxNumTasks: 2, + maxParseExceptions: 0, + queryId: '5aa683e2-a6ee-4655-a834-a643e91055b1', + scanSignature: + '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', + sqlInsertSegmentGranularity: '{"type":"all"}', + sqlQueryId: '5aa683e2-a6ee-4655-a834-a643e91055b1', + sqlReplaceTimeChunks: 'all', + }, + granularity: { + type: 'all', }, }, - signature: [ - { name: '__boost', type: 'LONG' }, - { name: 'agent_type', type: 'STRING' }, - { name: 'v0', type: 'LONG' }, - ], - shuffleSpec: { - type: 'targetSize', - clusterBy: { columns: [{ columnName: '__boost', order: 'ASCENDING' }] }, - targetSize: 3000000, + }, + signature: [ + { + name: '__boost', + type: 'LONG', + }, + { + name: 'agent_type', + type: 'STRING', + }, + { + name: 'v0', + type: 'LONG', + }, + ], + shuffleSpec: { + type: 'targetSize', + clusterBy: { + columns: [ + { + columnName: '__boost', + order: 'ASCENDING', + }, + ], }, - maxWorkerCount: 1, - shuffleCheckHasMultipleValues: true, - maxInputBytesPerWorker: 10737418240, + targetSize: 3000000, }, - phase: 'FINISHED', - workerCount: 1, - partitionCount: 1, - startTime: '2023-03-27T22:17:02.792Z', - duration: 24236, - sort: true, + maxWorkerCount: 1, + shuffleCheckHasMultipleValues: true, }, - { - stageNumber: 1, - definition: { - id: '8984a4c0-89a0-4a0a-9eaa-bf03088da3e3_1', - input: [{ type: 'stage', stage: 0 }], - processor: { - type: 'segmentGenerator', - dataSchema: { - dataSource: 'kttm_simple', - timestampSpec: { column: '__time', format: 'millis', missingValue: null }, - dimensionsSpec: { - dimensions: [ - { - type: 'string', - name: 'agent_type', - multiValueHandling: 'SORTED_ARRAY', - createBitmapIndex: true, - }, - ], - dimensionExclusions: ['__time'], - includeAllDimensions: false, - useSchemaDiscovery: false, - }, - metricsSpec: [], - granularitySpec: { - type: 'arbitrary', - queryGranularity: { type: 'none' }, - rollup: false, - intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + phase: 'FINISHED', + workerCount: 1, + partitionCount: 1, + startTime: '2023-06-19T05:39:26.711Z', + duration: 20483, + sort: true, + }, + { + stageNumber: 1, + definition: { + id: '8af42220-2724-4a76-b39f-c2f98df2de69_1', + input: [ + { + type: 'stage', + stage: 0, + }, + ], + processor: { + type: 'segmentGenerator', + dataSchema: { + dataSource: 'kttm_simple', + timestampSpec: { + column: '__time', + format: 'millis', + missingValue: null, + }, + dimensionsSpec: { + dimensions: [ + { + type: 'string', + name: 'agent_type', + multiValueHandling: 'SORTED_ARRAY', + createBitmapIndex: true, + }, + ], + dimensionExclusions: ['__time'], + includeAllDimensions: false, + useSchemaDiscovery: false, + }, + metricsSpec: [], + granularitySpec: { + type: 'arbitrary', + queryGranularity: { + type: 'none', }, - transformSpec: { filter: null, transforms: [] }, + rollup: false, + intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], }, - columnMappings: [ - { queryColumn: 'v0', outputColumn: '__time' }, - { queryColumn: 'agent_type', outputColumn: 'agent_type' }, - ], - tuningConfig: { - maxNumWorkers: 1, - maxRowsInMemory: 100000, - rowsPerSegment: 3000000, + transformSpec: { + filter: null, + transforms: [], }, }, - signature: [], - maxWorkerCount: 1, - maxInputBytesPerWorker: 10737418240, + columnMappings: [ + { + queryColumn: 'v0', + outputColumn: '__time', + }, + { + queryColumn: 'agent_type', + outputColumn: 'agent_type', + }, + ], + tuningConfig: { + maxNumWorkers: 1, + maxRowsInMemory: 100000, + rowsPerSegment: 3000000, + }, }, - phase: 'FINISHED', - workerCount: 1, - partitionCount: 1, - startTime: '2023-03-27T22:17:26.978Z', - duration: 4276, + signature: [], + maxWorkerCount: 1, }, - ], - counters: { + phase: 'FINISHED', + workerCount: 1, + partitionCount: 1, + startTime: '2023-06-19T05:39:47.166Z', + duration: 2381, + }, + ], + counters: { + '0': { '0': { - '0': { - input0: { - type: 'channel', - rows: [465346], - bytes: [360464067], - files: [1], - totalFiles: [1], + input0: { + type: 'channel', + rows: [465346], + bytes: [360464067], + files: [1], + totalFiles: [1], + }, + output: { + type: 'channel', + rows: [465346], + bytes: [25430674], + frames: [4], + }, + shuffle: { + type: 'channel', + rows: [465346], + bytes: [23570446], + frames: [38], + }, + sortProgress: { + type: 'sortProgress', + totalMergingLevels: 3, + levelToTotalBatches: { + '0': 1, + '1': 1, + '2': 1, }, - output: { type: 'channel', rows: [465346], bytes: [25430674], frames: [4] }, - shuffle: { type: 'channel', rows: [465346], bytes: [23570446], frames: [38] }, - sortProgress: { - type: 'sortProgress', - totalMergingLevels: 3, - levelToTotalBatches: { '0': 1, '1': 1, '2': 1 }, - levelToMergedBatches: { '0': 1, '1': 1, '2': 1 }, - totalMergersForUltimateLevel: 1, - progressDigest: 1.0, + levelToMergedBatches: { + '0': 1, + '1': 1, + '2': 1, }, + totalMergersForUltimateLevel: 1, + progressDigest: 1.0, }, }, - '1': { - '0': { - input0: { type: 'channel', rows: [465346], bytes: [23570446], frames: [38] }, - segmentGenerationProgress: { - type: 'segmentGenerationProgress', - rowsProcessed: 465346, - rowsPersisted: 465346, - rowsMerged: 465346, - rowsPushed: 465346, - }, + }, + '1': { + '0': { + input0: { + type: 'channel', + rows: [465346], + bytes: [23570446], + frames: [38], + }, + segmentGenerationProgress: { + type: 'segmentGenerationProgress', + rowsProcessed: 465346, + rowsPersisted: 465346, + rowsMerged: 465346, + rowsPushed: 465346, + }, + }, + }, + }, + }, + }, +}).updateWithTaskPayload({ + task: 'query-5aa683e2-a6ee-4655-a834-a643e91055b1', + payload: { + type: 'query_controller', + id: 'query-5aa683e2-a6ee-4655-a834-a643e91055b1', + spec: { + query: { + queryType: 'scan', + dataSource: { + type: 'external', + inputSource: { + type: 'http', + uris: ['https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz'], + }, + inputFormat: { + type: 'json', + keepNullColumns: false, + assumeNewlineDelimited: false, + useJsonNodeReader: false, + }, + signature: [ + { + name: 'timestamp', + type: 'STRING', + }, + { + name: 'agent_type', + type: 'STRING', }, + ], + }, + intervals: { + type: 'intervals', + intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + }, + virtualColumns: [ + { + type: 'expression', + name: 'v0', + expression: 'timestamp_parse("timestamp",null,\'UTC\')', + outputType: 'LONG', }, + ], + resultFormat: 'compactedList', + columns: ['agent_type', 'v0'], + legacy: false, + context: { + __user: 'allowAll', + finalize: false, + finalizeAggregations: false, + groupByEnableMultiValueUnnesting: false, + maxNumTasks: 2, + maxParseExceptions: 0, + queryId: '5aa683e2-a6ee-4655-a834-a643e91055b1', + scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', + sqlInsertSegmentGranularity: '{"type":"all"}', + sqlQueryId: '5aa683e2-a6ee-4655-a834-a643e91055b1', + sqlReplaceTimeChunks: 'all', + }, + granularity: { + type: 'all', + }, + }, + columnMappings: [ + { + queryColumn: 'v0', + outputColumn: '__time', + }, + { + queryColumn: 'agent_type', + outputColumn: 'agent_type', }, + ], + destination: { + type: 'dataSource', + dataSource: 'kttm_simple', + segmentGranularity: { + type: 'all', + }, + replaceTimeChunks: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + }, + assignmentStrategy: 'max', + tuningConfig: { + maxNumWorkers: 1, + maxRowsInMemory: 100000, + rowsPerSegment: 3000000, }, }, + sqlQuery: + 'REPLACE INTO "kttm_simple" OVERWRITE ALL\nSELECT\n TIME_PARSE("timestamp") AS "__time",\n "agent_type"\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-v2/kttm-v2-2019-08-25.json.gz"]}\',\n \'{"type":"json"}\'\n )\n) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)\nPARTITIONED BY ALL TIME', + sqlQueryContext: { + finalizeAggregations: false, + sqlQueryId: '5aa683e2-a6ee-4655-a834-a643e91055b1', + groupByEnableMultiValueUnnesting: false, + sqlInsertSegmentGranularity: '{"type":"all"}', + maxNumTasks: 2, + sqlReplaceTimeChunks: 'all', + queryId: '5aa683e2-a6ee-4655-a834-a643e91055b1', + }, + sqlResultsContext: { + timeZone: 'UTC', + serializeComplexValues: true, + stringifyArrays: true, + }, + sqlTypeNames: ['TIMESTAMP', 'VARCHAR'], + context: { + forceTimeChunkLock: true, + useLineageBasedSegmentAllocation: true, + }, + groupId: 'query-5aa683e2-a6ee-4655-a834-a643e91055b1', + dataSource: 'kttm_simple', + resource: { + availabilityGroup: 'query-5aa683e2-a6ee-4655-a834-a643e91055b1', + requiredCapacity: 1, + }, }, -); +}); diff --git a/web-console/src/druid-models/execution/execution-ingest-error.mock.ts b/web-console/src/druid-models/execution/execution-ingest-error.mock.ts index 485256c6b4a5..15715a1a0e08 100644 --- a/web-console/src/druid-models/execution/execution-ingest-error.mock.ts +++ b/web-console/src/druid-models/execution/execution-ingest-error.mock.ts @@ -41,152 +41,91 @@ PARTITIONED BY DAY } */ -export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport( - { - task: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765', +export const EXECUTION_INGEST_ERROR = Execution.fromTaskReport({ + multiStageQuery: { + type: 'multiStageQuery', + taskId: 'query-8f889312-e989-4b4c-9895-485a1fe796d3', payload: { - type: 'query_controller', - id: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765', - spec: { - query: { - queryType: 'scan', - dataSource: { - type: 'external', - inputSource: { - type: 'http', - uris: ['https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json'], - }, - inputFormat: { - type: 'json', - keepNullColumns: false, - assumeNewlineDelimited: false, - useJsonNodeReader: false, - }, - signature: [ - { name: 'timestamp', type: 'STRING' }, - { name: 'agent_type', type: 'STRING' }, - ], - }, - intervals: { - type: 'intervals', - intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], - }, - virtualColumns: [ - { - type: 'expression', - name: 'v0', - expression: 'timestamp_parse("timestamp",null,\'UTC\')', - outputType: 'LONG', - }, - ], - resultFormat: 'compactedList', - columns: ['agent_type', 'v0'], - legacy: false, - context: { - finalize: false, - finalizeAggregations: false, - groupByEnableMultiValueUnnesting: false, - maxNumTasks: 2, - maxParseExceptions: 2, - queryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765', - scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', - sqlInsertSegmentGranularity: '"DAY"', - sqlQueryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765', - sqlReplaceTimeChunks: 'all', + status: { + status: 'FAILED', + errorReport: { + taskId: 'query-8f889312-e989-4b4c-9895-485a1fe796d3-worker0_0', + host: 'localhost', + error: { + errorCode: 'TooManyWarnings', + maxWarnings: 2, + rootErrorCode: 'CannotParseExternalData', + errorMessage: 'Too many warnings of type CannotParseExternalData generated (max = 2)', }, - granularity: { type: 'all' }, - }, - columnMappings: [ - { queryColumn: 'v0', outputColumn: '__time' }, - { queryColumn: 'agent_type', outputColumn: 'agent_type' }, - ], - destination: { - type: 'dataSource', - dataSource: 'kttm-blank-lines', - segmentGranularity: 'DAY', - replaceTimeChunks: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], }, - assignmentStrategy: 'max', - tuningConfig: { maxNumWorkers: 1, maxRowsInMemory: 100000, rowsPerSegment: 3000000 }, - }, - sqlQuery: - 'REPLACE INTO "kttm-blank-lines" OVERWRITE ALL\nSELECT\n TIME_PARSE("timestamp") AS "__time",\n "agent_type"\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json"]}\',\n \'{"type":"json"}\'\n )\n) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)\nPARTITIONED BY DAY', - sqlQueryContext: { - maxParseExceptions: 2, - finalizeAggregations: false, - sqlQueryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765', - groupByEnableMultiValueUnnesting: false, - sqlInsertSegmentGranularity: '"DAY"', - maxNumTasks: 2, - sqlReplaceTimeChunks: 'all', - queryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765', - }, - sqlTypeNames: ['TIMESTAMP', 'VARCHAR'], - context: { forceTimeChunkLock: true, useLineageBasedSegmentAllocation: true }, - groupId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765', - dataSource: 'kttm-blank-lines', - resource: { - availabilityGroup: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765', - requiredCapacity: 1, - }, - }, - }, - - { - multiStageQuery: { - type: 'multiStageQuery', - taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765', - payload: { - status: { - status: 'FAILED', - errorReport: { - taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0', - host: 'localhost', + warnings: [ + { + taskId: 'query-8f889312-e989-4b4c-9895-485a1fe796d3-worker0_0', + host: 'localhost:8101', + stageNumber: 0, error: { - errorCode: 'TooManyWarnings', - maxWarnings: 2, - rootErrorCode: 'CannotParseExternalData', - errorMessage: 'Too many warnings of type CannotParseExternalData generated (max = 2)', + errorCode: 'CannotParseExternalData', + errorMessage: + 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)', }, + exceptionStackTrace: + 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:183)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:246)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:173)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:159)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:138)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:820)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', }, - warnings: [ - { - taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0', - host: 'localhost:8101', - stageNumber: 0, - error: { - errorCode: 'CannotParseExternalData', - errorMessage: - 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)', - }, - exceptionStackTrace: - 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 3, Line: 3)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:182)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:248)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:175)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:140)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:801)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', - }, - { - taskId: 'query-614dc100-a4b9-40a3-95ce-1227fa7ea765-worker0_0', - host: 'localhost:8101', - stageNumber: 0, - error: { - errorCode: 'CannotParseExternalData', - errorMessage: - 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)', - }, - exceptionStackTrace: - 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:182)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:248)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:175)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:164)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:140)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:801)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', - }, - ], - startTime: '2023-03-27T22:11:24.945Z', - durationMs: 14106, - pendingTasks: 0, - runningTasks: 2, - }, - stages: [ { + taskId: 'query-8f889312-e989-4b4c-9895-485a1fe796d3-worker0_0', + host: 'localhost:8101', stageNumber: 0, - definition: { - id: '0f627be4-63b6-4249-ba3d-71cd4a78faa2_0', - input: [ - { + error: { + errorCode: 'CannotParseExternalData', + errorMessage: + 'Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)', + }, + exceptionStackTrace: + 'org.apache.druid.java.util.common.parsers.ParseException: Unable to parse row [] (Path: https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json, Record: 6, Line: 7)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:79)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.findNextIteratorIfNecessary(CloseableIterator.java:74)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$2.next(CloseableIterator.java:108)\n\tat org.apache.druid.java.util.common.parsers.CloseableIterator$1.next(CloseableIterator.java:52)\n\tat org.apache.druid.msq.input.external.ExternalInputSliceReader$1$1.hasNext(ExternalInputSliceReader.java:183)\n\tat org.apache.druid.java.util.common.guava.BaseSequence$1.next(BaseSequence.java:115)\n\tat org.apache.druid.segment.RowWalker.advance(RowWalker.java:70)\n\tat org.apache.druid.segment.RowBasedCursor.advanceUninterruptibly(RowBasedCursor.java:110)\n\tat org.apache.druid.segment.RowBasedCursor.advance(RowBasedCursor.java:103)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.populateFrameWriterAndFlushIfNeeded(ScanQueryFrameProcessor.java:246)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runWithSegment(ScanQueryFrameProcessor.java:173)\n\tat org.apache.druid.msq.querykit.BaseLeafFrameProcessor.runIncrementally(BaseLeafFrameProcessor.java:159)\n\tat org.apache.druid.msq.querykit.scan.ScanQueryFrameProcessor.runIncrementally(ScanQueryFrameProcessor.java:138)\n\tat org.apache.druid.frame.processor.FrameProcessors$1FrameProcessorWithBaggage.runIncrementally(FrameProcessors.java:75)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.runProcessorNow(FrameProcessorExecutor.java:229)\n\tat org.apache.druid.frame.processor.FrameProcessorExecutor$1ExecutorRunnable.run(FrameProcessorExecutor.java:137)\n\tat org.apache.druid.msq.exec.WorkerImpl$1$2.run(WorkerImpl.java:820)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat org.apache.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:251)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\nCaused by: com.fasterxml.jackson.databind.exc.MismatchedInputException: No content to map due to end-of-input\n at [Source: (String)""; line: 1, column: 0]\n\tat com.fasterxml.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:59)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4360)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4205)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3214)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3182)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:75)\n\tat org.apache.druid.data.input.impl.JsonLineReader.parseInputRows(JsonLineReader.java:48)\n\tat org.apache.druid.data.input.IntermediateRowParsingReader$1.hasNext(IntermediateRowParsingReader.java:71)\n\t... 22 more\n', + }, + ], + startTime: '2023-06-19T05:37:48.605Z', + durationMs: 14760, + pendingTasks: 0, + runningTasks: 2, + }, + stages: [ + { + stageNumber: 0, + definition: { + id: 'd337a3d8-e361-4795-8eaa-97ced72d9a7b_0', + input: [ + { + type: 'external', + inputSource: { + type: 'http', + uris: [ + 'https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json', + ], + }, + inputFormat: { + type: 'json', + keepNullColumns: false, + assumeNewlineDelimited: false, + useJsonNodeReader: false, + }, + signature: [ + { + name: 'timestamp', + type: 'STRING', + }, + { + name: 'agent_type', + type: 'STRING', + }, + ], + }, + ], + processor: { + type: 'scan', + query: { + queryType: 'scan', + dataSource: { type: 'external', inputSource: { type: 'http', @@ -201,141 +140,312 @@ export const EXECUTION_INGEST_ERROR = Execution.fromTaskPayloadAndReport( useJsonNodeReader: false, }, signature: [ - { name: 'timestamp', type: 'STRING' }, - { name: 'agent_type', type: 'STRING' }, - ], - }, - ], - processor: { - type: 'scan', - query: { - queryType: 'scan', - dataSource: { type: 'inputNumber', inputNumber: 0 }, - intervals: { - type: 'intervals', - intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], - }, - virtualColumns: [ { - type: 'expression', - name: 'v0', - expression: 'timestamp_parse("timestamp",null,\'UTC\')', - outputType: 'LONG', + name: 'timestamp', + type: 'STRING', + }, + { + name: 'agent_type', + type: 'STRING', }, ], - resultFormat: 'compactedList', - columns: ['agent_type', 'v0'], - legacy: false, - context: { - __timeColumn: 'v0', - finalize: false, - finalizeAggregations: false, - groupByEnableMultiValueUnnesting: false, - maxNumTasks: 2, - maxParseExceptions: 2, - queryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765', - scanSignature: - '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', - sqlInsertSegmentGranularity: '"DAY"', - sqlQueryId: '614dc100-a4b9-40a3-95ce-1227fa7ea765', - sqlReplaceTimeChunks: 'all', + }, + intervals: { + type: 'intervals', + intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + }, + virtualColumns: [ + { + type: 'expression', + name: 'v0', + expression: 'timestamp_parse("timestamp",null,\'UTC\')', + outputType: 'LONG', }, - granularity: { type: 'all' }, + ], + resultFormat: 'compactedList', + columns: ['agent_type', 'v0'], + legacy: false, + context: { + __timeColumn: 'v0', + __user: 'allowAll', + finalize: false, + finalizeAggregations: false, + groupByEnableMultiValueUnnesting: false, + maxNumTasks: 2, + maxParseExceptions: 2, + queryId: '8f889312-e989-4b4c-9895-485a1fe796d3', + scanSignature: + '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', + sqlInsertSegmentGranularity: '"DAY"', + sqlQueryId: '8f889312-e989-4b4c-9895-485a1fe796d3', + sqlReplaceTimeChunks: 'all', }, - }, - signature: [ - { name: '__bucket', type: 'LONG' }, - { name: '__boost', type: 'LONG' }, - { name: 'agent_type', type: 'STRING' }, - { name: 'v0', type: 'LONG' }, - ], - shuffleSpec: { - type: 'targetSize', - clusterBy: { - columns: [ - { columnName: '__bucket', order: 'ASCENDING' }, - { columnName: '__boost', order: 'ASCENDING' }, - ], - bucketByCount: 1, + granularity: { + type: 'all', }, - targetSize: 3000000, }, - maxWorkerCount: 1, - shuffleCheckHasMultipleValues: true, - maxInputBytesPerWorker: 10737418240, }, - phase: 'FAILED', - workerCount: 1, - startTime: '2023-03-27T22:11:25.310Z', - duration: 13741, - sort: true, - }, - { - stageNumber: 1, - definition: { - id: '0f627be4-63b6-4249-ba3d-71cd4a78faa2_1', - input: [{ type: 'stage', stage: 0 }], - processor: { - type: 'segmentGenerator', - dataSchema: { - dataSource: 'kttm-blank-lines', - timestampSpec: { column: '__time', format: 'millis', missingValue: null }, - dimensionsSpec: { - dimensions: [ - { - type: 'string', - name: 'agent_type', - multiValueHandling: 'SORTED_ARRAY', - createBitmapIndex: true, - }, - ], - dimensionExclusions: ['__time'], - includeAllDimensions: false, - useSchemaDiscovery: false, + signature: [ + { + name: '__bucket', + type: 'LONG', + }, + { + name: '__boost', + type: 'LONG', + }, + { + name: 'agent_type', + type: 'STRING', + }, + { + name: 'v0', + type: 'LONG', + }, + ], + shuffleSpec: { + type: 'targetSize', + clusterBy: { + columns: [ + { + columnName: '__bucket', + order: 'ASCENDING', }, - metricsSpec: [], - granularitySpec: { - type: 'arbitrary', - queryGranularity: { type: 'none' }, - rollup: false, - intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + { + columnName: '__boost', + order: 'ASCENDING', }, - transformSpec: { filter: null, transforms: [] }, - }, - columnMappings: [ - { queryColumn: 'v0', outputColumn: '__time' }, - { queryColumn: 'agent_type', outputColumn: 'agent_type' }, ], - tuningConfig: { - maxNumWorkers: 1, - maxRowsInMemory: 100000, - rowsPerSegment: 3000000, + bucketByCount: 1, + }, + targetSize: 3000000, + }, + maxWorkerCount: 1, + shuffleCheckHasMultipleValues: true, + }, + phase: 'FAILED', + workerCount: 1, + startTime: '2023-06-19T05:37:48.952Z', + duration: 14412, + sort: true, + }, + { + stageNumber: 1, + definition: { + id: 'd337a3d8-e361-4795-8eaa-97ced72d9a7b_1', + input: [ + { + type: 'stage', + stage: 0, + }, + ], + processor: { + type: 'segmentGenerator', + dataSchema: { + dataSource: 'kttm-blank-lines', + timestampSpec: { + column: '__time', + format: 'millis', + missingValue: null, + }, + dimensionsSpec: { + dimensions: [ + { + type: 'string', + name: 'agent_type', + multiValueHandling: 'SORTED_ARRAY', + createBitmapIndex: true, + }, + ], + dimensionExclusions: ['__time'], + includeAllDimensions: false, + useSchemaDiscovery: false, + }, + metricsSpec: [], + granularitySpec: { + type: 'arbitrary', + queryGranularity: { + type: 'none', + }, + rollup: false, + intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + }, + transformSpec: { + filter: null, + transforms: [], + }, + }, + columnMappings: [ + { + queryColumn: 'v0', + outputColumn: '__time', + }, + { + queryColumn: 'agent_type', + outputColumn: 'agent_type', }, + ], + tuningConfig: { + maxNumWorkers: 1, + maxRowsInMemory: 100000, + rowsPerSegment: 3000000, }, - signature: [], - maxWorkerCount: 1, - maxInputBytesPerWorker: 10737418240, }, + signature: [], + maxWorkerCount: 1, }, - ], - counters: { + }, + ], + counters: { + '0': { '0': { - '0': { - input0: { type: 'channel', rows: [10], bytes: [7658], files: [1], totalFiles: [1] }, - output: { type: 'channel', rows: [10], bytes: [712], frames: [1] }, - sortProgress: { - type: 'sortProgress', - totalMergingLevels: 3, - levelToTotalBatches: { '0': 1, '1': 1, '2': -1 }, - levelToMergedBatches: {}, - totalMergersForUltimateLevel: -1, - progressDigest: 0.0, + input0: { + type: 'channel', + rows: [10], + bytes: [7658], + files: [1], + totalFiles: [1], + }, + output: { + type: 'channel', + rows: [10], + bytes: [712], + frames: [1], + }, + sortProgress: { + type: 'sortProgress', + totalMergingLevels: 3, + levelToTotalBatches: { + '0': 1, + '1': 1, + '2': -1, }, - warnings: { type: 'warnings', CannotParseExternalData: 3 }, + levelToMergedBatches: {}, + totalMergersForUltimateLevel: -1, + progressDigest: 0.0, + }, + warnings: { + type: 'warnings', + CannotParseExternalData: 3, + }, + }, + }, + }, + }, + }, +}).updateWithTaskPayload({ + task: 'query-8f889312-e989-4b4c-9895-485a1fe796d3', + payload: { + type: 'query_controller', + id: 'query-8f889312-e989-4b4c-9895-485a1fe796d3', + spec: { + query: { + queryType: 'scan', + dataSource: { + type: 'external', + inputSource: { + type: 'http', + uris: ['https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json'], + }, + inputFormat: { + type: 'json', + keepNullColumns: false, + assumeNewlineDelimited: false, + useJsonNodeReader: false, + }, + signature: [ + { + name: 'timestamp', + type: 'STRING', + }, + { + name: 'agent_type', + type: 'STRING', }, + ], + }, + intervals: { + type: 'intervals', + intervals: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], + }, + virtualColumns: [ + { + type: 'expression', + name: 'v0', + expression: 'timestamp_parse("timestamp",null,\'UTC\')', + outputType: 'LONG', }, + ], + resultFormat: 'compactedList', + columns: ['agent_type', 'v0'], + legacy: false, + context: { + __user: 'allowAll', + finalize: false, + finalizeAggregations: false, + groupByEnableMultiValueUnnesting: false, + maxNumTasks: 2, + maxParseExceptions: 2, + queryId: '8f889312-e989-4b4c-9895-485a1fe796d3', + scanSignature: '[{"name":"agent_type","type":"STRING"},{"name":"v0","type":"LONG"}]', + sqlInsertSegmentGranularity: '"DAY"', + sqlQueryId: '8f889312-e989-4b4c-9895-485a1fe796d3', + sqlReplaceTimeChunks: 'all', + }, + granularity: { + type: 'all', + }, + }, + columnMappings: [ + { + queryColumn: 'v0', + outputColumn: '__time', + }, + { + queryColumn: 'agent_type', + outputColumn: 'agent_type', }, + ], + destination: { + type: 'dataSource', + dataSource: 'kttm-blank-lines', + segmentGranularity: 'DAY', + replaceTimeChunks: ['-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z'], }, + assignmentStrategy: 'max', + tuningConfig: { + maxNumWorkers: 1, + maxRowsInMemory: 100000, + rowsPerSegment: 3000000, + }, + }, + sqlQuery: + 'REPLACE INTO "kttm-blank-lines" OVERWRITE ALL\nSELECT\n TIME_PARSE("timestamp") AS "__time",\n "agent_type"\nFROM TABLE(\n EXTERN(\n \'{"type":"http","uris":["https://static.imply.io/example-data/kttm-with-issues/kttm-blank-lines.json"]}\',\n \'{"type":"json"}\'\n )\n) EXTEND ("timestamp" VARCHAR, "agent_type" VARCHAR)\nPARTITIONED BY DAY', + sqlQueryContext: { + maxParseExceptions: 2, + finalizeAggregations: false, + sqlQueryId: '8f889312-e989-4b4c-9895-485a1fe796d3', + groupByEnableMultiValueUnnesting: false, + sqlInsertSegmentGranularity: '"DAY"', + maxNumTasks: 2, + sqlReplaceTimeChunks: 'all', + queryId: '8f889312-e989-4b4c-9895-485a1fe796d3', + }, + sqlResultsContext: { + timeZone: 'UTC', + serializeComplexValues: true, + stringifyArrays: true, + }, + sqlTypeNames: ['TIMESTAMP', 'VARCHAR'], + context: { + forceTimeChunkLock: true, + useLineageBasedSegmentAllocation: true, + }, + groupId: 'query-8f889312-e989-4b4c-9895-485a1fe796d3', + dataSource: 'kttm-blank-lines', + resource: { + availabilityGroup: 'query-8f889312-e989-4b4c-9895-485a1fe796d3', + requiredCapacity: 1, }, }, -); +}); diff --git a/web-console/src/druid-models/execution/execution.spec.ts b/web-console/src/druid-models/execution/execution.spec.ts index f2d4b1e3038b..9a36c1036ce4 100644 --- a/web-console/src/druid-models/execution/execution.spec.ts +++ b/web-console/src/druid-models/execution/execution.spec.ts @@ -20,16 +20,13 @@ import { Execution } from './execution'; import { EXECUTION_INGEST_COMPLETE } from './execution-ingest-complete.mock'; describe('Execution', () => { - describe('.fromTaskDetail', () => { + describe('.fromTaskReport', () => { it('fails for bad status (error: null)', () => { expect(() => - Execution.fromTaskPayloadAndReport( - {} as any, - { - asyncResultId: 'multi-stage-query-sql-1392d806-c17f-4937-94ee-8fa0a3ce1566', - error: null, - } as any, - ), + Execution.fromTaskReport({ + asyncResultId: 'multi-stage-query-sql-1392d806-c17f-4937-94ee-8fa0a3ce1566', + error: null, + } as any), ).toThrowError('Invalid payload'); }); diff --git a/web-console/src/druid-models/execution/execution.ts b/web-console/src/druid-models/execution/execution.ts index fd2c0881dd75..9843f95b12c2 100644 --- a/web-console/src/druid-models/execution/execution.ts +++ b/web-console/src/druid-models/execution/execution.ts @@ -26,10 +26,17 @@ import { oneOf, pluralIfNeeded, } from '../../utils'; +import type { AsyncState, AsyncStatusResponse } from '../async-query/async-query'; import type { DruidEngine } from '../druid-engine/druid-engine'; import { validDruidEngine } from '../druid-engine/druid-engine'; import type { QueryContext } from '../query-context/query-context'; import { Stages } from '../stages/stages'; +import type { + MsqTaskPayloadResponse, + MsqTaskReportResponse, + TaskStatus, + TaskStatusResponse, +} from '../task/task'; const IGNORE_CONTEXT_KEYS = [ '__asyncIdentity__', @@ -67,7 +74,7 @@ type ExecutionDestination = | { type: 'taskReport'; } - | { type: 'dataSource'; dataSource: string; loaded?: boolean } + | { type: 'dataSource'; dataSource: string; numRows?: number; loaded?: boolean } | { type: 'download' }; export type ExecutionStatus = 'RUNNING' | 'FAILED' | 'SUCCESS'; @@ -171,31 +178,26 @@ export interface ExecutionValue { error?: ExecutionError; warnings?: ExecutionError[]; capacityInfo?: CapacityInfo; - _payload?: { payload: any; task: string }; + _payload?: MsqTaskPayloadResponse; } export class Execution { - static validAsyncStatus( - status: string | undefined, - ): status is 'INITIALIZED' | 'RUNNING' | 'COMPLETE' | 'FAILED' | 'UNDETERMINED' { - return oneOf(status, 'INITIALIZED', 'RUNNING', 'COMPLETE', 'FAILED', 'UNDETERMINED'); + static INLINE_DATASOURCE_MARKER = '__query_select'; + + static validAsyncState(status: string | undefined): status is AsyncState { + return oneOf(status, 'ACCEPTED', 'RUNNING', 'FINISHED', 'FAILED'); } - static validTaskStatus( - status: string | undefined, - ): status is 'WAITING' | 'PENDING' | 'RUNNING' | 'FAILED' | 'SUCCESS' { + static validTaskStatus(status: string | undefined): status is TaskStatus { return oneOf(status, 'WAITING', 'PENDING', 'RUNNING', 'FAILED', 'SUCCESS'); } - static normalizeAsyncStatus( - state: 'INITIALIZED' | 'RUNNING' | 'COMPLETE' | 'FAILED' | 'UNDETERMINED', - ): ExecutionStatus { + static normalizeAsyncState(state: AsyncState): ExecutionStatus { switch (state) { - case 'COMPLETE': + case 'FINISHED': return 'SUCCESS'; - case 'INITIALIZED': - case 'UNDETERMINED': + case 'ACCEPTED': return 'RUNNING'; default: @@ -204,9 +206,7 @@ export class Execution { } // Treat WAITING as PENDING since they are all the same as far as the UI is concerned - static normalizeTaskStatus( - status: 'WAITING' | 'PENDING' | 'RUNNING' | 'FAILED' | 'SUCCESS', - ): ExecutionStatus { + static normalizeTaskStatus(status: TaskStatus): ExecutionStatus { switch (status) { case 'SUCCESS': case 'FAILED': @@ -249,8 +249,50 @@ export class Execution { }); } + static fromAsyncStatus( + asyncSubmitResult: AsyncStatusResponse, + sqlQuery?: string, + queryContext?: QueryContext, + ): Execution { + const { schema, resultSetInformation } = asyncSubmitResult; + + let result: QueryResult | undefined; + if (resultSetInformation?.sampleRecords) { + result = new QueryResult({ + header: schema.map( + s => new Column({ name: s.name, sqlType: s.type, nativeType: s.nativeType }), + ), + rows: resultSetInformation.sampleRecords, + }).inflateDatesFromSqlTypes(); + } + + return new Execution({ + engine: 'sql-msq-task', + id: 'queryId' in asyncSubmitResult ? asyncSubmitResult.queryId : 'x', + startTime: new Date(asyncSubmitResult.createdAt), + duration: asyncSubmitResult.durationInMs, + status: Execution.normalizeAsyncState(asyncSubmitResult.state), + sqlQuery, + queryContext, + error: asyncSubmitResult.queryException as any, // ToDo: remove 'as any' + destination: + typeof resultSetInformation?.dataSource === 'string' + ? resultSetInformation.dataSource !== Execution.INLINE_DATASOURCE_MARKER + ? { + type: 'dataSource', + dataSource: resultSetInformation.dataSource, + numRows: resultSetInformation.numRows, + } + : { + type: 'taskReport', + } + : undefined, + result, + }); + } + static fromTaskStatus( - taskStatus: { status: any; task: string }, + taskStatus: TaskStatusResponse, sqlQuery?: string, queryContext?: QueryContext, ): Execution { @@ -282,13 +324,7 @@ export class Execution { }); } - static fromTaskPayloadAndReport( - taskPayload: { payload: any; task: string }, - taskReport: { - multiStageQuery: { type: string; payload: any; taskId: string }; - error?: any; - }, - ): Execution { + static fromTaskReport(taskReport: MsqTaskReportResponse): Execution { // Must have status set for a valid report const id = deepGet(taskReport, 'multiStageQuery.taskId'); const status = deepGet(taskReport, 'multiStageQuery.payload.status.status'); @@ -328,7 +364,7 @@ export class Execution { }).inflateDatesFromSqlTypes(); } - let res = new Execution({ + return new Execution({ engine: 'sql-msq-task', id, status: Execution.normalizeTaskStatus(status), @@ -342,21 +378,8 @@ export class Execution { : undefined, error, warnings: Array.isArray(warnings) ? warnings : undefined, - destination: deepGet(taskPayload, 'payload.spec.destination'), result, - nativeQuery: deepGet(taskPayload, 'payload.spec.query'), - - _payload: taskPayload, }); - - if (deepGet(taskPayload, 'payload.sqlQuery')) { - res = res.changeSqlQuery( - deepGet(taskPayload, 'payload.sqlQuery'), - deleteKeys(deepGet(taskPayload, 'payload.sqlQueryContext'), IGNORE_CONTEXT_KEYS), - ); - } - - return res; } static fromResult(engine: DruidEngine, result: QueryResult): Execution { @@ -480,16 +503,26 @@ export class Execution { }); } - public updateWith(newSummary: Execution): Execution { - let nextSummary = newSummary; - if (this.sqlQuery && !nextSummary.sqlQuery) { - nextSummary = nextSummary.changeSqlQuery(this.sqlQuery, this.queryContext); - } - if (this.destination && !nextSummary.destination) { - nextSummary = nextSummary.changeDestination(this.destination); + public updateWithTaskPayload(taskPayload: MsqTaskPayloadResponse): Execution { + const value = this.valueOf(); + + value._payload = taskPayload; + value.destination = { + ...value.destination, + ...(deepGet(taskPayload, 'payload.spec.destination') || {}), + }; + value.nativeQuery = deepGet(taskPayload, 'payload.spec.query'); + + let ret = new Execution(value); + + if (deepGet(taskPayload, 'payload.sqlQuery')) { + ret = ret.changeSqlQuery( + deepGet(taskPayload, 'payload.sqlQuery'), + deleteKeys(deepGet(taskPayload, 'payload.sqlQueryContext'), IGNORE_CONTEXT_KEYS), + ); } - return nextSummary; + return ret; } public attachErrorFromStatus(status: any): Execution { @@ -550,6 +583,22 @@ export class Execution { return destination.dataSource; } + public getIngestNumRows(): number | undefined { + const { destination, stages } = this; + + if (destination?.type === 'dataSource' && typeof destination.numRows === 'number') { + return destination.numRows; + } + + const lastStage = stages?.getLastStage(); + if (stages && lastStage && lastStage.definition.processor.type === 'segmentGenerator') { + // Assume input0 since we know the segmentGenerator will only ever have one stage input + return stages.getTotalCounterForStage(lastStage, 'input0', 'rows'); + } + + return; + } + public isSuccessfulInsert(): boolean { return Boolean( this.isFullyComplete() && this.getIngestDatasource() && this.status === 'SUCCESS', diff --git a/web-console/src/druid-models/index.ts b/web-console/src/druid-models/index.ts index 0b4ad6b65f70..16edb184fcb0 100644 --- a/web-console/src/druid-models/index.ts +++ b/web-console/src/druid-models/index.ts @@ -16,6 +16,7 @@ * limitations under the License. */ +export * from './async-query/async-query'; export * from './compaction-config/compaction-config'; export * from './compaction-status/compaction-status'; export * from './coordinator-dynamic-config/coordinator-dynamic-config'; @@ -35,6 +36,7 @@ export * from './metric-spec/metric-spec'; export * from './overlord-dynamic-config/overlord-dynamic-config'; export * from './query-context/query-context'; export * from './stages/stages'; +export * from './task/task'; export * from './time/time'; export * from './timestamp-spec/timestamp-spec'; export * from './transform-spec/transform-spec'; diff --git a/web-console/src/druid-models/stages/stages.ts b/web-console/src/druid-models/stages/stages.ts index 7383840cf7e2..0b980ff63abf 100644 --- a/web-console/src/druid-models/stages/stages.ts +++ b/web-console/src/druid-models/stages/stages.ts @@ -62,6 +62,7 @@ export interface StageDefinition { }; maxWorkerCount: number; shuffleCheckHasMultipleValues?: boolean; + maxInputBytesPerWorker?: number; }; phase?: 'NEW' | 'READING_INPUT' | 'POST_READING' | 'RESULTS_READY' | 'FINISHED' | 'FAILED'; workerCount?: number; @@ -74,7 +75,7 @@ export interface StageDefinition { export interface ClusterBy { columns: { columnName: string; - descending?: boolean; + order?: 'ASCENDING' | 'DESCENDING'; }[]; bucketByCount?: number; } @@ -94,7 +95,9 @@ export function formatClusterBy( } } - return columns.map(part => part.columnName + (part.descending ? ' DESC' : '')).join(', '); + return columns + .map(part => part.columnName + (part.order === 'DESCENDING' ? ' DESC' : '')) + .join(', '); } export interface StageWorkerCounter { diff --git a/web-console/src/druid-models/task/task.ts b/web-console/src/druid-models/task/task.ts new file mode 100644 index 000000000000..4bb04f0cd879 --- /dev/null +++ b/web-console/src/druid-models/task/task.ts @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { StageDefinition } from '../stages/stages'; + +export type TaskStatus = 'WAITING' | 'PENDING' | 'RUNNING' | 'FAILED' | 'SUCCESS'; +export type TaskStatusWithCanceled = TaskStatus | 'CANCELED'; + +export interface TaskStatusResponse { + task: string; + status: { + status: TaskStatus; + error?: any; + }; +} + +export interface MsqTaskPayloadResponse { + task: string; + payload: { + type: 'query_controller'; + id: string; + spec: { + query: Record; + columnMappings: { + queryColumn: string; + outputColumn: string; + }[]; + destination: + | { + type: 'taskReport'; + } + | { + type: 'dataSource'; + dataSource: string; + segmentGranularity: string | { type: string }; + replaceTimeChunks: string[]; + }; + assignmentStrategy: 'max' | 'auto'; + tuningConfig: Record; + }; + sqlQuery: string; + sqlQueryContext: Record; + sqlResultsContext: Record; + sqlTypeNames: string[]; + context: Record; + groupId: string; + dataSource: string; + resource: { + availabilityGroup: string; + requiredCapacity: number; + }; + }; +} + +export interface MsqTaskReportResponse { + multiStageQuery: { + type: 'multiStageQuery'; + taskId: string; + payload: { + status: { + status: string; + errorReport?: MsqTaskErrorReport; + warnings?: MsqTaskErrorReport[]; + startTime: string; + durationMs: number; + pendingTasks: number; + runningTasks: number; + }; + stages: StageDefinition[]; + counters: Record>; + }; + }; + error?: any; +} + +export interface MsqTaskErrorReport { + taskId: string; + host: string; + error: { + errorCode: string; + errorMessage: string; + maxWarnings?: number; + rootErrorCode?: string; + }; + stageNumber?: number; + exceptionStackTrace?: string; +} diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index 53b974c87805..797fd119094e 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -71,8 +71,6 @@ export interface WorkbenchQueryValue { } export class WorkbenchQuery { - static INLINE_DATASOURCE_MARKER = '__query_select'; - private static enabledQueryEngines: DruidEngine[] = ['native', 'sql-native']; static blank(): WorkbenchQuery { @@ -622,6 +620,7 @@ export class WorkbenchQuery { } if (engine === 'sql-msq-task') { + apiQuery.context.executionMode ??= 'async'; apiQuery.context.finalizeAggregations ??= !ingestQuery; apiQuery.context.groupByEnableMultiValueUnnesting ??= !ingestQuery; } diff --git a/web-console/src/helpers/execution/sql-task-execution.ts b/web-console/src/helpers/execution/sql-task-execution.ts index c8afc1bf70ef..12a58451dec7 100644 --- a/web-console/src/helpers/execution/sql-task-execution.ts +++ b/web-console/src/helpers/execution/sql-task-execution.ts @@ -19,7 +19,7 @@ import type { AxiosResponse, CancelToken } from 'axios'; import { L, QueryResult } from 'druid-query-toolkit'; -import type { QueryContext } from '../../druid-models'; +import type { AsyncStatusResponse, QueryContext } from '../../druid-models'; import { Execution } from '../../druid-models'; import { Api } from '../../singletons'; import { @@ -31,6 +31,7 @@ import { } from '../../utils'; import { maybeGetClusterCapacity } from '../capacity'; +const USE_TASK_REPORTS = false; const WAIT_FOR_SEGMENT_METADATA_TIMEOUT = 180000; // 3 minutes to wait until segments appear in the metadata const WAIT_FOR_SEGMENT_LOAD_TIMEOUT = 540000; // 9 minutes to wait for segments to load at all @@ -85,27 +86,32 @@ export async function submitTaskQuery( } } - let sqlTaskResp: AxiosResponse; - + let sqlAsyncResp: AxiosResponse; try { - sqlTaskResp = await Api.instance.post(`/druid/v2/sql/task`, jsonQuery, { cancelToken }); + sqlAsyncResp = await Api.instance.post( + `/druid/v2/sql/statements`, + jsonQuery, + { + cancelToken, + }, + ); } catch (e) { - const druidError = deepGet(e, 'response.data.error'); + const druidError = deepGet(e, 'response.data'); if (!druidError) throw e; throw new DruidError(druidError, prefixLines); } - const sqlTaskPayload = sqlTaskResp.data; + const sqlAsyncStatus = sqlAsyncResp.data; - if (!sqlTaskPayload.taskId) { - if (!Array.isArray(sqlTaskPayload)) throw new Error('unexpected task payload'); + if (!sqlAsyncStatus.queryId) { + if (!Array.isArray(sqlAsyncStatus)) throw new Error('unexpected task payload'); return Execution.fromResult( 'sql-msq-task', - QueryResult.fromRawResult(sqlTaskPayload, false, true, true, true), + QueryResult.fromRawResult(sqlAsyncStatus, false, true, true, true), ); } - let execution = Execution.fromTaskSubmit(sqlTaskPayload, sqlQuery, context); + let execution = Execution.fromAsyncStatus(sqlAsyncStatus, sqlQuery, context); if (onSubmitted) { onSubmitted(execution.id); @@ -161,9 +167,7 @@ export async function updateExecutionWithTaskIfNeeded( if (!execution.isWaitingForQuery()) return execution; // Inherit old payload so as not to re-query it - return execution.updateWith( - await getTaskExecution(execution.id, execution._payload, cancelToken), - ); + return await getTaskExecution(execution.id, execution._payload, cancelToken); } export async function getTaskExecution( @@ -173,59 +177,68 @@ export async function getTaskExecution( ): Promise { const encodedId = Api.encodePath(id); - let taskPayloadResp: AxiosResponse | undefined; - if (!taskPayloadOverride) { + let execution: Execution | undefined; + + if (USE_TASK_REPORTS) { + let taskReport: any; try { - taskPayloadResp = await Api.instance.get(`/druid/indexer/v1/task/${encodedId}`, { - cancelToken, - }); + taskReport = ( + await Api.instance.get(`/druid/indexer/v1/task/${encodedId}/reports`, { + cancelToken, + }) + ).data; } catch (e) { if (Api.isNetworkError(e)) throw e; } + if (taskReport) { + try { + execution = Execution.fromTaskReport(taskReport); + } catch { + // We got a bad payload, wait a bit and try to get the payload again (also log it) + // This whole catch block is a hack, and we should make the detail route more robust + console.error( + `Got unusable response from the reports endpoint (/druid/indexer/v1/task/${encodedId}/reports) going to retry`, + ); + console.log('Report response:', taskReport); + } + } } - let taskReportResp: AxiosResponse | undefined; - try { - taskReportResp = await Api.instance.get(`/druid/indexer/v1/task/${encodedId}/reports`, { - cancelToken, - }); - } catch (e) { - if (Api.isNetworkError(e)) throw e; + if (!execution) { + const statusResp = await Api.instance.get( + `/druid/v2/sql/statements/${encodedId}`, + { + cancelToken, + }, + ); + + execution = Execution.fromAsyncStatus(statusResp.data); } - if ((taskPayloadResp || taskPayloadOverride) && taskReportResp) { - let execution: Execution | undefined; + let taskPayload: any = taskPayloadOverride; + if (!taskPayload) { try { - execution = Execution.fromTaskPayloadAndReport( - taskPayloadResp ? taskPayloadResp.data : taskPayloadOverride, - taskReportResp.data, - ); - } catch { - // We got a bad payload, wait a bit and try to get the payload again (also log it) - // This whole catch block is a hack, and we should make the detail route more robust - console.error( - `Got unusable response from the reports endpoint (/druid/indexer/v1/task/${encodedId}/reports) going to retry`, - ); - console.log('Report response:', taskReportResp.data); + taskPayload = ( + await Api.instance.get(`/druid/indexer/v1/task/${encodedId}`, { + cancelToken, + }) + ).data; + } catch (e) { + if (Api.isNetworkError(e)) throw e; } + } + if (taskPayload) { + execution = execution.updateWithTaskPayload(taskPayload); + } - if (execution) { - if (execution?.hasPotentiallyStuckStage()) { - const capacityInfo = await maybeGetClusterCapacity(); - if (capacityInfo) { - execution = execution.changeCapacityInfo(capacityInfo); - } - } - - return execution; + if (execution.hasPotentiallyStuckStage()) { + const capacityInfo = await maybeGetClusterCapacity(); + if (capacityInfo) { + execution = execution.changeCapacityInfo(capacityInfo); } } - const statusResp = await Api.instance.get(`/druid/indexer/v1/task/${encodedId}/status`, { - cancelToken, - }); - - return Execution.fromTaskStatus(statusResp.data); + return execution; } export async function updateExecutionWithDatasourceLoadedIfNeeded( diff --git a/web-console/src/views/workbench-view/execution-submit-dialog/execution-submit-dialog.tsx b/web-console/src/views/workbench-view/execution-submit-dialog/execution-submit-dialog.tsx index e197e2d87cb6..182c47ff56c9 100644 --- a/web-console/src/views/workbench-view/execution-submit-dialog/execution-submit-dialog.tsx +++ b/web-console/src/views/workbench-view/execution-submit-dialog/execution-submit-dialog.tsx @@ -59,7 +59,9 @@ export const ExecutionSubmitDialog = React.memo(function ExecutionSubmitDialog( if (typeof detailArchiveVersion === 'number') { try { if (detailArchiveVersion === 2) { - execution = Execution.fromTaskPayloadAndReport(parsed.payload, parsed.reports); + execution = Execution.fromTaskReport(parsed.reports).updateWithTaskPayload( + parsed.payload, + ); } else { AppToaster.show({ intent: Intent.DANGER, @@ -76,7 +78,7 @@ export const ExecutionSubmitDialog = React.memo(function ExecutionSubmitDialog( } } else if (typeof parsed.multiStageQuery === 'object') { try { - execution = Execution.fromTaskPayloadAndReport({} as any, parsed); + execution = Execution.fromTaskReport(parsed); } catch (e) { AppToaster.show({ intent: Intent.DANGER, diff --git a/web-console/src/views/workbench-view/ingest-success-pane/ingest-success-pane.tsx b/web-console/src/views/workbench-view/ingest-success-pane/ingest-success-pane.tsx index 39a52798c5ec..b3b7b1328c02 100644 --- a/web-console/src/views/workbench-view/ingest-success-pane/ingest-success-pane.tsx +++ b/web-console/src/views/workbench-view/ingest-success-pane/ingest-success-pane.tsx @@ -39,24 +39,18 @@ export const IngestSuccessPane = React.memo(function IngestSuccessPane( const datasource = execution.getIngestDatasource(); if (!datasource) return null; - - const { stages } = execution; - const lastStage = stages?.getLastStage(); - - const rows = - stages && lastStage && lastStage.definition.processor.type === 'segmentGenerator' - ? stages.getTotalCounterForStage(lastStage, 'input0', 'rows') // Assume input0 since we know the segmentGenerator will only ever have one stage input - : -1; - const table = T(datasource); + const rows = execution.getIngestNumRows(); - const warnings = stages?.getWarningCount() || 0; + const warnings = execution.stages?.getWarningCount() || 0; const duration = execution.duration; return (

- {`${rows < 0 ? 'Data' : pluralIfNeeded(rows, 'row')} inserted into ${T(datasource)}.`} + {`${typeof rows === 'number' ? pluralIfNeeded(rows, 'row') : 'Data'} inserted into ${T( + datasource, + )}.`} {warnings > 0 && ( <> {' '} diff --git a/web-console/src/views/workbench-view/recent-query-task-panel/recent-query-task-panel.tsx b/web-console/src/views/workbench-view/recent-query-task-panel/recent-query-task-panel.tsx index d57557e96c36..d216adf36940 100644 --- a/web-console/src/views/workbench-view/recent-query-task-panel/recent-query-task-panel.tsx +++ b/web-console/src/views/workbench-view/recent-query-task-panel/recent-query-task-panel.tsx @@ -27,8 +27,8 @@ import React, { useCallback, useState } from 'react'; import { useStore } from 'zustand'; import { Loader } from '../../../components'; -import type { Execution } from '../../../druid-models'; -import { WorkbenchQuery } from '../../../druid-models'; +import type { TaskStatusWithCanceled } from '../../../druid-models'; +import { Execution, WorkbenchQuery } from '../../../druid-models'; import { cancelTaskExecution, getTaskExecution } from '../../../helpers'; import { useClock, useInterval, useQueryManager } from '../../../hooks'; import { AppToaster } from '../../../singletons'; @@ -38,9 +38,7 @@ import { workStateStore } from '../work-state-store'; import './recent-query-task-panel.scss'; -type TaskStatus = 'RUNNING' | 'WAITING' | 'PENDING' | 'SUCCESS' | 'FAILED' | 'CANCELED'; - -function statusToIconAndColor(status: TaskStatus): [IconName, string] { +function statusToIconAndColor(status: TaskStatusWithCanceled): [IconName, string] { switch (status) { case 'RUNNING': return [IconNames.REFRESH, '#2167d5']; @@ -59,7 +57,7 @@ function statusToIconAndColor(status: TaskStatus): [IconName, string] { } interface RecentQueryEntry { - taskStatus: TaskStatus; + taskStatus: TaskStatusWithCanceled; taskId: string; datasource: string; createdTime: string; @@ -69,7 +67,7 @@ interface RecentQueryEntry { function formatDetail(entry: RecentQueryEntry): string | undefined { const lines: string[] = []; - if (entry.datasource !== WorkbenchQuery.INLINE_DATASOURCE_MARKER) { + if (entry.datasource !== Execution.INLINE_DATASOURCE_MARKER) { lines.push(`Datasource: ${entry.datasource}`); } if (entry.errorMessage) { @@ -191,7 +189,7 @@ LIMIT 100`, }} /> {w.taskStatus === 'SUCCESS' && - w.datasource !== WorkbenchQuery.INLINE_DATASOURCE_MARKER && ( + w.datasource !== Execution.INLINE_DATASOURCE_MARKER && (

- {w.datasource === WorkbenchQuery.INLINE_DATASOURCE_MARKER + {w.datasource === Execution.INLINE_DATASOURCE_MARKER ? 'data in report' : w.datasource}
From a9a3ca9b6029aafa305d5c0b057f68e59cfca7a1 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 5 Jul 2023 15:57:08 -0700 Subject: [PATCH 10/18] adjust to API changes --- .../async-query/async-query.mock.ts | 97 +++++++++++++++++++ .../druid-models/async-query/async-query.ts | 24 ++--- .../druid-models/execution/execution.spec.ts | 93 ++++++++++++++++++ .../src/druid-models/execution/execution.ts | 37 ++++--- web-console/src/druid-models/mocks.ts | 1 + .../helpers/execution/sql-task-execution.ts | 5 +- web-console/src/utils/druid-query.ts | 1 + 7 files changed, 226 insertions(+), 32 deletions(-) create mode 100644 web-console/src/druid-models/async-query/async-query.mock.ts diff --git a/web-console/src/druid-models/async-query/async-query.mock.ts b/web-console/src/druid-models/async-query/async-query.mock.ts new file mode 100644 index 000000000000..2aa24067e52e --- /dev/null +++ b/web-console/src/druid-models/async-query/async-query.mock.ts @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { AsyncStatusResponse } from './async-query'; + +/* +SELECT + "channel", + COUNT(*) AS "Count" +FROM "wikipedia" +GROUP BY 1 +ORDER BY 2 DESC +LIMIT 2 + */ + +export const SUCCESS_ASYNC_STATUS: AsyncStatusResponse = { + queryId: 'query-ad84d20a-c331-4ee9-ac59-83024e369cf1', + state: 'SUCCESS', + createdAt: '2023-07-05T21:33:19.147Z', + schema: [ + { + name: 'channel', + type: 'VARCHAR', + nativeType: 'STRING', + }, + { + name: 'Count', + type: 'BIGINT', + nativeType: 'LONG', + }, + ], + durationMs: 29168, + result: { + numTotalRows: 2, + totalSizeInBytes: 116, + dataSource: '__query_select', + sampleRecords: [ + ['#en.wikipedia', 6650], + ['#sh.wikipedia', 3969], + ], + pages: [ + { + numRows: 2, + sizeInBytes: 116, + id: 0, + }, + ], + }, +}; + +/* +REPLACE INTO "k" OVERWRITE ALL +WITH "ext" AS (SELECT * +FROM TABLE( + EXTERN( + '{"type":"local","filter":"blah.json_","baseDir":"/"}', + '{"type":"json"}' + ) +) EXTEND ("timestamp" VARCHAR, "session" VARCHAR)) +SELECT + TIME_PARSE("timestamp") AS "__time", + "session" +FROM "ext" +PARTITIONED BY DAY + */ + +export const FAILED_ASYNC_STATUS: AsyncStatusResponse = { + queryId: 'query-36ea273a-bd6d-48de-b890-2d853d879bf8', + state: 'FAILED', + createdAt: '2023-07-05T21:40:39.986Z', + durationMs: 11217, + errorDetails: { + error: 'druidException', + errorCode: 'UnknownError', + persona: 'USER', + category: 'UNCATEGORIZED', + errorMessage: 'java.io.UncheckedIOException: /', + context: { + message: 'java.io.UncheckedIOException: /', + }, + }, +}; diff --git a/web-console/src/druid-models/async-query/async-query.ts b/web-console/src/druid-models/async-query/async-query.ts index 81469643fc95..401569f558b5 100644 --- a/web-console/src/druid-models/async-query/async-query.ts +++ b/web-console/src/druid-models/async-query/async-query.ts @@ -16,26 +16,22 @@ * limitations under the License. */ -export type AsyncState = 'ACCEPTED' | 'RUNNING' | 'FINISHED' | 'FAILED'; +import type { ErrorResponse } from '../../utils'; + +export type AsyncState = 'ACCEPTED' | 'RUNNING' | 'SUCCESS' | 'FAILED'; export interface AsyncStatusResponse { queryId: string; state: AsyncState; createdAt: string; - durationInMs: number; - schema: { name: string; type: string; nativeType: string }[]; - resultSetInformation?: { + durationMs: number; + schema?: { name: string; type: string; nativeType: string }[]; + result?: { dataSource: string; sampleRecords: any[][]; - numRows: number; - sizeInBytes: number; + numTotalRows: number; + totalSizeInBytes: number; + pages: any[]; }; - queryException?: AsyncError; -} - -export interface AsyncError { - error: string; - errorMessage: string; - errorClass: string; - host?: string; + errorDetails?: ErrorResponse; } diff --git a/web-console/src/druid-models/execution/execution.spec.ts b/web-console/src/druid-models/execution/execution.spec.ts index 9a36c1036ce4..3a95b856f753 100644 --- a/web-console/src/druid-models/execution/execution.spec.ts +++ b/web-console/src/druid-models/execution/execution.spec.ts @@ -16,6 +16,8 @@ * limitations under the License. */ +import { FAILED_ASYNC_STATUS, SUCCESS_ASYNC_STATUS } from '../async-query/async-query.mock'; + import { Execution } from './execution'; import { EXECUTION_INGEST_COMPLETE } from './execution-ingest-complete.mock'; @@ -548,4 +550,95 @@ describe('Execution', () => { `); }); }); + + describe('.fromAsyncStatus', () => { + it('works on SUCCESS', () => { + expect(Execution.fromAsyncStatus(SUCCESS_ASYNC_STATUS)).toMatchInlineSnapshot(` + Execution { + "_payload": undefined, + "capacityInfo": undefined, + "destination": Object { + "type": "taskReport", + }, + "duration": 29168, + "engine": "sql-msq-task", + "error": undefined, + "id": "query-ad84d20a-c331-4ee9-ac59-83024e369cf1", + "nativeQuery": undefined, + "queryContext": undefined, + "result": _QueryResult { + "header": Array [ + Column { + "name": "channel", + "nativeType": "STRING", + "sqlType": "VARCHAR", + }, + Column { + "name": "Count", + "nativeType": "LONG", + "sqlType": "BIGINT", + }, + ], + "query": undefined, + "queryDuration": undefined, + "queryId": undefined, + "resultContext": undefined, + "rows": Array [ + Array [ + "#en.wikipedia", + 6650, + ], + Array [ + "#sh.wikipedia", + 3969, + ], + ], + "sqlQuery": undefined, + "sqlQueryId": undefined, + }, + "sqlQuery": undefined, + "stages": undefined, + "startTime": 2023-07-05T21:33:19.147Z, + "status": "SUCCESS", + "usageInfo": undefined, + "warnings": undefined, + } + `); + }); + + it('works on FAILED', () => { + expect(Execution.fromAsyncStatus(FAILED_ASYNC_STATUS)).toMatchInlineSnapshot(` + Execution { + "_payload": undefined, + "capacityInfo": undefined, + "destination": undefined, + "duration": 11217, + "engine": "sql-msq-task", + "error": Object { + "error": Object { + "category": "UNCATEGORIZED", + "context": Object { + "message": "java.io.UncheckedIOException: /", + }, + "error": "druidException", + "errorCode": "UnknownError", + "errorMessage": "java.io.UncheckedIOException: /", + "persona": "USER", + }, + "taskId": "query-36ea273a-bd6d-48de-b890-2d853d879bf8", + }, + "id": "query-36ea273a-bd6d-48de-b890-2d853d879bf8", + "nativeQuery": undefined, + "queryContext": undefined, + "result": undefined, + "sqlQuery": undefined, + "stages": undefined, + "startTime": 2023-07-05T21:40:39.986Z, + "status": "FAILED", + "usageInfo": undefined, + "warnings": undefined, + } + `); + }); + }); }); diff --git a/web-console/src/druid-models/execution/execution.ts b/web-console/src/druid-models/execution/execution.ts index 9843f95b12c2..7da8f9eef961 100644 --- a/web-console/src/druid-models/execution/execution.ts +++ b/web-console/src/druid-models/execution/execution.ts @@ -194,9 +194,6 @@ export class Execution { static normalizeAsyncState(state: AsyncState): ExecutionStatus { switch (state) { - case 'FINISHED': - return 'SUCCESS'; - case 'ACCEPTED': return 'RUNNING'; @@ -254,40 +251,48 @@ export class Execution { sqlQuery?: string, queryContext?: QueryContext, ): Execution { - const { schema, resultSetInformation } = asyncSubmitResult; + const { queryId, schema, result, errorDetails } = asyncSubmitResult; - let result: QueryResult | undefined; - if (resultSetInformation?.sampleRecords) { - result = new QueryResult({ + let queryResult: QueryResult | undefined; + if (schema && result?.sampleRecords) { + queryResult = new QueryResult({ header: schema.map( s => new Column({ name: s.name, sqlType: s.type, nativeType: s.nativeType }), ), - rows: resultSetInformation.sampleRecords, + rows: result.sampleRecords, }).inflateDatesFromSqlTypes(); } + let executionError: ExecutionError | undefined; + if (errorDetails) { + executionError = { + taskId: queryId, + error: errorDetails as any, + }; + } + return new Execution({ engine: 'sql-msq-task', - id: 'queryId' in asyncSubmitResult ? asyncSubmitResult.queryId : 'x', + id: queryId, startTime: new Date(asyncSubmitResult.createdAt), - duration: asyncSubmitResult.durationInMs, + duration: asyncSubmitResult.durationMs, status: Execution.normalizeAsyncState(asyncSubmitResult.state), sqlQuery, queryContext, - error: asyncSubmitResult.queryException as any, // ToDo: remove 'as any' + error: executionError, destination: - typeof resultSetInformation?.dataSource === 'string' - ? resultSetInformation.dataSource !== Execution.INLINE_DATASOURCE_MARKER + typeof result?.dataSource === 'string' + ? result.dataSource !== Execution.INLINE_DATASOURCE_MARKER ? { type: 'dataSource', - dataSource: resultSetInformation.dataSource, - numRows: resultSetInformation.numRows, + dataSource: result.dataSource, + numRows: result.numTotalRows, } : { type: 'taskReport', } : undefined, - result, + result: queryResult, }); } diff --git a/web-console/src/druid-models/mocks.ts b/web-console/src/druid-models/mocks.ts index 9c357043441d..38c924033d12 100644 --- a/web-console/src/druid-models/mocks.ts +++ b/web-console/src/druid-models/mocks.ts @@ -16,6 +16,7 @@ * limitations under the License. */ +export * from './async-query/async-query.mock'; export * from './execution/execution-ingest-complete.mock'; export * from './execution/execution-ingest-error.mock'; export * from './stages/stages.mock'; diff --git a/web-console/src/helpers/execution/sql-task-execution.ts b/web-console/src/helpers/execution/sql-task-execution.ts index 12a58451dec7..b3dfd0bce011 100644 --- a/web-console/src/helpers/execution/sql-task-execution.ts +++ b/web-console/src/helpers/execution/sql-task-execution.ts @@ -31,7 +31,8 @@ import { } from '../../utils'; import { maybeGetClusterCapacity } from '../capacity'; -const USE_TASK_REPORTS = false; +const USE_TASK_PAYLOAD = true; +const USE_TASK_REPORTS = true; const WAIT_FOR_SEGMENT_METADATA_TIMEOUT = 180000; // 3 minutes to wait until segments appear in the metadata const WAIT_FOR_SEGMENT_LOAD_TIMEOUT = 540000; // 9 minutes to wait for segments to load at all @@ -216,7 +217,7 @@ export async function getTaskExecution( } let taskPayload: any = taskPayloadOverride; - if (!taskPayload) { + if (USE_TASK_PAYLOAD && !taskPayload) { try { taskPayload = ( await Api.instance.get(`/druid/indexer/v1/task/${encodedId}`, { diff --git a/web-console/src/utils/druid-query.ts b/web-console/src/utils/druid-query.ts index efdfd1bd6e77..4f8e3fe55d6a 100644 --- a/web-console/src/utils/druid-query.ts +++ b/web-console/src/utils/druid-query.ts @@ -42,6 +42,7 @@ export type ErrorResponseCategory = export interface ErrorResponse { persona: ErrorResponsePersona; category: ErrorResponseCategory; + errorCode?: string; errorMessage: string; // a message for the intended audience context?: Record; // a map of extra context values that might be helpful From 57680bd6558858ab02ad56c5405661023081fa88 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Thu, 6 Jul 2023 20:31:26 -0700 Subject: [PATCH 11/18] final fixes --- .../src/components/auto-form/auto-form.tsx | 15 ++-- .../fancy-numeric-input.tsx | 11 +-- .../compaction-dynamic-config-dialog.tsx | 28 +++++-- web-console/src/utils/load-rule.ts | 16 ++-- web-console/src/utils/query-manager.tsx | 10 ++- .../datasources-view/datasources-view.tsx | 75 +++++++++++-------- 6 files changed, 96 insertions(+), 59 deletions(-) diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx index 505e1a7fe92a..60ddf448c6f4 100644 --- a/web-console/src/components/auto-form/auto-form.tsx +++ b/web-console/src/components/auto-form/auto-form.tsx @@ -83,6 +83,11 @@ export interface Field { }) => JSX.Element; } +function toNumberOrUndefined(n: unknown): number | undefined { + const r = Number(n); + return isNaN(r) ? undefined : r; +} + interface ComputedFieldValues { required: boolean; defaultValue?: any; @@ -137,7 +142,7 @@ export class AutoForm> extends React.PureComponent ): R { if (!model || functor == null) return defaultValue; if (typeof functor === 'function') { - return (functor as any)(model); + return (functor as any)(model) ?? defaultValue; } else { return functor; } @@ -252,8 +257,8 @@ export class AutoForm> extends React.PureComponent return ( { this.fieldChange( field, @@ -283,8 +288,8 @@ export class AutoForm> extends React.PureComponent return ( { this.fieldChange( field, diff --git a/web-console/src/components/fancy-numeric-input/fancy-numeric-input.tsx b/web-console/src/components/fancy-numeric-input/fancy-numeric-input.tsx index 8b70b4085c22..16866e1d7273 100644 --- a/web-console/src/components/fancy-numeric-input/fancy-numeric-input.tsx +++ b/web-console/src/components/fancy-numeric-input/fancy-numeric-input.tsx @@ -48,7 +48,8 @@ function evaluateSqlSimple(sql: SqlExpression): number | undefined { } } -function numberToShown(n: number): string { +function numberToShown(n: number | undefined): string { + if (typeof n === 'undefined') return ''; return String(n); } @@ -69,8 +70,8 @@ export interface FancyNumericInputProps { placeholder?: string; onBlur?: InputGroupProps2['onBlur']; - value: number; - defaultValue: number; + value: number | undefined; + defaultValue?: number; onValueChange(value: number): void; min?: number; @@ -130,8 +131,8 @@ export const FancyNumericInput = React.memo(function FancyNumericInput( ); const effectiveDisabled = disabled || readOnly; - const isIncrementDisabled = max !== undefined && +value >= max; - const isDecrementDisabled = min !== undefined && +value <= min; + const isIncrementDisabled = max !== undefined && value !== undefined && +value >= max; + const isDecrementDisabled = min !== undefined && value !== undefined && +value <= min; function changeValue(newValue: number): void { onValueChange(roundAndClamp(newValue)); diff --git a/web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-dialog.tsx b/web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-dialog.tsx index 79e288bdf6bc..4a91337d32e1 100644 --- a/web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-dialog.tsx +++ b/web-console/src/dialogs/compaction-dynamic-config-dialog/compaction-dynamic-config-dialog.tsx @@ -16,7 +16,7 @@ * limitations under the License. */ -import { Button, Classes, Dialog, Intent } from '@blueprintjs/core'; +import { Button, Classes, Code, Dialog, Intent } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import React, { useState } from 'react'; @@ -32,18 +32,21 @@ interface CompactionDynamicConfig { maxCompactionTaskSlots: number; } +const DEFAULT_RATIO = 0.1; +const DEFAULT_MAX = 2147483647; const COMPACTION_DYNAMIC_CONFIG_FIELDS: Field[] = [ { name: 'compactionTaskSlotRatio', type: 'ratio', - defaultValue: 0.1, + defaultValue: DEFAULT_RATIO, info: <>The ratio of the total task slots to the compaction task slots., }, { name: 'maxCompactionTaskSlots', type: 'number', - defaultValue: 2147483647, + defaultValue: DEFAULT_MAX, info: <>The maximum number of task slots for compaction tasks, + min: 1, }, ]; @@ -65,8 +68,8 @@ export const CompactionDynamicConfigDialog = React.memo(function CompactionDynam try { const c = (await Api.instance.get('/druid/coordinator/v1/config/compaction')).data; setDynamicConfig({ - compactionTaskSlotRatio: c.compactionTaskSlotRatio ?? 0.1, - maxCompactionTaskSlots: c.maxCompactionTaskSlots ?? 2147483647, + compactionTaskSlotRatio: c.compactionTaskSlotRatio ?? DEFAULT_RATIO, + maxCompactionTaskSlots: c.maxCompactionTaskSlots ?? DEFAULT_MAX, }); } catch (e) { AppToaster.show({ @@ -85,7 +88,9 @@ export const CompactionDynamicConfigDialog = React.memo(function CompactionDynam try { // This API is terrible. https://druid.apache.org/docs/latest/operations/api-reference.html#automatic-compaction-configuration await Api.instance.post( - `/druid/coordinator/v1/config/compaction/taskslots?ratio=${dynamicConfig.compactionTaskSlotRatio}&max=${dynamicConfig.maxCompactionTaskSlots}`, + `/druid/coordinator/v1/config/compaction/taskslots?ratio=${ + dynamicConfig.compactionTaskSlotRatio ?? DEFAULT_RATIO + }&max=${dynamicConfig.maxCompactionTaskSlots ?? DEFAULT_MAX}`, {}, ); } catch (e) { @@ -125,6 +130,15 @@ export const CompactionDynamicConfigDialog = React.memo(function CompactionDynam .

+

+ The maximum number of task slots used for compaction will be{' '} + {`clamp(floor(${ + dynamicConfig.compactionTaskSlotRatio ?? DEFAULT_RATIO + } * total_task_slots), 1, ${ + dynamicConfig.maxCompactionTaskSlots ?? DEFAULT_MAX + })`} + . +