diff --git a/web-console/src/components/auto-form/auto-form.scss b/web-console/src/components/auto-form/auto-form.scss index 5523f0f8173f..c303abc294f5 100644 --- a/web-console/src/components/auto-form/auto-form.scss +++ b/web-console/src/components/auto-form/auto-form.scss @@ -16,16 +16,8 @@ * limitations under the License. */ -@import '../../variables'; - .auto-form { - // Popover in info label - label.#{$bp-ns}-label { - position: relative; - - .#{$bp-ns}-text-muted { - position: absolute; - right: 0; - } + .custom-input input { + cursor: pointer; } } diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx index 1e56ef2b725d..146de61b6272 100644 --- a/web-console/src/components/auto-form/auto-form.tsx +++ b/web-console/src/components/auto-form/auto-form.tsx @@ -16,7 +16,14 @@ * limitations under the License. */ -import { Button, ButtonGroup, FormGroup, Intent, NumericInput } from '@blueprintjs/core'; +import { + Button, + ButtonGroup, + FormGroup, + InputGroup, + Intent, + NumericInput, +} from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import React from 'react'; @@ -46,7 +53,8 @@ export interface Field { | 'boolean' | 'string-array' | 'json' - | 'interval'; + | 'interval' + | 'custom'; defaultValue?: any; emptyValue?: any; suggestions?: Functor; @@ -64,6 +72,13 @@ export interface Field { valueAdjustment?: (value: any) => any; adjustment?: (model: Partial) => Partial; issueWithValue?: (value: any) => string | undefined; + + customSummary?: (v: any) => string; + customDialog?: (o: { + value: any; + onValueChange: (v: any) => void; + onClose: () => void; + }) => JSX.Element; } interface ComputedFieldValues { @@ -84,6 +99,7 @@ export interface AutoFormProps { export interface AutoFormState { showMore: boolean; + customDialog?: JSX.Element; } export class AutoForm> extends React.PureComponent< @@ -395,6 +411,36 @@ export class AutoForm> extends React.PureComponent ); } + private renderCustomInput(field: Field): JSX.Element { + const { model } = this.props; + const { required, defaultValue, modelValue } = AutoForm.computeFieldValues(model, field); + const effectiveValue = modelValue || defaultValue; + + const onEdit = () => { + this.setState({ + customDialog: field.customDialog?.({ + value: effectiveValue, + onValueChange: v => this.fieldChange(field, v), + onClose: () => { + this.setState({ customDialog: undefined }); + }, + }), + }); + }; + + return ( + } + onClick={onEdit} + /> + ); + } + renderFieldInput(field: Field) { switch (field.type) { case 'number': @@ -413,6 +459,8 @@ export class AutoForm> extends React.PureComponent return this.renderJsonInput(field); case 'interval': return this.renderIntervalInput(field); + case 'custom': + return this.renderCustomInput(field); default: throw new Error(`unknown field type '${field.type}'`); } @@ -464,7 +512,7 @@ export class AutoForm> extends React.PureComponent render(): JSX.Element { const { fields, model, showCustom } = this.props; - const { showMore } = this.state; + const { showMore, customDialog } = this.state; let shouldShowMore = false; const shownFields = fields.filter(field => { @@ -489,6 +537,7 @@ export class AutoForm> extends React.PureComponent {model && shownFields.map(this.renderField)} {model && showCustom && showCustom(model) && this.renderCustom()} {shouldShowMore && this.renderMoreOrLess()} + {customDialog} ); } diff --git a/web-console/src/components/form-group-with-info/form-group-with-info.scss b/web-console/src/components/form-group-with-info/form-group-with-info.scss index c9587cb088fa..a64c6d29274c 100644 --- a/web-console/src/components/form-group-with-info/form-group-with-info.scss +++ b/web-console/src/components/form-group-with-info/form-group-with-info.scss @@ -19,6 +19,20 @@ @import '../../variables'; .form-group-with-info { + label.#{$bp-ns}-label { + position: relative; + + .#{$bp-ns}-text-muted { + position: absolute; + right: 0; + + // This is only needed because BP4 alerts are too agro in setting CSS on icons + .#{$bp-ns}-icon { + margin-right: 0; + } + } + } + .#{$bp-ns}-text-muted .#{$bp-ns}-popover2-target { margin-top: 0; } diff --git a/web-console/src/components/table-clickable-cell/table-clickable-cell.scss b/web-console/src/components/table-clickable-cell/table-clickable-cell.scss index d6f6f8b2d7f7..5c5991df54e2 100644 --- a/web-console/src/components/table-clickable-cell/table-clickable-cell.scss +++ b/web-console/src/components/table-clickable-cell/table-clickable-cell.scss @@ -24,6 +24,10 @@ overflow: hidden; text-overflow: ellipsis; + &.disabled { + cursor: not-allowed; + } + .hover-icon { position: absolute; top: $table-cell-v-padding; diff --git a/web-console/src/components/table-clickable-cell/table-clickable-cell.tsx b/web-console/src/components/table-clickable-cell/table-clickable-cell.tsx index cc8cfd71e56d..7e4c66fdd5e5 100644 --- a/web-console/src/components/table-clickable-cell/table-clickable-cell.tsx +++ b/web-console/src/components/table-clickable-cell/table-clickable-cell.tsx @@ -27,18 +27,23 @@ export interface TableClickableCellProps { onClick: MouseEventHandler; hoverIcon?: IconName; title?: string; + disabled?: boolean; children?: ReactNode; } export const TableClickableCell = React.memo(function TableClickableCell( props: TableClickableCellProps, ) { - const { className, onClick, hoverIcon, title, children } = props; + const { className, onClick, hoverIcon, title, disabled, children } = props; return ( -
+
{children} - {hoverIcon && } + {hoverIcon && !disabled && }
); }); diff --git a/web-console/src/components/warning-checklist/warning-checklist.tsx b/web-console/src/components/warning-checklist/warning-checklist.tsx index 449ad970045d..5c74cbdb08a0 100644 --- a/web-console/src/components/warning-checklist/warning-checklist.tsx +++ b/web-console/src/components/warning-checklist/warning-checklist.tsx @@ -17,29 +17,31 @@ */ import { Switch } from '@blueprintjs/core'; -import React, { useState } from 'react'; +import React, { ReactNode, useState } from 'react'; export interface WarningChecklistProps { - checks: string[]; - onChange: (allChecked: boolean) => void; + checks: ReactNode[]; + onChange(allChecked: boolean): void; } export const WarningChecklist = React.memo(function WarningChecklist(props: WarningChecklistProps) { const { checks, onChange } = props; - const [checked, setChecked] = useState>({}); + const [checked, setChecked] = useState>({}); - function doCheck(check: string) { + function doCheck(checkIndex: number) { const newChecked = { ...checked }; - newChecked[check] = !newChecked[check]; + newChecked[checkIndex] = !newChecked[checkIndex]; setChecked(newChecked); - onChange(checks.every(check => newChecked[check])); + onChange(checks.every((_, i) => newChecked[i])); } return (
{checks.map((check, i) => ( - doCheck(check)} /> + doCheck(i)}> + {check} + ))}
); diff --git a/web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx b/web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx index f892936bab77..0d8cf385a5b6 100644 --- a/web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx +++ b/web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx @@ -47,7 +47,7 @@ export interface AsyncActionDialogProps { intent?: Intent; successText: string; failText: string; - warningChecks?: string[]; + warningChecks?: ReactNode[]; children?: ReactNode; } diff --git a/web-console/src/dialogs/index-spec-dialog/__snapshots__/index-spec-dialog.spec.tsx.snap b/web-console/src/dialogs/index-spec-dialog/__snapshots__/index-spec-dialog.spec.tsx.snap new file mode 100644 index 000000000000..57d989621b70 --- /dev/null +++ b/web-console/src/dialogs/index-spec-dialog/__snapshots__/index-spec-dialog.spec.tsx.snap @@ -0,0 +1,317 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`IndexSpecDialog matches snapshot with indexSpec 1`] = ` + + +
+ + Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns. + , + "label": "String dictionary encoding", + "name": "stringDictionaryEncoding.type", + "suggestions": Array [ + "utf8", + "frontCoded", + ], + "type": "string", + }, + Object { + "defaultValue": 4, + "defined": [Function], + "info": + The number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. + , + "label": "String dictionary encoding bucket size", + "max": 128, + "min": 1, + "name": "stringDictionaryEncoding.bucketSize", + "type": "number", + }, + Object { + "defaultValue": "roaring", + "info": + Compression format for bitmap indexes. + , + "label": "Bitmap type", + "name": "bitmap.type", + "suggestions": Array [ + "roaring", + "concise", + ], + "type": "string", + }, + Object { + "defaultValue": true, + "defined": [Function], + "info": + Controls whether or not run-length encoding will be used when it is determined to be more space-efficient. + , + "label": "Bitmap compress run on serialization", + "name": "bitmap.compressRunOnSerialization", + "type": "boolean", + }, + Object { + "defaultValue": "lz4", + "info": + Compression format for dimension columns. + , + "name": "dimensionCompression", + "suggestions": Array [ + "lz4", + "lzf", + "zstd", + "uncompressed", + ], + "type": "string", + }, + Object { + "defaultValue": "longs", + "info": + Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. + + auto + + encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. + + longs + + stores the value as-is with 8 bytes each. + , + "name": "longEncoding", + "suggestions": Array [ + "longs", + "auto", + ], + "type": "string", + }, + Object { + "defaultValue": "lz4", + "info": + Compression format for primitive type metric columns. + , + "name": "metricCompression", + "suggestions": Array [ + "lz4", + "lzf", + "zstd", + "uncompressed", + ], + "type": "string", + }, + Object { + "defaultValue": "lz4", + "info": + Compression format to use for nested column raw data. + , + "label": "JSON compression", + "name": "jsonCompression", + "suggestions": Array [ + "lz4", + "lzf", + "zstd", + "uncompressed", + ], + "type": "string", + }, + ] + } + model={ + Object { + "dimensionCompression": "lzf", + } + } + onChange={[Function]} + /> +
+
+
+ + +
+
+
+`; + +exports[`IndexSpecDialog matches snapshot without compactionConfig 1`] = ` + + +
+ + Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns. + , + "label": "String dictionary encoding", + "name": "stringDictionaryEncoding.type", + "suggestions": Array [ + "utf8", + "frontCoded", + ], + "type": "string", + }, + Object { + "defaultValue": 4, + "defined": [Function], + "info": + The number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. + , + "label": "String dictionary encoding bucket size", + "max": 128, + "min": 1, + "name": "stringDictionaryEncoding.bucketSize", + "type": "number", + }, + Object { + "defaultValue": "roaring", + "info": + Compression format for bitmap indexes. + , + "label": "Bitmap type", + "name": "bitmap.type", + "suggestions": Array [ + "roaring", + "concise", + ], + "type": "string", + }, + Object { + "defaultValue": true, + "defined": [Function], + "info": + Controls whether or not run-length encoding will be used when it is determined to be more space-efficient. + , + "label": "Bitmap compress run on serialization", + "name": "bitmap.compressRunOnSerialization", + "type": "boolean", + }, + Object { + "defaultValue": "lz4", + "info": + Compression format for dimension columns. + , + "name": "dimensionCompression", + "suggestions": Array [ + "lz4", + "lzf", + "zstd", + "uncompressed", + ], + "type": "string", + }, + Object { + "defaultValue": "longs", + "info": + Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. + + auto + + encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. + + longs + + stores the value as-is with 8 bytes each. + , + "name": "longEncoding", + "suggestions": Array [ + "longs", + "auto", + ], + "type": "string", + }, + Object { + "defaultValue": "lz4", + "info": + Compression format for primitive type metric columns. + , + "name": "metricCompression", + "suggestions": Array [ + "lz4", + "lzf", + "zstd", + "uncompressed", + ], + "type": "string", + }, + Object { + "defaultValue": "lz4", + "info": + Compression format to use for nested column raw data. + , + "label": "JSON compression", + "name": "jsonCompression", + "suggestions": Array [ + "lz4", + "lzf", + "zstd", + "uncompressed", + ], + "type": "string", + }, + ] + } + model={Object {}} + onChange={[Function]} + /> +
+
+
+ + +
+
+
+`; diff --git a/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.scss b/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.scss new file mode 100644 index 000000000000..e7cc53ee47dd --- /dev/null +++ b/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.scss @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import '../../variables'; + +.index-spec-dialog { + &.#{$bp-ns}-dialog { + height: 70vh; + } + + .form-json-selector { + margin: 15px; + } + + .content { + margin: 0 15px 10px 0; + padding: 0 5px 0 15px; + flex: 1; + overflow: auto; + } +} diff --git a/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.spec.tsx b/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.spec.tsx new file mode 100644 index 000000000000..68f7f56b885b --- /dev/null +++ b/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.spec.tsx @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { shallow } from 'enzyme'; +import React from 'react'; + +import { IndexSpecDialog } from './index-spec-dialog'; + +describe('IndexSpecDialog', () => { + it('matches snapshot without compactionConfig', () => { + const compactionDialog = shallow( + {}} onSave={() => {}} indexSpec={undefined} />, + ); + expect(compactionDialog).toMatchSnapshot(); + }); + + it('matches snapshot with indexSpec', () => { + const compactionDialog = shallow( + {}} + onSave={() => {}} + indexSpec={{ + dimensionCompression: 'lzf', + }} + />, + ); + expect(compactionDialog).toMatchSnapshot(); + }); +}); diff --git a/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.tsx b/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.tsx new file mode 100644 index 000000000000..4c870df45af2 --- /dev/null +++ b/web-console/src/dialogs/index-spec-dialog/index-spec-dialog.tsx @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Button, Classes, Dialog, Intent } from '@blueprintjs/core'; +import React, { useState } from 'react'; + +import { AutoForm, FormJsonSelector, FormJsonTabs, JsonInput } from '../../components'; +import { INDEX_SPEC_FIELDS, IndexSpec } from '../../druid-models'; + +import './index-spec-dialog.scss'; + +export interface IndexSpecDialogProps { + title?: string; + onClose: () => void; + onSave: (indexSpec: IndexSpec) => void; + indexSpec: IndexSpec | undefined; +} + +export const IndexSpecDialog = React.memo(function IndexSpecDialog(props: IndexSpecDialogProps) { + const { title, indexSpec, onSave, onClose } = props; + + const [currentTab, setCurrentTab] = useState('form'); + const [currentIndexSpec, setCurrentIndexSpec] = useState(indexSpec || {}); + const [jsonError, setJsonError] = useState(); + + const issueWithCurrentIndexSpec = AutoForm.issueWithModel(currentIndexSpec, INDEX_SPEC_FIELDS); + + return ( + + +
+ {currentTab === 'form' ? ( + setCurrentIndexSpec(m)} + /> + ) : ( + { + setCurrentIndexSpec(v); + setJsonError(undefined); + }} + onError={setJsonError} + issueWithValue={value => AutoForm.issueWithModel(value, INDEX_SPEC_FIELDS)} + height="100%" + /> + )} +
+
+
+
+
+
+ ); +}); diff --git a/web-console/src/dialogs/index.ts b/web-console/src/dialogs/index.ts index 9509442c8bd2..588257c84e74 100644 --- a/web-console/src/dialogs/index.ts +++ b/web-console/src/dialogs/index.ts @@ -24,6 +24,7 @@ export * from './diff-dialog/diff-dialog'; export * from './doctor-dialog/doctor-dialog'; export * from './edit-context-dialog/edit-context-dialog'; export * from './history-dialog/history-dialog'; +export * from './kill-datasource-dialog/kill-datasource-dialog'; export * from './lookup-edit-dialog/lookup-edit-dialog'; export * from './numeric-input-dialog/numeric-input-dialog'; export * from './overlord-dynamic-config-dialog/overlord-dynamic-config-dialog'; diff --git a/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx b/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx new file mode 100644 index 000000000000..3eb7e9fdf243 --- /dev/null +++ b/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Code, Intent } from '@blueprintjs/core'; +import React, { useState } from 'react'; + +import { FormGroupWithInfo, PopoverText } from '../../components'; +import { SuggestibleInput } from '../../components/suggestible-input/suggestible-input'; +import { Api } from '../../singletons'; +import { uniq } from '../../utils'; +import { AsyncActionDialog } from '../async-action-dialog/async-action-dialog'; + +function getSuggestions(): string[] { + // Default to a data 24h ago so as not to cause a conflict between streaming ingestion and kill tasks + const end = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); + const startOfDay = end.slice(0, 10); + const startOfMonth = end.slice(0, 7) + '-01'; + const startOfYear = end.slice(0, 4) + '-01-01'; + + return uniq([ + `1000-01-01/${startOfDay}`, + `1000-01-01/${startOfMonth}`, + `1000-01-01/${startOfYear}`, + '1000-01-01/3000-01-01', + ]); +} + +export interface KillDatasourceDialogProps { + datasource: string; + onClose(): void; + onSuccess(): void; +} + +export const KillDatasourceDialog = function KillDatasourceDialog( + props: KillDatasourceDialogProps, +) { + const { datasource, onClose, onSuccess } = props; + const suggestions = getSuggestions(); + const [interval, setInterval] = useState(suggestions[0]); + + return ( + { + const resp = await Api.instance.delete( + `/druid/coordinator/v1/datasources/${Api.encodePath( + datasource, + )}?kill=true&interval=${Api.encodePath(interval)}`, + {}, + ); + return resp.data; + }} + confirmButtonText="Permanently delete unused segments" + successText="Kill task was issued. Unused segments in datasource will be deleted" + failText="Failed submit kill task" + intent={Intent.DANGER} + onClose={onClose} + onSuccess={onSuccess} + warningChecks={[ + <> + I understand that this operation will delete all metadata about the unused segments of{' '} + {datasource} and removes them from deep storage. + , + 'I understand that this operation cannot be undone.', + ]} + > +

+ Are you sure you want to permanently delete unused segments in {datasource}? +

+

This action is not reversible and the data deleted will be lost.

+ +

+ The range of time over which to delete unused segments specified in ISO8601 interval + format. +

+

+ If you have streaming ingestion running make sure that your interval range doe not + overlap with intervals where streaming data is being added - otherwise the kill task + will not start. +

+ + } + > + setInterval(s || '')} + suggestions={suggestions} + /> +
+
+ ); +}; diff --git a/web-console/src/druid-models/compaction-status/compaction-status.spec.ts b/web-console/src/druid-models/compaction-status/compaction-status.spec.ts index 8ed0c5141366..9d1254090bf5 100644 --- a/web-console/src/druid-models/compaction-status/compaction-status.spec.ts +++ b/web-console/src/druid-models/compaction-status/compaction-status.spec.ts @@ -18,11 +18,7 @@ import { CompactionConfig } from '../compaction-config/compaction-config'; -import { - CompactionStatus, - formatCompactionConfigAndStatus, - zeroCompactionStatus, -} from './compaction-status'; +import { CompactionStatus, formatCompactionInfo, zeroCompactionStatus } from './compaction-status'; describe('compaction status', () => { const BASIC_CONFIG: CompactionConfig = {}; @@ -61,27 +57,30 @@ describe('compaction status', () => { }); it('formatCompactionConfigAndStatus', () => { - expect(formatCompactionConfigAndStatus(undefined, undefined)).toEqual('Not enabled'); + expect(formatCompactionInfo({})).toEqual('Not enabled'); - expect(formatCompactionConfigAndStatus(BASIC_CONFIG, undefined)).toEqual('Awaiting first run'); + expect(formatCompactionInfo({ config: BASIC_CONFIG })).toEqual('Awaiting first run'); - expect(formatCompactionConfigAndStatus(undefined, ZERO_STATUS)).toEqual('Not enabled'); + expect(formatCompactionInfo({ status: ZERO_STATUS })).toEqual('Not enabled'); - expect(formatCompactionConfigAndStatus(BASIC_CONFIG, ZERO_STATUS)).toEqual('Running'); + expect(formatCompactionInfo({ config: BASIC_CONFIG, status: ZERO_STATUS })).toEqual('Running'); expect( - formatCompactionConfigAndStatus(BASIC_CONFIG, { - dataSource: 'tbl', - scheduleStatus: 'RUNNING', - bytesAwaitingCompaction: 0, - bytesCompacted: 100, - bytesSkipped: 0, - segmentCountAwaitingCompaction: 0, - segmentCountCompacted: 10, - segmentCountSkipped: 0, - intervalCountAwaitingCompaction: 0, - intervalCountCompacted: 10, - intervalCountSkipped: 0, + formatCompactionInfo({ + config: BASIC_CONFIG, + status: { + dataSource: 'tbl', + scheduleStatus: 'RUNNING', + bytesAwaitingCompaction: 0, + bytesCompacted: 100, + bytesSkipped: 0, + segmentCountAwaitingCompaction: 0, + segmentCountCompacted: 10, + segmentCountSkipped: 0, + intervalCountAwaitingCompaction: 0, + intervalCountCompacted: 10, + intervalCountSkipped: 0, + }, }), ).toEqual('Fully compacted'); }); diff --git a/web-console/src/druid-models/compaction-status/compaction-status.ts b/web-console/src/druid-models/compaction-status/compaction-status.ts index 2982d9b69e17..d17f2c44fda4 100644 --- a/web-console/src/druid-models/compaction-status/compaction-status.ts +++ b/web-console/src/druid-models/compaction-status/compaction-status.ts @@ -50,19 +50,19 @@ export function zeroCompactionStatus(compactionStatus: CompactionStatus): boolea ); } -export function formatCompactionConfigAndStatus( - compactionConfig: CompactionConfig | undefined, - compactionStatus: CompactionStatus | undefined, -) { - if (compactionConfig) { - if (compactionStatus) { - if ( - compactionStatus.bytesAwaitingCompaction === 0 && - !zeroCompactionStatus(compactionStatus) - ) { +export interface CompactionInfo { + config?: CompactionConfig; + status?: CompactionStatus; +} + +export function formatCompactionInfo(compaction: CompactionInfo) { + const { config, status } = compaction; + if (config) { + if (status) { + if (status.bytesAwaitingCompaction === 0 && !zeroCompactionStatus(status)) { return 'Fully compacted'; } else { - return capitalizeFirst(compactionStatus.scheduleStatus); + return capitalizeFirst(status.scheduleStatus); } } else { return 'Awaiting first run'; diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx index eeb25db09c4b..ca957309ff83 100644 --- a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx +++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx @@ -69,20 +69,9 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field[ ), }, - { - name: 'killAllDataSources', - type: 'boolean', - defaultValue: false, - info: ( - <> - Send kill tasks for ALL dataSources if property druid.coordinator.kill.on is - true. If this is set to true then killDataSourceWhitelist must not be specified - or be empty list. - - ), - }, { name: 'killDataSourceWhitelist', + label: 'Kill datasource whitelist', type: 'string-array', emptyValue: [], info: ( diff --git a/web-console/src/druid-models/index-spec/index-spec.tsx b/web-console/src/druid-models/index-spec/index-spec.tsx new file mode 100644 index 000000000000..1a4246299661 --- /dev/null +++ b/web-console/src/druid-models/index-spec/index-spec.tsx @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Code } from '@blueprintjs/core'; +import React from 'react'; + +import { Field } from '../../components'; +import { deepGet } from '../../utils'; + +export interface IndexSpec { + bitmap?: Bitmap; + dimensionCompression?: string; + stringDictionaryEncoding?: { type: 'utf8' | 'frontCoded'; bucketSize: number }; + metricCompression?: string; + longEncoding?: string; + jsonCompression?: string; +} + +export interface Bitmap { + type: string; + compressRunOnSerialization?: boolean; +} + +export function summarizeIndexSpec(indexSpec: IndexSpec | undefined): string { + if (!indexSpec) return ''; + + const { stringDictionaryEncoding, bitmap, longEncoding } = indexSpec; + + const ret: string[] = []; + if (stringDictionaryEncoding) { + switch (stringDictionaryEncoding.type) { + case 'frontCoded': + ret.push(`frontCoded(${stringDictionaryEncoding.bucketSize || 4})`); + break; + + default: + ret.push(stringDictionaryEncoding.type); + break; + } + } + + if (bitmap) { + ret.push(bitmap.type); + } + + if (longEncoding) { + ret.push(longEncoding); + } + + return ret.join('; '); +} + +export const INDEX_SPEC_FIELDS: Field[] = [ + { + name: 'stringDictionaryEncoding.type', + label: 'String dictionary encoding', + type: 'string', + defaultValue: 'utf8', + suggestions: ['utf8', 'frontCoded'], + info: ( + <> + Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> + columns. + + ), + }, + { + name: 'stringDictionaryEncoding.bucketSize', + label: 'String dictionary encoding bucket size', + type: 'number', + defaultValue: 4, + min: 1, + max: 128, + defined: spec => deepGet(spec, 'stringDictionaryEncoding.type') === 'frontCoded', + info: ( + <> + The number of values to place in a bucket to perform delta encoding. Must be a power of 2, + maximum is 128. + + ), + }, + + { + name: 'bitmap.type', + label: 'Bitmap type', + type: 'string', + defaultValue: 'roaring', + suggestions: ['roaring', 'concise'], + info: <>Compression format for bitmap indexes., + }, + { + name: 'bitmap.compressRunOnSerialization', + label: 'Bitmap compress run on serialization', + type: 'boolean', + defaultValue: true, + defined: spec => (deepGet(spec, 'bitmap.type') || 'roaring') === 'roaring', + info: ( + <> + Controls whether or not run-length encoding will be used when it is determined to be more + space-efficient. + + ), + }, + + { + name: 'dimensionCompression', + type: 'string', + defaultValue: 'lz4', + suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'], + info: <>Compression format for dimension columns., + }, + + { + name: 'longEncoding', + type: 'string', + defaultValue: 'longs', + suggestions: ['longs', 'auto'], + info: ( + <> + Encoding format for long-typed columns. Applies regardless of whether they are dimensions or + metrics. auto encodes the values using offset or lookup table depending on + column cardinality, and store them with variable size. longs stores the value + as-is with 8 bytes each. + + ), + }, + { + name: 'metricCompression', + type: 'string', + defaultValue: 'lz4', + suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'], + info: <>Compression format for primitive type metric columns., + }, + + { + name: 'jsonCompression', + label: 'JSON compression', + type: 'string', + defaultValue: 'lz4', + suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'], + info: <>Compression format to use for nested column raw data. , + }, +]; diff --git a/web-console/src/druid-models/index.ts b/web-console/src/druid-models/index.ts index 359ba70440b8..0b4ad6b65f70 100644 --- a/web-console/src/druid-models/index.ts +++ b/web-console/src/druid-models/index.ts @@ -25,6 +25,7 @@ export * from './execution/execution'; export * from './external-config/external-config'; export * from './filter/filter'; export * from './flatten-spec/flatten-spec'; +export * from './index-spec/index-spec'; export * from './ingest-query-pattern/ingest-query-pattern'; export * from './ingestion-spec/ingestion-spec'; export * from './input-format/input-format'; diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx index b68052abd471..f144f4c2ad29 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx @@ -21,6 +21,7 @@ import { range } from 'd3-array'; import React from 'react'; import { AutoForm, ExternalLink, Field } from '../../components'; +import { IndexSpecDialog } from '../../dialogs/index-spec-dialog/index-spec-dialog'; import { getLink } from '../../links'; import { allowKeys, @@ -44,6 +45,7 @@ import { getDimensionSpecs, getDimensionSpecType, } from '../dimension-spec/dimension-spec'; +import { IndexSpec, summarizeIndexSpec } from '../index-spec/index-spec'; import { InputFormat, issueWithInputFormat } from '../input-format/input-format'; import { FILTER_SUGGESTIONS, @@ -1379,6 +1381,7 @@ export interface TuningConfig { partitionsSpec?: PartitionsSpec; maxPendingPersists?: number; indexSpec?: IndexSpec; + indexSpecForIntermediatePersists?: IndexSpec; forceExtendableShardSpecs?: boolean; forceGuaranteedRollup?: boolean; reportParseExceptions?: boolean; @@ -1869,103 +1872,38 @@ const TUNING_FORM_FIELDS: Field[] = [ }, { - name: 'spec.tuningConfig.indexSpec.bitmap.type', - label: 'Index bitmap type', - type: 'string', - defaultValue: 'roaring', - suggestions: ['concise', 'roaring'], + name: 'spec.tuningConfig.indexSpec', + type: 'custom', hideInMore: true, - info: <>Compression format for bitmap indexes., - }, - { - name: 'spec.tuningConfig.indexSpec.bitmap.compressRunOnSerialization', - type: 'boolean', - defaultValue: true, - defined: spec => deepGet(spec, 'spec.tuningConfig.indexSpec.bitmap.type') === 'roaring', - info: ( - <> - Controls whether or not run-length encoding will be used when it is determined to be more - space-efficient. - + info: <>Defines segment storage format options to use at indexing time., + placeholder: 'Default index spec', + customSummary: summarizeIndexSpec, + customDialog: ({ value, onValueChange, onClose }) => ( + ), }, - - { - name: 'spec.tuningConfig.indexSpec.dimensionCompression', - label: 'Index dimension compression', - type: 'string', - defaultValue: 'lz4', - suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'], - hideInMore: true, - info: <>Compression format for dimension columns., - }, - { - name: 'spec.tuningConfig.indexSpec.stringDictionaryEncoding.type', - label: 'Index string dictionary encoding', - type: 'string', - defaultValue: 'utf8', - suggestions: ['utf8', 'frontCoded'], + name: 'spec.tuningConfig.indexSpecForIntermediatePersists', + type: 'custom', hideInMore: true, info: ( <> - Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> - columns. + Defines segment storage format options to use at indexing time for intermediate persisted + temporary segments. ), - }, - { - name: 'spec.tuningConfig.indexSpec.stringDictionaryEncoding.bucketSize', - label: 'Index string dictionary encoding bucket size', - type: 'number', - defaultValue: 4, - min: 1, - max: 128, - defined: spec => - deepGet(spec, 'spec.tuningConfig.indexSpec.stringDictionaryEncoding.type') === 'frontCoded', - hideInMore: true, - info: ( - <> - The number of values to place in a bucket to perform delta encoding. Must be a power of 2, - maximum is 128. - + placeholder: 'Default index spec', + customSummary: summarizeIndexSpec, + customDialog: ({ value, onValueChange, onClose }) => ( + ), }, - { - name: 'spec.tuningConfig.indexSpec.metricCompression', - label: 'Index metric compression', - type: 'string', - defaultValue: 'lz4', - suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'], - hideInMore: true, - info: <>Compression format for primitive type metric columns., - }, - { - name: 'spec.tuningConfig.indexSpec.longEncoding', - label: 'Index long encoding', - type: 'string', - defaultValue: 'longs', - suggestions: ['longs', 'auto'], - hideInMore: true, - info: ( - <> - Encoding format for long-typed columns. Applies regardless of whether they are dimensions or - metrics. auto encodes the values using offset or lookup table depending on - column cardinality, and store them with variable size. longs stores the value - as-is with 8 bytes each. - - ), - }, - { - name: 'spec.tuningConfig.indexSpec.jsonCompression', - label: 'Index JSON compression', - type: 'string', - defaultValue: 'lz4', - suggestions: ['lz4', 'lzf', 'zstd', 'uncompressed'], - hideInMore: true, - info: <>Compression format to use for nested column raw data. , - }, { name: 'spec.tuningConfig.splitHintSpec.maxSplitSize', type: 'number', @@ -2172,18 +2110,6 @@ export function getTuningFormFields() { return TUNING_FORM_FIELDS; } -export interface IndexSpec { - bitmap?: Bitmap; - dimensionCompression?: string; - metricCompression?: string; - longEncoding?: string; -} - -export interface Bitmap { - type: string; - compressRunOnSerialization?: boolean; -} - // -------------- export function updateIngestionType( diff --git a/web-console/src/druid-models/workbench-query/workbench-query-part.ts b/web-console/src/druid-models/workbench-query/workbench-query-part.ts index 604cfb013213..5e4afb453f09 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query-part.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query-part.ts @@ -62,10 +62,10 @@ export class WorkbenchQueryPart { static getIngestDatasourceFromQueryFragment(queryFragment: string): string | undefined { // Assuming the queryFragment is no parsable find the prefix that look like: // REPLACEINTOSELECT - const matchInsertReplaceIndex = queryFragment.match(/(?:INSERT|REPLACE)\s+INTO/)?.index; + const matchInsertReplaceIndex = queryFragment.match(/(?:INSERT|REPLACE)\s+INTO/i)?.index; if (typeof matchInsertReplaceIndex !== 'number') return; - const matchEnd = queryFragment.match(/\b(?:SELECT|WITH)\b|$/); + const matchEnd = queryFragment.match(/\b(?:SELECT|WITH)\b|$/i); const fragmentQuery = SqlQuery.maybeParse( queryFragment.substring(matchInsertReplaceIndex, matchEnd?.index) + ' SELECT * FROM t', ); diff --git a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts index 5d7e2615098c..9af0fb240700 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts @@ -465,7 +465,7 @@ describe('WorkbenchQuery', () => { it('works with INSERT (unparsable)', () => { const sql = sane` -- Some comment - INSERT INTO trips2 + INSERT into trips2 SELECT TIME_PARSE(pickup_datetime) AS __time, * diff --git a/web-console/src/helpers/spec-conversion.spec.ts b/web-console/src/helpers/spec-conversion.spec.ts index 0239da185415..2f6aa59f51e2 100644 --- a/web-console/src/helpers/spec-conversion.spec.ts +++ b/web-console/src/helpers/spec-conversion.spec.ts @@ -106,6 +106,9 @@ describe('spec conversion', () => { partitionDimension: 'isRobot', targetRowsPerSegment: 150000, }, + indexSpec: { + dimensionCompression: 'lzf', + }, forceGuaranteedRollup: true, maxNumConcurrentSubTasks: 4, maxParseExceptions: 3, @@ -159,6 +162,9 @@ describe('spec conversion', () => { maxParseExceptions: 3, finalizeAggregations: false, maxNumTasks: 5, + indexSpec: { + dimensionCompression: 'lzf', + }, }); }); diff --git a/web-console/src/helpers/spec-conversion.ts b/web-console/src/helpers/spec-conversion.ts index d35433917f59..9b76e787dd61 100644 --- a/web-console/src/helpers/spec-conversion.ts +++ b/web-console/src/helpers/spec-conversion.ts @@ -70,6 +70,11 @@ export function convertSpecToSql(spec: any): QueryWithContext { groupByEnableMultiValueUnnesting: false, }; + const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec'); + if (indexSpec) { + context.indexSpec = indexSpec; + } + const lines: string[] = []; const rollup = deepGet(spec, 'spec.dataSchema.granularitySpec.rollup') ?? true; diff --git a/web-console/src/views/datasources-view/datasources-view.tsx b/web-console/src/views/datasources-view/datasources-view.tsx index 7f384fc8bc2a..c38a42b63938 100644 --- a/web-console/src/views/datasources-view/datasources-view.tsx +++ b/web-console/src/views/datasources-view/datasources-view.tsx @@ -36,12 +36,18 @@ import { TableColumnSelector, ViewControlBar, } from '../../components'; -import { AsyncActionDialog, CompactionDialog, RetentionDialog } from '../../dialogs'; +import { + AsyncActionDialog, + CompactionDialog, + KillDatasourceDialog, + RetentionDialog, +} from '../../dialogs'; import { DatasourceTableActionDialog } from '../../dialogs/datasource-table-action-dialog/datasource-table-action-dialog'; import { CompactionConfig, + CompactionInfo, CompactionStatus, - formatCompactionConfigAndStatus, + formatCompactionInfo, QueryWithContext, zeroCompactionStatus, } from '../../druid-models'; @@ -208,9 +214,8 @@ function segmentGranularityCountsToRank(row: DatasourceQueryResultRow): number { } interface Datasource extends DatasourceQueryResultRow { - readonly rules: Rule[]; - readonly compactionConfig?: CompactionConfig; - readonly compactionStatus?: CompactionStatus; + readonly rules?: Rule[]; + readonly compaction?: CompactionInfo; readonly unused?: boolean; } @@ -220,7 +225,7 @@ function makeUnusedDatasource(datasource: string): Datasource { interface DatasourcesAndDefaultRules { readonly datasources: Datasource[]; - readonly defaultRules: Rule[]; + readonly defaultRules?: Rule[]; } interface RetentionDialogOpenOn { @@ -433,43 +438,85 @@ ORDER BY 1`; let unused: string[] = []; if (showUnused) { - const unusedResp = await Api.instance.get( - '/druid/coordinator/v1/metadata/datasources?includeUnused', + try { + unused = ( + await Api.instance.get( + '/druid/coordinator/v1/metadata/datasources?includeUnused', + ) + ).data.filter(d => !seen[d]); + } catch { + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'Could not get the list of unused datasources', + }); + } + } + + let rules: Record = {}; + try { + rules = (await Api.instance.get>('/druid/coordinator/v1/rules')) + .data; + } catch { + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'Could not get load rules', + }); + } + + let compactionConfigs: Record | undefined; + try { + const compactionConfigsResp = await Api.instance.get<{ + compactionConfigs: CompactionConfig[]; + }>('/druid/coordinator/v1/config/compaction'); + compactionConfigs = lookupBy( + compactionConfigsResp.data.compactionConfigs || [], + c => c.dataSource, ); - unused = unusedResp.data.filter(d => !seen[d]); + } catch { + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'Could not get compaction configs', + }); } - const rulesResp = await Api.instance.get>( - '/druid/coordinator/v1/rules', - ); - const rules = rulesResp.data; - - const compactionConfigsResp = await Api.instance.get<{ - compactionConfigs: CompactionConfig[]; - }>('/druid/coordinator/v1/config/compaction'); - const compactionConfigs = lookupBy( - compactionConfigsResp.data.compactionConfigs || [], - c => c.dataSource, - ); - - const compactionStatusesResp = await Api.instance.get<{ latestStatus: CompactionStatus[] }>( - '/druid/coordinator/v1/compaction/status', - ); - const compactionStatuses = lookupBy( - compactionStatusesResp.data.latestStatus || [], - c => c.dataSource, - ); + let compactionStatuses: Record | undefined; + if (compactionConfigs) { + // Don't bother getting the statuses if we can not even get the configs + try { + const compactionStatusesResp = await Api.instance.get<{ + latestStatus: CompactionStatus[]; + }>('/druid/coordinator/v1/compaction/status'); + compactionStatuses = lookupBy( + compactionStatusesResp.data.latestStatus || [], + c => c.dataSource, + ); + } catch { + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'Could not get compaction statuses', + }); + } + } return { datasources: datasources.concat(unused.map(makeUnusedDatasource)).map(ds => { return { ...ds, - rules: rules[ds.datasource] || [], - compactionConfig: compactionConfigs[ds.datasource], - compactionStatus: compactionStatuses[ds.datasource], + rules: rules[ds.datasource], + compaction: + compactionConfigs && compactionStatuses + ? { + config: compactionConfigs[ds.datasource], + status: compactionStatuses[ds.datasource], + } + : undefined, }; }), - defaultRules: rules[DEFAULT_RULES_KEY] || [], + defaultRules: rules[DEFAULT_RULES_KEY], }; }, onStateChange: datasourcesAndDefaultRulesState => { @@ -633,36 +680,15 @@ ORDER BY 1`; if (!killDatasource) return; return ( - { - const resp = await Api.instance.delete( - `/druid/coordinator/v1/datasources/${Api.encodePath( - killDatasource, - )}?kill=true&interval=1000/3000`, - {}, - ); - return resp.data; - }} - confirmButtonText="Permanently delete unused segments" - successText="Kill task was issued. Unused segments in datasource will be deleted" - failText="Failed submit kill task" - intent={Intent.DANGER} + { this.setState({ killDatasource: undefined }); }} onSuccess={() => { this.fetchDatasourceData(); }} - warningChecks={[ - `I understand that this operation will delete all metadata about the unused segments of ${killDatasource} and removes them from deep storage.`, - 'I understand that this operation cannot be undone.', - ]} - > -

- {`Are you sure you want to permanently delete unused segments in '${killDatasource}'?`} -

-

This action is not reversible and the data deleted will be lost.

-
+ /> ); } @@ -756,20 +782,20 @@ ORDER BY 1`; this.setState({ retentionDialogOpenOn: undefined }); setTimeout(() => { this.setState(state => { - const datasourcesAndDefaultRules = state.datasourcesAndDefaultRulesState.data; - if (!datasourcesAndDefaultRules) return {}; + const defaultRules = state.datasourcesAndDefaultRulesState.data?.defaultRules; + if (!defaultRules) return {}; return { retentionDialogOpenOn: { datasource: '_default', - rules: datasourcesAndDefaultRules.defaultRules, + rules: defaultRules, }, }; }); }, 50); }; - private readonly saveCompaction = async (compactionConfig: any) => { + private readonly saveCompaction = async (compactionConfig: CompactionConfig) => { if (!compactionConfig) return; try { await Api.instance.post(`/druid/coordinator/v1/config/compaction`, compactionConfig); @@ -819,8 +845,8 @@ ORDER BY 1`; getDatasourceActions( datasource: string, unused: boolean | undefined, - rules: Rule[], - compactionConfig: CompactionConfig | undefined, + rules: Rule[] | undefined, + compactionInfo: CompactionInfo | undefined, ): BasicAction[] { const { goToQuery, goToTask, capabilities } = this.props; @@ -863,82 +889,83 @@ ORDER BY 1`; }, ]; } else { - return goToActions.concat([ - { - icon: IconNames.AUTOMATIC_UPDATES, - title: 'Edit retention rules', - onAction: () => { - this.setState({ - retentionDialogOpenOn: { - datasource, - rules, - }, - }); + return goToActions.concat( + compact([ + { + icon: IconNames.AUTOMATIC_UPDATES, + title: 'Edit retention rules', + onAction: () => { + this.setState({ + retentionDialogOpenOn: { + datasource, + rules: rules || [], + }, + }); + }, }, - }, - { - icon: IconNames.REFRESH, - title: 'Mark as used all segments (will lead to reapplying retention rules)', - onAction: () => - this.setState({ - datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn: datasource, - }), - }, - { - icon: IconNames.COMPRESSED, - title: 'Edit compaction configuration', - onAction: () => { - this.setState({ - compactionDialogOpenOn: { - datasource, - compactionConfig, - }, - }); + { + icon: IconNames.REFRESH, + title: 'Mark as used all segments (will lead to reapplying retention rules)', + onAction: () => + this.setState({ + datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn: datasource, + }), }, - }, - { - icon: IconNames.EXPORT, - title: 'Mark as used segments by interval', - - onAction: () => - this.setState({ - datasourceToMarkSegmentsByIntervalIn: datasource, - useUnuseAction: 'use', - }), - }, - { - icon: IconNames.IMPORT, - title: 'Mark as unused segments by interval', - - onAction: () => - this.setState({ - datasourceToMarkSegmentsByIntervalIn: datasource, - useUnuseAction: 'unuse', - }), - }, - { - icon: IconNames.IMPORT, - title: 'Mark as unused all segments', - intent: Intent.DANGER, - onAction: () => this.setState({ datasourceToMarkAsUnusedAllSegmentsIn: datasource }), - }, - { - icon: IconNames.TRASH, - title: 'Delete unused segments (issue kill task)', - intent: Intent.DANGER, - onAction: () => this.setState({ killDatasource: datasource }), - }, - ]); + compactionInfo + ? { + icon: IconNames.COMPRESSED, + title: 'Edit compaction configuration', + onAction: () => { + this.setState({ + compactionDialogOpenOn: { + datasource, + compactionConfig: compactionInfo.config, + }, + }); + }, + } + : undefined, + { + icon: IconNames.EXPORT, + title: 'Mark as used segments by interval', + + onAction: () => + this.setState({ + datasourceToMarkSegmentsByIntervalIn: datasource, + useUnuseAction: 'use', + }), + }, + { + icon: IconNames.IMPORT, + title: 'Mark as unused segments by interval', + + onAction: () => + this.setState({ + datasourceToMarkSegmentsByIntervalIn: datasource, + useUnuseAction: 'unuse', + }), + }, + { + icon: IconNames.IMPORT, + title: 'Mark as unused all segments', + intent: Intent.DANGER, + onAction: () => this.setState({ datasourceToMarkAsUnusedAllSegmentsIn: datasource }), + }, + { + icon: IconNames.TRASH, + title: 'Delete unused segments (issue kill task)', + intent: Intent.DANGER, + onAction: () => this.setState({ killDatasource: datasource }), + }, + ]), + ); } } private renderRetentionDialog(): JSX.Element | undefined { const { retentionDialogOpenOn, tiersState, datasourcesAndDefaultRulesState } = this.state; - const { defaultRules } = datasourcesAndDefaultRulesState.data || { - datasources: [], - defaultRules: [], - }; - if (!retentionDialogOpenOn) return; + const defaultRules = datasourcesAndDefaultRulesState.data?.defaultRules; + if (!retentionDialogOpenOn || !defaultRules) return; return ( !d.unused); @@ -1009,8 +1034,8 @@ ORDER BY 1`; const replicatedSizeValues = datasources.map(d => formatReplicatedSize(d.replicated_size)); const leftToBeCompactedValues = datasources.map(d => - d.compactionStatus - ? formatLeftToBeCompacted(d.compactionStatus.bytesAwaitingCompaction) + d.compaction?.status + ? formatLeftToBeCompacted(d.compaction?.status.bytesAwaitingCompaction) : '-', ); @@ -1297,24 +1322,26 @@ ORDER BY 1`; Header: 'Compaction', show: capabilities.hasCoordinatorAccess() && visibleColumns.shown('Compaction'), id: 'compactionStatus', - accessor: row => Boolean(row.compactionStatus), + accessor: row => Boolean(row.compaction?.status), filterable: false, width: 150, Cell: ({ original }) => { - const { datasource, compactionConfig, compactionStatus } = original as Datasource; + const { datasource, compaction } = original as Datasource; return ( + disabled={!compaction} + onClick={() => { + if (!compaction) return; this.setState({ compactionDialogOpenOn: { datasource, - compactionConfig, + compactionConfig: compaction.config, }, - }) - } + }); + }} hoverIcon={IconNames.EDIT} > - {formatCompactionConfigAndStatus(compactionConfig, compactionStatus)} + {compaction ? formatCompactionInfo(compaction) : 'Could not get compaction info'} ); }, @@ -1324,17 +1351,22 @@ ORDER BY 1`; show: capabilities.hasCoordinatorAccess() && visibleColumns.shown('% Compacted'), id: 'percentCompacted', width: 200, - accessor: ({ compactionStatus }) => - compactionStatus && compactionStatus.bytesCompacted - ? compactionStatus.bytesCompacted / - (compactionStatus.bytesAwaitingCompaction + compactionStatus.bytesCompacted) - : 0, + accessor: ({ compaction }) => { + const status = compaction?.status; + return status?.bytesCompacted + ? status.bytesCompacted / (status.bytesAwaitingCompaction + status.bytesCompacted) + : 0; + }, filterable: false, className: 'padded', Cell: ({ original }) => { - const { compactionStatus } = original as Datasource; + const { compaction } = original as Datasource; + if (!compaction) { + return 'Could not get compaction info'; + } - if (!compactionStatus || zeroCompactionStatus(compactionStatus)) { + const { status } = compaction; + if (!status || zeroCompactionStatus(status)) { return ( <>  {' '} @@ -1348,20 +1380,14 @@ ORDER BY 1`; <> {' '}  {' '} {' '} @@ -1369,8 +1395,8 @@ ORDER BY 1`; - (compactionStatus && compactionStatus.bytesAwaitingCompaction) || 0, + accessor: ({ compaction }) => { + const status = compaction?.status; + return status?.bytesAwaitingCompaction || 0; + }, filterable: false, className: 'padded', Cell: ({ original }) => { - const { compactionStatus } = original as Datasource; + const { compaction } = original as Datasource; + if (!compaction) { + return 'Could not get compaction info'; + } - if (!compactionStatus) { + const { status } = compaction; + if (!status) { return ; } return ( ); @@ -1408,26 +1440,30 @@ ORDER BY 1`; Header: 'Retention', show: capabilities.hasCoordinatorAccess() && visibleColumns.shown('Retention'), id: 'retention', - accessor: row => row.rules.length, + accessor: row => row.rules?.length || 0, filterable: false, width: 200, Cell: ({ original }) => { const { datasource, rules } = original as Datasource; return ( + disabled={!defaultRules} + onClick={() => { + if (!defaultRules) return; this.setState({ retentionDialogOpenOn: { datasource, - rules, + rules: rules || [], }, - }) - } + }); + }} hoverIcon={IconNames.EDIT} > - {rules.length + {rules?.length ? DatasourcesView.formatRules(rules) - : `Cluster default: ${DatasourcesView.formatRules(defaultRules)}`} + : defaultRules + ? `Cluster default: ${DatasourcesView.formatRules(defaultRules)}` + : 'Could not get default rules'} ); }, @@ -1440,12 +1476,12 @@ ORDER BY 1`; width: ACTION_COLUMN_WIDTH, filterable: false, Cell: ({ value: datasource, original }) => { - const { unused, rules, compactionConfig } = original as Datasource; + const { unused, rules, compaction } = original as Datasource; const datasourceActions = this.getDatasourceActions( datasource, unused, rules, - compactionConfig, + compaction, ); return ( 0) { return formatDuration(value); } diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx index aa2a934533e7..9dff93c8ab7d 100644 --- a/web-console/src/views/services-view/services-view.tsx +++ b/web-console/src/views/services-view/services-view.tsx @@ -36,11 +36,12 @@ import { import { AsyncActionDialog } from '../../dialogs'; import { QueryWithContext } from '../../druid-models'; import { STANDARD_TABLE_PAGE_SIZE, STANDARD_TABLE_PAGE_SIZE_OPTIONS } from '../../react-table'; -import { Api } from '../../singletons'; +import { Api, AppToaster } from '../../singletons'; import { Capabilities, CapabilitiesMode, deepGet, + filterMap, formatBytes, formatBytesCompact, hasPopoverOpen, @@ -117,7 +118,7 @@ export interface ServicesViewState { visibleColumns: LocalStorageBackedVisibility; } -interface ServiceQueryResultRow { +interface ServiceResultRow { readonly service: string; readonly service_type: string; readonly tier: string; @@ -127,16 +128,18 @@ interface ServiceQueryResultRow { readonly max_size: NumberLike; readonly plaintext_port: number; readonly tls_port: number; + loadQueueInfo?: LoadQueueInfo; + workerInfo?: WorkerInfo; } -interface LoadQueueStatus { +interface LoadQueueInfo { readonly segmentsToDrop: NumberLike; readonly segmentsToDropSize: NumberLike; readonly segmentsToLoad: NumberLike; readonly segmentsToLoadSize: NumberLike; } -interface MiddleManagerQueryResultRow { +interface WorkerInfo { readonly availabilityGroups: string[]; readonly blacklistedUntil: string | null; readonly currCapacityUsed: NumberLike; @@ -153,11 +156,6 @@ interface MiddleManagerQueryResultRow { }; } -interface ServiceResultRow - extends ServiceQueryResultRow, - Partial, - Partial {} - export class ServicesView extends React.PureComponent { private readonly serviceQueryManager: QueryManager; @@ -198,7 +196,7 @@ ORDER BY ) DESC, "service" DESC`; - static async getServices(): Promise { + static async getServices(): Promise { const allServiceResp = await Api.instance.get('/druid/coordinator/v1/servers?simple'); const allServices = allServiceResp.data; return allServices.map((s: any) => { @@ -228,7 +226,7 @@ ORDER BY this.serviceQueryManager = new QueryManager({ processQuery: async capabilities => { - let services: ServiceQueryResultRow[]; + let services: ServiceResultRow[]; if (capabilities.hasSql()) { services = await queryDruidSql({ query: ServicesView.SERVICE_SQL }); } else if (capabilities.hasCoordinatorAccess()) { @@ -238,50 +236,49 @@ ORDER BY } if (capabilities.hasCoordinatorAccess()) { - const loadQueueResponse = await Api.instance.get( - '/druid/coordinator/v1/loadqueue?simple', - ); - const loadQueues: Record = loadQueueResponse.data; - services = services.map(s => { - const loadQueueInfo = loadQueues[s.service]; - if (loadQueueInfo) { - s = { ...s, ...loadQueueInfo }; - } - return s; - }); + try { + const loadQueueInfos = ( + await Api.instance.get>( + '/druid/coordinator/v1/loadqueue?simple', + ) + ).data; + services.forEach(s => { + s.loadQueueInfo = loadQueueInfos[s.service]; + }); + } catch { + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'There was an error getting the load queue info', + }); + } } if (capabilities.hasOverlordAccess()) { - let middleManagers: MiddleManagerQueryResultRow[]; try { - const middleManagerResponse = await Api.instance.get('/druid/indexer/v1/workers'); - middleManagers = middleManagerResponse.data; + const workerInfos = (await Api.instance.get('/druid/indexer/v1/workers')) + .data; + + const workerInfoLookup: Record = lookupBy( + workerInfos, + m => m.worker?.host, + ); + + services.forEach(s => { + s.workerInfo = workerInfoLookup[s.service]; + }); } catch (e) { + // Swallow this error because it simply a reflection of a local task runner. if ( - e.response && - typeof e.response.data === 'object' && - e.response.data.error === 'Task Runner does not support worker listing' + deepGet(e, 'response.data.error') !== 'Task Runner does not support worker listing' ) { - // Swallow this error because it simply a reflection of a local task runner. - middleManagers = []; - } else { - // Otherwise re-throw. - throw e; + AppToaster.show({ + icon: IconNames.ERROR, + intent: Intent.DANGER, + message: 'There was an error getting the worker info', + }); } } - - const middleManagersLookup: Record = lookupBy( - middleManagers, - m => m.worker.host, - ); - - services = services.map(s => { - const middleManagerInfo = middleManagersLookup[s.service]; - if (middleManagerInfo) { - s = { ...s, ...middleManagerInfo }; - } - return s; - }); } return services; @@ -372,7 +369,8 @@ ORDER BY id: 'tier', width: 180, accessor: row => { - return row.tier ? row.tier : row.worker ? row.worker.category : null; + if (row.tier) return row.tier; + return deepGet(row, 'workerInfo.worker.category'); }, Cell: this.renderFilterableCell('tier'), }, @@ -451,9 +449,11 @@ ORDER BY className: 'padded', accessor: row => { if (oneOf(row.service_type, 'middle_manager', 'indexer')) { - return row.worker - ? (Number(row.currCapacityUsed) || 0) / Number(row.worker.capacity) - : null; + const { workerInfo } = row; + if (!workerInfo) return 0; + return ( + (Number(workerInfo.currCapacityUsed) || 0) / Number(workerInfo.worker?.capacity) + ); } else { return row.max_size ? Number(row.curr_size) / Number(row.max_size) : null; } @@ -469,15 +469,21 @@ ORDER BY case 'indexer': case 'middle_manager': { - const originalMiddleManagers: ServiceResultRow[] = row.subRows.map( - r => r._original, + const workerInfos: WorkerInfo[] = filterMap( + row.subRows, + r => r._original.workerInfo, ); + + if (!workerInfos.length) { + return 'Could not get worker infos'; + } + const totalCurrCapacityUsed = sum( - originalMiddleManagers, - s => Number(s.currCapacityUsed) || 0, + workerInfos, + w => Number(w.currCapacityUsed) || 0, ); const totalWorkerCapacity = sum( - originalMiddleManagers, + workerInfos, s => deepGet(s, 'worker.capacity') || 0, ); return `${totalCurrCapacityUsed} / ${totalWorkerCapacity} (total slots)`; @@ -496,8 +502,12 @@ ORDER BY case 'indexer': case 'middle_manager': { - const currCapacityUsed = deepGet(row, 'original.currCapacityUsed') || 0; - const capacity = deepGet(row, 'original.worker.capacity'); + if (!deepGet(row, 'original.workerInfo')) { + return 'Could not get capacity info'; + } + const currCapacityUsed = + deepGet(row, 'original.workerInfo.currCapacityUsed') || 0; + const capacity = deepGet(row, 'original.workerInfo.worker.capacity'); if (typeof capacity === 'number') { return `Slots used: ${currCapacityUsed} of ${capacity}`; } else { @@ -518,30 +528,58 @@ ORDER BY filterable: false, className: 'padded', accessor: row => { - if (oneOf(row.service_type, 'middle_manager', 'indexer')) { - if (deepGet(row, 'worker.version') === '') return 'Disabled'; + switch (row.service_type) { + case 'middle_manager': + case 'indexer': { + if (deepGet(row, 'worker.version') === '') return 'Disabled'; + const { workerInfo } = row; + if (!workerInfo) { + return 'Could not get detail info'; + } - const details: string[] = []; - if (row.lastCompletedTaskTime) { - details.push(`Last completed task: ${row.lastCompletedTaskTime}`); + const details: string[] = []; + if (workerInfo.lastCompletedTaskTime) { + details.push(`Last completed task: ${workerInfo.lastCompletedTaskTime}`); + } + if (workerInfo.blacklistedUntil) { + details.push(`Blacklisted until: ${workerInfo.blacklistedUntil}`); + } + return details.join(' '); } - if (row.blacklistedUntil) { - details.push(`Blacklisted until: ${row.blacklistedUntil}`); + + case 'coordinator': + case 'overlord': + return row.is_leader === 1 ? 'Leader' : ''; + + case 'historical': { + const { loadQueueInfo } = row; + if (!loadQueueInfo) return 0; + return ( + (Number(loadQueueInfo.segmentsToLoad) || 0) + + (Number(loadQueueInfo.segmentsToDrop) || 0) + ); } - return details.join(' '); - } else if (oneOf(row.service_type, 'coordinator', 'overlord')) { - return row.is_leader === 1 ? 'Leader' : ''; - } else { - return (Number(row.segmentsToLoad) || 0) + (Number(row.segmentsToDrop) || 0); + + default: + return 0; } }, Cell: row => { if (row.aggregated) return ''; const { service_type } = row.original; switch (service_type) { + case 'middle_manager': + case 'indexer': + case 'coordinator': + case 'overlord': + return row.value; + case 'historical': { + const { loadQueueInfo } = row.original; + if (!loadQueueInfo) return 'Could not get load queue info'; + const { segmentsToLoad, segmentsToLoadSize, segmentsToDrop, segmentsToDropSize } = - row.original; + loadQueueInfo; return formatQueues( segmentsToLoad, segmentsToLoadSize, @@ -550,23 +588,31 @@ ORDER BY ); } - case 'indexer': - case 'middle_manager': - case 'coordinator': - case 'overlord': - return row.value; - default: return ''; } }, Aggregated: row => { if (row.row._pivotVal !== 'historical') return ''; - const originals: ServiceResultRow[] = row.subRows.map(r => r._original); - const segmentsToLoad = sum(originals, s => Number(s.segmentsToLoad) || 0); - const segmentsToLoadSize = sum(originals, s => Number(s.segmentsToLoadSize) || 0); - const segmentsToDrop = sum(originals, s => Number(s.segmentsToDrop) || 0); - const segmentsToDropSize = sum(originals, s => Number(s.segmentsToDropSize) || 0); + const loadQueueInfos: LoadQueueInfo[] = filterMap( + row.subRows, + r => r._original.loadQueueInfo, + ); + + if (!loadQueueInfos.length) { + return 'Could not get load queue infos'; + } + + const segmentsToLoad = sum(loadQueueInfos, s => Number(s.segmentsToLoad) || 0); + const segmentsToLoadSize = sum( + loadQueueInfos, + s => Number(s.segmentsToLoadSize) || 0, + ); + const segmentsToDrop = sum(loadQueueInfos, s => Number(s.segmentsToDrop) || 0); + const segmentsToDropSize = sum( + loadQueueInfos, + s => Number(s.segmentsToDropSize) || 0, + ); return formatQueues( segmentsToLoad, segmentsToLoadSize, @@ -580,13 +626,14 @@ ORDER BY show: capabilities.hasOverlordAccess() && visibleColumns.shown(ACTION_COLUMN_LABEL), id: ACTION_COLUMN_ID, width: ACTION_COLUMN_WIDTH, - accessor: row => row.worker, + accessor: row => row.workerInfo, filterable: false, Cell: ({ value, aggregated }) => { if (aggregated) return ''; if (!value) return null; - const disabled = value.version === ''; - const workerActions = this.getWorkerActions(value.host, disabled); + const { worker } = value; + const disabled = worker.version === ''; + const workerActions = this.getWorkerActions(worker.host, disabled); return ; }, Aggregated: () => '', diff --git a/web-console/src/views/workbench-view/input-source-step/example-inputs.ts b/web-console/src/views/workbench-view/input-source-step/example-inputs.ts index e58dfacca35e..a74f1754b179 100644 --- a/web-console/src/views/workbench-view/input-source-step/example-inputs.ts +++ b/web-console/src/views/workbench-view/input-source-step/example-inputs.ts @@ -16,15 +16,74 @@ * limitations under the License. */ -import { InputSource } from '../../../druid-models'; +import { InputFormat, InputSource } from '../../../druid-models'; -export interface ExampleInputSource { +export interface ExampleInput { name: string; description: string; inputSource: InputSource; + inputFormat?: InputFormat; } -export const EXAMPLE_INPUT_SOURCES: ExampleInputSource[] = [ +const TRIPS_INPUT_FORMAT: InputFormat = { + type: 'csv', + findColumnsFromHeader: false, + columns: [ + 'trip_id', + 'vendor_id', + 'pickup_datetime', + 'dropoff_datetime', + 'store_and_fwd_flag', + 'rate_code_id', + 'pickup_longitude', + 'pickup_latitude', + 'dropoff_longitude', + 'dropoff_latitude', + 'passenger_count', + 'trip_distance', + 'fare_amount', + 'extra', + 'mta_tax', + 'tip_amount', + 'tolls_amount', + 'ehail_fee', + 'improvement_surcharge', + 'total_amount', + 'payment_type', + 'trip_type', + 'pickup', + 'dropoff', + 'cab_type', + 'precipitation', + 'snow_depth', + 'snowfall', + 'max_temperature', + 'min_temperature', + 'average_wind_speed', + 'pickup_nyct2010_gid', + 'pickup_ctlabel', + 'pickup_borocode', + 'pickup_boroname', + 'pickup_ct2010', + 'pickup_boroct2010', + 'pickup_cdeligibil', + 'pickup_ntacode', + 'pickup_ntaname', + 'pickup_puma', + 'dropoff_nyct2010_gid', + 'dropoff_ctlabel', + 'dropoff_borocode', + 'dropoff_boroname', + 'dropoff_ct2010', + 'dropoff_boroct2010', + 'dropoff_cdeligibil', + 'dropoff_ntacode', + 'dropoff_ntaname', + 'dropoff_puma', + ], +}; + +export const EXAMPLE_INPUTS: ExampleInput[] = [ { name: 'Wikipedia', description: 'One day of wikipedia edits (JSON)', @@ -62,6 +121,7 @@ export const EXAMPLE_INPUT_SOURCES: ExampleInputSource[] = [ 'https://static.imply.io/example-data/trips/trips_xac.csv.gz', ], }, + inputFormat: TRIPS_INPUT_FORMAT, }, { name: 'NYC Taxi cabs (all files)', @@ -145,6 +205,7 @@ export const EXAMPLE_INPUT_SOURCES: ExampleInputSource[] = [ 'https://static.imply.io/example-data/trips/trips_xcv.csv.gz', ], }, + inputFormat: TRIPS_INPUT_FORMAT, }, { name: 'FlightCarrierOnTime (1 month)', diff --git a/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx b/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx index 211271c62c3d..f144e8f975d5 100644 --- a/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx +++ b/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx @@ -55,7 +55,7 @@ import { UrlBaser } from '../../../singletons'; import { filterMap, IntermediateQueryState } from '../../../utils'; import { postToSampler, SampleSpec } from '../../../utils/sampler'; -import { EXAMPLE_INPUT_SOURCES } from './example-inputs'; +import { EXAMPLE_INPUTS } from './example-inputs'; import { InputSourceInfo } from './input-source-info'; import './input-source-step.scss'; @@ -81,16 +81,15 @@ export const InputSourceStep = React.memo(function InputSourceStep(props: InputS const [inputSource, setInputSource] = useState | string | undefined>( initInputSource, ); - const exampleInputSource = EXAMPLE_INPUT_SOURCES.find( - ({ name }) => name === inputSource, - )?.inputSource; + const exampleInput = EXAMPLE_INPUTS.find(({ name }) => name === inputSource); const [guessedInputFormatState, connectQueryManager] = useQueryManager< - InputSource, + { inputSource: InputSource; suggestedInputFormat?: InputFormat }, InputFormat, Execution >({ - processQuery: async (inputSource: InputSource, cancelToken) => { + processQuery: async ({ inputSource, suggestedInputFormat }, cancelToken) => { + let guessedInputFormat: InputFormat | undefined; if (mode === 'sampler') { const sampleSpec: SampleSpec = { type: 'index_parallel', @@ -127,7 +126,7 @@ export const InputSourceStep = React.memo(function InputSourceStep(props: InputS ); if (!sampleLines.length) throw new Error('No data returned from sampler'); - return guessInputFormat(sampleLines); + guessedInputFormat = guessInputFormat(sampleLines); } else { const tableExpression = externalConfigToTableExpression({ inputSource, @@ -151,8 +150,14 @@ export const InputSourceStep = React.memo(function InputSourceStep(props: InputS ); if (result instanceof IntermediateQueryState) return result; - return resultToInputFormat(result); + guessedInputFormat = resultToInputFormat(result); } + + if (suggestedInputFormat?.type === guessedInputFormat.type) { + return suggestedInputFormat; + } + + return guessedInputFormat; }, backgroundStatusCheck: async (execution, query, cancelToken) => { const result = await executionBackgroundResultStatusCheck(execution, query, cancelToken); @@ -164,7 +169,7 @@ export const InputSourceStep = React.memo(function InputSourceStep(props: InputS useEffect(() => { const guessedInputFormat = guessedInputFormatState.data; if (!guessedInputFormat) return; - onSet(exampleInputSource || (inputSource as any), guessedInputFormat); + onSet(exampleInput?.inputSource || (inputSource as any), guessedInputFormat); // eslint-disable-next-line react-hooks/exhaustive-deps }, [guessedInputFormatState]); @@ -217,7 +222,7 @@ export const InputSourceStep = React.memo(function InputSourceStep(props: InputS selectedValue={inputSource} onChange={e => setInputSource(e.currentTarget.value)} > - {EXAMPLE_INPUT_SOURCES.map((e, i) => ( + {EXAMPLE_INPUTS.map((e, i) => ( { - if (!exampleInputSource) return; - connectQueryManager.runQuery(exampleInputSource); + if (!exampleInput) return; + connectQueryManager.runQuery({ + inputSource: exampleInput.inputSource, + suggestedInputFormat: exampleInput.inputFormat, + }); }} /> ) : inputSource ? ( @@ -324,7 +332,7 @@ export const InputSourceStep = React.memo(function InputSourceStep(props: InputS } onClick={() => { if (!AutoForm.isValidModel(inputSource, INPUT_SOURCE_FIELDS)) return; - connectQueryManager.runQuery(inputSource); + connectQueryManager.runQuery({ inputSource }); }} /> ) : undefined} diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.tsx b/web-console/src/views/workbench-view/run-panel/run-panel.tsx index 572120c9e730..7299760b4621 100644 --- a/web-console/src/views/workbench-view/run-panel/run-panel.tsx +++ b/web-console/src/views/workbench-view/run-panel/run-panel.tsx @@ -33,6 +33,7 @@ import React, { useCallback, useMemo, useState } from 'react'; import { MenuCheckbox, MenuTristate } from '../../../components'; import { EditContextDialog, StringInputDialog } from '../../../dialogs'; +import { IndexSpecDialog } from '../../../dialogs/index-spec-dialog/index-spec-dialog'; import { changeDurableShuffleStorage, changeFinalizeAggregations, @@ -51,9 +52,12 @@ import { getUseApproximateCountDistinct, getUseApproximateTopN, getUseCache, + IndexSpec, + QueryContext, + summarizeIndexSpec, WorkbenchQuery, } from '../../../druid-models'; -import { pluralIfNeeded, tickIcon } from '../../../utils'; +import { deepGet, pluralIfNeeded, tickIcon } from '../../../utils'; import { MaxTasksButton } from '../max-tasks-button/max-tasks-button'; import './run-panel.scss'; @@ -94,6 +98,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { const { query, onQueryChange, onRun, moreMenu, loading, small, queryEngines } = props; const [editContextDialogOpen, setEditContextDialogOpen] = useState(false); const [customTimezoneDialogOpen, setCustomTimezoneDialogOpen] = useState(false); + const [indexSpecDialogSpec, setIndexSpecDialogSpec] = useState(); const emptyQuery = query.isEmptyQuery(); const ingestMode = query.isIngestQuery(); @@ -104,6 +109,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { const finalizeAggregations = getFinalizeAggregations(queryContext); const groupByEnableMultiValueUnnesting = getGroupByEnableMultiValueUnnesting(queryContext); const durableShuffleStorage = getDurableShuffleStorage(queryContext); + const indexSpec: IndexSpec | undefined = deepGet(queryContext, 'indexSpec'); const useApproximateCountDistinct = getUseApproximateCountDistinct(queryContext); const useApproximateTopN = getUseApproximateTopN(queryContext); const useCache = getUseCache(queryContext); @@ -157,6 +163,10 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { ); } + function changeQueryContext(queryContext: QueryContext) { + onQueryChange(query.changeQueryContext(queryContext)); + } + const availableEngines = ([undefined] as (DruidEngine | undefined)[]).concat(queryEngines); function offsetOptions(): JSX.Element[] { @@ -170,9 +180,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { icon={tickIcon(offset === timezone)} text={offset} shouldDismissPopover={false} - onClick={() => { - onQueryChange(query.changeQueryContext(changeTimezone(queryContext, offset))); - }} + onClick={() => changeQueryContext(changeTimezone(queryContext, offset))} />, ); } @@ -233,11 +241,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { icon={tickIcon(!timezone)} text="Default" shouldDismissPopover={false} - onClick={() => { - onQueryChange( - query.changeQueryContext(changeTimezone(queryContext, undefined)), - ); - }} + onClick={() => changeQueryContext(changeTimezone(queryContext, undefined))} /> {NAMED_TIMEZONES.map(namedTimezone => ( @@ -246,11 +250,9 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { icon={tickIcon(namedTimezone === timezone)} text={namedTimezone} shouldDismissPopover={false} - onClick={() => { - onQueryChange( - query.changeQueryContext(changeTimezone(queryContext, namedTimezone)), - ); - }} + onClick={() => + changeQueryContext(changeTimezone(queryContext, namedTimezone)) + } /> ))} @@ -276,11 +278,9 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { key={String(v)} icon={tickIcon(v === maxParseExceptions)} text={v === -1 ? '∞ (-1)' : String(v)} - onClick={() => { - onQueryChange( - query.changeQueryContext(changeMaxParseExceptions(queryContext, v)), - ); - }} + onClick={() => + changeQueryContext(changeMaxParseExceptions(queryContext, v)) + } shouldDismissPopover={false} /> ))} @@ -290,35 +290,36 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { text="Finalize aggregations" value={finalizeAggregations} undefinedEffectiveValue={!ingestMode} - onValueChange={v => { - onQueryChange( - query.changeQueryContext(changeFinalizeAggregations(queryContext, v)), - ); - }} + onValueChange={v => + changeQueryContext(changeFinalizeAggregations(queryContext, v)) + } /> { - onQueryChange( - query.changeQueryContext( - changeGroupByEnableMultiValueUnnesting(queryContext, v), - ), - ); + onValueChange={v => + changeQueryContext(changeGroupByEnableMultiValueUnnesting(queryContext, v)) + } + /> + { + setIndexSpecDialogSpec(indexSpec || {}); }} /> { - onQueryChange( - query.changeQueryContext( - changeDurableShuffleStorage(queryContext, !durableShuffleStorage), - ), - ); - }} + onChange={() => + changeQueryContext( + changeDurableShuffleStorage(queryContext, !durableShuffleStorage), + ) + } /> ) : ( @@ -326,22 +327,16 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { { - onQueryChange( - query.changeQueryContext(changeUseCache(queryContext, !useCache)), - ); - }} + onChange={() => changeQueryContext(changeUseCache(queryContext, !useCache))} /> { - onQueryChange( - query.changeQueryContext( - changeUseApproximateTopN(queryContext, !useApproximateTopN), - ), - ); - }} + onChange={() => + changeQueryContext( + changeUseApproximateTopN(queryContext, !useApproximateTopN), + ) + } /> )} @@ -349,16 +344,14 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { { - onQueryChange( - query.changeQueryContext( - changeUseApproximateCountDistinct( - queryContext, - !useApproximateCountDistinct, - ), + onChange={() => + changeQueryContext( + changeUseApproximateCountDistinct( + queryContext, + !useApproximateCountDistinct, ), - ); - }} + ) + } /> )} {effectiveEngine === 'sql-msq-task' && ( - - onQueryChange(query.changeQueryContext(queryContext)) - } - /> + )} )} @@ -399,10 +387,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { {editContextDialogOpen && ( { - if (!onQueryChange) return; - onQueryChange(query.changeQueryContext(newContext)); - }} + onQueryContextChange={changeQueryContext} onClose={() => { setEditContextDialogOpen(false); }} @@ -413,10 +398,17 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { title="Custom timezone" placeholder="Etc/UTC" maxLength={50} - onSubmit={tz => onQueryChange(query.changeQueryContext(changeTimezone(queryContext, tz)))} + onSubmit={tz => changeQueryContext(changeTimezone(queryContext, tz))} onClose={() => setCustomTimezoneDialogOpen(false)} /> )} + {indexSpecDialogSpec && ( + setIndexSpecDialogSpec(undefined)} + onSave={indexSpec => changeQueryContext({ ...queryContext, indexSpec })} + indexSpec={indexSpecDialogSpec} + /> + )}
); });