From 8c235208e6358ee0659029be198578bab75d5f37 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Thu, 3 Dec 2020 22:01:05 -0800 Subject: [PATCH 01/14] no need for intervals --- .../component/load-data/config/partition.ts | 1 - .../component/load-data/data-loader.ts | 5 +- web-console/e2e-tests/reindexing.spec.ts | 1 - web-console/e2e-tests/tutorial-batch.spec.ts | 1 - .../__snapshots__/auto-form.spec.tsx.snap | 12 +- .../src/components/auto-form/auto-form.tsx | 17 +-- .../numeric-input-with-default.spec.tsx.snap | 19 +++ .../numeric-input-with-default.spec.tsx | 30 ++++ .../numeric-input-with-default.tsx | 51 +++++++ .../src/druid-models/ingestion-spec.tsx | 130 +++++++++++------- .../views/load-data-view/load-data-view.tsx | 41 ++---- 11 files changed, 198 insertions(+), 110 deletions(-) create mode 100644 web-console/src/components/numeric-input-with-default/__snapshots__/numeric-input-with-default.spec.tsx.snap create mode 100644 web-console/src/components/numeric-input-with-default/numeric-input-with-default.spec.tsx create mode 100644 web-console/src/components/numeric-input-with-default/numeric-input-with-default.tsx diff --git a/web-console/e2e-tests/component/load-data/config/partition.ts b/web-console/e2e-tests/component/load-data/config/partition.ts index 375ad550745f..14197bd86533 100644 --- a/web-console/e2e-tests/component/load-data/config/partition.ts +++ b/web-console/e2e-tests/component/load-data/config/partition.ts @@ -164,7 +164,6 @@ export class PartitionConfig { interface PartitionConfigProps { readonly segmentGranularity: SegmentGranularity; - readonly timeIntervals: string | null; readonly partitionsSpec: PartitionsSpec | null; } diff --git a/web-console/e2e-tests/component/load-data/data-loader.ts b/web-console/e2e-tests/component/load-data/data-loader.ts index df16e7158753..8ee50a07ac3a 100644 --- a/web-console/e2e-tests/component/load-data/data-loader.ts +++ b/web-console/e2e-tests/component/load-data/data-loader.ts @@ -18,7 +18,7 @@ import * as playwright from 'playwright-chromium'; -import { clickButton, setLabeledInput, setLabeledTextarea } from '../../util/playwright'; +import { clickButton, setLabeledInput } from '../../util/playwright'; import { ConfigureSchemaConfig } from './config/configure-schema'; import { PartitionConfig } from './config/partition'; @@ -125,9 +125,6 @@ export class DataLoader { private async applyPartitionConfig(partitionConfig: PartitionConfig) { await setLabeledInput(this.page, 'Segment granularity', partitionConfig.segmentGranularity); - if (partitionConfig.timeIntervals) { - await setLabeledTextarea(this.page, 'Time intervals', partitionConfig.timeIntervals); - } if (partitionConfig.partitionsSpec != null) { await partitionConfig.partitionsSpec.apply(this.page); } diff --git a/web-console/e2e-tests/reindexing.spec.ts b/web-console/e2e-tests/reindexing.spec.ts index ae45b735965f..a935373b7833 100644 --- a/web-console/e2e-tests/reindexing.spec.ts +++ b/web-console/e2e-tests/reindexing.spec.ts @@ -67,7 +67,6 @@ describe('Reindexing from Druid', () => { const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false }); const partitionConfig = new PartitionConfig({ segmentGranularity: SegmentGranularity.DAY, - timeIntervals: interval, partitionsSpec: new SingleDimPartitionsSpec({ partitionDimension: 'channel', targetRowsPerSegment: 10_000, diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts b/web-console/e2e-tests/tutorial-batch.spec.ts index f4fa45054632..2f4e06b8011a 100644 --- a/web-console/e2e-tests/tutorial-batch.spec.ts +++ b/web-console/e2e-tests/tutorial-batch.spec.ts @@ -64,7 +64,6 @@ describe('Tutorial: Loading a file', () => { const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false }); const partitionConfig = new PartitionConfig({ segmentGranularity: SegmentGranularity.DAY, - timeIntervals: null, partitionsSpec: null, }); const publishConfig = new PublishConfig({ datasourceName: datasourceName }); diff --git a/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap b/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap index baf1011bdc8e..02a63a1b1074 100644 --- a/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap +++ b/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap @@ -8,23 +8,13 @@ exports[`AutoForm matches snapshot 1`] = ` key="testOne" label="Test one" > - > extends React.PureComponent private renderNumberInput(field: Field): JSX.Element { const { model, large, onFinalize } = this.props; - let modelValue = deepGet(model as any, field.name); - if (typeof modelValue !== 'number') modelValue = field.defaultValue; + const modelValue = deepGet(model as any, field.name); return ( - { - if (valueAsString === '' || isNaN(valueAsNumber)) return; - this.fieldChange( - field, - valueAsNumber === 0 && field.zeroMeansUndefined ? undefined : valueAsNumber, - ); + let newValue: number | undefined; + if (valueAsString !== '' && !isNaN(valueAsNumber)) { + newValue = valueAsNumber === 0 && field.zeroMeansUndefined ? undefined : valueAsNumber; + } + this.fieldChange(field, newValue); }} onBlur={e => { if (e.target.value === '') { diff --git a/web-console/src/components/numeric-input-with-default/__snapshots__/numeric-input-with-default.spec.tsx.snap b/web-console/src/components/numeric-input-with-default/__snapshots__/numeric-input-with-default.spec.tsx.snap new file mode 100644 index 000000000000..75335f79dcb5 --- /dev/null +++ b/web-console/src/components/numeric-input-with-default/__snapshots__/numeric-input-with-default.spec.tsx.snap @@ -0,0 +1,19 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`NumericInputWithDefault matches snapshot 1`] = ` + +`; diff --git a/web-console/src/components/numeric-input-with-default/numeric-input-with-default.spec.tsx b/web-console/src/components/numeric-input-with-default/numeric-input-with-default.spec.tsx new file mode 100644 index 000000000000..b0d2b61d2ce9 --- /dev/null +++ b/web-console/src/components/numeric-input-with-default/numeric-input-with-default.spec.tsx @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { shallow } from 'enzyme'; +import React from 'react'; + +import { NumericInputWithDefault } from './numeric-input-with-default'; + +describe('NumericInputWithDefault', () => { + it('matches snapshot', () => { + const numericInputWithDefault = shallow(); + + expect(numericInputWithDefault).toMatchSnapshot(); + }); +}); diff --git a/web-console/src/components/numeric-input-with-default/numeric-input-with-default.tsx b/web-console/src/components/numeric-input-with-default/numeric-input-with-default.tsx new file mode 100644 index 000000000000..f4d0b4e07990 --- /dev/null +++ b/web-console/src/components/numeric-input-with-default/numeric-input-with-default.tsx @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { HTMLInputProps, INumericInputProps, NumericInput } from '@blueprintjs/core'; +import React, { useState } from 'react'; + +export type NumericInputWithDefaultProps = HTMLInputProps & INumericInputProps; + +export const NumericInputWithDefault = React.memo(function NumericInputWithDefault( + props: NumericInputWithDefaultProps, +) { + const { value, defaultValue, onValueChange, onBlur, ...rest } = props; + const [hasChanged, setHasChanged] = useState(false); + + let effectiveValue = value; + if (effectiveValue == null) { + effectiveValue = hasChanged ? '' : defaultValue || ''; + } + + return ( + { + setHasChanged(true); + if (!onValueChange) return; + return onValueChange(valueAsNumber, valueAsString, inputElement); + }} + onBlur={e => { + setHasChanged(false); + if (!onBlur) return; + return onBlur(e); + }} + {...rest} + /> + ); +}); diff --git a/web-console/src/druid-models/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx index baa8b50729ea..bfae30420c1d 100644 --- a/web-console/src/druid-models/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec.tsx @@ -1340,33 +1340,31 @@ export interface PartitionsSpec { assumeGrouped?: boolean; } -export function adjustTuningConfig(tuningConfig: TuningConfig) { - const tuningConfigType = deepGet(tuningConfig, 'type'); - if (tuningConfigType !== 'index_parallel') return tuningConfig; +export function adjustTuningConfig(spec: IngestionSpec) { + const tuningConfigType = deepGet(spec, 'spec.tuningConfig.type'); + if (tuningConfigType !== 'index_parallel') return spec; - const partitionsSpecType = deepGet(tuningConfig, 'partitionsSpec.type') || 'dynamic'; + const partitionsSpecType = deepGet(spec, 'spec.tuningConfig.partitionsSpec.type') || 'dynamic'; if (partitionsSpecType === 'dynamic') { - tuningConfig = deepDelete(tuningConfig, 'forceGuaranteedRollup'); + spec = deepDelete(spec, 'spec.tuningConfig.forceGuaranteedRollup'); } else if (oneOf(partitionsSpecType, 'hashed', 'single_dim')) { - tuningConfig = deepSet(tuningConfig, 'forceGuaranteedRollup', true); + spec = deepSet(spec, 'spec.tuningConfig.forceGuaranteedRollup', true); } - return tuningConfig; + return spec; } -export function invalidTuningConfig(tuningConfig: TuningConfig, intervals: any): boolean { +export function invalidTuningConfig(tuningConfig: TuningConfig): boolean { if (tuningConfig.type !== 'index_parallel') return false; switch (deepGet(tuningConfig, 'partitionsSpec.type')) { case 'hashed': - if (!intervals) return true; return ( Boolean(deepGet(tuningConfig, 'partitionsSpec.targetRowsPerSegment')) && Boolean(deepGet(tuningConfig, 'partitionsSpec.numShards')) ); case 'single_dim': - if (!intervals) return true; if (!deepGet(tuningConfig, 'partitionsSpec.partitionDimension')) return true; const hasTargetRowsPerSegment = Boolean( deepGet(tuningConfig, 'partitionsSpec.targetRowsPerSegment'), @@ -1383,14 +1381,15 @@ export function invalidTuningConfig(tuningConfig: TuningConfig, intervals: any): } export function getPartitionRelatedTuningSpecFormFields( - specType: IngestionType, + spec: IngestionSpec, dimensionSuggestions: string[] | undefined, -): Field[] { +): Field[] { + const specType = getSpecType(spec) || 'index_parallel'; switch (specType) { case 'index_parallel': - return [ + const parallelFields: Field[] = [ { - name: 'partitionsSpec.type', + name: 'spec.tuningConfig.partitionsSpec.type', label: 'Partitioning type', type: 'string', required: true, @@ -1402,38 +1401,42 @@ export function getPartitionRelatedTuningSpecFormFields( single dimension). For best-effort rollup, you should use dynamic.

), - adjustment: (t: TuningConfig) => { - if (!Array.isArray(dimensionSuggestions) || !dimensionSuggestions.length) return t; - return deepSet(t, 'partitionsSpec.partitionDimension', dimensionSuggestions[0]); + adjustment: s => { + if (!Array.isArray(dimensionSuggestions) || !dimensionSuggestions.length) return s; + return deepSet( + s, + 'spec.tuningConfig.partitionsSpec.partitionDimension', + dimensionSuggestions[0], + ); }, }, // partitionsSpec type: dynamic { - name: 'partitionsSpec.maxRowsPerSegment', + name: 'spec.tuningConfig.partitionsSpec.maxRowsPerSegment', label: 'Max rows per segment', type: 'number', defaultValue: 5000000, - defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'dynamic', + defined: s => deepGet(s, 'spec.tuningConfig.partitionsSpec.type') === 'dynamic', info: <>Determines how many rows are in each segment., }, { - name: 'partitionsSpec.maxTotalRows', + name: 'spec.tuningConfig.partitionsSpec.maxTotalRows', label: 'Max total rows', type: 'number', defaultValue: 20000000, - defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'dynamic', + defined: s => deepGet(s, 'spec.tuningConfig.partitionsSpec.type') === 'dynamic', info: <>Total number of rows in segments waiting for being pushed., }, // partitionsSpec type: hashed { - name: 'partitionsSpec.targetRowsPerSegment', + name: 'spec.tuningConfig.partitionsSpec.targetRowsPerSegment', label: 'Target rows per segment', type: 'number', zeroMeansUndefined: true, defaultValue: 5000000, - defined: (t: TuningConfig) => - deepGet(t, 'partitionsSpec.type') === 'hashed' && - !deepGet(t, 'partitionsSpec.numShards'), + defined: s => + deepGet(s, 'spec.tuningConfig.partitionsSpec.type') === 'hashed' && + !deepGet(s, 'spec.tuningConfig.partitionsSpec.numShards'), info: ( <>

@@ -1449,14 +1452,14 @@ export function getPartitionRelatedTuningSpecFormFields( ), }, { - name: 'partitionsSpec.numShards', + name: 'spec.tuningConfig.partitionsSpec.numShards', label: 'Num shards', type: 'number', zeroMeansUndefined: true, hideInMore: true, - defined: (t: TuningConfig) => - deepGet(t, 'partitionsSpec.type') === 'hashed' && - !deepGet(t, 'partitionsSpec.targetRowsPerSegment'), + defined: s => + deepGet(s, 'spec.tuningConfig.partitionsSpec.type') === 'hashed' && + !deepGet(s, 'spec.tuningConfig.partitionsSpec.targetRowsPerSegment'), info: ( <>

@@ -1472,19 +1475,19 @@ export function getPartitionRelatedTuningSpecFormFields( ), }, { - name: 'partitionsSpec.partitionDimensions', + name: 'spec.tuningConfig.partitionsSpec.partitionDimensions', label: 'Partition dimensions', type: 'string-array', placeholder: '(all dimensions)', - defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'hashed', + defined: s => deepGet(s, 'spec.tuningConfig.partitionsSpec.type') === 'hashed', info:

The dimensions to partition on. Leave blank to select all dimensions.

, }, // partitionsSpec type: single_dim { - name: 'partitionsSpec.partitionDimension', + name: 'spec.tuningConfig.partitionsSpec.partitionDimension', label: 'Partition dimension', type: 'string', - defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'single_dim', + defined: s => deepGet(s, 'spec.tuningConfig.partitionsSpec.type') === 'single_dim', required: true, suggestions: dimensionSuggestions, info: ( @@ -1501,16 +1504,16 @@ export function getPartitionRelatedTuningSpecFormFields( ), }, { - name: 'partitionsSpec.targetRowsPerSegment', + name: 'spec.tuningConfig.partitionsSpec.targetRowsPerSegment', label: 'Target rows per segment', type: 'number', zeroMeansUndefined: true, - defined: (t: TuningConfig) => - deepGet(t, 'partitionsSpec.type') === 'single_dim' && - !deepGet(t, 'partitionsSpec.maxRowsPerSegment'), - required: (t: TuningConfig) => - !deepGet(t, 'partitionsSpec.targetRowsPerSegment') && - !deepGet(t, 'partitionsSpec.maxRowsPerSegment'), + defined: s => + deepGet(s, 'spec.tuningConfig.partitionsSpec.type') === 'single_dim' && + !deepGet(s, 'spec.tuningConfig.partitionsSpec.maxRowsPerSegment'), + required: s => + !deepGet(s, 'spec.tuningConfig.partitionsSpec.targetRowsPerSegment') && + !deepGet(s, 'spec.tuningConfig.partitionsSpec.maxRowsPerSegment'), info: (

Target number of rows to include in a partition, should be a number that targets @@ -1519,24 +1522,25 @@ export function getPartitionRelatedTuningSpecFormFields( ), }, { - name: 'partitionsSpec.maxRowsPerSegment', + name: 'spec.tuningConfig.partitionsSpec.maxRowsPerSegment', label: 'Max rows per segment', type: 'number', zeroMeansUndefined: true, - defined: (t: TuningConfig) => - deepGet(t, 'partitionsSpec.type') === 'single_dim' && - !deepGet(t, 'partitionsSpec.targetRowsPerSegment'), - required: (t: TuningConfig) => - !deepGet(t, 'partitionsSpec.targetRowsPerSegment') && - !deepGet(t, 'partitionsSpec.maxRowsPerSegment'), + defined: s => + deepGet(s, 'spec.tuningConfig.partitionsSpec.type') === 'single_dim' && + !deepGet(s, 'spec.tuningConfig.partitionsSpec.targetRowsPerSegment'), + required: s => + !deepGet(s, 'spec.tuningConfig.partitionsSpec.targetRowsPerSegment') && + !deepGet(s, 'spec.tuningConfig.partitionsSpec.maxRowsPerSegment'), info:

Maximum number of rows to include in a partition.

, }, { - name: 'partitionsSpec.assumeGrouped', + name: 'spec.tuningConfig.partitionsSpec.assumeGrouped', label: 'Assume grouped', type: 'boolean', defaultValue: false, - defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'single_dim', + hideInMore: true, + defined: s => deepGet(s, 'spec.tuningConfig.partitionsSpec.type') === 'single_dim', info: (

Assume that input data has already been grouped on time and dimensions. Ingestion will @@ -1546,17 +1550,41 @@ export function getPartitionRelatedTuningSpecFormFields( }, ]; + if (oneOf(deepGet(spec, 'spec.tuningConfig.partitionsSpec.type'), 'hashed', 'single_dim')) { + parallelFields.push({ + name: 'spec.dataSchema.granularitySpec.intervals', + label: 'Time intervals', + type: 'string-array', + placeholder: 'ex: 2018-01-01/2018-06-01', + hideInMore: true, + info: ( + <> +

A comma separated list of intervals for the raw data being ingested.

+

+ This list is used to determine the shards that will be created. If it is not + specified then then an additional job will run to automatically determine the data + intervals used. +

+ + ), + }); + } + + return parallelFields; + case 'kafka': case 'kinesis': return [ { - name: 'maxRowsPerSegment', + name: 'spec.tuningConfig.maxRowsPerSegment', + label: 'Max rows per segment', type: 'number', defaultValue: 5000000, info: <>Determines how many rows are in each segment., }, { - name: 'maxTotalRows', + name: 'spec.tuningConfig.maxTotalRows', + label: 'Max total rows', type: 'number', defaultValue: 20000000, info: <>Total number of rows in segments waiting for being pushed., diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 75b427a3e7de..31bb54ca4970 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -2214,7 +2214,7 @@ export class LoadDataView extends React.PureComponent this.updateSpec(s)} + onChange={this.updateSpec} /> this.updateSpec(s)} + onChange={this.updateSpec} /> )} this.updateSpec(deepSet(spec, 'spec.dataSchema.granularitySpec', g))} /> - {!isStreaming && ( - - ['hashed', 'single_dim'].includes( - deepGet(spec, 'spec.tuningConfig.partitionsSpec.type'), - ), - info: <>A comma separated list of intervals for the raw data being ingested., - }, - ]} - model={spec} - onChange={s => this.updateSpec(s)} - /> - )}
Secondary partitioning
this.updateSpec(deepSet(spec, 'spec.tuningConfig', t))} + onChange={this.updateSpec} />
@@ -3048,9 +3025,7 @@ export class LoadDataView extends React.PureComponent {this.renderNextBar({ - disabled: - !granularitySpec.segmentGranularity || - invalidTuningConfig(tuningConfig, granularitySpec.intervals), + disabled: !granularitySpec.segmentGranularity || invalidTuningConfig(tuningConfig), })} ); @@ -3148,7 +3123,7 @@ export class LoadDataView extends React.PureComponent this.updateSpec(s)} + onChange={this.updateSpec} />
@@ -3202,7 +3177,7 @@ export class LoadDataView extends React.PureComponent this.updateSpec(s)} + onChange={this.updateSpec} />
From 14b8f7f00040271e36f115b27944b39a72ef74bf Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Sun, 6 Dec 2020 15:26:27 -0800 Subject: [PATCH 02/14] don't set redundant fields --- .../src/druid-models/ingestion-spec.tsx | 88 +++++++++++-------- web-console/src/utils/object-change.ts | 5 ++ .../views/load-data-view/load-data-view.tsx | 63 ++++--------- 3 files changed, 73 insertions(+), 83 deletions(-) diff --git a/web-console/src/druid-models/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx index bfae30420c1d..2a93825b9aab 100644 --- a/web-console/src/druid-models/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec.tsx @@ -19,13 +19,14 @@ import { Code } from '@blueprintjs/core'; import React from 'react'; -import { ExternalLink, Field } from '../components'; +import { AutoForm, ExternalLink, Field } from '../components'; import { getLink } from '../links'; import { deepDelete, deepGet, deepMove, deepSet, + deepSetIfUnset, EMPTY_ARRAY, EMPTY_OBJECT, filterMap, @@ -290,11 +291,9 @@ export function normalizeSpec(spec: Partial): IngestionSpec { deepGet(spec, 'spec.tuningConfig.type'); if (!specType) return spec as IngestionSpec; - if (!deepGet(spec, 'type')) spec = deepSet(spec, 'type', specType); - if (!deepGet(spec, 'spec.ioConfig.type')) spec = deepSet(spec, 'spec.ioConfig.type', specType); - if (!deepGet(spec, 'spec.tuningConfig.type')) { - spec = deepSet(spec, 'spec.tuningConfig.type', specType); - } + spec = deepSetIfUnset(spec, 'type', specType); + spec = deepSetIfUnset(spec, 'spec.ioConfig.type', specType); + spec = deepSetIfUnset(spec, 'spec.tuningConfig.type', specType); return spec as IngestionSpec; } @@ -1340,9 +1339,8 @@ export interface PartitionsSpec { assumeGrouped?: boolean; } -export function adjustTuningConfig(spec: IngestionSpec) { - const tuningConfigType = deepGet(spec, 'spec.tuningConfig.type'); - if (tuningConfigType !== 'index_parallel') return spec; +export function adjustForceGuaranteedRollup(spec: IngestionSpec) { + if (getSpecType(spec) !== 'index_parallel') return spec; const partitionsSpecType = deepGet(spec, 'spec.tuningConfig.partitionsSpec.type') || 'dynamic'; if (partitionsSpecType === 'dynamic') { @@ -1354,37 +1352,38 @@ export function adjustTuningConfig(spec: IngestionSpec) { return spec; } -export function invalidTuningConfig(tuningConfig: TuningConfig): boolean { - if (tuningConfig.type !== 'index_parallel') return false; - - switch (deepGet(tuningConfig, 'partitionsSpec.type')) { - case 'hashed': - return ( - Boolean(deepGet(tuningConfig, 'partitionsSpec.targetRowsPerSegment')) && - Boolean(deepGet(tuningConfig, 'partitionsSpec.numShards')) - ); - - case 'single_dim': - if (!deepGet(tuningConfig, 'partitionsSpec.partitionDimension')) return true; - const hasTargetRowsPerSegment = Boolean( - deepGet(tuningConfig, 'partitionsSpec.targetRowsPerSegment'), - ); - const hasMaxRowsPerSegment = Boolean( - deepGet(tuningConfig, 'partitionsSpec.maxRowsPerSegment'), - ); - if (hasTargetRowsPerSegment === hasMaxRowsPerSegment) { - return true; - } - } - - return false; +export function invalidPartitionConfig(spec: IngestionSpec): boolean { + return ( + // Bad primary partitioning, or... + !deepGet(spec, 'spec.dataSchema.granularitySpec.segmentGranularity') || + // Bad secondary partitioning + Boolean(AutoForm.issueWithModel(spec, getSecondaryPartitionRelatedFormFields(spec, undefined))) + ); } -export function getPartitionRelatedTuningSpecFormFields( +export const PRIMARY_PARTITION_RELATED_FORM_FIELDS: Field[] = [ + { + name: 'spec.dataSchema.granularitySpec.segmentGranularity', + type: 'string', + suggestions: ['hour', 'day', 'week', 'month', 'year'], + defined: s => deepGet(s, 'spec.dataSchema.granularitySpec.type') === 'uniform', + required: true, + info: ( + <> + The granularity to create time chunks at. Multiple segments can be created per time chunk. + For example, with 'DAY' segmentGranularity, the events of the same day fall into the same + time chunk which can be optionally further partitioned into multiple segments based on other + configurations and input size. + + ), + }, +]; + +export function getSecondaryPartitionRelatedFormFields( spec: IngestionSpec, dimensionSuggestions: string[] | undefined, ): Field[] { - const specType = getSpecType(spec) || 'index_parallel'; + const specType = getSpecType(spec); switch (specType) { case 'index_parallel': const parallelFields: Field[] = [ @@ -1402,7 +1401,14 @@ export function getPartitionRelatedTuningSpecFormFields(

), adjustment: s => { - if (!Array.isArray(dimensionSuggestions) || !dimensionSuggestions.length) return s; + if ( + deepGet(s, 'spec.tuningConfig.partitionsSpec.type') !== 'single_dim' || + !Array.isArray(dimensionSuggestions) || + !dimensionSuggestions.length + ) { + return s; + } + return deepSet( s, 'spec.tuningConfig.partitionsSpec.partitionDimension', @@ -2172,6 +2178,16 @@ export function updateSchemaWithSample( newSpec = deepDelete(newSpec, 'spec.dataSchema.metricsSpec'); } + if (getSpecType(newSpec) === 'index_parallel') { + newSpec = adjustForceGuaranteedRollup( + deepSet( + newSpec, + 'spec.tuningConfig.partitionsSpec', + rollup ? { type: 'hashed' } : { type: 'dynamic' }, + ), + ); + } + newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.rollup', rollup); return newSpec; } diff --git a/web-console/src/utils/object-change.ts b/web-console/src/utils/object-change.ts index 7ff7d5e2fc8f..99e3166f9ca8 100644 --- a/web-console/src/utils/object-change.ts +++ b/web-console/src/utils/object-change.ts @@ -83,6 +83,11 @@ export function deepSet>(value: T, path: string, x return valueCopy; } +export function deepSetIfUnset>(value: T, path: string, x: any): T { + if (typeof deepGet(value, path) !== 'undefined') return value; + return deepSet(value, path, x); +} + export function deepSetMulti>( value: T, changes: Record, diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 31bb54ca4970..15e7df035fce 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -68,6 +68,7 @@ import { INPUT_FORMAT_FIELDS, issueWithSampleData, METRIC_SPEC_FIELDS, + PRIMARY_PARTITION_RELATED_FORM_FIELDS, removeTimestampTransform, TIMESTAMP_SPEC_FIELDS, TimestampSpec, @@ -76,7 +77,7 @@ import { updateSchemaWithSample, } from '../../druid-models'; import { - adjustTuningConfig, + adjustForceGuaranteedRollup, cleanSpec, computeFlattenPathsForData, DimensionMode, @@ -91,18 +92,17 @@ import { getIngestionTitle, getIoConfigFormFields, getIoConfigTuningFormFields, - getPartitionRelatedTuningSpecFormFields, getRequiredModule, getRollup, + getSecondaryPartitionRelatedFormFields, getSpecType, getTuningSpecFormFields, - GranularitySpec, IngestionComboTypeWithExtra, IngestionSpec, InputFormat, inputFormatCanFlatten, invalidIoConfig, - invalidTuningConfig, + invalidPartitionConfig, IoConfig, isDruidSource, isEmptyIngestionSpec, @@ -125,6 +125,7 @@ import { deepDelete, deepGet, deepSet, + deepSetIfUnset, deepSetMulti, EMPTY_ARRAY, EMPTY_OBJECT, @@ -509,9 +510,9 @@ export class LoadDataView extends React.PureComponent { - this.setState(({ specPreview }) => { + this.setState(({ spec, specPreview }) => { localStorageSet(LocalStorageKeys.INGESTION_SPEC, JSON.stringify(specPreview)); - return { spec: specPreview }; + return { spec: spec === specPreview ? Object.assign({}, specPreview) : specPreview }; // If applying again, make a shallow copy to force a refresh }); }; @@ -1886,17 +1887,16 @@ export class LoadDataView extends React.PureComponent {this.renderNextBar({ disabled: !schemaQueryState.data, - onNextStep: () => { - let newSpec = spec; - if (rollup) { - newSpec = deepSet(newSpec, 'spec.tuningConfig.partitionsSpec', { type: 'hashed' }); - newSpec = deepSet(newSpec, 'spec.tuningConfig.forceGuaranteedRollup', true); - } else { - newSpec = deepSet(newSpec, 'spec.tuningConfig.partitionsSpec', { type: 'dynamic' }); - newSpec = deepDelete(newSpec, 'spec.tuningConfig.forceGuaranteedRollup'); - } - - this.updateSpec(newSpec); - return true; - }, })} ); @@ -2920,8 +2907,6 @@ export class LoadDataView extends React.PureComponent
Primary partitioning (by time)
g.type === 'uniform', - required: true, - info: ( - <> - The granularity to create time chunks at. Multiple segments can be created per - time chunk. For example, with 'DAY' segmentGranularity, the events of the same - day fall into the same time chunk which can be optionally further partitioned - into multiple segments based on other configurations and input size. - - ), - }, - ]} - model={granularitySpec} - onChange={g => this.updateSpec(deepSet(spec, 'spec.dataSchema.granularitySpec', g))} + fields={PRIMARY_PARTITION_RELATED_FORM_FIELDS} + model={spec} + onChange={this.updateSpec} />
Secondary partitioning
@@ -3025,7 +2994,7 @@ export class LoadDataView extends React.PureComponent {this.renderNextBar({ - disabled: !granularitySpec.segmentGranularity || invalidTuningConfig(tuningConfig), + disabled: invalidPartitionConfig(spec), })} ); From 04a2a2fbadab42ec9f5edd53e74564de2eaa7806 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Sun, 6 Dec 2020 15:54:53 -0800 Subject: [PATCH 03/14] fix tests --- .../src/druid-models/ingestion-spec.spec.ts | 66 ++++++++++++++++++- .../src/druid-models/ingestion-spec.tsx | 7 +- .../views/load-data-view/load-data-view.tsx | 1 + 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/web-console/src/druid-models/ingestion-spec.spec.ts b/web-console/src/druid-models/ingestion-spec.spec.ts index fb6df068351f..1e061420918e 100644 --- a/web-console/src/druid-models/ingestion-spec.spec.ts +++ b/web-console/src/druid-models/ingestion-spec.spec.ts @@ -206,9 +206,14 @@ describe('spec utils', () => { }); it('updateSchemaWithSample', () => { - expect( - updateSchemaWithSample(ingestionSpec, { header: ['header'], rows: [] }, 'specific', true), - ).toMatchInlineSnapshot(` + const withRollup = updateSchemaWithSample( + ingestionSpec, + { header: ['header'], rows: [] }, + 'specific', + true, + ); + + expect(withRollup).toMatchInlineSnapshot(` Object { "spec": Object { "dataSchema": Object { @@ -248,6 +253,61 @@ describe('spec utils', () => { "type": "index_parallel", }, "tuningConfig": Object { + "forceGuaranteedRollup": true, + "partitionsSpec": Object { + "type": "hashed", + }, + "type": "index_parallel", + }, + }, + "type": "index_parallel", + } + `); + + const noRollup = updateSchemaWithSample( + ingestionSpec, + { header: ['header'], rows: [] }, + 'specific', + false, + ); + + expect(noRollup).toMatchInlineSnapshot(` + Object { + "spec": Object { + "dataSchema": Object { + "dataSource": "wikipedia", + "dimensionsSpec": Object { + "dimensions": Array [ + "header", + ], + }, + "granularitySpec": Object { + "queryGranularity": "none", + "rollup": false, + "segmentGranularity": "day", + "type": "uniform", + }, + "timestampSpec": Object { + "column": "timestamp", + "format": "iso", + }, + }, + "ioConfig": Object { + "inputFormat": Object { + "type": "json", + }, + "inputSource": Object { + "type": "http", + "uris": Array [ + "https://static.imply.io/data/wikipedia.json.gz", + ], + }, + "type": "index_parallel", + }, + "tuningConfig": Object { + "partitionsSpec": Object { + "type": "dynamic", + }, "type": "index_parallel", }, }, diff --git a/web-console/src/druid-models/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx index 2a93825b9aab..8efbddefade5 100644 --- a/web-console/src/druid-models/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec.tsx @@ -1364,6 +1364,7 @@ export function invalidPartitionConfig(spec: IngestionSpec): boolean { export const PRIMARY_PARTITION_RELATED_FORM_FIELDS: Field[] = [ { name: 'spec.dataSchema.granularitySpec.segmentGranularity', + label: 'Segment granularity', type: 'string', suggestions: ['hour', 'day', 'week', 'month', 'year'], defined: s => deepGet(s, 'spec.dataSchema.granularitySpec.type') === 'uniform', @@ -2149,6 +2150,7 @@ export function updateSchemaWithSample( headerAndRows: HeaderAndRows, dimensionMode: DimensionMode, rollup: boolean, + forcePartitionInitialization = false, ): IngestionSpec { const typeHints = getTypeHintsFromSpec(spec); @@ -2178,7 +2180,10 @@ export function updateSchemaWithSample( newSpec = deepDelete(newSpec, 'spec.dataSchema.metricsSpec'); } - if (getSpecType(newSpec) === 'index_parallel') { + if ( + getSpecType(newSpec) === 'index_parallel' && + (!deepGet(newSpec, 'spec.tuningConfig.partitionsSpec') || forcePartitionInitialization) + ) { newSpec = adjustForceGuaranteedRollup( deepSet( newSpec, diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 15e7df035fce..8f6339015b15 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -2577,6 +2577,7 @@ export class LoadDataView extends React.PureComponent Date: Mon, 7 Dec 2020 13:20:20 -0800 Subject: [PATCH 04/14] better filter control --- web-console/src/druid-models/filter.tsx | 51 +++++- .../filter-table/filter-table.spec.tsx | 1 - .../filter-table/filter-table.tsx | 18 +- .../views/load-data-view/load-data-view.tsx | 172 ++++++------------ 4 files changed, 105 insertions(+), 137 deletions(-) diff --git a/web-console/src/druid-models/filter.tsx b/web-console/src/druid-models/filter.tsx index 10791294063e..eea0647c327c 100644 --- a/web-console/src/druid-models/filter.tsx +++ b/web-console/src/druid-models/filter.tsx @@ -16,9 +16,14 @@ * limitations under the License. */ -import { Field } from '../components'; +import React from 'react'; + +import { ExternalLink, Field } from '../components'; +import { getLink } from '../links'; import { deepGet, EMPTY_ARRAY, oneOf } from '../utils'; +import { IngestionSpec } from './ingestion-spec'; + export type DruidFilter = Record; export interface DimensionFiltersWithRest { @@ -30,7 +35,9 @@ export function splitFilter(filter: DruidFilter | null): DimensionFiltersWithRes const inputAndFilters: DruidFilter[] = filter ? filter.type === 'and' && Array.isArray(filter.fields) ? filter.fields - : [filter] + : filter.type !== 'true' + ? [filter] + : EMPTY_ARRAY : EMPTY_ARRAY; const dimensionFilters: DruidFilter[] = inputAndFilters.filter( f => typeof f.dimension === 'string', @@ -119,3 +126,43 @@ export const FILTER_FIELDS: Field[] = [ df.type === 'not' && oneOf(deepGet(df, 'field.type'), 'regex', 'like'), }, ]; + +export const FILTERS_FIELDS: Field[] = [ + { + name: 'spec.dataSchema.granularitySpec.intervals', + label: 'Time intervals', + type: 'string-array', + placeholder: 'ex: 2020-01-01/2020-06-01', + info: ( + <> +

A comma separated list of intervals for the raw data being ingested.

+

+ Explicitly specifying the list of intervals contained in the data will make some ingestion + jobs run faster. +

+ + ), + }, + { + name: 'spec.dataSchema.transformSpec.filter', + label: 'Filter', + type: 'json', + height: '350px', + placeholder: '{ "type": "true" }', + info: ( + <> +

+ A Druid{' '} + + JSON filter expression + {' '} + to apply to the data. +

+

+ Note that only the value that match the filter will be included. If you want to remove + some data values you must negate the filter. +

+ + ), + }, +]; diff --git a/web-console/src/views/load-data-view/filter-table/filter-table.spec.tsx b/web-console/src/views/load-data-view/filter-table/filter-table.spec.tsx index 0c635b17947a..f023124877a4 100644 --- a/web-console/src/views/load-data-view/filter-table/filter-table.spec.tsx +++ b/web-console/src/views/load-data-view/filter-table/filter-table.spec.tsx @@ -39,7 +39,6 @@ describe('filter table', () => { columnFilter="" dimensionFilters={[]} selectedFilterName={undefined} - onShowGlobalFilter={() => {}} onFilterSelect={() => {}} /> ); diff --git a/web-console/src/views/load-data-view/filter-table/filter-table.tsx b/web-console/src/views/load-data-view/filter-table/filter-table.tsx index db0bddc3eedd..c64562a20556 100644 --- a/web-console/src/views/load-data-view/filter-table/filter-table.tsx +++ b/web-console/src/views/load-data-view/filter-table/filter-table.tsx @@ -42,19 +42,11 @@ export interface FilterTableProps { columnFilter: string; dimensionFilters: DruidFilter[]; selectedFilterName: string | undefined; - onShowGlobalFilter: () => void; onFilterSelect: (filter: DruidFilter, index: number) => void; } export const FilterTable = React.memo(function FilterTable(props: FilterTableProps) { - const { - sampleData, - columnFilter, - dimensionFilters, - selectedFilterName, - onShowGlobalFilter, - onFilterSelect, - } = props; + const { sampleData, columnFilter, dimensionFilters, selectedFilterName, onFilterSelect } = props; return ( { - if (timestamp) { - onShowGlobalFilter(); - } else if (filter) { + if (timestamp) return; + + if (filter) { onFilterSelect(filter, filterIndex); } else { onFilterSelect({ type: 'selector', dimension: columnName, value: '' }, -1); diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 8f6339015b15..9674b4e8e235 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -60,6 +60,7 @@ import { CONSTANT_TIMESTAMP_SPEC_FIELDS, DIMENSION_SPEC_FIELDS, FILTER_FIELDS, + FILTERS_FIELDS, FLATTEN_FIELD_FIELDS, getDimensionSpecName, getMetricSpecName, @@ -341,7 +342,6 @@ export interface LoadDataViewState { filterQueryState: QueryState; selectedFilterIndex: number; selectedFilter?: DruidFilter; - showGlobalFilter: boolean; newFilterValue?: Record; // for schema @@ -399,7 +399,6 @@ export class LoadDataView extends React.PureComponent )} @@ -2103,18 +2101,30 @@ export class LoadDataView extends React.PureComponent{mainFill}
- {!showGlobalFilter && this.renderColumnFilterControls()} - {!selectedFilter && this.renderGlobalFilterControls()} + {!selectedFilter && ( + <> + + {this.renderApplyButtonBar(filterQueryState, undefined)} + +
{this.renderNextBar({})} ); } - private onShowGlobalFilter = () => { - this.setState({ showGlobalFilter: true }); - }; - private onFilterSelect = (filter: DruidFilter, index: number) => { this.setState({ selectedFilterIndex: index, @@ -2124,6 +2134,7 @@ export class LoadDataView extends React.PureComponent { this.setState({ @@ -2132,129 +2143,48 @@ export class LoadDataView extends React.PureComponent - this.setState({ selectedFilter: f })} - showCustom={f => !oneOf(f.type, 'selector', 'in', 'regex', 'like', 'not')} - /> -
-
- - ); - } else { - return ( - + return ( +
+ this.setState({ selectedFilter: f })} + showCustom={f => !oneOf(f.type, 'selector', 'in', 'regex', 'like', 'not')} + /> +
+ )}
- ); - } else { - return ( - -