diff --git a/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx b/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx index e65f078d381f..1dd546886c94 100644 --- a/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx +++ b/web-console/src/dialogs/doctor-dialog/doctor-checks.tsx @@ -232,7 +232,6 @@ export const DOCTOR_CHECKS: DoctorCheck[] = [ { type: 'index_parallel', spec: { - type: 'index_parallel', ioConfig: { type: 'index_parallel', inputSource: { type: 'inline', data: '{"test":"Data"}' }, diff --git a/web-console/src/utils/__snapshots__/ingestion-spec.spec.ts.snap b/web-console/src/utils/__snapshots__/ingestion-spec.spec.ts.snap index 031d6872dd50..900672117484 100644 --- a/web-console/src/utils/__snapshots__/ingestion-spec.spec.ts.snap +++ b/web-console/src/utils/__snapshots__/ingestion-spec.spec.ts.snap @@ -2,74 +2,76 @@ exports[`ingestion-spec upgrades 1`] = ` Object { - "dataSchema": Object { - "dataSource": "wikipedia", - "dimensionsSpec": Object { - "dimensions": Array [ - "channel", - "cityName", - "comment", - ], - }, - "granularitySpec": Object { - "queryGranularity": "HOUR", - "rollup": true, - "segmentGranularity": "DAY", - "type": "uniform", - }, - "metricsSpec": Array [ - Object { - "name": "count", - "type": "count", - }, - Object { - "fieldName": "added", - "name": "sum_added", - "type": "longSum", + "spec": Object { + "dataSchema": Object { + "dataSource": "wikipedia", + "dimensionsSpec": Object { + "dimensions": Array [ + "channel", + "cityName", + "comment", + ], }, - ], - "timestampSpec": Object { - "column": "timestamp", - "format": "iso", - }, - "transformSpec": Object { - "filter": Object { - "dimension": "commentLength", - "type": "selector", - "value": "35", + "granularitySpec": Object { + "queryGranularity": "HOUR", + "rollup": true, + "segmentGranularity": "DAY", + "type": "uniform", }, - "transforms": Array [ + "metricsSpec": Array [ + Object { + "name": "count", + "type": "count", + }, Object { - "expression": "concat(\\"channel\\", 'lol')", - "name": "channel", - "type": "expression", + "fieldName": "added", + "name": "sum_added", + "type": "longSum", }, ], - }, - }, - "ioConfig": Object { - "inputFormat": Object { - "flattenSpec": Object { - "fields": Array [ + "timestampSpec": Object { + "column": "timestamp", + "format": "iso", + }, + "transformSpec": Object { + "filter": Object { + "dimension": "commentLength", + "type": "selector", + "value": "35", + }, + "transforms": Array [ Object { - "expr": "$.cityName", - "name": "cityNameAlt", - "type": "path", + "expression": "concat(\\"channel\\", 'lol')", + "name": "channel", + "type": "expression", }, ], }, - "type": "json", }, - "inputSource": Object { - "type": "http", - "uris": Array [ - "https://static.imply.io/data/wikipedia.json.gz", - ], + "ioConfig": Object { + "inputFormat": Object { + "flattenSpec": Object { + "fields": Array [ + Object { + "expr": "$.cityName", + "name": "cityNameAlt", + "type": "path", + }, + ], + }, + "type": "json", + }, + "inputSource": Object { + "type": "http", + "uris": Array [ + "https://static.imply.io/data/wikipedia.json.gz", + ], + }, + "type": "index_parallel", + }, + "tuningConfig": Object { + "type": "index_parallel", }, - "type": "index_parallel", - }, - "tuningConfig": Object { - "type": "index_parallel", }, "type": "index_parallel", } diff --git a/web-console/src/utils/druid-type.ts b/web-console/src/utils/druid-type.ts index 9f5a6f93980d..35e4cac7122c 100644 --- a/web-console/src/utils/druid-type.ts +++ b/web-console/src/utils/druid-type.ts @@ -88,28 +88,28 @@ export function updateSchemaWithSample( let newSpec = spec; if (dimensionMode === 'auto-detect') { - newSpec = deepSet(newSpec, 'dataSchema.dimensionsSpec.dimensions', []); + newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec.dimensions', []); } else { - newSpec = deepDelete(newSpec, 'dataSchema.dimensionsSpec.dimensionExclusions'); + newSpec = deepDelete(newSpec, 'spec.dataSchema.dimensionsSpec.dimensionExclusions'); const dimensions = getDimensionSpecs(headerAndRows, rollup); if (dimensions) { - newSpec = deepSet(newSpec, 'dataSchema.dimensionsSpec.dimensions', dimensions); + newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec.dimensions', dimensions); } } if (rollup) { - newSpec = deepSet(newSpec, 'dataSchema.granularitySpec.queryGranularity', 'HOUR'); + newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'HOUR'); const metrics = getMetricSecs(headerAndRows); if (metrics) { - newSpec = deepSet(newSpec, 'dataSchema.metricsSpec', metrics); + newSpec = deepSet(newSpec, 'spec.dataSchema.metricsSpec', metrics); } } else { - newSpec = deepSet(newSpec, 'dataSchema.granularitySpec.queryGranularity', 'NONE'); - newSpec = deepDelete(newSpec, 'dataSchema.metricsSpec'); + newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'NONE'); + newSpec = deepDelete(newSpec, 'spec.dataSchema.metricsSpec'); } - newSpec = deepSet(newSpec, 'dataSchema.granularitySpec.rollup', rollup); + newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.rollup', rollup); return newSpec; } diff --git a/web-console/src/utils/ingestion-spec.spec.ts b/web-console/src/utils/ingestion-spec.spec.ts index f56617ccdf79..1399aeb6557f 100644 --- a/web-console/src/utils/ingestion-spec.spec.ts +++ b/web-console/src/utils/ingestion-spec.spec.ts @@ -21,71 +21,73 @@ import { downgradeSpec, upgradeSpec } from './ingestion-spec'; describe('ingestion-spec', () => { const oldSpec = { type: 'index_parallel', - ioConfig: { - type: 'index_parallel', - firehose: { - type: 'http', - uris: ['https://static.imply.io/data/wikipedia.json.gz'], + spec: { + ioConfig: { + type: 'index_parallel', + firehose: { + type: 'http', + uris: ['https://static.imply.io/data/wikipedia.json.gz'], + }, }, - }, - tuningConfig: { - type: 'index_parallel', - }, - dataSchema: { - dataSource: 'wikipedia', - granularitySpec: { - type: 'uniform', - segmentGranularity: 'DAY', - queryGranularity: 'HOUR', - rollup: true, + tuningConfig: { + type: 'index_parallel', }, - parser: { - type: 'string', - parseSpec: { - format: 'json', - timestampSpec: { - column: 'timestamp', - format: 'iso', - }, - dimensionsSpec: { - dimensions: ['channel', 'cityName', 'comment'], + dataSchema: { + dataSource: 'wikipedia', + granularitySpec: { + type: 'uniform', + segmentGranularity: 'DAY', + queryGranularity: 'HOUR', + rollup: true, + }, + parser: { + type: 'string', + parseSpec: { + format: 'json', + timestampSpec: { + column: 'timestamp', + format: 'iso', + }, + dimensionsSpec: { + dimensions: ['channel', 'cityName', 'comment'], + }, + flattenSpec: { + fields: [ + { + type: 'path', + name: 'cityNameAlt', + expr: '$.cityName', + }, + ], + }, }, - flattenSpec: { - fields: [ - { - type: 'path', - name: 'cityNameAlt', - expr: '$.cityName', - }, - ], + }, + transformSpec: { + transforms: [ + { + type: 'expression', + name: 'channel', + expression: 'concat("channel", \'lol\')', + }, + ], + filter: { + type: 'selector', + dimension: 'commentLength', + value: '35', }, }, - }, - transformSpec: { - transforms: [ + metricsSpec: [ + { + name: 'count', + type: 'count', + }, { - type: 'expression', - name: 'channel', - expression: 'concat("channel", \'lol\')', + name: 'sum_added', + type: 'longSum', + fieldName: 'added', }, ], - filter: { - type: 'selector', - dimension: 'commentLength', - value: '35', - }, }, - metricsSpec: [ - { - name: 'count', - type: 'count', - }, - { - name: 'sum_added', - type: 'longSum', - fieldName: 'added', - }, - ], }, }; diff --git a/web-console/src/utils/ingestion-spec.tsx b/web-console/src/utils/ingestion-spec.tsx index bb01d8b5c492..ef07b345b1bd 100644 --- a/web-console/src/utils/ingestion-spec.tsx +++ b/web-console/src/utils/ingestion-spec.tsx @@ -40,7 +40,11 @@ export const EMPTY_ARRAY: any[] = []; const CURRENT_YEAR = new Date().getUTCFullYear(); export interface IngestionSpec { - type?: IngestionType; + type: IngestionType; + spec: IngestionSpecInner; +} + +export interface IngestionSpecInner { ioConfig: IoConfig; dataSchema: DataSchema; tuningConfig?: TuningConfig; @@ -69,8 +73,9 @@ export type IngestionComboType = export type IngestionComboTypeWithExtra = IngestionComboType | 'hadoop' | 'example' | 'other'; export function adjustIngestionSpec(spec: IngestionSpec) { - if (spec.tuningConfig) { - spec = deepSet(spec, 'tuningConfig', adjustTuningConfig(spec.tuningConfig)); + const tuningConfig = deepGet(spec, 'spec.tuningConfig'); + if (tuningConfig) { + spec = deepSet(spec, 'spec.tuningConfig', adjustTuningConfig(tuningConfig)); } return spec; } @@ -88,7 +93,7 @@ function ingestionTypeToIoAndTuningConfigType(ingestionType: IngestionType): str } export function getIngestionComboType(spec: IngestionSpec): IngestionComboType | undefined { - const ioConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT; + const ioConfig = deepGet(spec, 'spec.ioConfig') || EMPTY_OBJECT; switch (ioConfig.type) { case 'kafka': @@ -96,7 +101,7 @@ export function getIngestionComboType(spec: IngestionSpec): IngestionComboType | return ioConfig.type; case 'index_parallel': - const inputSource = deepGet(spec, 'ioConfig.inputSource') || EMPTY_OBJECT; + const inputSource = deepGet(spec, 'spec.ioConfig.inputSource') || EMPTY_OBJECT; switch (inputSource.type) { case 'local': case 'http': @@ -230,20 +235,20 @@ export interface InputFormat { export type DimensionMode = 'specific' | 'auto-detect'; export function getDimensionMode(spec: IngestionSpec): DimensionMode { - const dimensions = deepGet(spec, 'dataSchema.dimensionsSpec.dimensions') || EMPTY_ARRAY; + const dimensions = deepGet(spec, 'spec.dataSchema.dimensionsSpec.dimensions') || EMPTY_ARRAY; return Array.isArray(dimensions) && dimensions.length === 0 ? 'auto-detect' : 'specific'; } export function getRollup(spec: IngestionSpec): boolean { - const specRollup = deepGet(spec, 'dataSchema.granularitySpec.rollup'); + const specRollup = deepGet(spec, 'spec.dataSchema.granularitySpec.rollup'); return typeof specRollup === 'boolean' ? specRollup : true; } export function getSpecType(spec: Partial): IngestionType { return ( deepGet(spec, 'type') || - deepGet(spec, 'ioConfig.type') || - deepGet(spec, 'tuningConfig.type') || + deepGet(spec, 'spec.ioConfig.type') || + deepGet(spec, 'spec.tuningConfig.type') || 'index_parallel' ); } @@ -257,7 +262,7 @@ export function isTask(spec: IngestionSpec) { } export function isDruidSource(spec: IngestionSpec): boolean { - return deepGet(spec, 'ioConfig.inputSource.type') === 'druid'; + return deepGet(spec, 'spec.ioConfig.inputSource.type') === 'druid'; } /** @@ -271,14 +276,21 @@ export function normalizeSpec(spec: Partial): IngestionSpec { } // Make sure that if we actually get a task payload we extract the spec - if (typeof (spec as any).spec === 'object') spec = (spec as any).spec; + if (typeof spec.spec !== 'object' && typeof (spec as any).ioConfig === 'object') { + spec = { spec: spec as any }; + } const specType = - deepGet(spec, 'type') || deepGet(spec, 'ioConfig.type') || deepGet(spec, 'tuningConfig.type'); + deepGet(spec, 'type') || + deepGet(spec, 'spec.ioConfig.type') || + deepGet(spec, 'spec.tuningConfig.type'); + if (!specType) return spec as IngestionSpec; if (!deepGet(spec, 'type')) spec = deepSet(spec, 'type', specType); - if (!deepGet(spec, 'ioConfig.type')) spec = deepSet(spec, 'ioConfig.type', specType); - if (!deepGet(spec, 'tuningConfig.type')) spec = deepSet(spec, 'tuningConfig.type', specType); + if (!deepGet(spec, 'spec.ioConfig.type')) spec = deepSet(spec, 'spec.ioConfig.type', specType); + if (!deepGet(spec, 'spec.tuningConfig.type')) { + spec = deepSet(spec, 'spec.tuningConfig.type', specType); + } return spec as IngestionSpec; } @@ -1862,11 +1874,11 @@ function basenameFromFilename(filename: string): string | undefined { export function fillDataSourceNameIfNeeded(spec: IngestionSpec): IngestionSpec { const possibleName = guessDataSourceName(spec); if (!possibleName) return spec; - return deepSet(spec, 'dataSchema.dataSource', possibleName); + return deepSet(spec, 'spec.dataSchema.dataSource', possibleName); } export function guessDataSourceName(spec: IngestionSpec): string | undefined { - const ioConfig = deepGet(spec, 'ioConfig'); + const ioConfig = deepGet(spec, 'spec.ioConfig'); if (!ioConfig) return; switch (ioConfig.type) { @@ -2489,22 +2501,22 @@ export function updateIngestionType( let newSpec = spec; newSpec = deepSet(newSpec, 'type', ingestionType); - newSpec = deepSet(newSpec, 'ioConfig.type', ioAndTuningConfigType); - newSpec = deepSet(newSpec, 'tuningConfig.type', ioAndTuningConfigType); + newSpec = deepSet(newSpec, 'spec.ioConfig.type', ioAndTuningConfigType); + newSpec = deepSet(newSpec, 'spec.tuningConfig.type', ioAndTuningConfigType); if (inputSourceType) { - newSpec = deepSet(newSpec, 'ioConfig.inputSource', { type: inputSourceType }); + newSpec = deepSet(newSpec, 'spec.ioConfig.inputSource', { type: inputSourceType }); if (inputSourceType === 'local') { - newSpec = deepSet(newSpec, 'ioConfig.inputSource.filter', '*'); + newSpec = deepSet(newSpec, 'spec.ioConfig.inputSource.filter', '*'); } } - if (!deepGet(spec, 'dataSchema.dataSource')) { - newSpec = deepSet(newSpec, 'dataSchema.dataSource', 'new-data-source'); + if (!deepGet(spec, 'spec.dataSchema.dataSource')) { + newSpec = deepSet(newSpec, 'spec.dataSchema.dataSource', 'new-data-source'); } - if (!deepGet(spec, 'dataSchema.granularitySpec')) { + if (!deepGet(spec, 'spec.dataSchema.granularitySpec')) { const granularitySpec: GranularitySpec = { type: 'uniform', queryGranularity: 'HOUR', @@ -2513,22 +2525,22 @@ export function updateIngestionType( granularitySpec.segmentGranularity = 'HOUR'; } - newSpec = deepSet(newSpec, 'dataSchema.granularitySpec', granularitySpec); + newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec', granularitySpec); } - if (!deepGet(spec, 'dataSchema.timestampSpec')) { - newSpec = deepSet(newSpec, 'dataSchema.timestampSpec', getDummyTimestampSpec()); + if (!deepGet(spec, 'spec.dataSchema.timestampSpec')) { + newSpec = deepSet(newSpec, 'spec.dataSchema.timestampSpec', getDummyTimestampSpec()); } - if (!deepGet(spec, 'dataSchema.dimensionsSpec')) { - newSpec = deepSet(newSpec, 'dataSchema.dimensionsSpec', {}); + if (!deepGet(spec, 'spec.dataSchema.dimensionsSpec')) { + newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec', {}); } return newSpec; } export function fillInputFormat(spec: IngestionSpec, sampleData: string[]): IngestionSpec { - return deepSet(spec, 'ioConfig.inputFormat', guessInputFormat(sampleData)); + return deepSet(spec, 'spec.ioConfig.inputFormat', guessInputFormat(sampleData)); } function guessInputFormat(sampleData: string[]): InputFormat { @@ -2681,53 +2693,61 @@ export function getFilterFormFields() { } export function upgradeSpec(spec: any): any { - if (deepGet(spec, 'ioConfig.firehose')) { - switch (deepGet(spec, 'ioConfig.firehose.type')) { + if (deepGet(spec, 'spec.ioConfig.firehose')) { + switch (deepGet(spec, 'spec.ioConfig.firehose.type')) { case 'static-s3': - deepSet(spec, 'ioConfig.firehose.type', 's3'); + deepSet(spec, 'spec.ioConfig.firehose.type', 's3'); break; case 'static-google-blobstore': - deepSet(spec, 'ioConfig.firehose.type', 'google'); - deepMove(spec, 'ioConfig.firehose.blobs', 'ioConfig.firehose.objects'); + deepSet(spec, 'spec.ioConfig.firehose.type', 'google'); + deepMove(spec, 'spec.ioConfig.firehose.blobs', 'spec.ioConfig.firehose.objects'); break; } - spec = deepMove(spec, 'ioConfig.firehose', 'ioConfig.inputSource'); - spec = deepMove(spec, 'dataSchema.parser.parseSpec.timestampSpec', 'dataSchema.timestampSpec'); + spec = deepMove(spec, 'spec.ioConfig.firehose', 'spec.ioConfig.inputSource'); spec = deepMove( spec, - 'dataSchema.parser.parseSpec.dimensionsSpec', - 'dataSchema.dimensionsSpec', + 'spec.dataSchema.parser.parseSpec.timestampSpec', + 'spec.dataSchema.timestampSpec', ); - spec = deepMove(spec, 'dataSchema.parser.parseSpec', 'ioConfig.inputFormat'); - spec = deepDelete(spec, 'dataSchema.parser'); - spec = deepMove(spec, 'ioConfig.inputFormat.format', 'ioConfig.inputFormat.type'); + spec = deepMove( + spec, + 'spec.dataSchema.parser.parseSpec.dimensionsSpec', + 'spec.dataSchema.dimensionsSpec', + ); + spec = deepMove(spec, 'spec.dataSchema.parser.parseSpec', 'spec.ioConfig.inputFormat'); + spec = deepDelete(spec, 'spec.dataSchema.parser'); + spec = deepMove(spec, 'spec.ioConfig.inputFormat.format', 'spec.ioConfig.inputFormat.type'); } return spec; } export function downgradeSpec(spec: any): any { - if (deepGet(spec, 'ioConfig.inputSource')) { - spec = deepMove(spec, 'ioConfig.inputFormat.type', 'ioConfig.inputFormat.format'); - spec = deepSet(spec, 'dataSchema.parser', { type: 'string' }); - spec = deepMove(spec, 'ioConfig.inputFormat', 'dataSchema.parser.parseSpec'); + if (deepGet(spec, 'spec.ioConfig.inputSource')) { + spec = deepMove(spec, 'spec.ioConfig.inputFormat.type', 'spec.ioConfig.inputFormat.format'); + spec = deepSet(spec, 'spec.dataSchema.parser', { type: 'string' }); + spec = deepMove(spec, 'spec.ioConfig.inputFormat', 'spec.dataSchema.parser.parseSpec'); + spec = deepMove( + spec, + 'spec.dataSchema.dimensionsSpec', + 'spec.dataSchema.parser.parseSpec.dimensionsSpec', + ); spec = deepMove( spec, - 'dataSchema.dimensionsSpec', - 'dataSchema.parser.parseSpec.dimensionsSpec', + 'spec.dataSchema.timestampSpec', + 'spec.dataSchema.parser.parseSpec.timestampSpec', ); - spec = deepMove(spec, 'dataSchema.timestampSpec', 'dataSchema.parser.parseSpec.timestampSpec'); - spec = deepMove(spec, 'ioConfig.inputSource', 'ioConfig.firehose'); + spec = deepMove(spec, 'spec.ioConfig.inputSource', 'spec.ioConfig.firehose'); - switch (deepGet(spec, 'ioConfig.firehose.type')) { + switch (deepGet(spec, 'spec.ioConfig.firehose.type')) { case 's3': - deepSet(spec, 'ioConfig.firehose.type', 'static-s3'); + deepSet(spec, 'spec.ioConfig.firehose.type', 'static-s3'); break; case 'google': - deepSet(spec, 'ioConfig.firehose.type', 'static-google-blobstore'); - deepMove(spec, 'ioConfig.firehose.objects', 'ioConfig.firehose.blobs'); + deepSet(spec, 'spec.ioConfig.firehose.type', 'static-google-blobstore'); + deepMove(spec, 'spec.ioConfig.firehose.objects', 'spec.ioConfig.firehose.blobs'); break; } } diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts index 8b188ecb307b..464a8a9be3ac 100644 --- a/web-console/src/utils/sampler.ts +++ b/web-console/src/utils/sampler.ts @@ -44,11 +44,9 @@ const BASE_SAMPLER_CONFIG: SamplerConfig = { timeoutMs: 15000, }; -export interface SampleSpec { - type: string; - spec: IngestionSpec; +export type SampleSpec = IngestionSpec & { samplerConfig: SamplerConfig; -} +}; export interface SamplerConfig { numRows?: number; @@ -207,7 +205,7 @@ function makeSamplerIoConfig( This is a hack to deal with the fact that the sampler can not deal with the index_parallel type */ function fixSamplerTypes(sampleSpec: SampleSpec): SampleSpec { - let samplerType: string = getSpecType(sampleSpec.spec); + let samplerType: string = getSpecType(sampleSpec); if (samplerType === 'index_parallel') { samplerType = 'index'; } @@ -244,7 +242,7 @@ export async function sampleForConnect( ): Promise { const samplerType = getSpecType(spec); let ioConfig: IoConfig = makeSamplerIoConfig( - deepGet(spec, 'ioConfig'), + deepGet(spec, 'spec.ioConfig'), samplerType, sampleStrategy, ); @@ -308,7 +306,7 @@ export async function sampleForParser( ): Promise { const samplerType = getSpecType(spec); const ioConfig: IoConfig = makeSamplerIoConfig( - deepGet(spec, 'ioConfig'), + deepGet(spec, 'spec.ioConfig'), samplerType, sampleStrategy, ); @@ -316,7 +314,6 @@ export async function sampleForParser( const sampleSpec: SampleSpec = { type: samplerType, spec: { - type: samplerType, ioConfig, dataSchema: { dataSource: 'sample', @@ -335,15 +332,14 @@ export async function sampleForTimestamp( cacheRows: CacheRows, ): Promise { const samplerType = getSpecType(spec); - const timestampSpec: TimestampSpec = deepGet(spec, 'dataSchema.timestampSpec'); + const timestampSpec: TimestampSpec = deepGet(spec, 'spec.dataSchema.timestampSpec'); const columnTimestampSpec = isColumnTimestampSpec(timestampSpec); // First do a query with a static timestamp spec const sampleSpecColumns: SampleSpec = { type: samplerType, spec: { - type: samplerType, - ioConfig: deepGet(spec, 'ioConfig'), + ioConfig: deepGet(spec, 'spec.ioConfig'), dataSchema: { dataSource: 'sample', dimensionsSpec: {}, @@ -366,8 +362,7 @@ export async function sampleForTimestamp( const sampleSpec: SampleSpec = { type: samplerType, spec: { - type: samplerType, - ioConfig: deepGet(spec, 'ioConfig'), + ioConfig: deepGet(spec, 'spec.ioConfig'), dataSchema: { dataSource: 'sample', dimensionsSpec: {}, @@ -402,9 +397,9 @@ export async function sampleForTransform( cacheRows: CacheRows, ): Promise { const samplerType = getSpecType(spec); - const inputFormatColumns: string[] = deepGet(spec, 'ioConfig.inputFormat.columns') || []; - const timestampSpec: TimestampSpec = deepGet(spec, 'dataSchema.timestampSpec'); - const transforms: Transform[] = deepGet(spec, 'dataSchema.transformSpec.transforms') || []; + const inputFormatColumns: string[] = deepGet(spec, 'spec.ioConfig.inputFormat.columns') || []; + const timestampSpec: TimestampSpec = deepGet(spec, 'spec.dataSchema.timestampSpec'); + const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || []; // Extra step to simulate auto detecting dimension with transforms const specialDimensionSpec: DimensionsSpec = {}; @@ -412,8 +407,7 @@ export async function sampleForTransform( const sampleSpecHack: SampleSpec = { type: samplerType, spec: { - type: samplerType, - ioConfig: deepGet(spec, 'ioConfig'), + ioConfig: deepGet(spec, 'spec.ioConfig'), dataSchema: { dataSource: 'sample', timestampSpec, @@ -440,8 +434,7 @@ export async function sampleForTransform( const sampleSpec: SampleSpec = { type: samplerType, spec: { - type: samplerType, - ioConfig: deepGet(spec, 'ioConfig'), + ioConfig: deepGet(spec, 'spec.ioConfig'), dataSchema: { dataSource: 'sample', timestampSpec, @@ -462,10 +455,10 @@ export async function sampleForFilter( cacheRows: CacheRows, ): Promise { const samplerType = getSpecType(spec); - const inputFormatColumns: string[] = deepGet(spec, 'ioConfig.inputFormat.columns') || []; - const timestampSpec: TimestampSpec = deepGet(spec, 'dataSchema.timestampSpec'); - const transforms: Transform[] = deepGet(spec, 'dataSchema.transformSpec.transforms') || []; - const filter: any = deepGet(spec, 'dataSchema.transformSpec.filter'); + const inputFormatColumns: string[] = deepGet(spec, 'spec.ioConfig.inputFormat.columns') || []; + const timestampSpec: TimestampSpec = deepGet(spec, 'spec.dataSchema.timestampSpec'); + const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || []; + const filter: any = deepGet(spec, 'spec.dataSchema.transformSpec.filter'); // Extra step to simulate auto detecting dimension with transforms const specialDimensionSpec: DimensionsSpec = {}; @@ -473,8 +466,7 @@ export async function sampleForFilter( const sampleSpecHack: SampleSpec = { type: samplerType, spec: { - type: samplerType, - ioConfig: deepGet(spec, 'ioConfig'), + ioConfig: deepGet(spec, 'spec.ioConfig'), dataSchema: { dataSource: 'sample', timestampSpec, @@ -501,8 +493,7 @@ export async function sampleForFilter( const sampleSpec: SampleSpec = { type: samplerType, spec: { - type: samplerType, - ioConfig: deepGet(spec, 'ioConfig'), + ioConfig: deepGet(spec, 'spec.ioConfig'), dataSchema: { dataSource: 'sample', timestampSpec, @@ -524,19 +515,18 @@ export async function sampleForSchema( cacheRows: CacheRows, ): Promise { const samplerType = getSpecType(spec); - const timestampSpec: TimestampSpec = deepGet(spec, 'dataSchema.timestampSpec'); + const timestampSpec: TimestampSpec = deepGet(spec, 'spec.dataSchema.timestampSpec'); const transformSpec: TransformSpec = - deepGet(spec, 'dataSchema.transformSpec') || ({} as TransformSpec); - const dimensionsSpec: DimensionsSpec = deepGet(spec, 'dataSchema.dimensionsSpec'); - const metricsSpec: MetricSpec[] = deepGet(spec, 'dataSchema.metricsSpec') || []; + deepGet(spec, 'spec.dataSchema.transformSpec') || ({} as TransformSpec); + const dimensionsSpec: DimensionsSpec = deepGet(spec, 'spec.dataSchema.dimensionsSpec'); + const metricsSpec: MetricSpec[] = deepGet(spec, 'spec.dataSchema.metricsSpec') || []; const queryGranularity: string = - deepGet(spec, 'dataSchema.granularitySpec.queryGranularity') || 'NONE'; + deepGet(spec, 'spec.dataSchema.granularitySpec.queryGranularity') || 'NONE'; const sampleSpec: SampleSpec = { type: samplerType, spec: { - type: samplerType, - ioConfig: deepGet(spec, 'ioConfig'), + ioConfig: deepGet(spec, 'spec.ioConfig'), dataSchema: { dataSource: 'sample', timestampSpec, @@ -560,7 +550,6 @@ export async function sampleForExampleManifests( const exampleSpec: SampleSpec = { type: 'index_parallel', spec: { - type: 'index_parallel', ioConfig: { type: 'index_parallel', inputSource: { type: 'http', uris: [exampleManifestUrl] }, diff --git a/web-console/src/utils/utils.spec.ts b/web-console/src/utils/utils.spec.ts index b4d31ce0e613..736115d9671d 100644 --- a/web-console/src/utils/utils.spec.ts +++ b/web-console/src/utils/utils.spec.ts @@ -30,31 +30,33 @@ import { applyCache, headerFromSampleResponse } from './sampler'; describe('test-utils', () => { const ingestionSpec: IngestionSpec = { type: 'index_parallel', - ioConfig: { - type: 'index_parallel', - inputSource: { - type: 'http', - uris: ['https://static.imply.io/data/wikipedia.json.gz'], - }, - inputFormat: { - type: 'json', + spec: { + ioConfig: { + type: 'index_parallel', + inputSource: { + type: 'http', + uris: ['https://static.imply.io/data/wikipedia.json.gz'], + }, + inputFormat: { + type: 'json', + }, }, - }, - tuningConfig: { - type: 'index_parallel', - }, - dataSchema: { - dataSource: 'wikipedia', - granularitySpec: { - type: 'uniform', - segmentGranularity: 'DAY', - queryGranularity: 'HOUR', + tuningConfig: { + type: 'index_parallel', }, - timestampSpec: { - column: 'timestamp', - format: 'iso', + dataSchema: { + dataSource: 'wikipedia', + granularitySpec: { + type: 'uniform', + segmentGranularity: 'DAY', + queryGranularity: 'HOUR', + }, + timestampSpec: { + column: 'timestamp', + format: 'iso', + }, + dimensionsSpec: {}, }, - dimensionsSpec: {}, }, }; @@ -72,14 +74,12 @@ describe('test-utils', () => { it('spec-utils applyCache', () => { expect( applyCache( - { - type: 'index_parallel', - spec: ingestionSpec, + Object.assign({}, ingestionSpec, { samplerConfig: { numRows: 500, timeoutMs: 15000, }, - }, + }), [{ make: 'Honda', model: 'Accord' }, { make: 'Toyota', model: 'Prius' }], ), ).toMatchInlineSnapshot(` @@ -157,31 +157,33 @@ describe('test-utils', () => { describe('druid-type.ts', () => { const ingestionSpec: IngestionSpec = { type: 'index_parallel', - ioConfig: { - type: 'index_parallel', - inputSource: { - type: 'http', - uris: ['https://static.imply.io/data/wikipedia.json.gz'], + spec: { + ioConfig: { + type: 'index_parallel', + inputSource: { + type: 'http', + uris: ['https://static.imply.io/data/wikipedia.json.gz'], + }, + inputFormat: { + type: 'json', + }, }, - inputFormat: { - type: 'json', + tuningConfig: { + type: 'index_parallel', }, - }, - tuningConfig: { - type: 'index_parallel', - }, - dataSchema: { - dataSource: 'wikipedia', - granularitySpec: { - type: 'uniform', - segmentGranularity: 'DAY', - queryGranularity: 'HOUR', - }, - timestampSpec: { - column: 'timestamp', - format: 'iso', + dataSchema: { + dataSource: 'wikipedia', + granularitySpec: { + type: 'uniform', + segmentGranularity: 'DAY', + queryGranularity: 'HOUR', + }, + timestampSpec: { + column: 'timestamp', + format: 'iso', + }, + dimensionsSpec: {}, }, - dimensionsSpec: {}, }, }; @@ -219,44 +221,46 @@ describe('druid-type.ts', () => { updateSchemaWithSample(ingestionSpec, { header: ['header'], rows: [] }, 'specific', true), ).toMatchInlineSnapshot(` Object { - "dataSchema": Object { - "dataSource": "wikipedia", - "dimensionsSpec": Object { - "dimensions": Array [ - "header", + "spec": Object { + "dataSchema": Object { + "dataSource": "wikipedia", + "dimensionsSpec": Object { + "dimensions": Array [ + "header", + ], + }, + "granularitySpec": Object { + "queryGranularity": "HOUR", + "rollup": true, + "segmentGranularity": "DAY", + "type": "uniform", + }, + "metricsSpec": Array [ + Object { + "name": "count", + "type": "count", + }, ], - }, - "granularitySpec": Object { - "queryGranularity": "HOUR", - "rollup": true, - "segmentGranularity": "DAY", - "type": "uniform", - }, - "metricsSpec": Array [ - Object { - "name": "count", - "type": "count", + "timestampSpec": Object { + "column": "timestamp", + "format": "iso", }, - ], - "timestampSpec": Object { - "column": "timestamp", - "format": "iso", }, - }, - "ioConfig": Object { - "inputFormat": Object { - "type": "json", + "ioConfig": Object { + "inputFormat": Object { + "type": "json", + }, + "inputSource": Object { + "type": "http", + "uris": Array [ + "https://static.imply.io/data/wikipedia.json.gz", + ], + }, + "type": "index_parallel", }, - "inputSource": Object { - "type": "http", - "uris": Array [ - "https://static.imply.io/data/wikipedia.json.gz", - ], + "tuningConfig": Object { + "type": "index_parallel", }, - "type": "index_parallel", - }, - "tuningConfig": Object { - "type": "index_parallel", }, "type": "index_parallel", } diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 02c529ed94c4..ebb68debbd13 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -419,7 +419,7 @@ export class LoadDataView extends React.PureComponent = { spec: newSpec, specPreview: newSpec }; - if (!deepGet(newSpec, 'ioConfig.type')) { + if (!deepGet(newSpec, 'spec.ioConfig.type')) { deltaState.cacheRows = undefined; } this.setState(deltaState as LoadDataViewState); @@ -961,7 +961,7 @@ export class LoadDataView extends React.PureComponent { const stringValue = e.target.value.substr(0, MAX_INLINE_DATA_LENGTH); - this.updateSpecPreview(deepSet(spec, 'ioConfig.inputSource.data', stringValue)); + this.updateSpecPreview(deepSet(spec, 'spec.ioConfig.inputSource.data', stringValue)); }} /> ); @@ -1078,18 +1078,18 @@ export class LoadDataView extends React.PureComponent this.updateSpecPreview(deepSet(spec, 'ioConfig', c))} + onChange={c => this.updateSpecPreview(deepSet(spec, 'spec.ioConfig', c))} /> ) : ( this.updateSpecPreview(deepSet(spec, 'ioConfig', c))} + onChange={c => this.updateSpecPreview(deepSet(spec, 'spec.ioConfig', c))} height="300px" /> )} - {deepGet(spec, 'ioConfig.inputSource.type') === 'local' && ( + {deepGet(spec, 'spec.ioConfig.inputSource.type') === 'local' && ( This path must be available on the local filesystem of all Druid services. @@ -1117,19 +1117,23 @@ export class LoadDataView extends React.PureComponent k !== '__time' && !aggregators[k]) .map(k => ({ @@ -1151,7 +1155,7 @@ export class LoadDataView extends React.PureComponent this.updateSpecPreview(deepSet(spec, 'ioConfig.inputFormat', p))} + onChange={p => + this.updateSpecPreview(deepSet(spec, 'spec.ioConfig.inputFormat', p)) + } /> {this.renderApplyButtonBar()} @@ -1325,7 +1331,11 @@ export class LoadDataView extends React.PureComponent { this.updateSpec( - deepSet(spec, 'ioConfig.inputFormat.flattenSpec.fields', sugestedFlattenFields), + deepSet( + spec, + 'spec.ioConfig.inputFormat.flattenSpec.fields', + sugestedFlattenFields, + ), ); }} /> @@ -1351,7 +1361,7 @@ export class LoadDataView extends React.PureComponent { @@ -1397,7 +1407,7 @@ export class LoadDataView extends React.PureComponent