diff --git a/web-console/package-lock.json b/web-console/package-lock.json index 6ac0e3a2cd9d..e8609e31f4c9 100644 --- a/web-console/package-lock.json +++ b/web-console/package-lock.json @@ -1,6 +1,6 @@ { "name": "web-console", - "version": "24.0.0", + "version": "24.0.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/web-console/package.json b/web-console/package.json index 0158b3bd620c..241cafca6b13 100644 --- a/web-console/package.json +++ b/web-console/package.json @@ -1,6 +1,6 @@ { "name": "web-console", - "version": "24.0.0", + "version": "24.0.1", "description": "A web console for Apache Druid", "author": "Apache Druid Developers ", "license": "Apache-2.0", diff --git a/web-console/script/create-sql-docs.js b/web-console/script/create-sql-docs.js index 4258a0e99e4d..6af65006f8ef 100755 --- a/web-console/script/create-sql-docs.js +++ b/web-console/script/create-sql-docs.js @@ -23,7 +23,7 @@ const snarkdown = require('snarkdown'); const writefile = 'lib/sql-docs.js'; -const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 158; +const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 162; const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 14; function hasHtmlTags(str) { @@ -90,15 +90,15 @@ const readDoc = async () => { // Make sure there are enough functions found const numFunction = Object.keys(functionDocs).length; - if (numFunction < MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS) { + if (!(MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS <= numFunction)) { throw new Error( `Did not find enough function entries did the structure of '${readfile}' change? (found ${numFunction} but expected at least ${MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS})`, ); } // Make sure there are at least 10 data types for sanity - const numDataTypes = dataTypeDocs.length; - if (numDataTypes < MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES) { + const numDataTypes = Object.keys(dataTypeDocs).length; + if (!(MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES <= numDataTypes)) { throw new Error( `Did not find enough data type entries did the structure of '${readfile}' change? (found ${numDataTypes} but expected at least ${MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES})`, ); diff --git a/web-console/src/ace-modes/dsql.js b/web-console/src/ace-modes/dsql.js index c54970b17c67..3c743b7c4952 100644 --- a/web-console/src/ace-modes/dsql.js +++ b/web-console/src/ace-modes/dsql.js @@ -63,6 +63,10 @@ ace.define( this.$rules = { start: [ + { + token: 'comment.issue', + regex: '--:ISSUE:.*$', + }, { token: 'comment', regex: '--.*$', @@ -73,17 +77,13 @@ ace.define( end: '\\*/', }, { - token: 'string', // " string + token: 'variable.column', // " quoted reference regex: '".*?"', }, { - token: 'string', // ' string + token: 'string', // ' string literal regex: "'.*?'", }, - { - token: 'string', // ` string (apache drill) - regex: '`.*?`', - }, { token: 'constant.numeric', // float regex: '[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b', diff --git a/web-console/src/bootstrap/ace.scss b/web-console/src/bootstrap/ace.scss index ebe74d4c4121..e764348a8e32 100644 --- a/web-console/src/bootstrap/ace.scss +++ b/web-console/src/bootstrap/ace.scss @@ -25,6 +25,18 @@ .ace-solarized-dark { background-color: rgba($dark-gray1, 0.5); + // START: Custom code styles + .ace_variable.ace_column { + color: #2ceefb; + } + + .ace_comment.ace_issue { + color: #cb3116; + text-decoration: underline; + text-decoration-style: wavy; + } + // END: Custom code styles + &.no-background { background-color: transparent; } diff --git a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap index 08619b242768..66b902596cf4 100644 --- a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap +++ b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap @@ -326,7 +326,7 @@ exports[`HeaderBar matches snapshot 1`] = ` `; +exports[`TableCell matches snapshot array mixed 1`] = ` +
+ ["a",{"v":"b"},"c"] +
+`; + exports[`TableCell matches snapshot array short 1`] = `
{ expect(container.firstChild).toMatchSnapshot(); }); + it('matches snapshot array mixed', () => { + const tableCell = ; + + const { container } = render(tableCell); + expect(container.firstChild).toMatchSnapshot(); + }); + it('matches snapshot object', () => { const tableCell = ; diff --git a/web-console/src/components/table-cell/table-cell.tsx b/web-console/src/components/table-cell/table-cell.tsx index 3895a805a702..78f080b306d6 100644 --- a/web-console/src/components/table-cell/table-cell.tsx +++ b/web-console/src/components/table-cell/table-cell.tsx @@ -21,6 +21,7 @@ import * as JSONBig from 'json-bigint-native'; import React, { useState } from 'react'; import { ShowValueDialog } from '../../dialogs/show-value-dialog/show-value-dialog'; +import { isSimpleArray } from '../../utils'; import { ActionIcon } from '../action-icon/action-icon'; import './table-cell.scss'; @@ -97,7 +98,7 @@ export const TableCell = React.memo(function TableCell(props: TableCellProps) { {isNaN(dateValue) ? 'Unusable date' : value.toISOString()}
); - } else if (Array.isArray(value)) { + } else if (isSimpleArray(value)) { return renderTruncated(`[${value.join(', ')}]`); } else if (typeof value === 'object') { return renderTruncated(JSONBig.stringify(value)); diff --git a/web-console/src/components/table-filterable-cell/table-filterable-cell.tsx b/web-console/src/components/table-filterable-cell/table-filterable-cell.tsx index d3387fd37e81..14f568013f09 100644 --- a/web-console/src/components/table-filterable-cell/table-filterable-cell.tsx +++ b/web-console/src/components/table-filterable-cell/table-filterable-cell.tsx @@ -34,14 +34,14 @@ export interface TableFilterableCellProps { value: string; filters: Filter[]; onFiltersChange(filters: Filter[]): void; - disableComparisons?: boolean; + enableComparisons?: boolean; children?: ReactNode; } export const TableFilterableCell = React.memo(function TableFilterableCell( props: TableFilterableCellProps, ) { - const { field, value, children, filters, disableComparisons, onFiltersChange } = props; + const { field, value, children, filters, enableComparisons, onFiltersChange } = props; return ( ( - {(disableComparisons ? FILTER_MODES_NO_COMPARISONS : FILTER_MODES).map((mode, i) => ( + {(enableComparisons ? FILTER_MODES : FILTER_MODES_NO_COMPARISONS).map((mode, i) => ( documentation diff --git a/web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap b/web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap index 14d240ec01ed..ac6e6eaf8a4d 100644 --- a/web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap @@ -11,7 +11,7 @@ exports[`OverlordDynamicConfigDialog matches snapshot 1`] = ` Edit the overlord dynamic configuration on the fly. For more information please refer to the documentation diff --git a/web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap b/web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap index c8b55ae5940c..668128421ac5 100644 --- a/web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap @@ -63,7 +63,7 @@ exports[`RetentionDialog matches snapshot 1`] = ` Druid uses rules to determine what data should be retained in the cluster. The rules are evaluated in order from top to bottom. For more information please refer to the diff --git a/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts b/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts index a35c1853810e..822dd48eba30 100644 --- a/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts +++ b/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts @@ -22,7 +22,7 @@ describe('flatten-spec', () => { describe('computeFlattenExprsForData', () => { const data = [ { - context: { host: 'cla', topic: 'moon', bonus: { foo: 'bar' } }, + context: { host: 'cla', topic: 'moon', bonus: { 'fo.o': 'bar' } }, tags: ['a', 'b', 'c'], messages: [ { metric: 'request/time', value: 122 }, @@ -32,7 +32,7 @@ describe('flatten-spec', () => { value: 5, }, { - context: { host: 'piv', popic: 'sun' }, + context: { 'host': 'piv', '1pic': 'sun' }, tags: ['a', 'd'], messages: [ { metric: 'request/time', value: 44 }, @@ -41,7 +41,7 @@ describe('flatten-spec', () => { value: 4, }, { - context: { host: 'imp', dopik: 'fun' }, + context: { 'host': 'imp', "d\\o\npi'c'": 'fun' }, tags: ['x', 'y'], messages: [ { metric: 'request/time', value: 4 }, @@ -53,22 +53,12 @@ describe('flatten-spec', () => { ]; it('works for path, ignore-arrays', () => { - expect(computeFlattenExprsForData(data, 'path', 'ignore-arrays')).toEqual([ - '$.context.bonus.foo', - '$.context.dopik', + expect(computeFlattenExprsForData(data, 'ignore-arrays')).toEqual([ + "$.context.bonus['fo.o']", '$.context.host', - '$.context.popic', '$.context.topic', - ]); - }); - - it('works for jq, ignore-arrays', () => { - expect(computeFlattenExprsForData(data, 'jq', 'ignore-arrays')).toEqual([ - '.context.bonus.foo', - '.context.dopik', - '.context.host', - '.context.popic', - '.context.topic', + "$.context['1pic']", + "$.context['d\\\\o\npi\\'c\\'']", ]); }); }); diff --git a/web-console/src/druid-models/flatten-spec/flatten-spec.tsx b/web-console/src/druid-models/flatten-spec/flatten-spec.tsx index 3845a83916dd..5cf1d7c06c61 100644 --- a/web-console/src/druid-models/flatten-spec/flatten-spec.tsx +++ b/web-console/src/druid-models/flatten-spec/flatten-spec.tsx @@ -61,18 +61,22 @@ export const FLATTEN_FIELD_FIELDS: Field[] = [ }, ]; -export type ExprType = 'path' | 'jq'; export type ArrayHandling = 'ignore-arrays' | 'include-arrays'; +function escapePathKey(pathKey: string): string { + return /^[a-z]\w*$/i.test(pathKey) + ? `.${pathKey}` + : `['${pathKey.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}']`; +} + export function computeFlattenPathsForData( data: Record[], - exprType: ExprType, arrayHandling: ArrayHandling, ): FlattenField[] { - return computeFlattenExprsForData(data, exprType, arrayHandling).map(expr => { + return computeFlattenExprsForData(data, arrayHandling).map(expr => { return { - name: expr.replace(/^\$?\./, ''), - type: exprType, + name: expr.replace(/^\$\./, '').replace(/['\]]/g, '').replace(/\[/g, '.'), + type: 'path', expr, }; }); @@ -80,7 +84,6 @@ export function computeFlattenPathsForData( export function computeFlattenExprsForData( data: Record[], - exprType: ExprType, arrayHandling: ArrayHandling, includeTopLevel = false, ): string[] { @@ -91,12 +94,7 @@ export function computeFlattenExprsForData( for (const datumKey of datumKeys) { const datumValue = datum[datumKey]; if (includeTopLevel || isNested(datumValue)) { - addPath( - seenPaths, - exprType === 'path' ? `$.${datumKey}` : `.${datumKey}`, - datumValue, - arrayHandling, - ); + addPath(seenPaths, `$${escapePathKey(datumKey)}`, datumValue, arrayHandling); } } } @@ -114,7 +112,7 @@ function addPath( if (!Array.isArray(value)) { const valueKeys = Object.keys(value); for (const valueKey of valueKeys) { - addPath(paths, `${path}.${valueKey}`, value[valueKey], arrayHandling); + addPath(paths, `${path}${escapePathKey(valueKey)}`, value[valueKey], arrayHandling); } } else if (arrayHandling === 'include-arrays') { for (let i = 0; i < value.length; i++) { diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts b/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts index a846d29a8188..3f4be422c79f 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts @@ -725,6 +725,15 @@ describe('spec utils', () => { it('works for multi-value', () => { expect(guessColumnTypeFromInput(['a', ['b'], 'c'], false)).toEqual('string'); expect(guessColumnTypeFromInput([1, [2], 3], false)).toEqual('string'); + expect(guessColumnTypeFromInput([true, [true, 7, false], false, 'x'], false)).toEqual( + 'string', + ); + }); + + it('works for complex arrays', () => { + expect(guessColumnTypeFromInput([{ type: 'Dogs' }, { type: 'JavaScript' }], false)).toEqual( + 'COMPLEX', + ); }); it('works for strange json', () => { diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx index 4ce36c655dc3..f7492d35501a 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx @@ -32,6 +32,7 @@ import { EMPTY_ARRAY, EMPTY_OBJECT, filterMap, + isSimpleArray, oneOf, parseCsvLine, typeIs, @@ -2309,7 +2310,7 @@ export function guessIsArrayFromHeaderAndRows( headerAndRows: SampleHeaderAndRows, column: string, ): boolean { - return headerAndRows.rows.some(r => Array.isArray(r.input?.[column])); + return headerAndRows.rows.some(r => isSimpleArray(r.input?.[column])); } export function guessColumnTypeFromInput( @@ -2322,7 +2323,7 @@ export function guessColumnTypeFromInput( if (!definedValues.length) return 'string'; // If we see any arrays in the input this is a multi-value dimension that must be a string - if (definedValues.some(v => Array.isArray(v))) return 'string'; + if (definedValues.some(v => isSimpleArray(v))) return 'string'; // If we see any JSON objects in the input assume COMPLEX if (definedValues.some(v => v && typeof v === 'object')) return 'COMPLEX'; diff --git a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts index 008947994f58..8f6f2581e1d2 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts @@ -423,6 +423,20 @@ describe('WorkbenchQuery', () => { sqlPrefixLines: 0, }); }); + + it('works with sql with ISSUE comment', () => { + const sql = sane` + SELECT * + --:ISSUE: There is something wrong with this query. + FROM wikipedia + `; + + const workbenchQuery = WorkbenchQuery.blank().changeQueryString(sql); + + expect(() => workbenchQuery.getApiQuery(makeQueryId)).toThrow( + `This query contains an ISSUE comment: There is something wrong with this query. (Please resolve the issue in the comment, delete the ISSUE comment and re-run the query.)`, + ); + }); }); describe('#getIngestDatasource', () => { diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index 4200874aaf55..d30ea7d4cf9f 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -576,10 +576,15 @@ export class WorkbenchQuery { apiQuery.query = queryPrepend + apiQuery.query + queryAppend; } - const m = /(--:context\s.+)(?:\n|$)/.exec(apiQuery.query); + const m = /--:ISSUE:(.+)(?:\n|$)/.exec(apiQuery.query); if (m) { throw new Error( - `This query contains a context comment '${m[1]}'. Context comments have been deprecated. Please rewrite the context comment as a context parameter. The context parameter editor is located in the "Engine" dropdown.`, + `This query contains an ISSUE comment: ${m[1] + .trim() + .replace( + /\.$/, + '', + )}. (Please resolve the issue in the comment, delete the ISSUE comment and re-run the query.)`, ); } diff --git a/web-console/src/helpers/spec-conversion.spec.ts b/web-console/src/helpers/spec-conversion.spec.ts index 9b1e421e939d..43e7d31fac58 100644 --- a/web-console/src/helpers/spec-conversion.spec.ts +++ b/web-console/src/helpers/spec-conversion.spec.ts @@ -449,4 +449,262 @@ describe('spec conversion', () => { finalizeAggregations: false, }); }); + + it('converts with issue when there is a __time transform', () => { + const converted = convertSpecToSql({ + type: 'index_parallel', + spec: { + ioConfig: { + type: 'index_parallel', + inputSource: { + type: 'http', + uris: ['https://druid.apache.org/data/wikipedia.json.gz'], + }, + inputFormat: { + type: 'json', + }, + }, + dataSchema: { + granularitySpec: { + segmentGranularity: 'hour', + queryGranularity: 'none', + rollup: false, + }, + dataSource: 'wikipedia', + transformSpec: { + transforms: [{ name: '__time', expression: '_some_time_parse_expression_' }], + }, + timestampSpec: { + column: 'timestamp', + format: 'auto', + }, + dimensionsSpec: { + dimensions: [ + 'isRobot', + 'channel', + 'flags', + 'isUnpatrolled', + 'page', + 'diffUrl', + { + type: 'long', + name: 'added', + }, + 'comment', + { + type: 'long', + name: 'commentLength', + }, + 'isNew', + 'isMinor', + { + type: 'long', + name: 'delta', + }, + 'isAnonymous', + 'user', + { + type: 'long', + name: 'deltaBucket', + }, + { + type: 'long', + name: 'deleted', + }, + 'namespace', + 'cityName', + 'countryName', + 'regionIsoCode', + 'metroCode', + 'countryIsoCode', + 'regionName', + ], + }, + }, + tuningConfig: { + type: 'index_parallel', + partitionsSpec: { + type: 'single_dim', + partitionDimension: 'isRobot', + targetRowsPerSegment: 150000, + }, + forceGuaranteedRollup: true, + maxNumConcurrentSubTasks: 4, + maxParseExceptions: 3, + }, + }, + }); + + expect(converted.queryString).toEqual(sane` + -- This SQL query was auto generated from an ingestion spec + REPLACE INTO wikipedia OVERWRITE ALL + WITH source AS (SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', + '{"type":"json"}', + '[{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]' + ) + )) + SELECT + --:ISSUE: The spec contained transforms that could not be automatically converted. + REWRITE_[_some_time_parse_expression_]_TO_SQL AS __time, --:ISSUE: Transform for __time could not be converted + "isRobot", + "channel", + "flags", + "isUnpatrolled", + "page", + "diffUrl", + "added", + "comment", + "commentLength", + "isNew", + "isMinor", + "delta", + "isAnonymous", + "user", + "deltaBucket", + "deleted", + "namespace", + "cityName", + "countryName", + "regionIsoCode", + "metroCode", + "countryIsoCode", + "regionName" + FROM source + PARTITIONED BY HOUR + CLUSTERED BY "isRobot" + `); + }); + + it('converts with issue when there is a dimension transform and strange filter', () => { + const converted = convertSpecToSql({ + type: 'index_parallel', + spec: { + ioConfig: { + type: 'index_parallel', + inputSource: { + type: 'http', + uris: ['https://druid.apache.org/data/wikipedia.json.gz'], + }, + inputFormat: { + type: 'json', + }, + }, + dataSchema: { + granularitySpec: { + segmentGranularity: 'hour', + queryGranularity: 'none', + rollup: false, + }, + dataSource: 'wikipedia', + transformSpec: { + transforms: [{ name: 'comment', expression: '_some_expression_' }], + filter: { + type: 'strange', + }, + }, + timestampSpec: { + column: 'timestamp', + format: 'auto', + }, + dimensionsSpec: { + dimensions: [ + 'isRobot', + 'channel', + 'flags', + 'isUnpatrolled', + 'page', + 'diffUrl', + { + type: 'long', + name: 'added', + }, + 'comment', + { + type: 'long', + name: 'commentLength', + }, + 'isNew', + 'isMinor', + { + type: 'long', + name: 'delta', + }, + 'isAnonymous', + 'user', + { + type: 'long', + name: 'deltaBucket', + }, + { + type: 'long', + name: 'deleted', + }, + 'namespace', + 'cityName', + 'countryName', + 'regionIsoCode', + 'metroCode', + 'countryIsoCode', + 'regionName', + ], + }, + }, + tuningConfig: { + type: 'index_parallel', + partitionsSpec: { + type: 'single_dim', + partitionDimension: 'isRobot', + targetRowsPerSegment: 150000, + }, + forceGuaranteedRollup: true, + maxNumConcurrentSubTasks: 4, + maxParseExceptions: 3, + }, + }, + }); + + expect(converted.queryString).toEqual(sane` + -- This SQL query was auto generated from an ingestion spec + REPLACE INTO wikipedia OVERWRITE ALL + WITH source AS (SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', + '{"type":"json"}', + '[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]' + ) + )) + SELECT + --:ISSUE: The spec contained transforms that could not be automatically converted. + CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time, + "isRobot", + "channel", + "flags", + "isUnpatrolled", + "page", + "diffUrl", + "added", + REWRITE_[_some_expression_]_TO_SQL AS "comment", --:ISSUE: Transform for dimension could not be converted + "commentLength", + "isNew", + "isMinor", + "delta", + "isAnonymous", + "user", + "deltaBucket", + "deleted", + "namespace", + "cityName", + "countryName", + "regionIsoCode", + "metroCode", + "countryIsoCode", + "regionName" + FROM source + WHERE REWRITE_[{"type":"strange"}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually + PARTITIONED BY HOUR + CLUSTERED BY "isRobot" + `); + }); }); diff --git a/web-console/src/helpers/spec-conversion.ts b/web-console/src/helpers/spec-conversion.ts index 498c908595b6..9eedcce9f797 100644 --- a/web-console/src/helpers/spec-conversion.ts +++ b/web-console/src/helpers/spec-conversion.ts @@ -102,45 +102,55 @@ export function convertSpecToSql(spec: any): QueryWithContext { ); } + const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || []; + if (!Array.isArray(transforms)) { + throw new Error(`spec.dataSchema.transformSpec.transforms is not an array`); + } + let timeExpression: string; const column = timestampSpec.column || 'timestamp'; const columnRef = SqlRef.column(column); const format = timestampSpec.format || 'auto'; - switch (format) { - case 'auto': - columns.unshift({ name: column, type: 'string' }); - timeExpression = `CASE WHEN CAST(${columnRef} AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST(${columnRef} AS BIGINT)) ELSE TIME_PARSE(${columnRef}) END`; - break; - - case 'iso': - columns.unshift({ name: column, type: 'string' }); - timeExpression = `TIME_PARSE(${columnRef})`; - break; - - case 'posix': - columns.unshift({ name: column, type: 'long' }); - timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} * 1000)`; - break; - - case 'millis': - columns.unshift({ name: column, type: 'long' }); - timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef})`; - break; - - case 'micro': - columns.unshift({ name: column, type: 'long' }); - timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} / 1000)`; - break; - - case 'nano': - columns.unshift({ name: column, type: 'long' }); - timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} / 1000000)`; - break; - - default: - columns.unshift({ name: column, type: 'string' }); - timeExpression = `TIME_PARSE(${columnRef}, ${SqlLiteral.create(format)})`; - break; + const timeTransform = transforms.find(t => t.name === '__time'); + if (timeTransform) { + timeExpression = `REWRITE_[${timeTransform.expression}]_TO_SQL`; + } else { + switch (format) { + case 'auto': + columns.unshift({ name: column, type: 'string' }); + timeExpression = `CASE WHEN CAST(${columnRef} AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST(${columnRef} AS BIGINT)) ELSE TIME_PARSE(${columnRef}) END`; + break; + + case 'iso': + columns.unshift({ name: column, type: 'string' }); + timeExpression = `TIME_PARSE(${columnRef})`; + break; + + case 'posix': + columns.unshift({ name: column, type: 'long' }); + timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} * 1000)`; + break; + + case 'millis': + columns.unshift({ name: column, type: 'long' }); + timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef})`; + break; + + case 'micro': + columns.unshift({ name: column, type: 'long' }); + timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} / 1000)`; + break; + + case 'nano': + columns.unshift({ name: column, type: 'long' }); + timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} / 1000000)`; + break; + + default: + columns.unshift({ name: column, type: 'string' }); + timeExpression = `TIME_PARSE(${columnRef}, ${SqlLiteral.create(format)})`; + break; + } } if (timestampSpec.missingValue) { @@ -238,19 +248,24 @@ export function convertSpecToSql(spec: any): QueryWithContext { lines.push(`SELECT`); - const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || []; - if (!Array.isArray(transforms)) - throw new Error(`spec.dataSchema.transformSpec.transforms is not an array`); if (transforms.length) { - lines.push(` -- The spec contained transforms that could not be automatically converted.`); + lines.push( + ` --:ISSUE: The spec contained transforms that could not be automatically converted.`, + ); } - const dimensionExpressions = [` ${timeExpression} AS __time,`].concat( + const dimensionExpressions = [ + ` ${timeExpression} AS __time,${ + timeTransform ? ` --:ISSUE: Transform for __time could not be converted` : '' + }`, + ].concat( dimensions.flatMap((dimension: DimensionSpec) => { const dimensionName = dimension.name; const relevantTransform = transforms.find(t => t.name === dimensionName); - return ` ${SqlRef.columnWithQuotes(dimensionName)},${ - relevantTransform ? ` -- Relevant transform: ${JSONBig.stringify(relevantTransform)}` : '' + return ` ${ + relevantTransform ? `REWRITE_[${relevantTransform.expression}]_TO_SQL AS ` : '' + }${SqlRef.columnWithQuotes(dimensionName)},${ + relevantTransform ? ` --:ISSUE: Transform for dimension could not be converted` : '' }`; }), ); @@ -275,9 +290,9 @@ export function convertSpecToSql(spec: any): QueryWithContext { lines.push(`WHERE ${convertFilter(filter)}`); } catch { lines.push( - `-- The spec contained a filter that could not be automatically converted: ${JSONBig.stringify( + `WHERE REWRITE_[${JSONBig.stringify( filter, - )}`, + )}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually`, ); } } diff --git a/web-console/src/links.ts b/web-console/src/links.ts index d083a517c155..096b2f3eed95 100644 --- a/web-console/src/links.ts +++ b/web-console/src/links.ts @@ -19,7 +19,7 @@ import hasOwnProp from 'has-own-prop'; // This is set to the latest available version and should be updated to the next version before release -const DRUID_DOCS_VERSION = '24.0.0'; +const DRUID_DOCS_VERSION = '24.0.1'; function fillVersion(str: string): string { return str.replace(/\{\{VERSION}}/g, DRUID_DOCS_VERSION); @@ -63,6 +63,7 @@ export type LinkNames = | 'DOCS_SQL' | 'DOCS_RUNE' | 'DOCS_API' + | 'DOCS_MSQ_ERROR' | 'COMMUNITY' | 'SLACK' | 'USER_GROUP' @@ -82,6 +83,8 @@ export function getLink(linkName: LinkNames): string { return `${links.docsHref}/querying/querying.html`; case 'DOCS_API': return `${links.docsHref}/operations/api-reference.html`; + case 'DOCS_MSQ_ERROR': + return `${links.docsHref}/multi-stage-query/concepts.html#error-codes`; case 'COMMUNITY': return links.communityHref; case 'SLACK': diff --git a/web-console/src/react-table/react-table-inputs.tsx b/web-console/src/react-table/react-table-inputs.tsx index d7e02c3dc37e..7dbb69b18f3e 100644 --- a/web-console/src/react-table/react-table-inputs.tsx +++ b/web-console/src/react-table/react-table-inputs.tsx @@ -43,7 +43,7 @@ export function GenericFilterInput({ column, filter, onChange, key }: FilterRend const [menuOpen, setMenuOpen] = useState(false); const [focused, setFocused] = useState(false); - const disableComparisons = String(column.headerClassName).includes('disable-comparisons'); + const enableComparisons = String(column.headerClassName).includes('enable-comparisons'); const { mode, needle } = (filter ? parseFilterModeAndNeedle(filter, true) : undefined) || { mode: '~', @@ -64,7 +64,7 @@ export function GenericFilterInput({ column, filter, onChange, key }: FilterRend onInteraction={setMenuOpen} content={ - {(disableComparisons ? FILTER_MODES_NO_COMPARISON : FILTER_MODES).map((m, i) => ( + {(enableComparisons ? FILTER_MODES : FILTER_MODES_NO_COMPARISON).map((m, i) => ( { + const t = typeof x; + return t === 'string' || t === 'number' || t === 'boolean'; + }) + ); +} + export function wait(ms: number): Promise { return new Promise(resolve => { setTimeout(resolve, ms); diff --git a/web-console/src/views/load-data-view/info-messages.tsx b/web-console/src/views/load-data-view/info-messages.tsx index 7d7235bc3d36..2d151ea995fb 100644 --- a/web-console/src/views/load-data-view/info-messages.tsx +++ b/web-console/src/views/load-data-view/info-messages.tsx @@ -16,12 +16,13 @@ * limitations under the License. */ -import { Callout, Code, FormGroup } from '@blueprintjs/core'; +import { Button, Callout, Code, FormGroup, Intent } from '@blueprintjs/core'; import React from 'react'; import { ExternalLink, LearnMore } from '../../components'; import { DimensionMode, getIngestionDocLink, IngestionSpec } from '../../druid-models'; import { getLink } from '../../links'; +import { deepGet, deepSet } from '../../utils'; export interface ConnectMessageProps { inlineMode: boolean; @@ -216,3 +217,48 @@ export const SpecMessage = React.memo(function SpecMessage() { ); }); + +export interface AppendToExistingIssueProps { + spec: Partial; + onChangeSpec(newSpec: Partial): void; +} + +export const AppendToExistingIssue = React.memo(function AppendToExistingIssue( + props: AppendToExistingIssueProps, +) { + const { spec, onChangeSpec } = props; + + const partitionsSpecType = deepGet(spec, 'spec.tuningConfig.partitionsSpec.type'); + if ( + partitionsSpecType === 'dynamic' || + deepGet(spec, 'spec.ioConfig.appendToExisting') !== true + ) { + return null; + } + + const dynamicPartitionSpec = { + type: 'dynamic', + maxRowsPerSegment: + deepGet(spec, 'spec.tuningConfig.partitionsSpec.maxRowsPerSegment') || + deepGet(spec, 'spec.tuningConfig.partitionsSpec.targetRowsPerSegment'), + }; + + return ( + + +

+ Only dynamic partitioning supports appendToExisting: true. You + have currently selected {partitionsSpecType} partitioning. +

+ + + + ); +}); diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 7afc13aa7504..e4322e34e5b0 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -168,6 +168,7 @@ import { ExamplePicker } from './example-picker/example-picker'; import { FilterTable, filterTableSelectedColumnName } from './filter-table/filter-table'; import { FormEditor } from './form-editor/form-editor'; import { + AppendToExistingIssue, ConnectMessage, FilterMessage, ParserMessage, @@ -1490,7 +1491,6 @@ export class LoadDataView extends React.PureComponent r.input), - 'path', 'ignore-arrays', ); } @@ -3003,6 +3003,7 @@ export class LoadDataView extends React.PureComponent {nonsensicalSingleDimPartitioningMessage} + {this.renderNextBar({ disabled: invalidPartitionConfig(spec), @@ -3096,8 +3097,8 @@ export class LoadDataView extends React.PureComponent - deepGet(spec, 'spec.tuningConfig.partitionsSpec.type') === 'dynamic', + // appendToExisting can only be set on 'dynamic' portioning. + // We chose to show it always and instead have a specific message, separate from this form, to notify the user of the issue. info: ( <> Creates segments as additional shards of the latest version, effectively @@ -3166,6 +3167,7 @@ export class LoadDataView extends React.PureComponent
+
{this.renderNextBar({})} @@ -3234,6 +3236,7 @@ export class LoadDataView extends React.PureComponent{`There is an issue with the spec: ${issueWithSpec}`} )} +
{!isEmptyIngestionSpec(spec) && ( diff --git a/web-console/src/views/query-view/query-input/query-input.tsx b/web-console/src/views/query-view/query-input/query-input.tsx index d4fd3fe6ae35..9f85c9de08aa 100644 --- a/web-console/src/views/query-view/query-input/query-input.tsx +++ b/web-console/src/views/query-view/query-input/query-input.tsx @@ -20,6 +20,7 @@ import { ResizeEntry } from '@blueprintjs/core'; import { ResizeSensor2 } from '@blueprintjs/popover2'; import type { Ace } from 'ace-builds'; import ace from 'ace-builds'; +import { SqlRef, SqlTableRef } from 'druid-query-toolkit'; import escape from 'lodash.escape'; import React from 'react'; import AceEditor from 'react-ace'; @@ -150,7 +151,7 @@ export class QueryInput extends React.PureComponent d.TABLE_SCHEMA)).map(v => ({ - value: v, + value: SqlTableRef.create(v).toString(), score: 10, meta: 'schema', })), @@ -159,7 +160,7 @@ export class QueryInput extends React.PureComponent (currentSchema ? d.TABLE_SCHEMA === currentSchema : true)) .map(d => d.TABLE_NAME), ).map(v => ({ - value: v, + value: SqlTableRef.create(v).toString(), score: 49, meta: 'datasource', })), @@ -172,7 +173,7 @@ export class QueryInput extends React.PureComponent d.COLUMN_NAME), ).map(v => ({ - value: v, + value: SqlRef.column(v).toString(), score: 50, meta: 'column', })), diff --git a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap index 6ac125bfdf43..5d273a286915 100755 --- a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap +++ b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap @@ -169,6 +169,7 @@ exports[`SegmentsView matches snapshot 1`] = ` "accessor": "start", "defaultSortDesc": true, "filterable": true, + "headerClassName": "enable-comparisons", "show": true, "sortable": true, "width": 160, @@ -179,6 +180,7 @@ exports[`SegmentsView matches snapshot 1`] = ` "accessor": "end", "defaultSortDesc": true, "filterable": true, + "headerClassName": "enable-comparisons", "show": true, "sortable": true, "width": 160, @@ -206,7 +208,6 @@ exports[`SegmentsView matches snapshot 1`] = ` "Cell": [Function], "Header": "Shard type", "accessor": [Function], - "headerClassName": "disable-comparisons", "id": "shard_type", "show": true, "sortable": false, diff --git a/web-console/src/views/segments-view/segments-view.tsx b/web-console/src/views/segments-view/segments-view.tsx index b0327182be13..85d027962268 100644 --- a/web-console/src/views/segments-view/segments-view.tsx +++ b/web-console/src/views/segments-view/segments-view.tsx @@ -485,7 +485,7 @@ END AS "time_span"`, }); } - private renderFilterableCell(field: string, disableComparisons = false) { + private renderFilterableCell(field: string, enableComparisons = false) { const { segmentFilter } = this.state; return (row: { value: any }) => ( @@ -494,7 +494,7 @@ END AS "time_span"`, value={row.value} filters={segmentFilter} onFiltersChange={filters => this.setState({ segmentFilter: filters })} - disableComparisons={disableComparisons} + enableComparisons={enableComparisons} > {row.value} @@ -582,21 +582,23 @@ END AS "time_span"`, Header: 'Start', show: visibleColumns.shown('Start'), accessor: 'start', + headerClassName: 'enable-comparisons', width: 160, sortable: hasSql, defaultSortDesc: true, filterable: allowGeneralFilter, - Cell: this.renderFilterableCell('start'), + Cell: this.renderFilterableCell('start', true), }, { Header: 'End', show: visibleColumns.shown('End'), accessor: 'end', + headerClassName: 'enable-comparisons', width: 160, sortable: hasSql, defaultSortDesc: true, filterable: allowGeneralFilter, - Cell: this.renderFilterableCell('end'), + Cell: this.renderFilterableCell('end', true), }, { Header: 'Version', @@ -623,7 +625,6 @@ END AS "time_span"`, id: 'shard_type', width: 100, sortable: false, - headerClassName: 'disable-comparisons', accessor: d => { let v: any; try { diff --git a/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap b/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap index cf8bccd618b4..cf7ed02ec721 100644 --- a/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap @@ -8,7 +8,12 @@ exports[`ExecutionErrorPane matches snapshot 1`] = `

- TooManyWarnings: + + TooManyWarnings + + : Too many warnings of type CannotParseExternalData generated (max = 10)

diff --git a/web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx b/web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx index da7e3173b35d..b0368534cc95 100644 --- a/web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx +++ b/web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx @@ -20,9 +20,10 @@ import { Callout } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import React, { useState } from 'react'; -import { ClickToCopy } from '../../../components'; +import { ClickToCopy, ExternalLink } from '../../../components'; import { ShowValueDialog } from '../../../dialogs/show-value-dialog/show-value-dialog'; import { Execution } from '../../../druid-models'; +import { getLink } from '../../../links'; import { downloadQueryDetailArchive } from '../../../utils'; import './execution-error-pane.scss'; @@ -43,7 +44,12 @@ export const ExecutionErrorPane = React.memo(function ExecutionErrorPane( return (

- {error.errorCode && <>{`${error.errorCode}: `}} + {error.errorCode && ( + <> + {error.errorCode} + {': '} + + )} {error.errorMessage || (exceptionStackTrace || '').split('\n')[0]} {exceptionStackTrace && ( <> diff --git a/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx b/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx index c83865cdb195..1559e7981c84 100644 --- a/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx +++ b/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx @@ -21,6 +21,7 @@ import { ResizeSensor2 } from '@blueprintjs/popover2'; import type { Ace } from 'ace-builds'; import ace from 'ace-builds'; import classNames from 'classnames'; +import { SqlRef, SqlTableRef } from 'druid-query-toolkit'; import escape from 'lodash.escape'; import React from 'react'; import AceEditor from 'react-ace'; @@ -163,7 +164,7 @@ export class FlexibleQueryInput extends React.PureComponent< ) { const completions = ([] as any[]).concat( uniq(columnMetadata.map(d => d.TABLE_SCHEMA)).map(v => ({ - value: v, + value: SqlTableRef.create(v).toString(), score: 10, meta: 'schema', })), @@ -172,7 +173,7 @@ export class FlexibleQueryInput extends React.PureComponent< .filter(d => (currentSchema ? d.TABLE_SCHEMA === currentSchema : true)) .map(d => d.TABLE_NAME), ).map(v => ({ - value: v, + value: SqlTableRef.create(v).toString(), score: 49, meta: 'datasource', })), @@ -185,7 +186,7 @@ export class FlexibleQueryInput extends React.PureComponent< ) .map(d => d.COLUMN_NAME), ).map(v => ({ - value: v, + value: SqlRef.column(v).toString(), score: 50, meta: 'column', })), diff --git a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx index e4b78b5af738..ba8b11001a48 100644 --- a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx +++ b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx @@ -82,7 +82,7 @@ function jsonValue(ex: SqlExpression, path: string): SqlExpression { } function getJsonPaths(jsons: Record[]): string[] { - return ['$.'].concat(computeFlattenExprsForData(jsons, 'path', 'include-arrays', true)); + return ['$.'].concat(computeFlattenExprsForData(jsons, 'include-arrays', true)); } function isComparable(x: unknown): boolean { diff --git a/web-console/unified-console.html b/web-console/unified-console.html index 19fa9d10bf67..ab0f71739229 100644 --- a/web-console/unified-console.html +++ b/web-console/unified-console.html @@ -71,6 +71,6 @@ }; - +