diff --git a/web-console/src/ace-modes/dsql.js b/web-console/src/ace-modes/dsql.js index c54970b17c67..3c743b7c4952 100644 --- a/web-console/src/ace-modes/dsql.js +++ b/web-console/src/ace-modes/dsql.js @@ -63,6 +63,10 @@ ace.define( this.$rules = { start: [ + { + token: 'comment.issue', + regex: '--:ISSUE:.*$', + }, { token: 'comment', regex: '--.*$', @@ -73,17 +77,13 @@ ace.define( end: '\\*/', }, { - token: 'string', // " string + token: 'variable.column', // " quoted reference regex: '".*?"', }, { - token: 'string', // ' string + token: 'string', // ' string literal regex: "'.*?'", }, - { - token: 'string', // ` string (apache drill) - regex: '`.*?`', - }, { token: 'constant.numeric', // float regex: '[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b', diff --git a/web-console/src/bootstrap/ace.scss b/web-console/src/bootstrap/ace.scss index ebe74d4c4121..e764348a8e32 100644 --- a/web-console/src/bootstrap/ace.scss +++ b/web-console/src/bootstrap/ace.scss @@ -25,6 +25,18 @@ .ace-solarized-dark { background-color: rgba($dark-gray1, 0.5); + // START: Custom code styles + .ace_variable.ace_column { + color: #2ceefb; + } + + .ace_comment.ace_issue { + color: #cb3116; + text-decoration: underline; + text-decoration-style: wavy; + } + // END: Custom code styles + &.no-background { background-color: transparent; } diff --git a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts index 008947994f58..8f6f2581e1d2 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts @@ -423,6 +423,20 @@ describe('WorkbenchQuery', () => { sqlPrefixLines: 0, }); }); + + it('works with sql with ISSUE comment', () => { + const sql = sane` + SELECT * + --:ISSUE: There is something wrong with this query. + FROM wikipedia + `; + + const workbenchQuery = WorkbenchQuery.blank().changeQueryString(sql); + + expect(() => workbenchQuery.getApiQuery(makeQueryId)).toThrow( + `This query contains an ISSUE comment: There is something wrong with this query. (Please resolve the issue in the comment, delete the ISSUE comment and re-run the query.)`, + ); + }); }); describe('#getIngestDatasource', () => { diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index bf7ca26b29b1..d30ea7d4cf9f 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -576,6 +576,18 @@ export class WorkbenchQuery { apiQuery.query = queryPrepend + apiQuery.query + queryAppend; } + const m = /--:ISSUE:(.+)(?:\n|$)/.exec(apiQuery.query); + if (m) { + throw new Error( + `This query contains an ISSUE comment: ${m[1] + .trim() + .replace( + /\.$/, + '', + )}. (Please resolve the issue in the comment, delete the ISSUE comment and re-run the query.)`, + ); + } + const ingestQuery = this.isIngestQuery(); if (!unlimited && !ingestQuery) { apiQuery.context ||= {}; diff --git a/web-console/src/helpers/spec-conversion.spec.ts b/web-console/src/helpers/spec-conversion.spec.ts index 9b1e421e939d..43e7d31fac58 100644 --- a/web-console/src/helpers/spec-conversion.spec.ts +++ b/web-console/src/helpers/spec-conversion.spec.ts @@ -449,4 +449,262 @@ describe('spec conversion', () => { finalizeAggregations: false, }); }); + + it('converts with issue when there is a __time transform', () => { + const converted = convertSpecToSql({ + type: 'index_parallel', + spec: { + ioConfig: { + type: 'index_parallel', + inputSource: { + type: 'http', + uris: ['https://druid.apache.org/data/wikipedia.json.gz'], + }, + inputFormat: { + type: 'json', + }, + }, + dataSchema: { + granularitySpec: { + segmentGranularity: 'hour', + queryGranularity: 'none', + rollup: false, + }, + dataSource: 'wikipedia', + transformSpec: { + transforms: [{ name: '__time', expression: '_some_time_parse_expression_' }], + }, + timestampSpec: { + column: 'timestamp', + format: 'auto', + }, + dimensionsSpec: { + dimensions: [ + 'isRobot', + 'channel', + 'flags', + 'isUnpatrolled', + 'page', + 'diffUrl', + { + type: 'long', + name: 'added', + }, + 'comment', + { + type: 'long', + name: 'commentLength', + }, + 'isNew', + 'isMinor', + { + type: 'long', + name: 'delta', + }, + 'isAnonymous', + 'user', + { + type: 'long', + name: 'deltaBucket', + }, + { + type: 'long', + name: 'deleted', + }, + 'namespace', + 'cityName', + 'countryName', + 'regionIsoCode', + 'metroCode', + 'countryIsoCode', + 'regionName', + ], + }, + }, + tuningConfig: { + type: 'index_parallel', + partitionsSpec: { + type: 'single_dim', + partitionDimension: 'isRobot', + targetRowsPerSegment: 150000, + }, + forceGuaranteedRollup: true, + maxNumConcurrentSubTasks: 4, + maxParseExceptions: 3, + }, + }, + }); + + expect(converted.queryString).toEqual(sane` + -- This SQL query was auto generated from an ingestion spec + REPLACE INTO wikipedia OVERWRITE ALL + WITH source AS (SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', + '{"type":"json"}', + '[{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]' + ) + )) + SELECT + --:ISSUE: The spec contained transforms that could not be automatically converted. + REWRITE_[_some_time_parse_expression_]_TO_SQL AS __time, --:ISSUE: Transform for __time could not be converted + "isRobot", + "channel", + "flags", + "isUnpatrolled", + "page", + "diffUrl", + "added", + "comment", + "commentLength", + "isNew", + "isMinor", + "delta", + "isAnonymous", + "user", + "deltaBucket", + "deleted", + "namespace", + "cityName", + "countryName", + "regionIsoCode", + "metroCode", + "countryIsoCode", + "regionName" + FROM source + PARTITIONED BY HOUR + CLUSTERED BY "isRobot" + `); + }); + + it('converts with issue when there is a dimension transform and strange filter', () => { + const converted = convertSpecToSql({ + type: 'index_parallel', + spec: { + ioConfig: { + type: 'index_parallel', + inputSource: { + type: 'http', + uris: ['https://druid.apache.org/data/wikipedia.json.gz'], + }, + inputFormat: { + type: 'json', + }, + }, + dataSchema: { + granularitySpec: { + segmentGranularity: 'hour', + queryGranularity: 'none', + rollup: false, + }, + dataSource: 'wikipedia', + transformSpec: { + transforms: [{ name: 'comment', expression: '_some_expression_' }], + filter: { + type: 'strange', + }, + }, + timestampSpec: { + column: 'timestamp', + format: 'auto', + }, + dimensionsSpec: { + dimensions: [ + 'isRobot', + 'channel', + 'flags', + 'isUnpatrolled', + 'page', + 'diffUrl', + { + type: 'long', + name: 'added', + }, + 'comment', + { + type: 'long', + name: 'commentLength', + }, + 'isNew', + 'isMinor', + { + type: 'long', + name: 'delta', + }, + 'isAnonymous', + 'user', + { + type: 'long', + name: 'deltaBucket', + }, + { + type: 'long', + name: 'deleted', + }, + 'namespace', + 'cityName', + 'countryName', + 'regionIsoCode', + 'metroCode', + 'countryIsoCode', + 'regionName', + ], + }, + }, + tuningConfig: { + type: 'index_parallel', + partitionsSpec: { + type: 'single_dim', + partitionDimension: 'isRobot', + targetRowsPerSegment: 150000, + }, + forceGuaranteedRollup: true, + maxNumConcurrentSubTasks: 4, + maxParseExceptions: 3, + }, + }, + }); + + expect(converted.queryString).toEqual(sane` + -- This SQL query was auto generated from an ingestion spec + REPLACE INTO wikipedia OVERWRITE ALL + WITH source AS (SELECT * FROM TABLE( + EXTERN( + '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}', + '{"type":"json"}', + '[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]' + ) + )) + SELECT + --:ISSUE: The spec contained transforms that could not be automatically converted. + CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time, + "isRobot", + "channel", + "flags", + "isUnpatrolled", + "page", + "diffUrl", + "added", + REWRITE_[_some_expression_]_TO_SQL AS "comment", --:ISSUE: Transform for dimension could not be converted + "commentLength", + "isNew", + "isMinor", + "delta", + "isAnonymous", + "user", + "deltaBucket", + "deleted", + "namespace", + "cityName", + "countryName", + "regionIsoCode", + "metroCode", + "countryIsoCode", + "regionName" + FROM source + WHERE REWRITE_[{"type":"strange"}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually + PARTITIONED BY HOUR + CLUSTERED BY "isRobot" + `); + }); }); diff --git a/web-console/src/helpers/spec-conversion.ts b/web-console/src/helpers/spec-conversion.ts index 498c908595b6..9eedcce9f797 100644 --- a/web-console/src/helpers/spec-conversion.ts +++ b/web-console/src/helpers/spec-conversion.ts @@ -102,45 +102,55 @@ export function convertSpecToSql(spec: any): QueryWithContext { ); } + const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || []; + if (!Array.isArray(transforms)) { + throw new Error(`spec.dataSchema.transformSpec.transforms is not an array`); + } + let timeExpression: string; const column = timestampSpec.column || 'timestamp'; const columnRef = SqlRef.column(column); const format = timestampSpec.format || 'auto'; - switch (format) { - case 'auto': - columns.unshift({ name: column, type: 'string' }); - timeExpression = `CASE WHEN CAST(${columnRef} AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST(${columnRef} AS BIGINT)) ELSE TIME_PARSE(${columnRef}) END`; - break; - - case 'iso': - columns.unshift({ name: column, type: 'string' }); - timeExpression = `TIME_PARSE(${columnRef})`; - break; - - case 'posix': - columns.unshift({ name: column, type: 'long' }); - timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} * 1000)`; - break; - - case 'millis': - columns.unshift({ name: column, type: 'long' }); - timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef})`; - break; - - case 'micro': - columns.unshift({ name: column, type: 'long' }); - timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} / 1000)`; - break; - - case 'nano': - columns.unshift({ name: column, type: 'long' }); - timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} / 1000000)`; - break; - - default: - columns.unshift({ name: column, type: 'string' }); - timeExpression = `TIME_PARSE(${columnRef}, ${SqlLiteral.create(format)})`; - break; + const timeTransform = transforms.find(t => t.name === '__time'); + if (timeTransform) { + timeExpression = `REWRITE_[${timeTransform.expression}]_TO_SQL`; + } else { + switch (format) { + case 'auto': + columns.unshift({ name: column, type: 'string' }); + timeExpression = `CASE WHEN CAST(${columnRef} AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST(${columnRef} AS BIGINT)) ELSE TIME_PARSE(${columnRef}) END`; + break; + + case 'iso': + columns.unshift({ name: column, type: 'string' }); + timeExpression = `TIME_PARSE(${columnRef})`; + break; + + case 'posix': + columns.unshift({ name: column, type: 'long' }); + timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} * 1000)`; + break; + + case 'millis': + columns.unshift({ name: column, type: 'long' }); + timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef})`; + break; + + case 'micro': + columns.unshift({ name: column, type: 'long' }); + timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} / 1000)`; + break; + + case 'nano': + columns.unshift({ name: column, type: 'long' }); + timeExpression = `MILLIS_TO_TIMESTAMP(${columnRef} / 1000000)`; + break; + + default: + columns.unshift({ name: column, type: 'string' }); + timeExpression = `TIME_PARSE(${columnRef}, ${SqlLiteral.create(format)})`; + break; + } } if (timestampSpec.missingValue) { @@ -238,19 +248,24 @@ export function convertSpecToSql(spec: any): QueryWithContext { lines.push(`SELECT`); - const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || []; - if (!Array.isArray(transforms)) - throw new Error(`spec.dataSchema.transformSpec.transforms is not an array`); if (transforms.length) { - lines.push(` -- The spec contained transforms that could not be automatically converted.`); + lines.push( + ` --:ISSUE: The spec contained transforms that could not be automatically converted.`, + ); } - const dimensionExpressions = [` ${timeExpression} AS __time,`].concat( + const dimensionExpressions = [ + ` ${timeExpression} AS __time,${ + timeTransform ? ` --:ISSUE: Transform for __time could not be converted` : '' + }`, + ].concat( dimensions.flatMap((dimension: DimensionSpec) => { const dimensionName = dimension.name; const relevantTransform = transforms.find(t => t.name === dimensionName); - return ` ${SqlRef.columnWithQuotes(dimensionName)},${ - relevantTransform ? ` -- Relevant transform: ${JSONBig.stringify(relevantTransform)}` : '' + return ` ${ + relevantTransform ? `REWRITE_[${relevantTransform.expression}]_TO_SQL AS ` : '' + }${SqlRef.columnWithQuotes(dimensionName)},${ + relevantTransform ? ` --:ISSUE: Transform for dimension could not be converted` : '' }`; }), ); @@ -275,9 +290,9 @@ export function convertSpecToSql(spec: any): QueryWithContext { lines.push(`WHERE ${convertFilter(filter)}`); } catch { lines.push( - `-- The spec contained a filter that could not be automatically converted: ${JSONBig.stringify( + `WHERE REWRITE_[${JSONBig.stringify( filter, - )}`, + )}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually`, ); } }