Web console: better spec conversion with issues

web-console/src/ace-modes/dsql.js

@@ -63,6 +63,10 @@ ace.define(
 
       this.$rules = {
         start: [
+          {
+            token: 'comment.issue',
+            regex: '--:ISSUE:.*$',
+          },
           {
             token: 'comment',
             regex: '--.*$',
@@ -73,17 +77,13 @@ ace.define(
             end: '\\*/',
           },
           {
-            token: 'string', // " string
+            token: 'variable.column', // " quoted reference
             regex: '".*?"',
           },
           {
-            token: 'string', // ' string
+            token: 'string', // ' string literal
             regex: "'.*?'",
           },
-          {
-            token: 'string', // ` string (apache drill)
-            regex: '`.*?`',
-          },
           {
             token: 'constant.numeric', // float
             regex: '[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b',

web-console/src/bootstrap/ace.scss

@@ -25,6 +25,18 @@
 .ace-solarized-dark {
   background-color: rgba($dark-gray1, 0.5);
 
+  // START: Custom code styles
+  .ace_variable.ace_column {
+    color: #2ceefb;
+  }
+
+  .ace_comment.ace_issue {
+    color: #cb3116;
+    text-decoration: underline;
+    text-decoration-style: wavy;
+  }
+  // END: Custom code styles
+
   &.no-background {
     background-color: transparent;
   }

web-console/src/druid-models/workbench-query/workbench-query.spec.ts

@@ -423,6 +423,20 @@ describe('WorkbenchQuery', () => {
         sqlPrefixLines: 0,
       });
     });
+
+    it('works with sql with ISSUE comment', () => {
+      const sql = sane`
+        SELECT *
+        --:ISSUE: There is something wrong with this query.
+        FROM wikipedia
+      `;
+
+      const workbenchQuery = WorkbenchQuery.blank().changeQueryString(sql);
+
+      expect(() => workbenchQuery.getApiQuery(makeQueryId)).toThrow(
+        `This query contains an ISSUE comment: There is something wrong with this query. (Please resolve the issue in the comment, delete the ISSUE comment and re-run the query.)`,
+      );
+    });
   });
 
   describe('#getIngestDatasource', () => {

web-console/src/druid-models/workbench-query/workbench-query.ts

@@ -576,6 +576,18 @@ export class WorkbenchQuery {
       apiQuery.query = queryPrepend + apiQuery.query + queryAppend;
     }
 
+    const m = /--:ISSUE:(.+)(?:\n|$)/.exec(apiQuery.query);
+    if (m) {
+      throw new Error(
+        `This query contains an ISSUE comment: ${m[1]
+          .trim()
+          .replace(
+            /\.$/,
+            '',
+          )}. (Please resolve the issue in the comment, delete the ISSUE comment and re-run the query.)`,
+      );
+    }
+
     const ingestQuery = this.isIngestQuery();
     if (!unlimited && !ingestQuery) {
       apiQuery.context ||= {};

web-console/src/helpers/spec-conversion.spec.ts

@@ -449,4 +449,262 @@ describe('spec conversion', () => {
       finalizeAggregations: false,
     });
   });
+
+  it('converts with issue when there is a __time transform', () => {
+    const converted = convertSpecToSql({
+      type: 'index_parallel',
+      spec: {
+        ioConfig: {
+          type: 'index_parallel',
+          inputSource: {
+            type: 'http',
+            uris: ['https://druid.apache.org/data/wikipedia.json.gz'],
+          },
+          inputFormat: {
+            type: 'json',
+          },
+        },
+        dataSchema: {
+          granularitySpec: {
+            segmentGranularity: 'hour',
+            queryGranularity: 'none',
+            rollup: false,
+          },
+          dataSource: 'wikipedia',
+          transformSpec: {
+            transforms: [{ name: '__time', expression: '_some_time_parse_expression_' }],
+          },
+          timestampSpec: {
+            column: 'timestamp',
+            format: 'auto',
+          },
+          dimensionsSpec: {
+            dimensions: [
+              'isRobot',
+              'channel',
+              'flags',
+              'isUnpatrolled',
+              'page',
+              'diffUrl',
+              {
+                type: 'long',
+                name: 'added',
+              },
+              'comment',
+              {
+                type: 'long',
+                name: 'commentLength',
+              },
+              'isNew',
+              'isMinor',
+              {
+                type: 'long',
+                name: 'delta',
+              },
+              'isAnonymous',
+              'user',
+              {
+                type: 'long',
+                name: 'deltaBucket',
+              },
+              {
+                type: 'long',
+                name: 'deleted',
+              },
+              'namespace',
+              'cityName',
+              'countryName',
+              'regionIsoCode',
+              'metroCode',
+              'countryIsoCode',
+              'regionName',
+            ],
+          },
+        },
+        tuningConfig: {
+          type: 'index_parallel',
+          partitionsSpec: {
+            type: 'single_dim',
+            partitionDimension: 'isRobot',
+            targetRowsPerSegment: 150000,
+          },
+          forceGuaranteedRollup: true,
+          maxNumConcurrentSubTasks: 4,
+          maxParseExceptions: 3,
+        },
+      },
+    });
+
+    expect(converted.queryString).toEqual(sane`
+      -- This SQL query was auto generated from an ingestion spec
+      REPLACE INTO wikipedia OVERWRITE ALL
+      WITH source AS (SELECT * FROM TABLE(
+        EXTERN(
+          '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
+          '{"type":"json"}',
+          '[{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]'
+        )
+      ))
+      SELECT
+        --:ISSUE: The spec contained transforms that could not be automatically converted.
+        REWRITE_[_some_time_parse_expression_]_TO_SQL AS __time, --:ISSUE: Transform for __time could not be converted
+        "isRobot",
+        "channel",
+        "flags",
+        "isUnpatrolled",
+        "page",
+        "diffUrl",
+        "added",
+        "comment",
+        "commentLength",
+        "isNew",
+        "isMinor",
+        "delta",
+        "isAnonymous",
+        "user",
+        "deltaBucket",
+        "deleted",
+        "namespace",
+        "cityName",
+        "countryName",
+        "regionIsoCode",
+        "metroCode",
+        "countryIsoCode",
+        "regionName"
+      FROM source
+      PARTITIONED BY HOUR
+      CLUSTERED BY "isRobot"
+    `);
+  });
+
+  it('converts with issue when there is a dimension transform and strange filter', () => {
+    const converted = convertSpecToSql({
+      type: 'index_parallel',
+      spec: {
+        ioConfig: {
+          type: 'index_parallel',
+          inputSource: {
+            type: 'http',
+            uris: ['https://druid.apache.org/data/wikipedia.json.gz'],
+          },
+          inputFormat: {
+            type: 'json',
+          },
+        },
+        dataSchema: {
+          granularitySpec: {
+            segmentGranularity: 'hour',
+            queryGranularity: 'none',
+            rollup: false,
+          },
+          dataSource: 'wikipedia',
+          transformSpec: {
+            transforms: [{ name: 'comment', expression: '_some_expression_' }],
+            filter: {
+              type: 'strange',
+            },
+          },
+          timestampSpec: {
+            column: 'timestamp',
+            format: 'auto',
+          },
+          dimensionsSpec: {
+            dimensions: [
+              'isRobot',
+              'channel',
+              'flags',
+              'isUnpatrolled',
+              'page',
+              'diffUrl',
+              {
+                type: 'long',
+                name: 'added',
+              },
+              'comment',
+              {
+                type: 'long',
+                name: 'commentLength',
+              },
+              'isNew',
+              'isMinor',
+              {
+                type: 'long',
+                name: 'delta',
+              },
+              'isAnonymous',
+              'user',
+              {
+                type: 'long',
+                name: 'deltaBucket',
+              },
+              {
+                type: 'long',
+                name: 'deleted',
+              },
+              'namespace',
+              'cityName',
+              'countryName',
+              'regionIsoCode',
+              'metroCode',
+              'countryIsoCode',
+              'regionName',
+            ],
+          },
+        },
+        tuningConfig: {
+          type: 'index_parallel',
+          partitionsSpec: {
+            type: 'single_dim',
+            partitionDimension: 'isRobot',
+            targetRowsPerSegment: 150000,
+          },
+          forceGuaranteedRollup: true,
+          maxNumConcurrentSubTasks: 4,
+          maxParseExceptions: 3,
+        },
+      },
+    });
+
+    expect(converted.queryString).toEqual(sane`
+      -- This SQL query was auto generated from an ingestion spec
+      REPLACE INTO wikipedia OVERWRITE ALL
+      WITH source AS (SELECT * FROM TABLE(
+        EXTERN(
+          '{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
+          '{"type":"json"}',
+          '[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name":"isAnonymous","type":"string"},{"name":"user","type":"string"},{"name":"deltaBucket","type":"long"},{"name":"deleted","type":"long"},{"name":"namespace","type":"string"},{"name":"cityName","type":"string"},{"name":"countryName","type":"string"},{"name":"regionIsoCode","type":"string"},{"name":"metroCode","type":"string"},{"name":"countryIsoCode","type":"string"},{"name":"regionName","type":"string"}]'
+        )
+      ))
+      SELECT
+        --:ISSUE: The spec contained transforms that could not be automatically converted.
+        CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp") END AS __time,
+        "isRobot",
+        "channel",
+        "flags",
+        "isUnpatrolled",
+        "page",
+        "diffUrl",
+        "added",
+        REWRITE_[_some_expression_]_TO_SQL AS "comment", --:ISSUE: Transform for dimension could not be converted
+        "commentLength",
+        "isNew",
+        "isMinor",
+        "delta",
+        "isAnonymous",
+        "user",
+        "deltaBucket",
+        "deleted",
+        "namespace",
+        "cityName",
+        "countryName",
+        "regionIsoCode",
+        "metroCode",
+        "countryIsoCode",
+        "regionName"
+      FROM source
+      WHERE REWRITE_[{"type":"strange"}]_TO_SQL --:ISSUE: The spec contained a filter that could not be automatically converted, please convert it manually
+      PARTITIONED BY HOUR
+      CLUSTERED BY "isRobot"
+    `);
+  });
 });