[backport] Backport 24.0.1 web console issue fixes

web-console/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-console",
-  "version": "24.0.0",
+  "version": "24.0.1",
   "description": "A web console for Apache Druid",
   "author": "Apache Druid Developers <dev@druid.apache.org>",
   "license": "Apache-2.0",

web-console/script/create-sql-docs.js

@@ -23,7 +23,7 @@ const snarkdown = require('snarkdown');
 
 const writefile = 'lib/sql-docs.js';
 
-const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 158;
+const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 162;
 const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 14;
 
 function hasHtmlTags(str) {
@@ -90,15 +90,15 @@ const readDoc = async () => {
 
   // Make sure there are enough functions found
   const numFunction = Object.keys(functionDocs).length;
-  if (numFunction < MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS) {
+  if (!(MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS <= numFunction)) {
     throw new Error(
       `Did not find enough function entries did the structure of '${readfile}' change? (found ${numFunction} but expected at least ${MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS})`,
     );
   }
 
   // Make sure there are at least 10 data types for sanity
-  const numDataTypes = dataTypeDocs.length;
-  if (numDataTypes < MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES) {
+  const numDataTypes = Object.keys(dataTypeDocs).length;
+  if (!(MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES <= numDataTypes)) {
     throw new Error(
       `Did not find enough data type entries did the structure of '${readfile}' change? (found ${numDataTypes} but expected at least ${MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES})`,
     );

web-console/src/ace-modes/dsql.js

@@ -63,6 +63,10 @@ ace.define(
 
       this.$rules = {
         start: [
+          {
+            token: 'comment.issue',
+            regex: '--:ISSUE:.*$',
+          },
           {
             token: 'comment',
             regex: '--.*$',
@@ -73,17 +77,13 @@ ace.define(
             end: '\\*/',
           },
           {
-            token: 'string', // " string
+            token: 'variable.column', // " quoted reference
             regex: '".*?"',
           },
           {
-            token: 'string', // ' string
+            token: 'string', // ' string literal
             regex: "'.*?'",
           },
-          {
-            token: 'string', // ` string (apache drill)
-            regex: '`.*?`',
-          },
           {
             token: 'constant.numeric', // float
             regex: '[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b',

web-console/src/bootstrap/ace.scss

@@ -25,6 +25,18 @@
 .ace-solarized-dark {
   background-color: rgba($dark-gray1, 0.5);
 
+  // START: Custom code styles
+  .ace_variable.ace_column {
+    color: #2ceefb;
+  }
+
+  .ace_comment.ace_issue {
+    color: #cb3116;
+    text-decoration: underline;
+    text-decoration-style: wavy;
+  }
+  // END: Custom code styles
+
   &.no-background {
     background-color: transparent;
   }

web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap

@@ -326,7 +326,7 @@ exports[`HeaderBar matches snapshot 1`] = `
           <Blueprint4.MenuItem
             active={false}
             disabled={false}
-            href="https://druid.apache.org/docs/24.0.0"
+            href="https://druid.apache.org/docs/24.0.1"
             icon="th"
             multiline={false}
             popoverProps={Object {}}

web-console/src/components/table-cell/__snapshots__/table-cell.spec.tsx.snap

@@ -49,6 +49,14 @@ exports[`TableCell matches snapshot array long 1`] = `
 </div>
 `;
 
+exports[`TableCell matches snapshot array mixed 1`] = `
+<div
+  class="table-cell plain"
+>
+  ["a",{"v":"b"},"c"]
+</div>
+`;
+
 exports[`TableCell matches snapshot array short 1`] = `
 <div
   class="table-cell plain"

web-console/src/components/table-cell/table-cell.spec.tsx

@@ -64,6 +64,13 @@ describe('TableCell', () => {
     expect(container.firstChild).toMatchSnapshot();
   });
 
+  it('matches snapshot array mixed', () => {
+    const tableCell = <TableCell value={['a', { v: 'b' }, 'c']} />;
+
+    const { container } = render(tableCell);
+    expect(container.firstChild).toMatchSnapshot();
+  });
+
   it('matches snapshot object', () => {
     const tableCell = <TableCell value={{ hello: 'world' }} />;
 

web-console/src/components/table-cell/table-cell.tsx

@@ -21,6 +21,7 @@ import * as JSONBig from 'json-bigint-native';
 import React, { useState } from 'react';
 
 import { ShowValueDialog } from '../../dialogs/show-value-dialog/show-value-dialog';
+import { isSimpleArray } from '../../utils';
 import { ActionIcon } from '../action-icon/action-icon';
 
 import './table-cell.scss';
@@ -97,7 +98,7 @@ export const TableCell = React.memo(function TableCell(props: TableCellProps) {
           {isNaN(dateValue) ? 'Unusable date' : value.toISOString()}
         </div>
       );
-    } else if (Array.isArray(value)) {
+    } else if (isSimpleArray(value)) {
       return renderTruncated(`[${value.join(', ')}]`);
     } else if (typeof value === 'object') {
       return renderTruncated(JSONBig.stringify(value));

web-console/src/components/table-filterable-cell/table-filterable-cell.tsx

@@ -34,14 +34,14 @@ export interface TableFilterableCellProps {
   value: string;
   filters: Filter[];
   onFiltersChange(filters: Filter[]): void;
-  disableComparisons?: boolean;
+  enableComparisons?: boolean;
   children?: ReactNode;
 }
 
 export const TableFilterableCell = React.memo(function TableFilterableCell(
   props: TableFilterableCellProps,
 ) {
-  const { field, value, children, filters, disableComparisons, onFiltersChange } = props;
+  const { field, value, children, filters, enableComparisons, onFiltersChange } = props;
 
   return (
     <Popover2
@@ -51,7 +51,7 @@ export const TableFilterableCell = React.memo(function TableFilterableCell(
           content={() => (
             <Menu>
               <MenuDivider title="Filter" />
-              {(disableComparisons ? FILTER_MODES_NO_COMPARISONS : FILTER_MODES).map((mode, i) => (
+              {(enableComparisons ? FILTER_MODES : FILTER_MODES_NO_COMPARISONS).map((mode, i) => (
                 <MenuItem
                   key={i}
                   icon={filterModeToIcon(mode)}

web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/coordinator-dynamic-config-dialog.spec.tsx.snap

@@ -12,7 +12,7 @@ exports[`CoordinatorDynamicConfigDialog matches snapshot 1`] = `
     Edit the coordinator dynamic configuration on the fly. For more information please refer to the
 
     <Memo(ExternalLink)
-      href="https://druid.apache.org/docs/24.0.0/configuration/index.html#dynamic-configuration"
+      href="https://druid.apache.org/docs/24.0.1/configuration/index.html#dynamic-configuration"
     >
       documentation
     </Memo(ExternalLink)>

web-console/src/dialogs/overlord-dynamic-config-dialog/__snapshots__/overload-dynamic-config-dialog.spec.tsx.snap

@@ -11,7 +11,7 @@ exports[`OverlordDynamicConfigDialog matches snapshot 1`] = `
     Edit the overlord dynamic configuration on the fly. For more information please refer to the
 
     <Memo(ExternalLink)
-      href="https://druid.apache.org/docs/24.0.0/configuration/index.html#overlord-dynamic-configuration"
+      href="https://druid.apache.org/docs/24.0.1/configuration/index.html#overlord-dynamic-configuration"
     >
       documentation
     </Memo(ExternalLink)>

web-console/src/dialogs/retention-dialog/__snapshots__/retention-dialog.spec.tsx.snap

@@ -63,7 +63,7 @@ exports[`RetentionDialog matches snapshot 1`] = `
             Druid uses rules to determine what data should be retained in the cluster. The rules are evaluated in order from top to bottom. For more information please refer to the
 
             <a
-              href="https://druid.apache.org/docs/24.0.0/operations/rule-configuration.html"
+              href="https://druid.apache.org/docs/24.0.1/operations/rule-configuration.html"
               rel="noopener noreferrer"
               target="_blank"
             >

web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts

@@ -22,7 +22,7 @@ describe('flatten-spec', () => {
   describe('computeFlattenExprsForData', () => {
     const data = [
       {
-        context: { host: 'cla', topic: 'moon', bonus: { foo: 'bar' } },
+        context: { host: 'cla', topic: 'moon', bonus: { 'fo.o': 'bar' } },
         tags: ['a', 'b', 'c'],
         messages: [
           { metric: 'request/time', value: 122 },
@@ -32,7 +32,7 @@ describe('flatten-spec', () => {
         value: 5,
       },
       {
-        context: { host: 'piv', popic: 'sun' },
+        context: { 'host': 'piv', '1pic': 'sun' },
         tags: ['a', 'd'],
         messages: [
           { metric: 'request/time', value: 44 },
@@ -41,7 +41,7 @@ describe('flatten-spec', () => {
         value: 4,
       },
       {
-        context: { host: 'imp', dopik: 'fun' },
+        context: { 'host': 'imp', "d\\o\npi'c'": 'fun' },
         tags: ['x', 'y'],
         messages: [
           { metric: 'request/time', value: 4 },
@@ -53,22 +53,12 @@ describe('flatten-spec', () => {
     ];
 
     it('works for path, ignore-arrays', () => {
-      expect(computeFlattenExprsForData(data, 'path', 'ignore-arrays')).toEqual([
-        '$.context.bonus.foo',
-        '$.context.dopik',
+      expect(computeFlattenExprsForData(data, 'ignore-arrays')).toEqual([
+        "$.context.bonus['fo.o']",
         '$.context.host',
-        '$.context.popic',
         '$.context.topic',
-      ]);
-    });
-
-    it('works for jq, ignore-arrays', () => {
-      expect(computeFlattenExprsForData(data, 'jq', 'ignore-arrays')).toEqual([
-        '.context.bonus.foo',
-        '.context.dopik',
-        '.context.host',
-        '.context.popic',
-        '.context.topic',
+        "$.context['1pic']",
+        "$.context['d\\\\o\npi\\'c\\'']",
       ]);
     });
   });

web-console/src/druid-models/flatten-spec/flatten-spec.tsx

@@ -61,26 +61,29 @@ export const FLATTEN_FIELD_FIELDS: Field<FlattenField>[] = [
   },
 ];
 
-export type ExprType = 'path' | 'jq';
 export type ArrayHandling = 'ignore-arrays' | 'include-arrays';
 
+function escapePathKey(pathKey: string): string {
+  return /^[a-z]\w*$/i.test(pathKey)
+    ? `.${pathKey}`
+    : `['${pathKey.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}']`;
+}
+
 export function computeFlattenPathsForData(
   data: Record<string, any>[],
-  exprType: ExprType,
   arrayHandling: ArrayHandling,
 ): FlattenField[] {
-  return computeFlattenExprsForData(data, exprType, arrayHandling).map(expr => {
+  return computeFlattenExprsForData(data, arrayHandling).map(expr => {
     return {
-      name: expr.replace(/^\$?\./, ''),
-      type: exprType,
+      name: expr.replace(/^\$\./, '').replace(/['\]]/g, '').replace(/\[/g, '.'),
+      type: 'path',
       expr,
     };
   });
 }
 
 export function computeFlattenExprsForData(
   data: Record<string, any>[],
-  exprType: ExprType,
   arrayHandling: ArrayHandling,
   includeTopLevel = false,
 ): string[] {
@@ -91,12 +94,7 @@ export function computeFlattenExprsForData(
     for (const datumKey of datumKeys) {
       const datumValue = datum[datumKey];
       if (includeTopLevel || isNested(datumValue)) {
-        addPath(
-          seenPaths,
-          exprType === 'path' ? `$.${datumKey}` : `.${datumKey}`,
-          datumValue,
-          arrayHandling,
-        );
+        addPath(seenPaths, `$${escapePathKey(datumKey)}`, datumValue, arrayHandling);
       }
     }
   }
@@ -114,7 +112,7 @@ function addPath(
     if (!Array.isArray(value)) {
       const valueKeys = Object.keys(value);
       for (const valueKey of valueKeys) {
-        addPath(paths, `${path}.${valueKey}`, value[valueKey], arrayHandling);
+        addPath(paths, `${path}${escapePathKey(valueKey)}`, value[valueKey], arrayHandling);
       }
     } else if (arrayHandling === 'include-arrays') {
       for (let i = 0; i < value.length; i++) {

web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts

@@ -725,6 +725,15 @@ describe('spec utils', () => {
     it('works for multi-value', () => {
       expect(guessColumnTypeFromInput(['a', ['b'], 'c'], false)).toEqual('string');
       expect(guessColumnTypeFromInput([1, [2], 3], false)).toEqual('string');
+      expect(guessColumnTypeFromInput([true, [true, 7, false], false, 'x'], false)).toEqual(
+        'string',
+      );
+    });
+
+    it('works for complex arrays', () => {
+      expect(guessColumnTypeFromInput([{ type: 'Dogs' }, { type: 'JavaScript' }], false)).toEqual(
+        'COMPLEX<json>',
+      );
     });
 
     it('works for strange json', () => {

web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx

@@ -32,6 +32,7 @@ import {
   EMPTY_ARRAY,
   EMPTY_OBJECT,
   filterMap,
+  isSimpleArray,
   oneOf,
   parseCsvLine,
   typeIs,
@@ -2309,7 +2310,7 @@ export function guessIsArrayFromHeaderAndRows(
   headerAndRows: SampleHeaderAndRows,
   column: string,
 ): boolean {
-  return headerAndRows.rows.some(r => Array.isArray(r.input?.[column]));
+  return headerAndRows.rows.some(r => isSimpleArray(r.input?.[column]));
 }
 
 export function guessColumnTypeFromInput(
@@ -2322,7 +2323,7 @@ export function guessColumnTypeFromInput(
   if (!definedValues.length) return 'string';
 
   // If we see any arrays in the input this is a multi-value dimension that must be a string
-  if (definedValues.some(v => Array.isArray(v))) return 'string';
+  if (definedValues.some(v => isSimpleArray(v))) return 'string';
 
   // If we see any JSON objects in the input assume COMPLEX<json>
   if (definedValues.some(v => v && typeof v === 'object')) return 'COMPLEX<json>';

web-console/src/druid-models/workbench-query/workbench-query.spec.ts

@@ -423,6 +423,20 @@ describe('WorkbenchQuery', () => {
         sqlPrefixLines: 0,
       });
     });
+
+    it('works with sql with ISSUE comment', () => {
+      const sql = sane`
+        SELECT *
+        --:ISSUE: There is something wrong with this query.
+        FROM wikipedia
+      `;
+
+      const workbenchQuery = WorkbenchQuery.blank().changeQueryString(sql);
+
+      expect(() => workbenchQuery.getApiQuery(makeQueryId)).toThrow(
+        `This query contains an ISSUE comment: There is something wrong with this query. (Please resolve the issue in the comment, delete the ISSUE comment and re-run the query.)`,
+      );
+    });
   });
 
   describe('#getIngestDatasource', () => {

web-console/src/druid-models/workbench-query/workbench-query.ts

@@ -576,10 +576,15 @@ export class WorkbenchQuery {
       apiQuery.query = queryPrepend + apiQuery.query + queryAppend;
     }
 
-    const m = /(--:context\s.+)(?:\n|$)/.exec(apiQuery.query);
+    const m = /--:ISSUE:(.+)(?:\n|$)/.exec(apiQuery.query);
     if (m) {
       throw new Error(
-        `This query contains a context comment '${m[1]}'. Context comments have been deprecated. Please rewrite the context comment as a context parameter. The context parameter editor is located in the "Engine" dropdown.`,
+        `This query contains an ISSUE comment: ${m[1]
+          .trim()
+          .replace(
+            /\.$/,
+            '',
+          )}. (Please resolve the issue in the comment, delete the ISSUE comment and re-run the query.)`,
       );
     }