From a3f0d8dd0d185a4d4edbe75b7c63ef61d8cd5807 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 2 Oct 2024 23:05:19 -0700 Subject: [PATCH] explore QA (#17225) --- .../components/module-pane/module-pane.scss | 8 +++ .../src/views/explore-view/explore-view.tsx | 23 ++++-- .../views/explore-view/models/query-source.ts | 72 ++++++++++--------- .../module-repository/module-repository.ts | 3 +- .../explore-view/modules/bar-chart-module.tsx | 10 ++- .../modules/grouping-table-module.tsx | 68 +++++++++++------- .../modules/multi-axis-chart-module.tsx | 12 ++-- .../explore-view/modules/pie-chart-module.tsx | 10 ++- .../modules/record-table-module.tsx | 6 +- .../modules/time-chart-module.tsx | 15 +++- .../explore-view/query-macros/aggregate.ts | 2 +- web-console/webpack.config.js | 8 ++- 12 files changed, 154 insertions(+), 83 deletions(-) diff --git a/web-console/src/views/explore-view/components/module-pane/module-pane.scss b/web-console/src/views/explore-view/components/module-pane/module-pane.scss index d015fca9ecd3..9d180cb8da2b 100644 --- a/web-console/src/views/explore-view/components/module-pane/module-pane.scss +++ b/web-console/src/views/explore-view/components/module-pane/module-pane.scss @@ -35,9 +35,17 @@ & > .issue { position: absolute; + top: 0; + left: 0; width: 100%; height: 100%; } + + .loader { + position: absolute; + top: 0; + left: 0; + } } .tile-content { diff --git a/web-console/src/views/explore-view/explore-view.tsx b/web-console/src/views/explore-view/explore-view.tsx index e68dbd7db11e..773eb78e2ac5 100644 --- a/web-console/src/views/explore-view/explore-view.tsx +++ b/web-console/src/views/explore-view/explore-view.tsx @@ -22,6 +22,7 @@ import { Button, Intent, Menu, MenuDivider, MenuItem } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import type { Column, QueryResult, SqlExpression } from '@druid-toolkit/query'; import { QueryRunner, SqlQuery } from '@druid-toolkit/query'; +import type { CancelToken } from 'axios'; import classNames from 'classnames'; import copy from 'copy-to-clipboard'; import React, { useEffect, useMemo, useRef, useState } from 'react'; @@ -79,27 +80,33 @@ const queryRunner = new QueryRunner({ }, }); -async function runSqlQuery(query: string | SqlQuery): Promise { +async function runSqlQuery( + query: string | SqlQuery, + cancelToken?: CancelToken, +): Promise { try { return await queryRunner.runQuery({ query, defaultQueryContext: { sqlStringifyArrays: false, }, + cancelToken, }); } catch (e) { throw new DruidError(e); } } -async function introspectSource(source: string): Promise { +async function introspectSource(source: string, cancelToken?: CancelToken): Promise { const query = SqlQuery.parse(source); const introspectResult = await runSqlQuery(QuerySource.makeLimitZeroIntrospectionQuery(query)); + cancelToken?.throwIfRequested(); const baseIntrospectResult = QuerySource.isSingleStarQuery(query) ? introspectResult : await runSqlQuery( QuerySource.makeLimitZeroIntrospectionQuery(QuerySource.stripToBaseSource(query)), + cancelToken, ); return QuerySource.fromIntrospectResult( @@ -238,11 +245,15 @@ export const ExploreView = React.memo(function ExploreView() { const querySource = querySourceState.getSomeData(); const runSqlPlusQuery = useMemo(() => { - return async (query: string | SqlQuery) => { + return async (query: string | SqlQuery, cancelToken?: CancelToken) => { if (!querySource) throw new Error('no querySource'); - return await runSqlQuery( - await rewriteMaxDataTime(rewriteAggregate(SqlQuery.parse(query), querySource.measures)), - ); + const parsedQuery = SqlQuery.parse(query); + return ( + await runSqlQuery( + await rewriteMaxDataTime(rewriteAggregate(parsedQuery, querySource.measures)), + cancelToken, + ) + ).attachQuery({ query: '' }, parsedQuery); }; }, [querySource]); diff --git a/web-console/src/views/explore-view/models/query-source.ts b/web-console/src/views/explore-view/models/query-source.ts index 4ff83cf14d65..6224344093a4 100644 --- a/web-console/src/views/explore-view/models/query-source.ts +++ b/web-console/src/views/explore-view/models/query-source.ts @@ -44,31 +44,6 @@ export class QuerySource { ); } - static materializeStarIfNeeded(query: SqlQuery, columns: readonly Column[]): SqlQuery { - let columnsToExpand = columns.map(c => c.name); - const selectExpressions = query.getSelectExpressionsArray(); - let starCount = 0; - for (const selectExpression of selectExpressions) { - if (selectExpression instanceof SqlStar) { - starCount++; - continue; - } - const outputName = selectExpression.getOutputName(); - if (!outputName) continue; - columnsToExpand = columnsToExpand.filter(c => c !== outputName); - } - if (starCount === 0) return query; - if (starCount > 1) throw new Error('can not handle multiple stars'); - - return query - .changeSelectExpressions( - selectExpressions.flatMap(selectExpression => - selectExpression instanceof SqlStar ? columnsToExpand.map(c => C(c)) : selectExpression, - ), - ) - .prettify(); - } - static isSingleStarQuery(query: SqlQuery): boolean { const selectExpressions = query.getSelectExpressionsArray(); return selectExpressions.length === 1 && selectExpressions[0] instanceof SqlStar; @@ -151,6 +126,35 @@ export class QuerySource { }; } + private materializeStarIfNeeded(): SqlQuery { + const { query, columns, measures } = this; + let columnsToExpand = columns.map(c => c.name); + const selectExpressions = query.getSelectExpressionsArray(); + let starCount = 0; + for (const selectExpression of selectExpressions) { + if (selectExpression instanceof SqlStar) { + starCount++; + continue; + } + const outputName = selectExpression.getOutputName(); + if (!outputName) continue; + columnsToExpand = columnsToExpand.filter(c => c !== outputName); + } + if (starCount === 0) return query; + if (starCount > 1) throw new Error('can not handle multiple stars'); + + return Measure.addMeasuresToQuery( + query + .changeSelectExpressions( + selectExpressions.flatMap(selectExpression => + selectExpression instanceof SqlStar ? columnsToExpand.map(c => C(c)) : selectExpression, + ), + ) + .prettify(), + measures, + ); + } + public getFirstAggregateMeasure(): Measure | undefined { return this.measures[0]?.toAggregateBasedMeasure(); } @@ -226,12 +230,12 @@ export class QuerySource { } public addColumn(newExpression: SqlExpression): SqlQuery { - const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns); + const noStarQuery = this.materializeStarIfNeeded(); return noStarQuery.addSelect(newExpression); } public addColumnAfter(neighborName: string, ...newExpressions: SqlExpression[]): SqlQuery { - const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns); + const noStarQuery = this.materializeStarIfNeeded(); return noStarQuery.changeSelectExpressions( noStarQuery .getSelectExpressionsArray() @@ -240,7 +244,7 @@ export class QuerySource { } public changeColumn(oldName: string, newExpression: SqlExpression): SqlQuery { - const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns); + const noStarQuery = this.materializeStarIfNeeded(); return noStarQuery.changeSelectExpressions( noStarQuery .getSelectExpressionsArray() @@ -249,7 +253,7 @@ export class QuerySource { } public deleteColumn(outputName: string): SqlQuery { - const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns); + const noStarQuery = this.materializeStarIfNeeded(); return noStarQuery.changeSelectExpressions( noStarQuery.getSelectExpressionsArray().filter(ex => ex.getOutputName() !== outputName), ); @@ -260,7 +264,7 @@ export class QuerySource { } public applyColumnNameMap(columnNameMap: Map): SqlQuery { - const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns); + const noStarQuery = this.materializeStarIfNeeded(); return noStarQuery.changeSelectExpressions( noStarQuery.getSelectExpressionsArray().map(ex => { const outputName = ex.getOutputName(); @@ -275,12 +279,12 @@ export class QuerySource { // ------------------------------------ public addMeasure(measure: Measure): SqlQuery { - const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns); + const noStarQuery = this.materializeStarIfNeeded(); return Measure.addMeasuresToQuery(noStarQuery, this.measures.concat(measure)); } public addMeasureAfter(neighborName: string, newMeasure: Measure): SqlQuery { - const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns); + const noStarQuery = this.materializeStarIfNeeded(); return Measure.addMeasuresToQuery( noStarQuery, this.measures.flatMap(m => (m.name === neighborName ? [m, newMeasure] : m)), @@ -288,7 +292,7 @@ export class QuerySource { } public changeMeasure(oldName: string, newMeasure: Measure): SqlQuery { - const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns); + const noStarQuery = this.materializeStarIfNeeded(); return Measure.addMeasuresToQuery( noStarQuery, this.measures.map(m => (m.name === oldName ? newMeasure : m)), @@ -296,7 +300,7 @@ export class QuerySource { } public deleteMeasure(measureName: string): SqlQuery { - const noStarQuery = QuerySource.materializeStarIfNeeded(this.query, this.columns); + const noStarQuery = this.materializeStarIfNeeded(); return Measure.addMeasuresToQuery( noStarQuery, this.measures.filter(m => m.name !== measureName), diff --git a/web-console/src/views/explore-view/module-repository/module-repository.ts b/web-console/src/views/explore-view/module-repository/module-repository.ts index 74feae4b4616..0d90a8b69336 100644 --- a/web-console/src/views/explore-view/module-repository/module-repository.ts +++ b/web-console/src/views/explore-view/module-repository/module-repository.ts @@ -17,6 +17,7 @@ */ import type { QueryResult, SqlExpression, SqlQuery } from '@druid-toolkit/query'; +import type { CancelToken } from 'axios'; import type { ParameterDefinition, QuerySource, Stage } from '../models'; @@ -34,7 +35,7 @@ interface ModuleComponentProps

{ setWhere(where: SqlExpression): void; parameterValues: P; setParameterValues: (parameters: Partial

) => void; - runSqlQuery(query: string | SqlQuery): Promise; + runSqlQuery(query: string | SqlQuery, cancelToken?: CancelToken): Promise; } export class ModuleRepository { diff --git a/web-console/src/views/explore-view/modules/bar-chart-module.tsx b/web-console/src/views/explore-view/modules/bar-chart-module.tsx index 3b1fa45001cf..a9ac2911ae0a 100644 --- a/web-console/src/views/explore-view/modules/bar-chart-module.tsx +++ b/web-console/src/views/explore-view/modules/bar-chart-module.tsx @@ -21,6 +21,7 @@ import type { ECharts } from 'echarts'; import * as echarts from 'echarts'; import React, { useEffect, useMemo, useRef } from 'react'; +import { Loader } from '../../../components'; import { useQueryManager } from '../../../hooks'; import { formatEmpty } from '../../../utils'; import { Issue } from '../components'; @@ -90,10 +91,10 @@ ModuleRepository.registerModule({ .changeLimitValue(limit); }, [querySource, where, splitColumn, measure, measureToSort, limit]); - const [sourceDataState] = useQueryManager({ + const [sourceDataState, queryManager] = useQueryManager({ query: dataQuery, - processQuery: async (query: SqlQuery) => { - return (await runSqlQuery(query)).toObjectArray(); + processQuery: async (query, cancelToken) => { + return (await runSqlQuery(query, cancelToken)).toObjectArray(); }, }); @@ -203,6 +204,9 @@ ModuleRepository.registerModule({ }} /> {errorMessage && } + {sourceDataState.loading && ( + queryManager.cancelCurrent()} /> + )} ); }, diff --git a/web-console/src/views/explore-view/modules/grouping-table-module.tsx b/web-console/src/views/explore-view/modules/grouping-table-module.tsx index e2cba7cef6d7..af778b0aa215 100644 --- a/web-console/src/views/explore-view/modules/grouping-table-module.tsx +++ b/web-console/src/views/explore-view/modules/grouping-table-module.tsx @@ -17,7 +17,7 @@ */ import { Button } from '@blueprintjs/core'; -import type { SqlOrderByDirection } from '@druid-toolkit/query'; +import type { SqlExpression, SqlOrderByDirection } from '@druid-toolkit/query'; import { C, F, SqlQuery } from '@druid-toolkit/query'; import React, { useMemo } from 'react'; @@ -43,6 +43,11 @@ import './grouping-table-module.scss'; // when ordering on non __time is more robust const NEEDS_GROUPING_TO_ORDER = true; +interface QueryAndMore { + originalWhere: SqlExpression; + queryAndHints: QueryAndHints; +} + interface GroupingTableParameterValues { splitColumns: ExpressionMeta[]; timeBucket: string; @@ -216,14 +221,14 @@ ModuleRepository.registerModule({ .changeLimitValue(maxPivotValues); }, [querySource.query, parameterValues]); - const [pivotValueState] = useQueryManager({ + const [pivotValueState, queryManager] = useQueryManager({ query: pivotValueQuery, processQuery: async (pivotValueQuery: SqlQuery) => { return (await runSqlQuery(pivotValueQuery)).getColumnByName('v') as string[]; }, }); - const queryAndHints = useMemo((): QueryAndHints | undefined => { + const queryAndMore = useMemo((): QueryAndMore | undefined => { const pivotValues = pivotValueState.data; if (parameterValues.pivotColumn && !pivotValues) return; const { orderByColumn, orderByDirection } = parameterValues; @@ -231,32 +236,43 @@ ModuleRepository.registerModule({ ? C(orderByColumn).toOrderByExpression(orderByDirection) : undefined; - return makeTableQueryAndHints({ - source: querySource.query, - where, - splitColumns: parameterValues.splitColumns, - timeBucket: parameterValues.timeBucket, - showColumns: parameterValues.showColumns, - multipleValueMode: parameterValues.multipleValueMode, - pivotColumn: parameterValues.pivotColumn, - pivotValues, - measures: parameterValues.measures, - compares: parameterValues.compares || [], - compareStrategy: parameterValues.compareStrategy, - compareTypes: parameterValues.compareTypes, - restrictTop: parameterValues.restrictTop, - maxRows: parameterValues.maxRows, - orderBy, - useGroupingToOrderSubQueries: NEEDS_GROUPING_TO_ORDER, - }); + return { + originalWhere: where, + queryAndHints: makeTableQueryAndHints({ + source: querySource.query, + where, + splitColumns: parameterValues.splitColumns, + timeBucket: parameterValues.timeBucket, + showColumns: parameterValues.showColumns, + multipleValueMode: parameterValues.multipleValueMode, + pivotColumn: parameterValues.pivotColumn, + pivotValues, + measures: parameterValues.measures, + compares: parameterValues.compares || [], + compareStrategy: parameterValues.compareStrategy, + compareTypes: parameterValues.compareTypes, + restrictTop: parameterValues.restrictTop, + maxRows: parameterValues.maxRows, + orderBy, + useGroupingToOrderSubQueries: NEEDS_GROUPING_TO_ORDER, + }), + }; }, [querySource.query, where, parameterValues, pivotValueState.data]); const [resultState] = useQueryManager({ - query: queryAndHints, - processQuery: async (queryAndHints: QueryAndHints) => { + query: queryAndMore, + processQuery: async (queryAndMore, cancelToken) => { + const { originalWhere, queryAndHints } = queryAndMore; const { query, columnHints } = queryAndHints; + let result = await runSqlQuery(query, cancelToken); + if (result.sqlQuery) { + result = result.attachQuery( + { query: '' }, + result.sqlQuery.changeWhereExpression(originalWhere), + ); + } return { - result: await runSqlQuery(query), + result, columnHints, }; }, @@ -297,7 +313,9 @@ ModuleRepository.registerModule({ initPageSize={calculateInitPageSize(stage.height)} /> ) : undefined} - {resultState.loading && } + {resultState.loading && ( + queryManager.cancelCurrent()} /> + )} ); }, diff --git a/web-console/src/views/explore-view/modules/multi-axis-chart-module.tsx b/web-console/src/views/explore-view/modules/multi-axis-chart-module.tsx index 67a007cbba1b..d448352fcb4c 100644 --- a/web-console/src/views/explore-view/modules/multi-axis-chart-module.tsx +++ b/web-console/src/views/explore-view/modules/multi-axis-chart-module.tsx @@ -21,6 +21,7 @@ import type { ECharts } from 'echarts'; import * as echarts from 'echarts'; import React, { useEffect, useMemo, useRef } from 'react'; +import { Loader } from '../../../components'; import { useQueryManager } from '../../../hooks'; import { formatInteger, @@ -91,16 +92,16 @@ ModuleRepository.registerModule({ direction: 'ASC', }) .applyForEach(measures, (q, measure) => q.addSelect(measure.expression.as(measure.name))); - }, [querySource, where, timeGranularity, measures]); + }, [querySource, where, timeColumnName, timeGranularity, measures]); - const [sourceDataState] = useQueryManager({ + const [sourceDataState, queryManager] = useQueryManager({ query: dataQuery, - processQuery: async (query: SqlQuery) => { + processQuery: async (query: SqlQuery, cancelToken) => { if (!timeColumnName) { throw new Error(`Must have a column of type TIMESTAMP for the multi-axis chart to work`); } - return (await runSqlQuery(query)).toObjectArray(); + return (await runSqlQuery(query, cancelToken)).toObjectArray(); }, }); @@ -327,6 +328,9 @@ ModuleRepository.registerModule({ }} /> {errorMessage && } + {sourceDataState.loading && ( + queryManager.cancelCurrent()} /> + )} ); }, diff --git a/web-console/src/views/explore-view/modules/pie-chart-module.tsx b/web-console/src/views/explore-view/modules/pie-chart-module.tsx index f70a167b10df..985754087386 100644 --- a/web-console/src/views/explore-view/modules/pie-chart-module.tsx +++ b/web-console/src/views/explore-view/modules/pie-chart-module.tsx @@ -21,6 +21,7 @@ import type { ECharts } from 'echarts'; import * as echarts from 'echarts'; import React, { useEffect, useMemo, useRef } from 'react'; +import { Loader } from '../../../components'; import { useQueryManager } from '../../../hooks'; import { formatEmpty, formatNumber } from '../../../utils'; import { Issue } from '../components'; @@ -113,10 +114,10 @@ ModuleRepository.registerModule({ }; }, [querySource, where, splitColumn, measure, limit, showOthers]); - const [sourceDataState] = useQueryManager({ + const [sourceDataState, queryManager] = useQueryManager({ query: dataQueries, - processQuery: async ({ mainQuery, splitExpression, othersPartialQuery }) => { - const result = await runSqlQuery(mainQuery); + processQuery: async ({ mainQuery, splitExpression, othersPartialQuery }, cancelToken) => { + const result = await runSqlQuery(mainQuery, cancelToken); const data = result.toObjectArray(); if (splitExpression && othersPartialQuery) { @@ -251,6 +252,9 @@ ModuleRepository.registerModule({ }} /> {errorMessage && } + {sourceDataState.loading && ( + queryManager.cancelCurrent()} /> + )} ); }, diff --git a/web-console/src/views/explore-view/modules/record-table-module.tsx b/web-console/src/views/explore-view/modules/record-table-module.tsx index 38e2cfb6fac3..43e84e60de95 100644 --- a/web-console/src/views/explore-view/modules/record-table-module.tsx +++ b/web-console/src/views/explore-view/modules/record-table-module.tsx @@ -79,7 +79,7 @@ ModuleRepository.registerModule({ .toString(); }, [querySource, where, parameterValues]); - const [resultState] = useQueryManager({ + const [resultState, queryManager] = useQueryManager({ query: query, processQuery: runSqlQuery, }); @@ -110,7 +110,9 @@ ModuleRepository.registerModule({ initPageSize={calculateInitPageSize(stage.height)} /> ) : undefined} - {resultState.loading && } + {resultState.loading && ( + queryManager.cancelCurrent()} /> + )} ); }, diff --git a/web-console/src/views/explore-view/modules/time-chart-module.tsx b/web-console/src/views/explore-view/modules/time-chart-module.tsx index ed3c9645228e..c4169e33028c 100644 --- a/web-console/src/views/explore-view/modules/time-chart-module.tsx +++ b/web-console/src/views/explore-view/modules/time-chart-module.tsx @@ -21,6 +21,7 @@ import type { ECharts } from 'echarts'; import * as echarts from 'echarts'; import React, { useEffect, useMemo, useRef } from 'react'; +import { Loader } from '../../../components'; import { useQueryManager } from '../../../hooks'; import { formatInteger, @@ -141,9 +142,12 @@ ModuleRepository.registerModule({ }; }, [querySource, where, measure, splitColumn, numberToStack, showOthers]); - const [sourceDataState] = useQueryManager({ + const [sourceDataState, queryManager] = useQueryManager({ query: dataQuery, - processQuery: async ({ baseQuery, measure, splitExpression, numberToStack, showOthers }) => { + processQuery: async ( + { baseQuery, measure, splitExpression, numberToStack, showOthers }, + cancelToken, + ) => { if (!timeColumnName) { throw new Error(`Must have a column of type TIMESTAMP for the time chart to work`); } @@ -155,10 +159,13 @@ ModuleRepository.registerModule({ .addSelect(splitExpression.as('v'), { addToGroupBy: 'end' }) .changeOrderByExpression(measure.expression.toOrderByExpression('DESC')) .changeLimitValue(numberToStack), + cancelToken, ) ).getColumnByIndex(0)! : undefined; + cancelToken.throwIfRequested(); + const dataset = ( await runSqlQuery( baseQuery @@ -181,6 +188,7 @@ ModuleRepository.registerModule({ ); }) .addSelect(measure.expression.as(METRIC_NAME)), + cancelToken, ) ).toObjectArray(); @@ -430,6 +438,9 @@ ModuleRepository.registerModule({ }} /> {errorMessage && } + {sourceDataState.loading && ( + queryManager.cancelCurrent()} /> + )} ); }, diff --git a/web-console/src/views/explore-view/query-macros/aggregate.ts b/web-console/src/views/explore-view/query-macros/aggregate.ts index 377c7f5b02d0..eedf544d12a5 100644 --- a/web-console/src/views/explore-view/query-macros/aggregate.ts +++ b/web-console/src/views/explore-view/query-macros/aggregate.ts @@ -55,7 +55,7 @@ export function rewriteAggregate(query: SqlQuery, measures: Measure[]): SqlQuery filterMap(queryMeasures, queryMeasure => usedMeasures.get(queryMeasure.name) ? queryMeasure.expression : undefined, ).flatMap(ex => ex.getUsedColumnNames()), - ).filter(columnName => !ex.getSelectIndexForOutputColumn(columnName)), + ).filter(columnName => ex.getSelectIndexForOutputColumn(columnName) === -1), (q, columnName) => q.addSelect(C(columnName)), ); } diff --git a/web-console/webpack.config.js b/web-console/webpack.config.js index d6518623a478..1735cd3dfd8f 100644 --- a/web-console/webpack.config.js +++ b/web-console/webpack.config.js @@ -32,8 +32,12 @@ function friendlyErrorFormatter(e) { module.exports = env => { let druidUrl = (env || {}).druid_host || process.env.druid_host || 'localhost'; - if (!druidUrl.startsWith('http')) druidUrl = 'http://' + druidUrl; - if (!/:\d+$/.test(druidUrl)) druidUrl += ':8888'; + if (!druidUrl.startsWith('http')) { + druidUrl = (druidUrl.endsWith(':9088') ? 'https://' : 'http://') + druidUrl; + } + if (!/:\d+$/.test(druidUrl)) { + druidUrl += druidUrl.startsWith('https://') ? ':9088' : ':8888'; + } const proxyTarget = { target: druidUrl,