From 789fa12bc8e232662547f13d39cc856d2a6d5575 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Mon, 30 Sep 2024 09:37:01 -0700 Subject: [PATCH 01/16] make record table able to hide column --- web-console/src/utils/table-helpers.ts | 1 + .../generic-output-table.tsx | 4 ++- .../modules/record-table-module.tsx | 26 ++++++++++++++++--- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/web-console/src/utils/table-helpers.ts b/web-console/src/utils/table-helpers.ts index 90df7fa10648..45e8758bf6f8 100644 --- a/web-console/src/utils/table-helpers.ts +++ b/web-console/src/utils/table-helpers.ts @@ -35,6 +35,7 @@ export function changePage(pagination: Pagination, page: number): Pagination { export interface ColumnHint { displayName?: string; group?: string; + hidden?: boolean; expressionForWhere?: SqlExpression; formatter?: (x: any) => string; } diff --git a/web-console/src/views/explore-view/components/generic-output-table/generic-output-table.tsx b/web-console/src/views/explore-view/components/generic-output-table/generic-output-table.tsx index 0050a34122c6..b557d99b4fde 100644 --- a/web-console/src/views/explore-view/components/generic-output-table/generic-output-table.tsx +++ b/web-console/src/views/explore-view/components/generic-output-table/generic-output-table.tsx @@ -428,6 +428,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable( columns={columnNester( queryResult.header.map((column, i) => { const h = column.name; + const hint = columnHints?.get(h); const icon = showTypeIcons ? columnToIcon(column) : undefined; return { @@ -446,9 +447,10 @@ export const GenericOutputTable = React.memo(function GenericOutputTable( }, headerClassName: getHeaderClassName(h), accessor: String(i), + show: !hint?.hidden, Cell(row) { const value = row.value; - const formatter = columnHints?.get(h)?.formatter || formatNumber; + const formatter = hint?.formatter || formatNumber; return (
getCellMenu(column, i, value)} />}> diff --git a/web-console/src/views/explore-view/modules/record-table-module.tsx b/web-console/src/views/explore-view/modules/record-table-module.tsx index b272a4dfabe1..cba34eb3ad39 100644 --- a/web-console/src/views/explore-view/modules/record-table-module.tsx +++ b/web-console/src/views/explore-view/modules/record-table-module.tsx @@ -21,10 +21,9 @@ import React, { useMemo } from 'react'; import { Loader } from '../../../components'; import { useQueryManager } from '../../../hooks'; -import { - calculateInitPageSize, - GenericOutputTable, -} from '../components/generic-output-table/generic-output-table'; +import type { ColumnHint } from '../../../utils'; +import { filterMap } from '../../../utils'; +import { calculateInitPageSize, GenericOutputTable } from '../components'; import { ModuleRepository } from '../module-repository/module-repository'; import './record-table-module.scss'; @@ -33,6 +32,7 @@ interface RecordTableParameterValues { maxRows: number; ascending: boolean; showTypeIcons: boolean; + hideNullColumns: boolean; } ModuleRepository.registerModule({ @@ -55,6 +55,11 @@ ModuleRepository.registerModule({ type: 'boolean', defaultValue: true, }, + hideNullColumns: { + type: 'boolean', + label: 'Hide all null column', + defaultValue: false, + }, }, component: function RecordTableModule(props) { const { stage, querySource, where, setWhere, parameterValues, runSqlQuery } = props; @@ -77,6 +82,18 @@ ModuleRepository.registerModule({ }); const resultData = resultState.getSomeData(); + + let columnHints: Map | undefined; + if (parameterValues.hideNullColumns && resultData) { + columnHints = new Map( + filterMap(resultData.header, (column, i) => + resultData.getColumnByIndex(i)?.every(v => v == null) + ? [column.name, { hidden: true }] + : undefined, + ), + ); + } + return (
{resultState.error ? ( @@ -84,6 +101,7 @@ ModuleRepository.registerModule({ ) : resultData ? ( Date: Mon, 30 Sep 2024 13:28:17 -0700 Subject: [PATCH 02/16] stickyness --- web-console/src/utils/local-storage-keys.tsx | 1 + .../src/views/explore-view/explore-view.tsx | 36 +++++++++++++++++-- .../views/explore-view/models/parameter.ts | 1 + .../modules/grouping-table-module.tsx | 1 + .../modules/record-table-module.tsx | 5 ++- .../modules/time-chart-module.tsx | 1 + 6 files changed, 41 insertions(+), 4 deletions(-) diff --git a/web-console/src/utils/local-storage-keys.tsx b/web-console/src/utils/local-storage-keys.tsx index d4efec06e224..fac8fab02d55 100644 --- a/web-console/src/utils/local-storage-keys.tsx +++ b/web-console/src/utils/local-storage-keys.tsx @@ -57,6 +57,7 @@ export const LocalStorageKeys = { SQL_DATA_LOADER_CONTENT: 'sql-data-loader-content' as const, EXPLORE_STATE: 'explore-state' as const, + EXPLORE_STICKY: 'explore-sticky' as const, }; export type LocalStorageKeys = (typeof LocalStorageKeys)[keyof typeof LocalStorageKeys]; diff --git a/web-console/src/views/explore-view/explore-view.tsx b/web-console/src/views/explore-view/explore-view.tsx index 9e80f3ae06ad..f781186837c7 100644 --- a/web-console/src/views/explore-view/explore-view.tsx +++ b/web-console/src/views/explore-view/explore-view.tsx @@ -30,7 +30,15 @@ import { useStore } from 'zustand'; import { ShowValueDialog } from '../../dialogs/show-value-dialog/show-value-dialog'; import { useHashAndLocalStorageHybridState, useQueryManager } from '../../hooks'; import { Api, AppToaster } from '../../singletons'; -import { DruidError, LocalStorageKeys, queryDruidSql } from '../../utils'; +import { + DruidError, + isEmpty, + localStorageGetJson, + LocalStorageKeys, + localStorageSetJson, + mapRecord, + queryDruidSql, +} from '../../utils'; import { ControlPane, @@ -77,6 +85,12 @@ function getFormattedQueryHistory(): string { // --------------------------------------- +function getStickyParameterValuesForModule(moduleId: string): ParameterValues { + return localStorageGetJson(LocalStorageKeys.EXPLORE_STICKY)?.[moduleId] || {}; +} + +// --------------------------------------- + const queryRunner = new QueryRunner({ inflateDateStrategy: 'fromSqlTypes', executor: async (sqlQueryPayload, isSql, cancelToken) => { @@ -193,10 +207,25 @@ export const ExploreView = React.memo(function ExploreView() { } function resetParameterValues() { - setParameterValues({}); + setParameterValues(getStickyParameterValuesForModule(moduleId)); } function updateParameterValues(newParameterValues: ParameterValues) { + // Evaluate sticky-ness + if (module) { + const currentExploreSticky = localStorageGetJson(LocalStorageKeys.EXPLORE_STICKY) || {}; + const currentModuleSticky = currentExploreSticky[moduleId] || {}; + const newModuleSticky = { + ...currentModuleSticky, + ...mapRecord(newParameterValues, (v, k) => (module.parameters[k]?.sticky ? v : undefined)), + }; + + localStorageSetJson(LocalStorageKeys.EXPLORE_STICKY, { + ...currentExploreSticky, + [moduleId]: isEmpty(newModuleSticky) ? undefined : newModuleSticky, + }); + } + setParameterValues({ ...parameterValues, ...newParameterValues }); } @@ -311,7 +340,8 @@ export const ExploreView = React.memo(function ExploreView() { ]} selectedModuleId={moduleId} onSelectedModuleIdChange={newModuleId => { - const newParameterValues: ParameterValues = {}; + const newParameterValues = getStickyParameterValuesForModule(newModuleId); + const oldModule = ModuleRepository.getModule(moduleId); const newModule = ModuleRepository.getModule(newModuleId); if (oldModule && newModule) { diff --git a/web-console/src/views/explore-view/models/parameter.ts b/web-console/src/views/explore-view/models/parameter.ts index b7a952a14006..f4a5f622d737 100644 --- a/web-console/src/views/explore-view/models/parameter.ts +++ b/web-console/src/views/explore-view/models/parameter.ts @@ -85,6 +85,7 @@ export type TypedParameterDefinition = TypedE | ParameterTypes[Type] | ((querySource: QuerySource) => ParameterTypes[Type] | undefined); + sticky?: boolean; required?: ModuleFunctor; description?: ModuleFunctor; placeholder?: string; diff --git a/web-console/src/views/explore-view/modules/grouping-table-module.tsx b/web-console/src/views/explore-view/modules/grouping-table-module.tsx index c8aa74922eca..e2cba7cef6d7 100644 --- a/web-console/src/views/explore-view/modules/grouping-table-module.tsx +++ b/web-console/src/views/explore-view/modules/grouping-table-module.tsx @@ -105,6 +105,7 @@ ModuleRepository.registerModule({ count: `Show ' values'`, }, defaultValue: 'null', + sticky: true, visible: ({ parameterValues }) => Boolean((parameterValues.showColumns || []).length), }, pivotColumn: { diff --git a/web-console/src/views/explore-view/modules/record-table-module.tsx b/web-console/src/views/explore-view/modules/record-table-module.tsx index cba34eb3ad39..38e2cfb6fac3 100644 --- a/web-console/src/views/explore-view/modules/record-table-module.tsx +++ b/web-console/src/views/explore-view/modules/record-table-module.tsx @@ -50,15 +50,18 @@ ModuleRepository.registerModule({ ascending: { type: 'boolean', defaultValue: false, + sticky: true, }, showTypeIcons: { type: 'boolean', defaultValue: true, + sticky: true, }, hideNullColumns: { type: 'boolean', - label: 'Hide all null column', + label: 'Hide all null columns', defaultValue: false, + sticky: true, }, }, component: function RecordTableModule(props) { diff --git a/web-console/src/views/explore-view/modules/time-chart-module.tsx b/web-console/src/views/explore-view/modules/time-chart-module.tsx index f62aa518f4cf..e2379dfb54c9 100644 --- a/web-console/src/views/explore-view/modules/time-chart-module.tsx +++ b/web-console/src/views/explore-view/modules/time-chart-module.tsx @@ -116,6 +116,7 @@ ModuleRepository.registerModule({ type: 'boolean', label: 'Snap highlight to nearest dates', defaultValue: true, + sticky: true, }, }, component: function TimeChartModule(props) { From 39f075e2a26205c2172effbc4b6098e80fc52ee4 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Mon, 30 Sep 2024 15:16:29 -0700 Subject: [PATCH 03/16] refactor query log --- .../src/views/explore-view/explore-view.tsx | 35 +++----------- .../src/views/explore-view/utils/index.ts | 1 + .../src/views/explore-view/utils/query-log.ts | 48 +++++++++++++++++++ 3 files changed, 56 insertions(+), 28 deletions(-) create mode 100644 web-console/src/views/explore-view/utils/query-log.ts diff --git a/web-console/src/views/explore-view/explore-view.tsx b/web-console/src/views/explore-view/explore-view.tsx index f781186837c7..f70c45146f13 100644 --- a/web-console/src/views/explore-view/explore-view.tsx +++ b/web-console/src/views/explore-view/explore-view.tsx @@ -58,32 +58,11 @@ import { QuerySource } from './models'; import { ModuleRepository } from './module-repository/module-repository'; import { rewriteAggregate, rewriteMaxDataTime } from './query-macros'; import type { Rename } from './utils'; -import { adjustTransferValue, normalizeType } from './utils'; +import { adjustTransferValue, normalizeType, QueryLog } from './utils'; import './explore-view.scss'; -// --------------------------------------- - -interface QueryHistoryEntry { - time: Date; - sqlQuery: string; -} - -const MAX_PAST_QUERIES = 10; -const QUERY_HISTORY: QueryHistoryEntry[] = []; - -function addQueryToHistory(sqlQuery: string): void { - QUERY_HISTORY.unshift({ time: new Date(), sqlQuery }); - while (QUERY_HISTORY.length > MAX_PAST_QUERIES) QUERY_HISTORY.pop(); -} - -function getFormattedQueryHistory(): string { - return QUERY_HISTORY.map( - ({ time, sqlQuery }) => `At ${time.toISOString()} ran query:\n\n${sqlQuery}`, - ).join('\n\n-----------------------------------------------------\n\n'); -} - -// --------------------------------------- +const QUERY_LOG = new QueryLog(); function getStickyParameterValuesForModule(moduleId: string): ParameterValues { return localStorageGetJson(LocalStorageKeys.EXPLORE_STICKY)?.[moduleId] || {}; @@ -95,7 +74,7 @@ const queryRunner = new QueryRunner({ inflateDateStrategy: 'fromSqlTypes', executor: async (sqlQueryPayload, isSql, cancelToken) => { if (!isSql) throw new Error('should never get here'); - addQueryToHistory(sqlQueryPayload.query); + QUERY_LOG.addQuery(sqlQueryPayload.query); return Api.instance.post('/druid/v2/sql', sqlQueryPayload, { cancelToken }); }, }); @@ -379,9 +358,9 @@ export const ExploreView = React.memo(function ExploreView() { { - copy(QUERY_HISTORY[0]?.sqlQuery, { format: 'text/plain' }); + copy(QUERY_LOG.getLastQuery()!, { format: 'text/plain' }); AppToaster.show({ message: `Copied query to clipboard`, intent: Intent.SUCCESS, @@ -390,9 +369,9 @@ export const ExploreView = React.memo(function ExploreView() { /> { - setShownText(getFormattedQueryHistory()); + setShownText(QUERY_LOG.getFormatted()); }} /> MAX_QUERIES_TO_LOG) queryLog.pop(); + } + + public getLastQuery(): string | undefined { + return this.queryLog[0]?.sqlQuery; + } + + public getFormatted(): string { + return this.queryLog + .map(({ time, sqlQuery }) => `At ${time.toISOString()} ran query:\n\n${sqlQuery}`) + .join('\n\n-----------------------------------------------------\n\n'); + } +} From 9b71a1b835685dc5f82d6a649d70c0acf1fea363 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Mon, 30 Sep 2024 15:41:05 -0700 Subject: [PATCH 04/16] fix measure drag --- .../components/control-pane/control-pane.tsx | 22 ++++++++++--------- .../control-pane/named-expressions-input.tsx | 1 + .../src/views/explore-view/models/measure.ts | 10 ++++----- .../modules/time-chart-module.tsx | 2 +- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/web-console/src/views/explore-view/components/control-pane/control-pane.tsx b/web-console/src/views/explore-view/components/control-pane/control-pane.tsx index 7fcc6352f293..faa7576bb67a 100644 --- a/web-console/src/views/explore-view/components/control-pane/control-pane.tsx +++ b/web-console/src/views/explore-view/components/control-pane/control-pane.tsx @@ -194,7 +194,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) { }; return { element: ( - allowReordering values={effectiveValue ? [effectiveValue] : []} onValuesChange={vs => onValueChange(vs[0])} @@ -223,7 +223,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) { ); return { element: ( - allowReordering values={effectiveValue as ExpressionMeta[]} onValuesChange={onValueChange} @@ -266,7 +266,7 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) { case 'measure': { return { element: ( - values={effectiveValue ? [effectiveValue] : []} onValuesChange={vs => onValueChange(vs[0])} singleton @@ -284,9 +284,11 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) { /> ), onDropColumn: column => { - const measures = Measure.getPossibleMeasuresForColumn(column); - if (!measures.length) return; - onValueChange(measures[0]); + const candidateMeasures = Measure.getPossibleMeasuresForColumn(column).filter( + p => !effectiveValue || effectiveValue.name !== p.name, + ); + if (!candidateMeasures.length) return; + onValueChange(candidateMeasures[0]); }, onDropMeasure: onValueChange, }; @@ -313,11 +315,11 @@ export const ControlPane = function ControlPane(props: ControlPaneProps) { /> ), onDropColumn: column => { - const measures = Measure.getPossibleMeasuresForColumn(column).filter( - p => !effectiveValue.some((v: ExpressionMeta) => v.name === p.name), + const candidateMeasures = Measure.getPossibleMeasuresForColumn(column).filter( + p => !effectiveValue.some((v: Measure) => v.name === p.name), ); - if (!measures.length) return; - onValueChange(effectiveValue.concat(measures[0])); + if (!candidateMeasures.length) return; + onValueChange(effectiveValue.concat(candidateMeasures[0])); }, onDropMeasure: measure => { onValueChange(effectiveValue.concat(measure)); diff --git a/web-console/src/views/explore-view/components/control-pane/named-expressions-input.tsx b/web-console/src/views/explore-view/components/control-pane/named-expressions-input.tsx index ae498341ecd6..d93a6c522cea 100644 --- a/web-console/src/views/explore-view/components/control-pane/named-expressions-input.tsx +++ b/web-console/src/views/explore-view/components/control-pane/named-expressions-input.tsx @@ -61,6 +61,7 @@ export const NamedExpressionsInput = function NamedExpressionsInput< const onDragOver = useCallback( (e: React.DragEvent, i: number) => { + if (dragIndex === -1) return; const targetRect = e.currentTarget.getBoundingClientRect(); const before = e.clientX - targetRect.left <= targetRect.width / 2; setDropBefore(before); diff --git a/web-console/src/views/explore-view/models/measure.ts b/web-console/src/views/explore-view/models/measure.ts index 7a469bdaced8..e7ead812a7e0 100644 --- a/web-console/src/views/explore-view/models/measure.ts +++ b/web-console/src/views/explore-view/models/measure.ts @@ -90,7 +90,7 @@ export class Measure extends ExpressionMeta { } switch (column.nativeType) { - case 'BIGINT': + case 'LONG': case 'FLOAT': case 'DOUBLE': return [ @@ -103,16 +103,16 @@ export class Measure extends ExpressionMeta { new Measure({ expression: F.min(C(column.name)), }), - new Measure({ - expression: SqlFunction.countDistinct(C(column.name)), - }), new Measure({ as: `P98 ${column.name}`, expression: F('APPROX_QUANTILE_DS', C(column.name), 0.98), }), + new Measure({ + expression: SqlFunction.countDistinct(C(column.name)), + }), ]; - case 'VARCHAR': + case 'STRING': case 'COMPLEX': case 'COMPLEX': return [ diff --git a/web-console/src/views/explore-view/modules/time-chart-module.tsx b/web-console/src/views/explore-view/modules/time-chart-module.tsx index e2379dfb54c9..ed3c9645228e 100644 --- a/web-console/src/views/explore-view/modules/time-chart-module.tsx +++ b/web-console/src/views/explore-view/modules/time-chart-module.tsx @@ -114,7 +114,7 @@ ModuleRepository.registerModule({ }, snappyHighlight: { type: 'boolean', - label: 'Snap highlight to nearest dates', + label: 'Snap highlight to granularity', defaultValue: true, sticky: true, }, From df5946491da10c37aa43da9743081e02836f6a95 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Mon, 30 Sep 2024 21:20:28 -0700 Subject: [PATCH 05/16] start nested column dialog --- .../column-dialog/column-dialog.tsx | 5 +- .../measure-dialog/measure-dialog.tsx | 5 +- .../nested-column-dialog.scss | 57 ++++++++++ .../nested-column-dialog.tsx | 106 ++++++++++++++++++ .../resource-pane/resource-pane.tsx | 48 ++++++-- .../src/views/explore-view/explore-view.tsx | 3 + .../views/explore-view/models/query-source.ts | 4 + 7 files changed, 212 insertions(+), 16 deletions(-) create mode 100644 web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss create mode 100644 web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx diff --git a/web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx b/web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx index 4cc4fcd674ce..ecd7af968b5a 100644 --- a/web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx +++ b/web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx @@ -48,9 +48,8 @@ export const ColumnDialog = React.memo(function ColumnDialog(props: ColumnDialog if (!expression) return; return SqlQuery.from(QuerySource.stripToBaseSource(querySource.query)) .addSelect(F.cast(expression, 'VARCHAR').as('v'), { addToGroupBy: 'end' }) - .applyIf( - querySource.baseColumns.find(column => column.isTimeColumn()), - q => q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`), + .applyIf(querySource.hasBaseTimeColumn(), q => + q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`), ) .changeLimitValue(100) .toString(); diff --git a/web-console/src/views/explore-view/components/resource-pane/measure-dialog/measure-dialog.tsx b/web-console/src/views/explore-view/components/resource-pane/measure-dialog/measure-dialog.tsx index f02facf61f09..b48638fe72f5 100644 --- a/web-console/src/views/explore-view/components/resource-pane/measure-dialog/measure-dialog.tsx +++ b/web-console/src/views/explore-view/components/resource-pane/measure-dialog/measure-dialog.tsx @@ -57,9 +57,8 @@ export const MeasureDialog = React.memo(function MeasureDialog(props: MeasureDia .changeWithParts([SqlWithPart.simple('t', QuerySource.stripToBaseSource(querySource.query))]) .addSelect(L('Overall').as('label')) .addSelect(expression.as('value')) - .applyIf( - querySource.baseColumns.find(column => column.isTimeColumn()), - q => q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`), + .applyIf(querySource.hasBaseTimeColumn(), q => + q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`), ) .toString(); }, [querySource.query, formula]); diff --git a/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss b/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss new file mode 100644 index 000000000000..820f1e0f16e3 --- /dev/null +++ b/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import '../../../../../variables'; + +.nested-column-dialog { + &.#{$bp-ns}-dialog { + width: 50vw; + } + + .#{$bp-ns}-dialog-body { + display: flex; + gap: 12px; + + .controls { + flex: 1; + display: flex; + flex-direction: column; + + .sql-expression-form-group { + flex: 1; + margin: 0; + + .#{$bp-ns}-form-content { + flex: 1; + + .flexible-query-input { + height: 100%; + } + } + } + } + + .preview-pane { + width: 300px; + } + } + + .#{$bp-ns}-dialog-footer { + margin-top: 0; + } +} diff --git a/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx b/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx new file mode 100644 index 000000000000..3ca125575308 --- /dev/null +++ b/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Button, Classes, Dialog, Intent, Menu } from '@blueprintjs/core'; +import type { SqlExpression } from '@druid-toolkit/query'; +import { type QueryResult, F, sql, SqlQuery } from '@druid-toolkit/query'; +import React, { useState } from 'react'; + +import { MenuCheckbox } from '../../../../../components'; +import { useQueryManager } from '../../../../../hooks'; +import { pluralIfNeeded, uniq } from '../../../../../utils'; +import { QuerySource } from '../../../models'; +import { toggle } from '../../../utils'; + +import './nested-column-dialog.scss'; + +export interface NestedColumnDialogProps { + nestedColumn: SqlExpression; + onApply(newQuery: SqlQuery): void; + querySource: QuerySource; + runSqlQuery(query: string | SqlQuery): Promise; + onClose(): void; +} + +export const NestedColumnDialog = React.memo(function NestedColumnDialog( + props: NestedColumnDialogProps, +) { + const { nestedColumn, querySource, runSqlQuery, onClose } = props; + const [selectedPaths, setSelectedPaths] = useState([]); + + const [pathsState] = useQueryManager({ + query: nestedColumn, + processQuery: async nestedColumn => { + const query = SqlQuery.from(QuerySource.stripToBaseSource(querySource.query)) + .addSelect(F('JSON_PATHS', nestedColumn).as('p')) + .applyIf(querySource.hasBaseTimeColumn(), q => + q.addWhere(sql`MAX_DATA_TIME() - INTERVAL '14' DAY <= __time`), + ) + .changeLimitValue(200); + + const pathResult = await runSqlQuery(query); + + const pathColumn = pathResult.getColumnByIndex(0); + if (!pathColumn) throw new Error('Could not get path column'); + + return uniq(([] as string[]).concat(...pathColumn)).sort(); + }, + }); + + const paths = pathsState.data; + return ( + +
+ {pathsState.getErrorMessage()} + {paths && ( + + {pathsState.data?.map((path, i) => { + return ( + setSelectedPaths(toggle(selectedPaths, path))} + text={path} + /> + ); + })} + + )} +
+
+
+
+
+
+
+ + ); +}); diff --git a/web-console/src/views/explore-view/components/resource-pane/resource-pane.tsx b/web-console/src/views/explore-view/components/resource-pane/resource-pane.tsx index f6489eb17e2c..d68c87cd2f04 100644 --- a/web-console/src/views/explore-view/components/resource-pane/resource-pane.tsx +++ b/web-console/src/views/explore-view/components/resource-pane/resource-pane.tsx @@ -40,6 +40,7 @@ import type { Rename } from '../../utils'; import { ColumnDialog } from './column-dialog/column-dialog'; import { MeasureDialog } from './measure-dialog/measure-dialog'; +import { NestedColumnDialog } from './nested-column-dialog/nested-column-dialog'; import './resource-pane.scss'; @@ -67,6 +68,9 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) { const [columnSearch, setColumnSearch] = useState(''); const [columnEditorOpenOn, setColumnEditorOpenOn] = useState(); + const [nestedColumnEditorOpenOn, setNestedColumnEditorOpenOn] = useState< + SqlExpression | undefined + >(); const [measureEditorOpenOn, setMeasureEditorOpenOn] = useState(); function applyUtil(nameTransform: (columnName: string) => string) { @@ -112,6 +116,7 @@ export const ResourcePane = function ResourcePane(props: ResourcePaneProps) {
{filterMap(querySource.columns, (column, i) => { const columnName = column.name; + const isNestedColumn = column.nativeType === 'COMPLEX'; if (!caseInsensitiveContains(columnName, columnSearch)) return; return ( - {onFilter && ( + {isNestedColumn ? ( onFilter(column)} + icon={IconNames.EXPAND_ALL} + text="Expload" + onClick={() => + setNestedColumnEditorOpenOn( + querySource.getSourceExpressionForColumn(columnName), + ) + } /> + ) : ( + <> + {onFilter && ( + onFilter(column)} + /> + )} + onShowColumn(column)} + /> + + )} - onShowColumn(column)} - /> - setColumnEditorOpenOn(undefined)} /> )} + {nestedColumnEditorOpenOn && ( + {}} + querySource={querySource} + runSqlQuery={runSqlQuery} + onClose={() => setNestedColumnEditorOpenOn(undefined)} + /> + )} {measureEditorOpenOn && ( { try { return await queryRunner.runQuery({ query, + defaultQueryContext: { + sqlStringifyArrays: false, + }, }); } catch (e) { throw new DruidError(e); diff --git a/web-console/src/views/explore-view/models/query-source.ts b/web-console/src/views/explore-view/models/query-source.ts index a8a5257e311e..c23f59a7373e 100644 --- a/web-console/src/views/explore-view/models/query-source.ts +++ b/web-console/src/views/explore-view/models/query-source.ts @@ -179,6 +179,10 @@ export class QuerySource { return this.measures.some(m => m.name === name); } + public hasBaseTimeColumn(): boolean { + return this.baseColumns.some(column => column.isTimeColumn()); + } + public getSourceExpressionForColumn(outputName: string): SqlExpression { const selectExpressionsArray = this.query.getSelectExpressionsArray(); From bd3a0a3c231549cd664ee7660871bbabcb0ec10f Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Mon, 30 Sep 2024 22:09:03 -0700 Subject: [PATCH 06/16] nested expand --- .../contains-filter-control.tsx | 1 + .../regexp-filter-control.tsx | 1 + .../values-filter-control.tsx | 83 +++++++++---------- .../column-dialog/column-dialog.tsx | 2 +- .../nested-column-dialog.scss | 30 ++----- .../nested-column-dialog.tsx | 78 +++++++++++++---- .../resource-pane/resource-pane.tsx | 6 +- .../views/explore-view/models/query-source.ts | 13 ++- 8 files changed, 129 insertions(+), 85 deletions(-) diff --git a/web-console/src/views/explore-view/components/filter-pane/filter-menu/contains-filter-control/contains-filter-control.tsx b/web-console/src/views/explore-view/components/filter-pane/filter-menu/contains-filter-control/contains-filter-control.tsx index 353179c905e4..8751aad7e600 100644 --- a/web-console/src/views/explore-view/components/filter-pane/filter-menu/contains-filter-control/contains-filter-control.tsx +++ b/web-console/src/views/explore-view/components/filter-pane/filter-menu/contains-filter-control/contains-filter-control.tsx @@ -52,6 +52,7 @@ export const ContainsFilterControl = React.memo(function ContainsFilterControl( ), ) .changeOrderByExpression(F.count().toOrderByExpression('DESC')) + .changeLimitValue(101) .toString(), // eslint-disable-next-line react-hooks/exhaustive-deps -- exclude 'makePattern' from deps [querySource.query, filter, column, contains, negated], diff --git a/web-console/src/views/explore-view/components/filter-pane/filter-menu/regexp-filter-control/regexp-filter-control.tsx b/web-console/src/views/explore-view/components/filter-pane/filter-menu/regexp-filter-control/regexp-filter-control.tsx index ea6bdf743858..dd9a90d4bf6f 100644 --- a/web-console/src/views/explore-view/components/filter-pane/filter-menu/regexp-filter-control/regexp-filter-control.tsx +++ b/web-console/src/views/explore-view/components/filter-pane/filter-menu/regexp-filter-control/regexp-filter-control.tsx @@ -58,6 +58,7 @@ export const RegexpFilterControl = React.memo(function RegexpFilterControl( SqlExpression.and(filter, regexp ? filterPatternToExpression(filterPattern) : undefined), ) .changeOrderByExpression(F.count().toOrderByExpression('DESC')) + .changeLimitValue(101) .toString(), // eslint-disable-next-line react-hooks/exhaustive-deps -- exclude 'makePattern' from deps [querySource.query, filter, column, regexp, negated], diff --git a/web-console/src/views/explore-view/components/filter-pane/filter-menu/values-filter-control/values-filter-control.tsx b/web-console/src/views/explore-view/components/filter-pane/filter-menu/values-filter-control/values-filter-control.tsx index a15d3daad762..3c726f2b1fff 100644 --- a/web-console/src/views/explore-view/components/filter-pane/filter-menu/values-filter-control/values-filter-control.tsx +++ b/web-console/src/views/explore-view/components/filter-pane/filter-menu/values-filter-control/values-filter-control.tsx @@ -18,12 +18,12 @@ import { FormGroup, InputGroup, Menu, MenuItem } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; -import type { QueryResult, SqlQuery, ValuesFilterPattern } from '@druid-toolkit/query'; -import { C, F, L, SqlExpression, SqlLiteral } from '@druid-toolkit/query'; +import type { QueryResult, ValuesFilterPattern } from '@druid-toolkit/query'; +import { C, F, SqlExpression, SqlQuery } from '@druid-toolkit/query'; import React, { useMemo, useState } from 'react'; import { useQueryManager } from '../../../../../../hooks'; -import { caseInsensitiveContains } from '../../../../../../utils'; +import { caseInsensitiveContains, filterMap } from '../../../../../../utils'; import type { QuerySource } from '../../../../models'; import { toggle } from '../../../../utils'; import { ColumnValue } from '../../column-value/column-value'; @@ -46,21 +46,21 @@ export const ValuesFilterControl = React.memo(function ValuesFilterControl( const [initValues] = useState(selectedValues); const [searchString, setSearchString] = useState(''); - const valuesQuery = useMemo(() => { - const columnRef = C(column); - const queryParts: string[] = [`SELECT ${columnRef.as('c')}`, `FROM (${querySource.query})`]; - - const filterEx = SqlExpression.and( - filter, - searchString ? F('ICONTAINS_STRING', columnRef, L(searchString)) : undefined, - ); - if (!(filterEx instanceof SqlLiteral)) { - queryParts.push(`WHERE ${filterEx}`); - } - - queryParts.push(`GROUP BY 1 ORDER BY COUNT(*) DESC LIMIT 101`); - return queryParts.join('\n'); - }, [querySource.query, filter, column, searchString]); + const valuesQuery = useMemo( + () => + SqlQuery.from(querySource.query) + .addSelect(C(column).as('c'), { addToGroupBy: 'end' }) + .changeWhereExpression( + SqlExpression.and( + filter, + searchString ? F('ICONTAINS_STRING', C(column), searchString) : undefined, + ), + ) + .changeOrderByExpression(F.count().toOrderByExpression('DESC')) + .changeLimitValue(101) + .toString(), + [querySource.query, filter, column, searchString], + ); const [valuesState] = useQueryManager({ query: valuesQuery, @@ -77,12 +77,8 @@ export const ValuesFilterControl = React.memo(function ValuesFilterControl( if (values) { valuesToShow = valuesToShow.concat(values.filter(v => !initValues.includes(v))); } - if (searchString) { - valuesToShow = valuesToShow.filter(v => caseInsensitiveContains(v, searchString)); - } const showSearch = querySource.columns.find(c => c.name === column)?.sqlType !== 'BOOLEAN'; - return ( {showSearch && ( @@ -93,26 +89,29 @@ export const ValuesFilterControl = React.memo(function ValuesFilterControl( /> )} - {valuesToShow.map((v, i) => ( - } - shouldDismissPopover={false} - onClick={e => { - setFilterPattern({ - ...filterPattern, - values: e.altKey ? [v] : toggle(selectedValues, v), - }); - }} - /> - ))} + {filterMap(valuesToShow, (v, i) => { + if (caseInsensitiveContains(v, searchString)) return; + return ( + } + shouldDismissPopover={false} + onClick={e => { + setFilterPattern({ + ...filterPattern, + values: e.altKey ? [v] : toggle(selectedValues, v), + }); + }} + /> + ); + })} {valuesState.loading && } diff --git a/web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx b/web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx index ecd7af968b5a..0e4cc3cb77af 100644 --- a/web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx +++ b/web-console/src/views/explore-view/components/resource-pane/column-dialog/column-dialog.tsx @@ -150,7 +150,7 @@ export const ColumnDialog = React.memo(function ColumnDialog(props: ColumnDialog } else { onApply( querySource.changeColumn(initExpressionName, newExpression), - new Map([[initExpression.getOutputName()!, newExpression.getOutputName()!]]), + new Map([[initExpressionName, newExpression.getOutputName()!]]), ); } } else { diff --git a/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss b/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss index 820f1e0f16e3..1863c5c949be 100644 --- a/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss +++ b/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.scss @@ -21,33 +21,21 @@ .nested-column-dialog { &.#{$bp-ns}-dialog { width: 50vw; + height: 60vh; } .#{$bp-ns}-dialog-body { display: flex; - gap: 12px; + flex-direction: column; - .controls { + .path-selector { flex: 1; - display: flex; - flex-direction: column; - - .sql-expression-form-group { - flex: 1; - margin: 0; - - .#{$bp-ns}-form-content { - flex: 1; - - .flexible-query-input { - height: 100%; - } - } - } - } - - .preview-pane { - width: 300px; + padding: 5px 0; + height: 300px; + overflow: auto; + border: 1px solid rgba(15, 19, 32, 0.4); + border-top: none; + margin-bottom: 15px; } } diff --git a/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx b/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx index 3ca125575308..248450ab756e 100644 --- a/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx +++ b/web-console/src/views/explore-view/components/resource-pane/nested-column-dialog/nested-column-dialog.tsx @@ -16,15 +16,24 @@ * limitations under the License. */ -import { Button, Classes, Dialog, Intent, Menu } from '@blueprintjs/core'; +import { + Button, + Classes, + Dialog, + FormGroup, + InputGroup, + Intent, + Menu, + Tag, +} from '@blueprintjs/core'; import type { SqlExpression } from '@druid-toolkit/query'; import { type QueryResult, F, sql, SqlQuery } from '@druid-toolkit/query'; import React, { useState } from 'react'; import { MenuCheckbox } from '../../../../../components'; import { useQueryManager } from '../../../../../hooks'; -import { pluralIfNeeded, uniq } from '../../../../../utils'; -import { QuerySource } from '../../../models'; +import { caseInsensitiveContains, filterMap, pluralIfNeeded, uniq } from '../../../../../utils'; +import { ExpressionMeta, QuerySource } from '../../../models'; import { toggle } from '../../../utils'; import './nested-column-dialog.scss'; @@ -40,8 +49,10 @@ export interface NestedColumnDialogProps { export const NestedColumnDialog = React.memo(function NestedColumnDialog( props: NestedColumnDialogProps, ) { - const { nestedColumn, querySource, runSqlQuery, onClose } = props; + const { nestedColumn, onApply, querySource, runSqlQuery, onClose } = props; + const [searchString, setSearchString] = useState(''); const [selectedPaths, setSelectedPaths] = useState([]); + const [namingScheme, setNamingScheme] = useState(`${nestedColumn.getFirstColumnName()}[%]`); const [pathsState] = useQueryManager({ query: nestedColumn, @@ -66,21 +77,41 @@ export const NestedColumnDialog = React.memo(function NestedColumnDialog( return (
+

+ Replace {String(nestedColumn)} with path expansions for the selected + paths. +

{pathsState.getErrorMessage()} {paths && ( - - {pathsState.data?.map((path, i) => { - return ( - setSelectedPaths(toggle(selectedPaths, path))} - text={path} - /> - ); - })} - + <> + setSearchString(e.target.value)} + placeholder="Search" + /> + + {filterMap(paths, (path, i) => { + if (!caseInsensitiveContains(path, searchString)) return; + return ( + setSelectedPaths(toggle(selectedPaths, path))} + text={path} + /> + ); + })} + + )} + + { + setNamingScheme(e.target.value.slice(0, ExpressionMeta.MAX_NAME_LENGTH)); + }} + /> +
@@ -89,12 +120,25 @@ export const NestedColumnDialog = React.memo(function NestedColumnDialog(