From ca1f20408331f2272015f0c794eecfea4999b404 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Fri, 16 Sep 2022 14:59:59 -0700 Subject: [PATCH 1/4] fix path generation --- .../flatten-spec/flatten-spec.spec.ts | 24 ++++++------------- .../flatten-spec/flatten-spec.tsx | 22 +++++++---------- .../views/load-data-view/load-data-view.tsx | 1 - .../result-table-pane/result-table-pane.tsx | 2 +- 4 files changed, 17 insertions(+), 32 deletions(-) diff --git a/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts b/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts index a35c1853810e..7de1ec87529d 100644 --- a/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts +++ b/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts @@ -22,7 +22,7 @@ describe('flatten-spec', () => { describe('computeFlattenExprsForData', () => { const data = [ { - context: { host: 'cla', topic: 'moon', bonus: { foo: 'bar' } }, + context: { host: 'cla', topic: 'moon', bonus: { 'fo.o': 'bar' } }, tags: ['a', 'b', 'c'], messages: [ { metric: 'request/time', value: 122 }, @@ -32,7 +32,7 @@ describe('flatten-spec', () => { value: 5, }, { - context: { host: 'piv', popic: 'sun' }, + context: { 'host': 'piv', '1pic': 'sun' }, tags: ['a', 'd'], messages: [ { metric: 'request/time', value: 44 }, @@ -41,7 +41,7 @@ describe('flatten-spec', () => { value: 4, }, { - context: { host: 'imp', dopik: 'fun' }, + context: { 'host': 'imp', 'do\npic': 'fun' }, tags: ['x', 'y'], messages: [ { metric: 'request/time', value: 4 }, @@ -53,22 +53,12 @@ describe('flatten-spec', () => { ]; it('works for path, ignore-arrays', () => { - expect(computeFlattenExprsForData(data, 'path', 'ignore-arrays')).toEqual([ - '$.context.bonus.foo', - '$.context.dopik', + expect(computeFlattenExprsForData(data, 'ignore-arrays')).toEqual([ + "$.context.bonus['fo.o']", '$.context.host', - '$.context.popic', '$.context.topic', - ]); - }); - - it('works for jq, ignore-arrays', () => { - expect(computeFlattenExprsForData(data, 'jq', 'ignore-arrays')).toEqual([ - '.context.bonus.foo', - '.context.dopik', - '.context.host', - '.context.popic', - '.context.topic', + "$.context['1pic']", + "$.context['do\npic']", ]); }); }); diff --git a/web-console/src/druid-models/flatten-spec/flatten-spec.tsx b/web-console/src/druid-models/flatten-spec/flatten-spec.tsx index 3845a83916dd..0353b0f042a9 100644 --- a/web-console/src/druid-models/flatten-spec/flatten-spec.tsx +++ b/web-console/src/druid-models/flatten-spec/flatten-spec.tsx @@ -61,18 +61,20 @@ export const FLATTEN_FIELD_FIELDS: Field[] = [ }, ]; -export type ExprType = 'path' | 'jq'; export type ArrayHandling = 'ignore-arrays' | 'include-arrays'; +function escapePathKey(pathKey: string): string { + return /^[a-z]\w*$/i.test(pathKey) ? `.${pathKey}` : `['${pathKey}']`; +} + export function computeFlattenPathsForData( data: Record[], - exprType: ExprType, arrayHandling: ArrayHandling, ): FlattenField[] { - return computeFlattenExprsForData(data, exprType, arrayHandling).map(expr => { + return computeFlattenExprsForData(data, arrayHandling).map(expr => { return { - name: expr.replace(/^\$?\./, ''), - type: exprType, + name: expr.replace(/^\$\./, '').replace(/['\]]/g, '').replace(/\[/g, '.'), + type: 'path', expr, }; }); @@ -80,7 +82,6 @@ export function computeFlattenPathsForData( export function computeFlattenExprsForData( data: Record[], - exprType: ExprType, arrayHandling: ArrayHandling, includeTopLevel = false, ): string[] { @@ -91,12 +92,7 @@ export function computeFlattenExprsForData( for (const datumKey of datumKeys) { const datumValue = datum[datumKey]; if (includeTopLevel || isNested(datumValue)) { - addPath( - seenPaths, - exprType === 'path' ? `$.${datumKey}` : `.${datumKey}`, - datumValue, - arrayHandling, - ); + addPath(seenPaths, `$${escapePathKey(datumKey)}`, datumValue, arrayHandling); } } } @@ -114,7 +110,7 @@ function addPath( if (!Array.isArray(value)) { const valueKeys = Object.keys(value); for (const valueKey of valueKeys) { - addPath(paths, `${path}.${valueKey}`, value[valueKey], arrayHandling); + addPath(paths, `${path}${escapePathKey(valueKey)}`, value[valueKey], arrayHandling); } } else if (arrayHandling === 'include-arrays') { for (let i = 0; i < value.length; i++) { diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 7afc13aa7504..989263a8808a 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -1490,7 +1490,6 @@ export class LoadDataView extends React.PureComponent r.input), - 'path', 'ignore-arrays', ); } diff --git a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx index e4b78b5af738..ba8b11001a48 100644 --- a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx +++ b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx @@ -82,7 +82,7 @@ function jsonValue(ex: SqlExpression, path: string): SqlExpression { } function getJsonPaths(jsons: Record[]): string[] { - return ['$.'].concat(computeFlattenExprsForData(jsons, 'path', 'include-arrays', true)); + return ['$.'].concat(computeFlattenExprsForData(jsons, 'include-arrays', true)); } function isComparable(x: unknown): boolean { From 0a1c57301e7bcf35a3dec222661b387e7be7b302 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Fri, 16 Sep 2022 18:31:40 -0700 Subject: [PATCH 2/4] do escape --- .../src/druid-models/flatten-spec/flatten-spec.spec.ts | 4 ++-- web-console/src/druid-models/flatten-spec/flatten-spec.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts b/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts index 7de1ec87529d..5927a746ff45 100644 --- a/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts +++ b/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts @@ -41,7 +41,7 @@ describe('flatten-spec', () => { value: 4, }, { - context: { 'host': 'imp', 'do\npic': 'fun' }, + context: { 'host': 'imp', "do\npi'c": 'fun' }, tags: ['x', 'y'], messages: [ { metric: 'request/time', value: 4 }, @@ -58,7 +58,7 @@ describe('flatten-spec', () => { '$.context.host', '$.context.topic', "$.context['1pic']", - "$.context['do\npic']", + "$.context['do\npi\\'c']", ]); }); }); diff --git a/web-console/src/druid-models/flatten-spec/flatten-spec.tsx b/web-console/src/druid-models/flatten-spec/flatten-spec.tsx index 0353b0f042a9..42919f89f46b 100644 --- a/web-console/src/druid-models/flatten-spec/flatten-spec.tsx +++ b/web-console/src/druid-models/flatten-spec/flatten-spec.tsx @@ -64,7 +64,7 @@ export const FLATTEN_FIELD_FIELDS: Field[] = [ export type ArrayHandling = 'ignore-arrays' | 'include-arrays'; function escapePathKey(pathKey: string): string { - return /^[a-z]\w*$/i.test(pathKey) ? `.${pathKey}` : `['${pathKey}']`; + return /^[a-z]\w*$/i.test(pathKey) ? `.${pathKey}` : `['${pathKey.replace(/'/, "\\'")}']`; } export function computeFlattenPathsForData( From 193aee5abe2b838fb2e7b4e32c6a99311ede455b Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Fri, 16 Sep 2022 21:04:58 -0700 Subject: [PATCH 3/4] fix replace --- .../src/druid-models/flatten-spec/flatten-spec.spec.ts | 4 ++-- web-console/src/druid-models/flatten-spec/flatten-spec.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts b/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts index 5927a746ff45..a9921e4e9c84 100644 --- a/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts +++ b/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts @@ -41,7 +41,7 @@ describe('flatten-spec', () => { value: 4, }, { - context: { 'host': 'imp', "do\npi'c": 'fun' }, + context: { 'host': 'imp', "do\npi'c'": 'fun' }, tags: ['x', 'y'], messages: [ { metric: 'request/time', value: 4 }, @@ -58,7 +58,7 @@ describe('flatten-spec', () => { '$.context.host', '$.context.topic', "$.context['1pic']", - "$.context['do\npi\\'c']", + "$.context['do\npi\\'c\\'']", ]); }); }); diff --git a/web-console/src/druid-models/flatten-spec/flatten-spec.tsx b/web-console/src/druid-models/flatten-spec/flatten-spec.tsx index 42919f89f46b..c4b486caf39d 100644 --- a/web-console/src/druid-models/flatten-spec/flatten-spec.tsx +++ b/web-console/src/druid-models/flatten-spec/flatten-spec.tsx @@ -64,7 +64,7 @@ export const FLATTEN_FIELD_FIELDS: Field[] = [ export type ArrayHandling = 'ignore-arrays' | 'include-arrays'; function escapePathKey(pathKey: string): string { - return /^[a-z]\w*$/i.test(pathKey) ? `.${pathKey}` : `['${pathKey.replace(/'/, "\\'")}']`; + return /^[a-z]\w*$/i.test(pathKey) ? `.${pathKey}` : `['${pathKey.replace(/'/g, "\\'")}']`; } export function computeFlattenPathsForData( From 787c898633ec76fe7524b86716250a208aed51fb Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Fri, 16 Sep 2022 21:16:37 -0700 Subject: [PATCH 4/4] fix replace for good --- .../src/druid-models/flatten-spec/flatten-spec.spec.ts | 4 ++-- web-console/src/druid-models/flatten-spec/flatten-spec.tsx | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts b/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts index a9921e4e9c84..822dd48eba30 100644 --- a/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts +++ b/web-console/src/druid-models/flatten-spec/flatten-spec.spec.ts @@ -41,7 +41,7 @@ describe('flatten-spec', () => { value: 4, }, { - context: { 'host': 'imp', "do\npi'c'": 'fun' }, + context: { 'host': 'imp', "d\\o\npi'c'": 'fun' }, tags: ['x', 'y'], messages: [ { metric: 'request/time', value: 4 }, @@ -58,7 +58,7 @@ describe('flatten-spec', () => { '$.context.host', '$.context.topic', "$.context['1pic']", - "$.context['do\npi\\'c\\'']", + "$.context['d\\\\o\npi\\'c\\'']", ]); }); }); diff --git a/web-console/src/druid-models/flatten-spec/flatten-spec.tsx b/web-console/src/druid-models/flatten-spec/flatten-spec.tsx index c4b486caf39d..5cf1d7c06c61 100644 --- a/web-console/src/druid-models/flatten-spec/flatten-spec.tsx +++ b/web-console/src/druid-models/flatten-spec/flatten-spec.tsx @@ -64,7 +64,9 @@ export const FLATTEN_FIELD_FIELDS: Field[] = [ export type ArrayHandling = 'ignore-arrays' | 'include-arrays'; function escapePathKey(pathKey: string): string { - return /^[a-z]\w*$/i.test(pathKey) ? `.${pathKey}` : `['${pathKey.replace(/'/g, "\\'")}']`; + return /^[a-z]\w*$/i.test(pathKey) + ? `.${pathKey}` + : `['${pathKey.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}']`; } export function computeFlattenPathsForData(