From a79caf445cd2ae603051dc826c24b78a960df50d Mon Sep 17 00:00:00 2001 From: Chi Cao Minh Date: Thu, 1 Oct 2020 15:14:41 -0700 Subject: [PATCH 1/2] Web console reindexing E2E test (#10453) Add an E2E test for the web console workflow of reindexing a Druid datasource to change the secondary partitioning type. The new test changes dynamic to single dim partitions since the autocompaction test already does dynamic to hashed partitions. Also, run the web console E2E tests in parallel to reduce CI time and change naming convention for test datasources to make it easier to map them to the corresponding test run. Main changes: 1) web-consolee2e-tests/reindexing.spec.ts - new E2E test 2) web-console/e2e-tests/component/load-data/data-connector/reindex.ts - new data loader connector for druid input source 3) web-console/e2e-tests/component/load-data/config/partition.ts - move partition spec definitions from compaction.ts - add new single dim partition spec definition --- web-console/e2e-tests/auto-compaction.spec.ts | 31 +-- .../component/datasources/compaction.ts | 41 +--- .../component/datasources/overview.ts | 20 +- .../component/load-data/config/partition.ts | 70 +++++++ .../data-connector/data-connector.ts | 9 + .../load-data/data-connector/local-file.ts | 33 ++- .../load-data/data-connector/reindex.ts | 53 +++++ .../component/load-data/data-loader.ts | 66 +++--- .../e2e-tests/component/query/overview.ts | 15 +- web-console/e2e-tests/reindexing.spec.ts | 193 ++++++++++++++++++ web-console/e2e-tests/tutorial-batch.spec.ts | 22 +- web-console/e2e-tests/util/druid.ts | 23 ++- web-console/e2e-tests/util/playwright.ts | 59 ++++++ web-console/package.json | 2 +- 14 files changed, 495 insertions(+), 142 deletions(-) create mode 100644 web-console/e2e-tests/component/load-data/data-connector/reindex.ts create mode 100644 web-console/e2e-tests/reindexing.spec.ts diff --git a/web-console/e2e-tests/auto-compaction.spec.ts b/web-console/e2e-tests/auto-compaction.spec.ts index edcf6a8d6ed4..512c2724b8c4 100644 --- a/web-console/e2e-tests/auto-compaction.spec.ts +++ b/web-console/e2e-tests/auto-compaction.spec.ts @@ -17,19 +17,18 @@ */ import axios from 'axios'; -import { execSync } from 'child_process'; import path from 'path'; import * as playwright from 'playwright-core'; -import { v4 as uuid } from 'uuid'; import { CompactionConfig } from './component/datasources/compaction'; -import { CompactionHashPartitionsSpec } from './component/datasources/compaction'; import { Datasource } from './component/datasources/datasource'; import { DatasourcesOverview } from './component/datasources/overview'; +import { HashedPartitionsSpec } from './component/load-data/config/partition'; import { saveScreenshotIfError } from './util/debug'; import { COORDINATOR_URL } from './util/druid'; -import { DRUID_DIR } from './util/druid'; +import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid'; import { UNIFIED_CONSOLE_URL } from './util/druid'; +import { runIndexTask } from './util/druid'; import { createBrowserNormal as createBrowser } from './util/playwright'; import { createPage } from './util/playwright'; import { retryIfJestAssertionError } from './util/retry'; @@ -57,17 +56,18 @@ describe('Auto-compaction', () => { }); it('Compacts segments from dynamic to hash partitions', async () => { - const datasourceName = uuid(); + const testName = 'autocompaction-dynamic-to-hash-'; + const datasourceName = testName + new Date().toISOString(); loadInitialData(datasourceName); - await saveScreenshotIfError('auto-compaction-', page, async () => { + await saveScreenshotIfError(testName, page, async () => { const uncompactedNumSegment = 3; const numRow = 1412; await validateDatasourceStatus(page, datasourceName, uncompactedNumSegment, numRow); const compactionConfig = new CompactionConfig({ skipOffsetFromLatest: 'PT0S', - partitionsSpec: new CompactionHashPartitionsSpec({ + partitionsSpec: new HashedPartitionsSpec({ numShards: null, }), }); @@ -88,25 +88,14 @@ describe('Auto-compaction', () => { }); function loadInitialData(datasourceName: string) { - const postIndexTask = path.join(DRUID_DIR, 'examples', 'bin', 'post-index-task'); const ingestionSpec = path.join( - DRUID_DIR, - 'examples', - 'quickstart', - 'tutorial', + DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, 'compaction-init-index.json', ); const setDatasourceName = `s/compaction-tutorial/${datasourceName}/`; const setIntervals = 's|2015-09-12/2015-09-13|2015-09-12/2015-09-12T02:00|'; // shorten to reduce test duration - execSync( - `${postIndexTask} \ - --file <(sed -e '${setDatasourceName}' -e '${setIntervals}' ${ingestionSpec}) \ - --url ${COORDINATOR_URL}`, - { - shell: 'bash', - timeout: 3 * 60 * 1000, - }, - ); + const sedCommands = [setDatasourceName, setIntervals]; + runIndexTask(ingestionSpec, sedCommands); } async function validateDatasourceStatus( diff --git a/web-console/e2e-tests/component/datasources/compaction.ts b/web-console/e2e-tests/component/datasources/compaction.ts index c22285dee1bb..946362069016 100644 --- a/web-console/e2e-tests/component/datasources/compaction.ts +++ b/web-console/e2e-tests/component/datasources/compaction.ts @@ -16,44 +16,7 @@ * limitations under the License. */ -import * as playwright from 'playwright-core'; - -/* tslint:disable max-classes-per-file */ - -const PARTITIONING_TYPE = 'Partitioning type'; - -interface CompactionPartitionsSpec { - readonly type: string; - apply(page: playwright.Page): Promise; -} - -export class CompactionHashPartitionsSpec implements CompactionPartitionsSpec { - readonly type: string; - - constructor(props: CompactionHashPartitionsSpecProps) { - Object.assign(this, props); - this.type = 'hashed'; - } - - async apply(page: playwright.Page): Promise { - await setInput(page, PARTITIONING_TYPE, this.type); - if (this.numShards != null) { - await setInput(page, 'Num shards', String(this.numShards)); - } - } -} - -async function setInput(page: playwright.Page, label: string, value: string): Promise { - const input = await page.$(`//*[text()="${label}"]/following-sibling::div//input`); - await input!.fill(''); - await input!.type(value); -} - -interface CompactionHashPartitionsSpecProps { - readonly numShards: number | null; -} - -export interface CompactionHashPartitionsSpec extends CompactionHashPartitionsSpecProps {} +import { PartitionsSpec } from '../load-data/config/partition'; /** * Datasource compaction configuration @@ -66,7 +29,7 @@ export class CompactionConfig { interface CompactionConfigProps { readonly skipOffsetFromLatest: string; - readonly partitionsSpec: CompactionPartitionsSpec; + readonly partitionsSpec: PartitionsSpec; } export interface CompactionConfig extends CompactionConfigProps {} diff --git a/web-console/e2e-tests/component/datasources/overview.ts b/web-console/e2e-tests/component/datasources/overview.ts index f5433116b270..54af58b7f0c6 100644 --- a/web-console/e2e-tests/component/datasources/overview.ts +++ b/web-console/e2e-tests/component/datasources/overview.ts @@ -18,6 +18,8 @@ import * as playwright from 'playwright-core'; +import { clickButton } from '../../util/playwright'; +import { setLabeledInput } from '../../util/playwright'; import { extractTable } from '../../util/table'; import { CompactionConfig } from './compaction'; @@ -79,10 +81,14 @@ export class DatasourcesOverview { await this.openEditActions(datasourceName); await this.page.click('"Edit compaction configuration"'); - await this.setInput('Skip offset from latest', compactionConfig.skipOffsetFromLatest); + await setLabeledInput( + this.page, + 'Skip offset from latest', + compactionConfig.skipOffsetFromLatest, + ); await compactionConfig.partitionsSpec.apply(this.page); - await this.clickButton('Submit'); + await clickButton(this.page, 'Submit'); } private async openEditActions(datasourceName: string): Promise { @@ -96,14 +102,4 @@ export class DatasourcesOverview { editActions[index].click(); await this.page.waitFor(5000); } - - private async setInput(label: string, value: string) { - const input = await this.page.$(`//*[text()="${label}"]/following-sibling::div//input`); - await input!.fill(''); - await input!.type(value); - } - - private async clickButton(text: string) { - await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any); - } } diff --git a/web-console/e2e-tests/component/load-data/config/partition.ts b/web-console/e2e-tests/component/load-data/config/partition.ts index 5ecba3200077..8431eccab983 100644 --- a/web-console/e2e-tests/component/load-data/config/partition.ts +++ b/web-console/e2e-tests/component/load-data/config/partition.ts @@ -16,6 +16,13 @@ * limitations under the License. */ +import * as playwright from 'playwright-core'; + +import { selectSuggestibleInput } from '../../../util/playwright'; +import { setLabeledInput } from '../../../util/playwright'; + +/* tslint:disable max-classes-per-file */ + /** * Possible values for partition step segment granularity. */ @@ -26,17 +33,80 @@ export enum SegmentGranularity { YEAR = 'YEAR', } +const PARTITIONING_TYPE = 'Partitioning type'; + +export interface PartitionsSpec { + readonly type: string; + apply(page: playwright.Page): Promise; +} + +export class HashedPartitionsSpec implements PartitionsSpec { + readonly type: string; + + constructor(props: HashedPartitionsSpecProps) { + Object.assign(this, props); + this.type = 'hashed'; + } + + async apply(page: playwright.Page): Promise { + await setLabeledInput(page, PARTITIONING_TYPE, this.type); + if (this.numShards != null) { + await setLabeledInput(page, 'Num shards', String(this.numShards)); + } + } +} + +interface HashedPartitionsSpecProps { + readonly numShards: number | null; +} + +export interface HashedPartitionsSpec extends HashedPartitionsSpecProps {} + +export class SingleDimPartitionsSpec implements PartitionsSpec { + readonly type: string; + + constructor(props: SingleDimPartitionsSpecProps) { + Object.assign(this, props); + this.type = 'single_dim'; + } + + async apply(page: playwright.Page): Promise { + await selectSuggestibleInput(page, PARTITIONING_TYPE, this.type); + await setLabeledInput(page, 'Partition dimension', this.partitionDimension); + if (this.targetRowsPerSegment) { + await setLabeledInput(page, 'Target rows per segment', String(this.targetRowsPerSegment)); + } + if (this.maxRowsPerSegment) { + await setLabeledInput(page, 'Max rows per segment', String(this.maxRowsPerSegment)); + } + } +} + +interface SingleDimPartitionsSpecProps { + readonly partitionDimension: string; + readonly targetRowsPerSegment: number | null; + readonly maxRowsPerSegment: number | null; +} + +export interface SingleDimPartitionsSpec extends SingleDimPartitionsSpecProps {} + /** * Data loader partition step configuration. */ export class PartitionConfig { + readonly forceGuaranteedRollupText: string; + constructor(props: PartitionConfigProps) { Object.assign(this, props); + this.forceGuaranteedRollupText = this.forceGuaranteedRollup ? 'True' : 'False'; } } interface PartitionConfigProps { readonly segmentGranularity: SegmentGranularity; + readonly timeIntervals: string | null; + readonly forceGuaranteedRollup: boolean | null; + readonly partitionsSpec: PartitionsSpec | null; } export interface PartitionConfig extends PartitionConfigProps {} diff --git a/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts b/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts index 50a9d292328f..7112f2649ebf 100644 --- a/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts +++ b/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts @@ -16,10 +16,19 @@ * limitations under the License. */ +import * as playwright from 'playwright-core'; + +import { clickButton } from '../../../util/playwright'; + /** * Connector for data loader input data. */ export interface DataConnector { readonly name: string; + readonly needParse: boolean; connect(): Promise; } + +export async function clickApplyButton(page: playwright.Page): Promise { + await clickButton(page, 'Apply'); +} diff --git a/web-console/e2e-tests/component/load-data/data-connector/local-file.ts b/web-console/e2e-tests/component/load-data/data-connector/local-file.ts index 4e2aa3666796..dfbaed59945d 100644 --- a/web-console/e2e-tests/component/load-data/data-connector/local-file.ts +++ b/web-console/e2e-tests/component/load-data/data-connector/local-file.ts @@ -18,6 +18,9 @@ import * as playwright from 'playwright-core'; +import { setLabeledInput } from '../../../util/playwright'; + +import { clickApplyButton } from './data-connector'; import { DataConnector } from './data-connector'; /** @@ -25,30 +28,26 @@ import { DataConnector } from './data-connector'; */ export class LocalFileDataConnector implements DataConnector { readonly name: string; + readonly needParse: boolean; private page: playwright.Page; - private baseDirectory: string; - private fileFilter: string; - constructor(page: playwright.Page, baseDirectory: string, fileFilter: string) { + constructor(page: playwright.Page, props: LocalFileDataConnectorProps) { + Object.assign(this, props); this.name = 'Local disk'; + this.needParse = true; this.page = page; - this.baseDirectory = baseDirectory; - this.fileFilter = fileFilter; } async connect() { - const baseDirectoryInput = await this.page.$('input[placeholder="/path/to/files/"]'); - await this.setInput(baseDirectoryInput!, this.baseDirectory); - - const fileFilterInput = await this.page.$('input[value="*"]'); - await this.setInput(fileFilterInput!, this.fileFilter); - - const applyButton = await this.page.$('"Apply"'); - await applyButton!.click(); + await setLabeledInput(this.page, 'Base directory', this.baseDirectory); + await setLabeledInput(this.page, 'File filter', this.fileFilter); + await clickApplyButton(this.page); } +} - private async setInput(input: playwright.ElementHandle, value: string) { - await input.fill(''); - await input.type(value); - } +interface LocalFileDataConnectorProps { + readonly baseDirectory: string; + readonly fileFilter: string; } + +export interface LocalFileDataConnector extends LocalFileDataConnectorProps {} diff --git a/web-console/e2e-tests/component/load-data/data-connector/reindex.ts b/web-console/e2e-tests/component/load-data/data-connector/reindex.ts new file mode 100644 index 000000000000..d519a5d3ca8b --- /dev/null +++ b/web-console/e2e-tests/component/load-data/data-connector/reindex.ts @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as playwright from 'playwright-core'; + +import { setLabeledInput } from '../../../util/playwright'; + +import { clickApplyButton } from './data-connector'; +import { DataConnector } from './data-connector'; + +/** + * Reindexing connector for data loader input data. + */ +export class ReindexDataConnector implements DataConnector { + readonly name: string; + readonly needParse: boolean; + private page: playwright.Page; + + constructor(page: playwright.Page, props: ReindexDataConnectorProps) { + Object.assign(this, props); + this.name = 'Reindex from Druid'; + this.needParse = false; + this.page = page; + } + + async connect() { + await setLabeledInput(this.page, 'Datasource', this.datasourceName); + await setLabeledInput(this.page, 'Interval', this.interval); + await clickApplyButton(this.page); + } +} + +interface ReindexDataConnectorProps { + readonly datasourceName: string; + readonly interval: string; +} + +export interface ReindexDataConnector extends ReindexDataConnectorProps {} diff --git a/web-console/e2e-tests/component/load-data/data-loader.ts b/web-console/e2e-tests/component/load-data/data-loader.ts index 0244c43a974d..4d3c97104b33 100644 --- a/web-console/e2e-tests/component/load-data/data-loader.ts +++ b/web-console/e2e-tests/component/load-data/data-loader.ts @@ -18,6 +18,11 @@ import * as playwright from 'playwright-core'; +import { clickButton } from '../../util/playwright'; +import { clickLabeledButton } from '../../util/playwright'; +import { setLabeledInput } from '../../util/playwright'; +import { setLabeledTextarea } from '../../util/playwright'; + import { ConfigureSchemaConfig } from './config/configure-schema'; import { PartitionConfig } from './config/partition'; import { PublishConfig } from './config/publish'; @@ -41,8 +46,10 @@ export class DataLoader { await this.page.goto(this.baseUrl); await this.start(); await this.connect(this.connector, this.connectValidator); - await this.parseData(); - await this.parseTime(); + if (this.connector.needParse) { + await this.parseData(); + await this.parseTime(); + } await this.transform(); await this.filter(); await this.configureSchema(this.configureSchemaConfig); @@ -54,13 +61,14 @@ export class DataLoader { private async start() { await this.page.click(`"${this.connector.name}"`); - await this.clickButton('Connect data'); + await clickButton(this.page, 'Connect data'); } private async connect(connector: DataConnector, validator: (preview: string) => void) { await connector.connect(); await this.validateConnect(validator); - await this.clickButton('Next: Parse data'); + const next = this.connector.needParse ? 'Parse data' : 'Transform'; + await clickButton(this.page, `Next: ${next}`); } private async validateConnect(validator: (preview: string) => void) { @@ -72,28 +80,28 @@ export class DataLoader { private async parseData() { await this.page.waitFor('.parse-data-table'); - await this.clickButton('Next: Parse time'); + await clickButton(this.page, 'Next: Parse time'); } private async parseTime() { await this.page.waitFor('.parse-time-table'); - await this.clickButton('Next: Transform'); + await clickButton(this.page, 'Next: Transform'); } private async transform() { await this.page.waitFor('.transform-table'); - await this.clickButton('Next: Filter'); + await clickButton(this.page, 'Next: Filter'); } private async filter() { await this.page.waitFor('.filter-table'); - await this.clickButton('Next: Configure schema'); + await clickButton(this.page, 'Next: Configure schema'); } private async configureSchema(configureSchemaConfig: ConfigureSchemaConfig) { await this.page.waitFor('.schema-table'); await this.applyConfigureSchemaConfig(configureSchemaConfig); - await this.clickButton('Next: Partition'); + await clickButton(this.page, 'Next: Partition'); } private async applyConfigureSchemaConfig(configureSchemaConfig: ConfigureSchemaConfig) { @@ -103,7 +111,7 @@ export class DataLoader { await rollup!.click(); const confirmationDialogSelector = '//*[contains(@class,"bp3-alert-body")]'; await this.page.waitFor(confirmationDialogSelector); - await this.clickButton('Yes'); + await clickButton(this.page, 'Yes'); const statusMessageSelector = '.recipe-toaster'; await this.page.waitFor(statusMessageSelector); await this.page.click(`${statusMessageSelector} button`); @@ -113,48 +121,44 @@ export class DataLoader { private async partition(partitionConfig: PartitionConfig) { await this.page.waitFor('div.load-data-view.partition'); await this.applyPartitionConfig(partitionConfig); - await this.clickButton('Next: Tune'); + await clickButton(this.page, 'Next: Tune'); } private async applyPartitionConfig(partitionConfig: PartitionConfig) { - const segmentGranularity = await this.page.$( - '//*[text()="Segment granularity"]/following-sibling::div//input', - ); - await this.setInput(segmentGranularity!, partitionConfig.segmentGranularity); + await setLabeledInput(this.page, 'Segment granularity', partitionConfig.segmentGranularity); + if (partitionConfig.forceGuaranteedRollup) { + await clickLabeledButton( + this.page, + 'Force guaranteed rollup', + partitionConfig.forceGuaranteedRollupText, + ); + await setLabeledTextarea(this.page, 'Time intervals', partitionConfig.timeIntervals!); + } + if (partitionConfig.partitionsSpec != null) { + await partitionConfig.partitionsSpec.apply(this.page); + } } private async tune() { await this.page.waitFor('div.load-data-view.tuning'); - await this.clickButton('Next: Publish'); + await clickButton(this.page, 'Next: Publish'); } private async publish(publishConfig: PublishConfig) { await this.page.waitFor('div.load-data-view.publish'); await this.applyPublishConfig(publishConfig); - await this.clickButton('Edit spec'); + await clickButton(this.page, 'Edit spec'); } private async applyPublishConfig(publishConfig: PublishConfig) { if (publishConfig.datasourceName != null) { - const datasourceName = await this.page.$( - '//*[text()="Datasource name"]/following-sibling::div//input', - ); - await this.setInput(datasourceName!, publishConfig.datasourceName); + await setLabeledInput(this.page, 'Datasource name', publishConfig.datasourceName); } } private async editSpec() { await this.page.waitFor('div.load-data-view.spec'); - await this.clickButton('Submit'); - } - - private async clickButton(text: string) { - await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any); - } - - private async setInput(input: playwright.ElementHandle, value: string) { - await input.fill(''); - await input.type(value); + await clickButton(this.page, 'Submit'); } } diff --git a/web-console/e2e-tests/component/query/overview.ts b/web-console/e2e-tests/component/query/overview.ts index 973d29a57894..3175a6d41093 100644 --- a/web-console/e2e-tests/component/query/overview.ts +++ b/web-console/e2e-tests/component/query/overview.ts @@ -18,6 +18,8 @@ import * as playwright from 'playwright-core'; +import { clickButton } from '../../util/playwright'; +import { setInput } from '../../util/playwright'; import { extractTable } from '../../util/table'; /** @@ -37,19 +39,10 @@ export class QueryOverview { await this.page.reload({ waitUntil: 'networkidle0' }); const input = await this.page.$('div.query-input textarea'); - await this.setInput(input!, query); - await this.clickButton('Run'); + await setInput(input!, query); + await clickButton(this.page, 'Run'); await this.page.waitFor('div.query-info'); return await extractTable(this.page, 'div.query-output div.rt-tr-group', 'div.rt-td'); } - - private async setInput(input: playwright.ElementHandle, value: string) { - await input.fill(''); - await input.type(value); - } - - private async clickButton(text: string) { - await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any); - } } diff --git a/web-console/e2e-tests/reindexing.spec.ts b/web-console/e2e-tests/reindexing.spec.ts new file mode 100644 index 000000000000..b7355783339b --- /dev/null +++ b/web-console/e2e-tests/reindexing.spec.ts @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import * as playwright from 'playwright-core'; + +import { DatasourcesOverview } from './component/datasources/overview'; +import { IngestionOverview } from './component/ingestion/overview'; +import { ConfigureSchemaConfig } from './component/load-data/config/configure-schema'; +import { PartitionConfig } from './component/load-data/config/partition'; +import { SegmentGranularity } from './component/load-data/config/partition'; +import { SingleDimPartitionsSpec } from './component/load-data/config/partition'; +import { PublishConfig } from './component/load-data/config/publish'; +import { ReindexDataConnector } from './component/load-data/data-connector/reindex'; +import { DataLoader } from './component/load-data/data-loader'; +import { saveScreenshotIfError } from './util/debug'; +import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid'; +import { UNIFIED_CONSOLE_URL } from './util/druid'; +import { runIndexTask } from './util/druid'; +import { createBrowserNormal as createBrowser } from './util/playwright'; +import { createPage } from './util/playwright'; +import { retryIfJestAssertionError } from './util/retry'; +import { waitTillWebConsoleReady } from './util/setup'; + +jest.setTimeout(5 * 60 * 1000); + +describe('Reindexing from Druid', () => { + let browser: playwright.Browser; + let page: playwright.Page; + + beforeAll(async () => { + await waitTillWebConsoleReady(); + browser = await createBrowser(); + }); + + beforeEach(async () => { + page = await createPage(browser); + }); + + afterAll(async () => { + await browser.close(); + }); + + it('Reindex datasource from dynamic to single dim partitions', async () => { + const testName = 'reindex-dynamic-to-single-dim-'; + const datasourceName = testName + new Date().toISOString(); + const interval = '2015-09-12/2015-09-13'; + const dataConnector = new ReindexDataConnector(page, { + datasourceName, + interval, + }); + const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false }); + const partitionConfig = new PartitionConfig({ + segmentGranularity: SegmentGranularity.DAY, + timeIntervals: interval, + forceGuaranteedRollup: true, + partitionsSpec: new SingleDimPartitionsSpec({ + partitionDimension: 'channel', + targetRowsPerSegment: 10_000, + maxRowsPerSegment: null, + }), + }); + const publishConfig = new PublishConfig({ datasourceName: datasourceName }); + + const dataLoader = new DataLoader({ + page: page, + unifiedConsoleUrl: UNIFIED_CONSOLE_URL, + connector: dataConnector, + connectValidator: validateConnectLocalData, + configureSchemaConfig: configureSchemaConfig, + partitionConfig: partitionConfig, + publishConfig: publishConfig, + }); + + loadInitialData(datasourceName); + + await saveScreenshotIfError(testName, page, async () => { + const numInitialSegment = 1; + await validateDatasourceStatus(page, datasourceName, numInitialSegment); + + await dataLoader.load(); + await validateTaskStatus(page, datasourceName); + + const numReindexedSegment = 4; // 39k rows into segments of ~10k rows + await validateDatasourceStatus(page, datasourceName, numReindexedSegment); + }); + }); +}); + +function loadInitialData(datasourceName: string) { + const ingestionSpec = path.join(DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, 'wikipedia-index.json'); + const setDatasourceName = `s/wikipedia/${datasourceName}/`; + const sedCommands = [setDatasourceName]; + runIndexTask(ingestionSpec, sedCommands); +} + +function validateConnectLocalData(preview: string) { + const lines = preview.split('\n'); + expect(lines.length).toBe(500); + const firstLine = lines[0]; + expect(firstLine).toBe( + 'Druid row: {' + + '"__time":1442018818771' + + ',"isRobot":"false"' + + ',"countryIsoCode":null' + + ',"added":"36"' + + ',"regionName":null' + + ',"channel":"#en.wikipedia"' + + ',"delta":"36"' + + ',"isUnpatrolled":"false"' + + ',"isNew":"false"' + + ',"isMinor":"false"' + + ',"isAnonymous":"false"' + + ',"deleted":"0"' + + ',"cityName":null' + + ',"metroCode":null' + + ',"namespace":"Talk"' + + ',"comment":"added project"' + + ',"countryName":null' + + ',"page":"Talk:Oswald Tilghman"' + + ',"user":"GELongstreet"' + + ',"regionIsoCode":null' + + '}', + ); + const lastLine = lines[lines.length - 1]; + expect(lastLine).toBe( + 'Druid row: {' + + '"__time":1442020314823' + + ',"isRobot":"false"' + + ',"countryIsoCode":null' + + ',"added":"1"' + + ',"regionName":null' + + ',"channel":"#en.wikipedia"' + + ',"delta":"1"' + + ',"isUnpatrolled":"false"' + + ',"isNew":"false"' + + ',"isMinor":"true"' + + ',"isAnonymous":"false"' + + ',"deleted":"0"' + + ',"cityName":null' + + ',"metroCode":null' + + ',"namespace":"Main"' + + ',"comment":"/* History */[[WP:AWB/T|Typo fixing]], [[WP:AWB/T|typo(s) fixed]]: nothern → northern using [[Project:AWB|AWB]]"' + + ',"countryName":null' + + ',"page":"Hapoel Katamon Jerusalem F.C."' + + ',"user":"The Quixotic Potato"' + + ',"regionIsoCode":null' + + '}', + ); +} + +async function validateTaskStatus(page: playwright.Page, datasourceName: string) { + const ingestionOverview = new IngestionOverview(page, UNIFIED_CONSOLE_URL); + + await retryIfJestAssertionError(async () => { + const tasks = await ingestionOverview.getTasks(); + const task = tasks.find(t => t.datasource === datasourceName); + expect(task).toBeDefined(); + expect(task!.status).toMatch('SUCCESS'); + }); +} + +async function validateDatasourceStatus( + page: playwright.Page, + datasourceName: string, + expectedNumSegment: number, +) { + const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL); + const numSegmentString = `${expectedNumSegment} segment` + (expectedNumSegment !== 1 ? 's' : ''); + + await retryIfJestAssertionError(async () => { + const datasources = await datasourcesOverview.getDatasources(); + const datasource = datasources.find(t => t.name === datasourceName); + expect(datasource).toBeDefined(); + expect(datasource!.availability).toMatch(`Fully available (${numSegmentString})`); + expect(datasource!.totalRows).toBe(39244); + }); +} diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts b/web-console/e2e-tests/tutorial-batch.spec.ts index 590893885f37..c65566323bd8 100644 --- a/web-console/e2e-tests/tutorial-batch.spec.ts +++ b/web-console/e2e-tests/tutorial-batch.spec.ts @@ -17,7 +17,6 @@ */ import * as playwright from 'playwright-core'; -import { v4 as uuid } from 'uuid'; import { DatasourcesOverview } from './component/datasources/overview'; import { IngestionOverview } from './component/ingestion/overview'; @@ -55,14 +54,19 @@ describe('Tutorial: Loading a file', () => { }); it('Loads data from local disk', async () => { - const datasourceName = uuid(); - const dataConnector = new LocalFileDataConnector( - page, - 'quickstart/tutorial/', - 'wikiticker-2015-09-12-sampled.json.gz', - ); + const testName = 'load-data-from-local-disk-'; + const datasourceName = testName + new Date().toISOString(); + const dataConnector = new LocalFileDataConnector(page, { + baseDirectory: 'quickstart/tutorial/', + fileFilter: 'wikiticker-2015-09-12-sampled.json.gz', + }); const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false }); - const partitionConfig = new PartitionConfig({ segmentGranularity: SegmentGranularity.DAY }); + const partitionConfig = new PartitionConfig({ + segmentGranularity: SegmentGranularity.DAY, + timeIntervals: null, + forceGuaranteedRollup: null, + partitionsSpec: null, + }); const publishConfig = new PublishConfig({ datasourceName: datasourceName }); const dataLoader = new DataLoader({ @@ -75,7 +79,7 @@ describe('Tutorial: Loading a file', () => { publishConfig: publishConfig, }); - await saveScreenshotIfError('load-data-from-local-disk-', page, async () => { + await saveScreenshotIfError(testName, page, async () => { await dataLoader.load(); await validateTaskStatus(page, datasourceName); await validateDatasourceStatus(page, datasourceName); diff --git a/web-console/e2e-tests/util/druid.ts b/web-console/e2e-tests/util/druid.ts index 617be859428b..87b5d4145d5a 100644 --- a/web-console/e2e-tests/util/druid.ts +++ b/web-console/e2e-tests/util/druid.ts @@ -16,6 +16,7 @@ * limitations under the License. */ +import { execSync } from 'child_process'; import path from 'path'; export const UNIFIED_CONSOLE_URL = 'http://localhost:8888/unified-console.html'; @@ -24,4 +25,24 @@ export const COORDINATOR_URL = 'http://localhost:8081'; const UTIL_DIR = __dirname; const E2E_TEST_DIR = path.dirname(UTIL_DIR); const WEB_CONSOLE_DIR = path.dirname(E2E_TEST_DIR); -export const DRUID_DIR = path.dirname(WEB_CONSOLE_DIR); +const DRUID_DIR = path.dirname(WEB_CONSOLE_DIR); +export const DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR = path.join( + DRUID_DIR, + 'examples', + 'quickstart', + 'tutorial', +); + +export function runIndexTask(ingestionSpecPath: string, sedCommands: Array) { + const postIndexTask = path.join(DRUID_DIR, 'examples', 'bin', 'post-index-task'); + const sedCommandsString = sedCommands.map(sedCommand => `-e '${sedCommand}'`).join(' '); + execSync( + `${postIndexTask} \ + --file <(sed ${sedCommandsString} ${ingestionSpecPath}) \ + --url ${COORDINATOR_URL}`, + { + shell: 'bash', + timeout: 3 * 60 * 1000, + }, + ); +} diff --git a/web-console/e2e-tests/util/playwright.ts b/web-console/e2e-tests/util/playwright.ts index 21cbd336740a..e31a31b4acc4 100644 --- a/web-console/e2e-tests/util/playwright.ts +++ b/web-console/e2e-tests/util/playwright.ts @@ -48,3 +48,62 @@ export async function createPage(browser: playwright.Browser): Promise { + return setLabeledElement(page, 'input', label, value); +} + +export async function setLabeledTextarea( + page: playwright.Page, + label: string, + value: string, +): Promise { + return setLabeledElement(page, 'textarea', label, value); +} + +async function setLabeledElement( + page: playwright.Page, + type: string, + label: string, + value: string, +): Promise { + const element = await page.$(`//*[text()="${label}"]/following-sibling::div//${type}`); + await setInput(element!, value); +} + +export async function setInput( + input: playwright.ElementHandle, + value: string, +): Promise { + await input.fill(''); + await input.type(value); +} + +function buttonSelector(text: string) { + return `//button/*[contains(text(),"${text}")]`; +} + +export async function clickButton(page: playwright.Page, text: string): Promise { + await page.click(buttonSelector(text)); +} + +export async function clickLabeledButton( + page: playwright.Page, + label: string, + text: string, +): Promise { + await page.click(`//*[text()="${label}"]/following-sibling::div${buttonSelector(text)}`); +} + +export async function selectSuggestibleInput( + page: playwright.Page, + label: string, + value: string, +): Promise { + await page.click(`//*[text()="${label}"]/following-sibling::div//button`); + await page.click(`"${value}"`); +} diff --git a/web-console/package.json b/web-console/package.json index bce8dc5e79ee..6d132e983f0f 100644 --- a/web-console/package.json +++ b/web-console/package.json @@ -39,7 +39,7 @@ "test-base": "npm run tslint && npm run sasslint && npm run jest", "test": "npm run test-base -- --silent 2>&1", "test-ci": "npm run test-base -- --coverage", - "test-e2e": "jest --config jest.e2e.config.js --detectOpenHandles e2e-tests", + "test-e2e": "jest --config jest.e2e.config.js e2e-tests", "codecov": "codecov --disable=gcov -p ..", "coverage": "jest --coverage src", "update-snapshots": "jest -u", From feeddc5609b561de12d29e845c13f631d32f2895 Mon Sep 17 00:00:00 2001 From: Chi Cao Minh Date: Thu, 1 Oct 2020 23:59:21 -0700 Subject: [PATCH 2/2] Fix UI datasources view edit action compaction (#10459) Restore the web console's ability to view a datasource's compaction configuration via the "action" menu. Refactoring done in https://github.com/apache/druid/pull/10438 introduced a regression that always caused the default compaction configuration to be shown via the "action" menu instead. Regression test is added in e2e-tests/auto-compaction.spec.ts. --- web-console/e2e-tests/auto-compaction.spec.ts | 5 ++ .../component/datasources/overview.ts | 22 +++++- .../component/load-data/config/partition.ts | 77 +++++++++++++++++-- web-console/e2e-tests/util/playwright.ts | 7 ++ .../views/datasource-view/datasource-view.tsx | 4 +- 5 files changed, 104 insertions(+), 11 deletions(-) diff --git a/web-console/e2e-tests/auto-compaction.spec.ts b/web-console/e2e-tests/auto-compaction.spec.ts index 512c2724b8c4..68cd00b0287a 100644 --- a/web-console/e2e-tests/auto-compaction.spec.ts +++ b/web-console/e2e-tests/auto-compaction.spec.ts @@ -126,6 +126,11 @@ async function configureCompaction( ) { const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL); await datasourcesOverview.setCompactionConfiguration(datasourceName, compactionConfig); + + const savedCompactionConfig = await datasourcesOverview.getCompactionConfiguration( + datasourceName, + ); + expect(savedCompactionConfig).toEqual(compactionConfig); } async function triggerCompaction() { diff --git a/web-console/e2e-tests/component/datasources/overview.ts b/web-console/e2e-tests/component/datasources/overview.ts index 54af58b7f0c6..11f44b48d410 100644 --- a/web-console/e2e-tests/component/datasources/overview.ts +++ b/web-console/e2e-tests/component/datasources/overview.ts @@ -19,8 +19,10 @@ import * as playwright from 'playwright-core'; import { clickButton } from '../../util/playwright'; +import { getLabeledInput } from '../../util/playwright'; import { setLabeledInput } from '../../util/playwright'; import { extractTable } from '../../util/table'; +import { readPartitionSpec } from '../load-data/config/partition'; import { CompactionConfig } from './compaction'; import { Datasource } from './datasource'; @@ -42,6 +44,9 @@ enum DatasourceColumn { ACTIONS, } +const EDIT_COMPACTION_CONFIGURATION = 'Edit compaction configuration'; +const SKIP_OFFSET_FROM_LATEST = 'Skip offset from latest'; + /** * Represents datasource overview tab. */ @@ -80,10 +85,10 @@ export class DatasourcesOverview { ): Promise { await this.openEditActions(datasourceName); - await this.page.click('"Edit compaction configuration"'); + await this.page.click(`"${EDIT_COMPACTION_CONFIGURATION}"`); await setLabeledInput( this.page, - 'Skip offset from latest', + SKIP_OFFSET_FROM_LATEST, compactionConfig.skipOffsetFromLatest, ); await compactionConfig.partitionsSpec.apply(this.page); @@ -91,6 +96,17 @@ export class DatasourcesOverview { await clickButton(this.page, 'Submit'); } + async getCompactionConfiguration(datasourceName: string): Promise { + await this.openEditActions(datasourceName); + + await this.page.click(`"${EDIT_COMPACTION_CONFIGURATION}"`); + const skipOffsetFromLatest = await getLabeledInput(this.page, SKIP_OFFSET_FROM_LATEST); + const partitionsSpec = await readPartitionSpec(this.page); + + await clickButton(this.page, 'Close'); + return new CompactionConfig({ skipOffsetFromLatest, partitionsSpec: partitionsSpec! }); + } + private async openEditActions(datasourceName: string): Promise { const datasources = await this.getDatasources(); const index = datasources.findIndex(t => t.name === datasourceName); @@ -100,6 +116,6 @@ export class DatasourcesOverview { const editActions = await this.page.$$('span[icon=wrench]'); editActions[index].click(); - await this.page.waitFor(5000); + await this.page.waitFor('ul.bp3-menu'); } } diff --git a/web-console/e2e-tests/component/load-data/config/partition.ts b/web-console/e2e-tests/component/load-data/config/partition.ts index 8431eccab983..51713c6efdaa 100644 --- a/web-console/e2e-tests/component/load-data/config/partition.ts +++ b/web-console/e2e-tests/component/load-data/config/partition.ts @@ -19,6 +19,7 @@ import * as playwright from 'playwright-core'; import { selectSuggestibleInput } from '../../../util/playwright'; +import { getLabeledInput } from '../../../util/playwright'; import { setLabeledInput } from '../../../util/playwright'; /* tslint:disable max-classes-per-file */ @@ -40,22 +41,49 @@ export interface PartitionsSpec { apply(page: playwright.Page): Promise; } +export async function readPartitionSpec(page: playwright.Page): Promise { + const type = await getLabeledInput(page, PARTITIONING_TYPE); + switch (type) { + case HashedPartitionsSpec.TYPE: + return HashedPartitionsSpec.read(page); + case SingleDimPartitionsSpec.TYPE: + return SingleDimPartitionsSpec.read(page); + } + return null; +} + export class HashedPartitionsSpec implements PartitionsSpec { + public static TYPE = 'hashed'; + private static NUM_SHARDS = 'Num shards'; + readonly type: string; + static async read(page: playwright.Page): Promise { + const numShards = await getLabeledInputAsNumber(page, HashedPartitionsSpec.NUM_SHARDS); + return new HashedPartitionsSpec({ numShards }); + } + constructor(props: HashedPartitionsSpecProps) { Object.assign(this, props); - this.type = 'hashed'; + this.type = HashedPartitionsSpec.TYPE; } async apply(page: playwright.Page): Promise { await setLabeledInput(page, PARTITIONING_TYPE, this.type); if (this.numShards != null) { - await setLabeledInput(page, 'Num shards', String(this.numShards)); + await setLabeledInput(page, HashedPartitionsSpec.NUM_SHARDS, String(this.numShards)); } } } +async function getLabeledInputAsNumber( + page: playwright.Page, + label: string, +): Promise { + const valueString = await getLabeledInput(page, label); + return valueString === '' ? null : Number(valueString); +} + interface HashedPartitionsSpecProps { readonly numShards: number | null; } @@ -63,21 +91,58 @@ interface HashedPartitionsSpecProps { export interface HashedPartitionsSpec extends HashedPartitionsSpecProps {} export class SingleDimPartitionsSpec implements PartitionsSpec { + public static TYPE = 'single_dim'; + private static PARTITION_DIMENSION = 'Partition dimension'; + private static TARGET_ROWS_PER_SEGMENT = 'Target rows per segment'; + private static MAX_ROWS_PER_SEGMENT = 'Max rows per segment'; + readonly type: string; + static async read(page: playwright.Page): Promise { + const partitionDimension = await getLabeledInput( + page, + SingleDimPartitionsSpec.PARTITION_DIMENSION, + ); + const targetRowsPerSegment = await getLabeledInputAsNumber( + page, + SingleDimPartitionsSpec.TARGET_ROWS_PER_SEGMENT, + ); + const maxRowsPerSegment = await getLabeledInputAsNumber( + page, + SingleDimPartitionsSpec.MAX_ROWS_PER_SEGMENT, + ); + return new SingleDimPartitionsSpec({ + partitionDimension, + targetRowsPerSegment, + maxRowsPerSegment, + }); + } + constructor(props: SingleDimPartitionsSpecProps) { Object.assign(this, props); - this.type = 'single_dim'; + this.type = SingleDimPartitionsSpec.TYPE; } async apply(page: playwright.Page): Promise { await selectSuggestibleInput(page, PARTITIONING_TYPE, this.type); - await setLabeledInput(page, 'Partition dimension', this.partitionDimension); + await setLabeledInput( + page, + SingleDimPartitionsSpec.PARTITION_DIMENSION, + this.partitionDimension, + ); if (this.targetRowsPerSegment) { - await setLabeledInput(page, 'Target rows per segment', String(this.targetRowsPerSegment)); + await setLabeledInput( + page, + SingleDimPartitionsSpec.TARGET_ROWS_PER_SEGMENT, + String(this.targetRowsPerSegment), + ); } if (this.maxRowsPerSegment) { - await setLabeledInput(page, 'Max rows per segment', String(this.maxRowsPerSegment)); + await setLabeledInput( + page, + SingleDimPartitionsSpec.MAX_ROWS_PER_SEGMENT, + String(this.maxRowsPerSegment), + ); } } } diff --git a/web-console/e2e-tests/util/playwright.ts b/web-console/e2e-tests/util/playwright.ts index e31a31b4acc4..1a421a9a4f77 100644 --- a/web-console/e2e-tests/util/playwright.ts +++ b/web-console/e2e-tests/util/playwright.ts @@ -49,6 +49,13 @@ export async function createPage(browser: playwright.Browser): Promise { + return await page.$eval( + `//*[text()="${label}"]/following-sibling::div//input`, + el => (el as HTMLInputElement).value, + ); +} + export async function setLabeledInput( page: playwright.Page, label: string, diff --git a/web-console/src/views/datasource-view/datasource-view.tsx b/web-console/src/views/datasource-view/datasource-view.tsx index 59688b04a658..9f81a03cb58b 100644 --- a/web-console/src/views/datasource-view/datasource-view.tsx +++ b/web-console/src/views/datasource-view/datasource-view.tsx @@ -1209,12 +1209,12 @@ GROUP BY 1`; width: ACTION_COLUMN_WIDTH, filterable: false, Cell: ({ value: datasource, original }) => { - const { unused, rules, compaction } = original; + const { unused, rules, compactionConfig } = original; const datasourceActions = this.getDatasourceActions( datasource, unused, rules, - compaction, + compactionConfig, ); return (