diff --git a/web-console/e2e-tests/auto-compaction.spec.ts b/web-console/e2e-tests/auto-compaction.spec.ts index edcf6a8d6ed4..68cd00b0287a 100644 --- a/web-console/e2e-tests/auto-compaction.spec.ts +++ b/web-console/e2e-tests/auto-compaction.spec.ts @@ -17,19 +17,18 @@ */ import axios from 'axios'; -import { execSync } from 'child_process'; import path from 'path'; import * as playwright from 'playwright-core'; -import { v4 as uuid } from 'uuid'; import { CompactionConfig } from './component/datasources/compaction'; -import { CompactionHashPartitionsSpec } from './component/datasources/compaction'; import { Datasource } from './component/datasources/datasource'; import { DatasourcesOverview } from './component/datasources/overview'; +import { HashedPartitionsSpec } from './component/load-data/config/partition'; import { saveScreenshotIfError } from './util/debug'; import { COORDINATOR_URL } from './util/druid'; -import { DRUID_DIR } from './util/druid'; +import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid'; import { UNIFIED_CONSOLE_URL } from './util/druid'; +import { runIndexTask } from './util/druid'; import { createBrowserNormal as createBrowser } from './util/playwright'; import { createPage } from './util/playwright'; import { retryIfJestAssertionError } from './util/retry'; @@ -57,17 +56,18 @@ describe('Auto-compaction', () => { }); it('Compacts segments from dynamic to hash partitions', async () => { - const datasourceName = uuid(); + const testName = 'autocompaction-dynamic-to-hash-'; + const datasourceName = testName + new Date().toISOString(); loadInitialData(datasourceName); - await saveScreenshotIfError('auto-compaction-', page, async () => { + await saveScreenshotIfError(testName, page, async () => { const uncompactedNumSegment = 3; const numRow = 1412; await validateDatasourceStatus(page, datasourceName, uncompactedNumSegment, numRow); const compactionConfig = new CompactionConfig({ skipOffsetFromLatest: 'PT0S', - partitionsSpec: new CompactionHashPartitionsSpec({ + partitionsSpec: new HashedPartitionsSpec({ numShards: null, }), }); @@ -88,25 +88,14 @@ describe('Auto-compaction', () => { }); function loadInitialData(datasourceName: string) { - const postIndexTask = path.join(DRUID_DIR, 'examples', 'bin', 'post-index-task'); const ingestionSpec = path.join( - DRUID_DIR, - 'examples', - 'quickstart', - 'tutorial', + DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, 'compaction-init-index.json', ); const setDatasourceName = `s/compaction-tutorial/${datasourceName}/`; const setIntervals = 's|2015-09-12/2015-09-13|2015-09-12/2015-09-12T02:00|'; // shorten to reduce test duration - execSync( - `${postIndexTask} \ - --file <(sed -e '${setDatasourceName}' -e '${setIntervals}' ${ingestionSpec}) \ - --url ${COORDINATOR_URL}`, - { - shell: 'bash', - timeout: 3 * 60 * 1000, - }, - ); + const sedCommands = [setDatasourceName, setIntervals]; + runIndexTask(ingestionSpec, sedCommands); } async function validateDatasourceStatus( @@ -137,6 +126,11 @@ async function configureCompaction( ) { const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL); await datasourcesOverview.setCompactionConfiguration(datasourceName, compactionConfig); + + const savedCompactionConfig = await datasourcesOverview.getCompactionConfiguration( + datasourceName, + ); + expect(savedCompactionConfig).toEqual(compactionConfig); } async function triggerCompaction() { diff --git a/web-console/e2e-tests/component/datasources/compaction.ts b/web-console/e2e-tests/component/datasources/compaction.ts index c22285dee1bb..946362069016 100644 --- a/web-console/e2e-tests/component/datasources/compaction.ts +++ b/web-console/e2e-tests/component/datasources/compaction.ts @@ -16,44 +16,7 @@ * limitations under the License. */ -import * as playwright from 'playwright-core'; - -/* tslint:disable max-classes-per-file */ - -const PARTITIONING_TYPE = 'Partitioning type'; - -interface CompactionPartitionsSpec { - readonly type: string; - apply(page: playwright.Page): Promise; -} - -export class CompactionHashPartitionsSpec implements CompactionPartitionsSpec { - readonly type: string; - - constructor(props: CompactionHashPartitionsSpecProps) { - Object.assign(this, props); - this.type = 'hashed'; - } - - async apply(page: playwright.Page): Promise { - await setInput(page, PARTITIONING_TYPE, this.type); - if (this.numShards != null) { - await setInput(page, 'Num shards', String(this.numShards)); - } - } -} - -async function setInput(page: playwright.Page, label: string, value: string): Promise { - const input = await page.$(`//*[text()="${label}"]/following-sibling::div//input`); - await input!.fill(''); - await input!.type(value); -} - -interface CompactionHashPartitionsSpecProps { - readonly numShards: number | null; -} - -export interface CompactionHashPartitionsSpec extends CompactionHashPartitionsSpecProps {} +import { PartitionsSpec } from '../load-data/config/partition'; /** * Datasource compaction configuration @@ -66,7 +29,7 @@ export class CompactionConfig { interface CompactionConfigProps { readonly skipOffsetFromLatest: string; - readonly partitionsSpec: CompactionPartitionsSpec; + readonly partitionsSpec: PartitionsSpec; } export interface CompactionConfig extends CompactionConfigProps {} diff --git a/web-console/e2e-tests/component/datasources/overview.ts b/web-console/e2e-tests/component/datasources/overview.ts index f5433116b270..11f44b48d410 100644 --- a/web-console/e2e-tests/component/datasources/overview.ts +++ b/web-console/e2e-tests/component/datasources/overview.ts @@ -18,7 +18,11 @@ import * as playwright from 'playwright-core'; +import { clickButton } from '../../util/playwright'; +import { getLabeledInput } from '../../util/playwright'; +import { setLabeledInput } from '../../util/playwright'; import { extractTable } from '../../util/table'; +import { readPartitionSpec } from '../load-data/config/partition'; import { CompactionConfig } from './compaction'; import { Datasource } from './datasource'; @@ -40,6 +44,9 @@ enum DatasourceColumn { ACTIONS, } +const EDIT_COMPACTION_CONFIGURATION = 'Edit compaction configuration'; +const SKIP_OFFSET_FROM_LATEST = 'Skip offset from latest'; + /** * Represents datasource overview tab. */ @@ -78,11 +85,26 @@ export class DatasourcesOverview { ): Promise { await this.openEditActions(datasourceName); - await this.page.click('"Edit compaction configuration"'); - await this.setInput('Skip offset from latest', compactionConfig.skipOffsetFromLatest); + await this.page.click(`"${EDIT_COMPACTION_CONFIGURATION}"`); + await setLabeledInput( + this.page, + SKIP_OFFSET_FROM_LATEST, + compactionConfig.skipOffsetFromLatest, + ); await compactionConfig.partitionsSpec.apply(this.page); - await this.clickButton('Submit'); + await clickButton(this.page, 'Submit'); + } + + async getCompactionConfiguration(datasourceName: string): Promise { + await this.openEditActions(datasourceName); + + await this.page.click(`"${EDIT_COMPACTION_CONFIGURATION}"`); + const skipOffsetFromLatest = await getLabeledInput(this.page, SKIP_OFFSET_FROM_LATEST); + const partitionsSpec = await readPartitionSpec(this.page); + + await clickButton(this.page, 'Close'); + return new CompactionConfig({ skipOffsetFromLatest, partitionsSpec: partitionsSpec! }); } private async openEditActions(datasourceName: string): Promise { @@ -94,16 +116,6 @@ export class DatasourcesOverview { const editActions = await this.page.$$('span[icon=wrench]'); editActions[index].click(); - await this.page.waitFor(5000); - } - - private async setInput(label: string, value: string) { - const input = await this.page.$(`//*[text()="${label}"]/following-sibling::div//input`); - await input!.fill(''); - await input!.type(value); - } - - private async clickButton(text: string) { - await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any); + await this.page.waitFor('ul.bp3-menu'); } } diff --git a/web-console/e2e-tests/component/load-data/config/partition.ts b/web-console/e2e-tests/component/load-data/config/partition.ts index 5ecba3200077..51713c6efdaa 100644 --- a/web-console/e2e-tests/component/load-data/config/partition.ts +++ b/web-console/e2e-tests/component/load-data/config/partition.ts @@ -16,6 +16,14 @@ * limitations under the License. */ +import * as playwright from 'playwright-core'; + +import { selectSuggestibleInput } from '../../../util/playwright'; +import { getLabeledInput } from '../../../util/playwright'; +import { setLabeledInput } from '../../../util/playwright'; + +/* tslint:disable max-classes-per-file */ + /** * Possible values for partition step segment granularity. */ @@ -26,17 +34,144 @@ export enum SegmentGranularity { YEAR = 'YEAR', } +const PARTITIONING_TYPE = 'Partitioning type'; + +export interface PartitionsSpec { + readonly type: string; + apply(page: playwright.Page): Promise; +} + +export async function readPartitionSpec(page: playwright.Page): Promise { + const type = await getLabeledInput(page, PARTITIONING_TYPE); + switch (type) { + case HashedPartitionsSpec.TYPE: + return HashedPartitionsSpec.read(page); + case SingleDimPartitionsSpec.TYPE: + return SingleDimPartitionsSpec.read(page); + } + return null; +} + +export class HashedPartitionsSpec implements PartitionsSpec { + public static TYPE = 'hashed'; + private static NUM_SHARDS = 'Num shards'; + + readonly type: string; + + static async read(page: playwright.Page): Promise { + const numShards = await getLabeledInputAsNumber(page, HashedPartitionsSpec.NUM_SHARDS); + return new HashedPartitionsSpec({ numShards }); + } + + constructor(props: HashedPartitionsSpecProps) { + Object.assign(this, props); + this.type = HashedPartitionsSpec.TYPE; + } + + async apply(page: playwright.Page): Promise { + await setLabeledInput(page, PARTITIONING_TYPE, this.type); + if (this.numShards != null) { + await setLabeledInput(page, HashedPartitionsSpec.NUM_SHARDS, String(this.numShards)); + } + } +} + +async function getLabeledInputAsNumber( + page: playwright.Page, + label: string, +): Promise { + const valueString = await getLabeledInput(page, label); + return valueString === '' ? null : Number(valueString); +} + +interface HashedPartitionsSpecProps { + readonly numShards: number | null; +} + +export interface HashedPartitionsSpec extends HashedPartitionsSpecProps {} + +export class SingleDimPartitionsSpec implements PartitionsSpec { + public static TYPE = 'single_dim'; + private static PARTITION_DIMENSION = 'Partition dimension'; + private static TARGET_ROWS_PER_SEGMENT = 'Target rows per segment'; + private static MAX_ROWS_PER_SEGMENT = 'Max rows per segment'; + + readonly type: string; + + static async read(page: playwright.Page): Promise { + const partitionDimension = await getLabeledInput( + page, + SingleDimPartitionsSpec.PARTITION_DIMENSION, + ); + const targetRowsPerSegment = await getLabeledInputAsNumber( + page, + SingleDimPartitionsSpec.TARGET_ROWS_PER_SEGMENT, + ); + const maxRowsPerSegment = await getLabeledInputAsNumber( + page, + SingleDimPartitionsSpec.MAX_ROWS_PER_SEGMENT, + ); + return new SingleDimPartitionsSpec({ + partitionDimension, + targetRowsPerSegment, + maxRowsPerSegment, + }); + } + + constructor(props: SingleDimPartitionsSpecProps) { + Object.assign(this, props); + this.type = SingleDimPartitionsSpec.TYPE; + } + + async apply(page: playwright.Page): Promise { + await selectSuggestibleInput(page, PARTITIONING_TYPE, this.type); + await setLabeledInput( + page, + SingleDimPartitionsSpec.PARTITION_DIMENSION, + this.partitionDimension, + ); + if (this.targetRowsPerSegment) { + await setLabeledInput( + page, + SingleDimPartitionsSpec.TARGET_ROWS_PER_SEGMENT, + String(this.targetRowsPerSegment), + ); + } + if (this.maxRowsPerSegment) { + await setLabeledInput( + page, + SingleDimPartitionsSpec.MAX_ROWS_PER_SEGMENT, + String(this.maxRowsPerSegment), + ); + } + } +} + +interface SingleDimPartitionsSpecProps { + readonly partitionDimension: string; + readonly targetRowsPerSegment: number | null; + readonly maxRowsPerSegment: number | null; +} + +export interface SingleDimPartitionsSpec extends SingleDimPartitionsSpecProps {} + /** * Data loader partition step configuration. */ export class PartitionConfig { + readonly forceGuaranteedRollupText: string; + constructor(props: PartitionConfigProps) { Object.assign(this, props); + this.forceGuaranteedRollupText = this.forceGuaranteedRollup ? 'True' : 'False'; } } interface PartitionConfigProps { readonly segmentGranularity: SegmentGranularity; + readonly timeIntervals: string | null; + readonly forceGuaranteedRollup: boolean | null; + readonly partitionsSpec: PartitionsSpec | null; } export interface PartitionConfig extends PartitionConfigProps {} diff --git a/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts b/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts index 50a9d292328f..7112f2649ebf 100644 --- a/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts +++ b/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts @@ -16,10 +16,19 @@ * limitations under the License. */ +import * as playwright from 'playwright-core'; + +import { clickButton } from '../../../util/playwright'; + /** * Connector for data loader input data. */ export interface DataConnector { readonly name: string; + readonly needParse: boolean; connect(): Promise; } + +export async function clickApplyButton(page: playwright.Page): Promise { + await clickButton(page, 'Apply'); +} diff --git a/web-console/e2e-tests/component/load-data/data-connector/local-file.ts b/web-console/e2e-tests/component/load-data/data-connector/local-file.ts index 4e2aa3666796..dfbaed59945d 100644 --- a/web-console/e2e-tests/component/load-data/data-connector/local-file.ts +++ b/web-console/e2e-tests/component/load-data/data-connector/local-file.ts @@ -18,6 +18,9 @@ import * as playwright from 'playwright-core'; +import { setLabeledInput } from '../../../util/playwright'; + +import { clickApplyButton } from './data-connector'; import { DataConnector } from './data-connector'; /** @@ -25,30 +28,26 @@ import { DataConnector } from './data-connector'; */ export class LocalFileDataConnector implements DataConnector { readonly name: string; + readonly needParse: boolean; private page: playwright.Page; - private baseDirectory: string; - private fileFilter: string; - constructor(page: playwright.Page, baseDirectory: string, fileFilter: string) { + constructor(page: playwright.Page, props: LocalFileDataConnectorProps) { + Object.assign(this, props); this.name = 'Local disk'; + this.needParse = true; this.page = page; - this.baseDirectory = baseDirectory; - this.fileFilter = fileFilter; } async connect() { - const baseDirectoryInput = await this.page.$('input[placeholder="/path/to/files/"]'); - await this.setInput(baseDirectoryInput!, this.baseDirectory); - - const fileFilterInput = await this.page.$('input[value="*"]'); - await this.setInput(fileFilterInput!, this.fileFilter); - - const applyButton = await this.page.$('"Apply"'); - await applyButton!.click(); + await setLabeledInput(this.page, 'Base directory', this.baseDirectory); + await setLabeledInput(this.page, 'File filter', this.fileFilter); + await clickApplyButton(this.page); } +} - private async setInput(input: playwright.ElementHandle, value: string) { - await input.fill(''); - await input.type(value); - } +interface LocalFileDataConnectorProps { + readonly baseDirectory: string; + readonly fileFilter: string; } + +export interface LocalFileDataConnector extends LocalFileDataConnectorProps {} diff --git a/web-console/e2e-tests/component/load-data/data-connector/reindex.ts b/web-console/e2e-tests/component/load-data/data-connector/reindex.ts new file mode 100644 index 000000000000..d519a5d3ca8b --- /dev/null +++ b/web-console/e2e-tests/component/load-data/data-connector/reindex.ts @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as playwright from 'playwright-core'; + +import { setLabeledInput } from '../../../util/playwright'; + +import { clickApplyButton } from './data-connector'; +import { DataConnector } from './data-connector'; + +/** + * Reindexing connector for data loader input data. + */ +export class ReindexDataConnector implements DataConnector { + readonly name: string; + readonly needParse: boolean; + private page: playwright.Page; + + constructor(page: playwright.Page, props: ReindexDataConnectorProps) { + Object.assign(this, props); + this.name = 'Reindex from Druid'; + this.needParse = false; + this.page = page; + } + + async connect() { + await setLabeledInput(this.page, 'Datasource', this.datasourceName); + await setLabeledInput(this.page, 'Interval', this.interval); + await clickApplyButton(this.page); + } +} + +interface ReindexDataConnectorProps { + readonly datasourceName: string; + readonly interval: string; +} + +export interface ReindexDataConnector extends ReindexDataConnectorProps {} diff --git a/web-console/e2e-tests/component/load-data/data-loader.ts b/web-console/e2e-tests/component/load-data/data-loader.ts index 0244c43a974d..4d3c97104b33 100644 --- a/web-console/e2e-tests/component/load-data/data-loader.ts +++ b/web-console/e2e-tests/component/load-data/data-loader.ts @@ -18,6 +18,11 @@ import * as playwright from 'playwright-core'; +import { clickButton } from '../../util/playwright'; +import { clickLabeledButton } from '../../util/playwright'; +import { setLabeledInput } from '../../util/playwright'; +import { setLabeledTextarea } from '../../util/playwright'; + import { ConfigureSchemaConfig } from './config/configure-schema'; import { PartitionConfig } from './config/partition'; import { PublishConfig } from './config/publish'; @@ -41,8 +46,10 @@ export class DataLoader { await this.page.goto(this.baseUrl); await this.start(); await this.connect(this.connector, this.connectValidator); - await this.parseData(); - await this.parseTime(); + if (this.connector.needParse) { + await this.parseData(); + await this.parseTime(); + } await this.transform(); await this.filter(); await this.configureSchema(this.configureSchemaConfig); @@ -54,13 +61,14 @@ export class DataLoader { private async start() { await this.page.click(`"${this.connector.name}"`); - await this.clickButton('Connect data'); + await clickButton(this.page, 'Connect data'); } private async connect(connector: DataConnector, validator: (preview: string) => void) { await connector.connect(); await this.validateConnect(validator); - await this.clickButton('Next: Parse data'); + const next = this.connector.needParse ? 'Parse data' : 'Transform'; + await clickButton(this.page, `Next: ${next}`); } private async validateConnect(validator: (preview: string) => void) { @@ -72,28 +80,28 @@ export class DataLoader { private async parseData() { await this.page.waitFor('.parse-data-table'); - await this.clickButton('Next: Parse time'); + await clickButton(this.page, 'Next: Parse time'); } private async parseTime() { await this.page.waitFor('.parse-time-table'); - await this.clickButton('Next: Transform'); + await clickButton(this.page, 'Next: Transform'); } private async transform() { await this.page.waitFor('.transform-table'); - await this.clickButton('Next: Filter'); + await clickButton(this.page, 'Next: Filter'); } private async filter() { await this.page.waitFor('.filter-table'); - await this.clickButton('Next: Configure schema'); + await clickButton(this.page, 'Next: Configure schema'); } private async configureSchema(configureSchemaConfig: ConfigureSchemaConfig) { await this.page.waitFor('.schema-table'); await this.applyConfigureSchemaConfig(configureSchemaConfig); - await this.clickButton('Next: Partition'); + await clickButton(this.page, 'Next: Partition'); } private async applyConfigureSchemaConfig(configureSchemaConfig: ConfigureSchemaConfig) { @@ -103,7 +111,7 @@ export class DataLoader { await rollup!.click(); const confirmationDialogSelector = '//*[contains(@class,"bp3-alert-body")]'; await this.page.waitFor(confirmationDialogSelector); - await this.clickButton('Yes'); + await clickButton(this.page, 'Yes'); const statusMessageSelector = '.recipe-toaster'; await this.page.waitFor(statusMessageSelector); await this.page.click(`${statusMessageSelector} button`); @@ -113,48 +121,44 @@ export class DataLoader { private async partition(partitionConfig: PartitionConfig) { await this.page.waitFor('div.load-data-view.partition'); await this.applyPartitionConfig(partitionConfig); - await this.clickButton('Next: Tune'); + await clickButton(this.page, 'Next: Tune'); } private async applyPartitionConfig(partitionConfig: PartitionConfig) { - const segmentGranularity = await this.page.$( - '//*[text()="Segment granularity"]/following-sibling::div//input', - ); - await this.setInput(segmentGranularity!, partitionConfig.segmentGranularity); + await setLabeledInput(this.page, 'Segment granularity', partitionConfig.segmentGranularity); + if (partitionConfig.forceGuaranteedRollup) { + await clickLabeledButton( + this.page, + 'Force guaranteed rollup', + partitionConfig.forceGuaranteedRollupText, + ); + await setLabeledTextarea(this.page, 'Time intervals', partitionConfig.timeIntervals!); + } + if (partitionConfig.partitionsSpec != null) { + await partitionConfig.partitionsSpec.apply(this.page); + } } private async tune() { await this.page.waitFor('div.load-data-view.tuning'); - await this.clickButton('Next: Publish'); + await clickButton(this.page, 'Next: Publish'); } private async publish(publishConfig: PublishConfig) { await this.page.waitFor('div.load-data-view.publish'); await this.applyPublishConfig(publishConfig); - await this.clickButton('Edit spec'); + await clickButton(this.page, 'Edit spec'); } private async applyPublishConfig(publishConfig: PublishConfig) { if (publishConfig.datasourceName != null) { - const datasourceName = await this.page.$( - '//*[text()="Datasource name"]/following-sibling::div//input', - ); - await this.setInput(datasourceName!, publishConfig.datasourceName); + await setLabeledInput(this.page, 'Datasource name', publishConfig.datasourceName); } } private async editSpec() { await this.page.waitFor('div.load-data-view.spec'); - await this.clickButton('Submit'); - } - - private async clickButton(text: string) { - await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any); - } - - private async setInput(input: playwright.ElementHandle, value: string) { - await input.fill(''); - await input.type(value); + await clickButton(this.page, 'Submit'); } } diff --git a/web-console/e2e-tests/component/query/overview.ts b/web-console/e2e-tests/component/query/overview.ts index 973d29a57894..3175a6d41093 100644 --- a/web-console/e2e-tests/component/query/overview.ts +++ b/web-console/e2e-tests/component/query/overview.ts @@ -18,6 +18,8 @@ import * as playwright from 'playwright-core'; +import { clickButton } from '../../util/playwright'; +import { setInput } from '../../util/playwright'; import { extractTable } from '../../util/table'; /** @@ -37,19 +39,10 @@ export class QueryOverview { await this.page.reload({ waitUntil: 'networkidle0' }); const input = await this.page.$('div.query-input textarea'); - await this.setInput(input!, query); - await this.clickButton('Run'); + await setInput(input!, query); + await clickButton(this.page, 'Run'); await this.page.waitFor('div.query-info'); return await extractTable(this.page, 'div.query-output div.rt-tr-group', 'div.rt-td'); } - - private async setInput(input: playwright.ElementHandle, value: string) { - await input.fill(''); - await input.type(value); - } - - private async clickButton(text: string) { - await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any); - } } diff --git a/web-console/e2e-tests/reindexing.spec.ts b/web-console/e2e-tests/reindexing.spec.ts new file mode 100644 index 000000000000..b7355783339b --- /dev/null +++ b/web-console/e2e-tests/reindexing.spec.ts @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import * as playwright from 'playwright-core'; + +import { DatasourcesOverview } from './component/datasources/overview'; +import { IngestionOverview } from './component/ingestion/overview'; +import { ConfigureSchemaConfig } from './component/load-data/config/configure-schema'; +import { PartitionConfig } from './component/load-data/config/partition'; +import { SegmentGranularity } from './component/load-data/config/partition'; +import { SingleDimPartitionsSpec } from './component/load-data/config/partition'; +import { PublishConfig } from './component/load-data/config/publish'; +import { ReindexDataConnector } from './component/load-data/data-connector/reindex'; +import { DataLoader } from './component/load-data/data-loader'; +import { saveScreenshotIfError } from './util/debug'; +import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid'; +import { UNIFIED_CONSOLE_URL } from './util/druid'; +import { runIndexTask } from './util/druid'; +import { createBrowserNormal as createBrowser } from './util/playwright'; +import { createPage } from './util/playwright'; +import { retryIfJestAssertionError } from './util/retry'; +import { waitTillWebConsoleReady } from './util/setup'; + +jest.setTimeout(5 * 60 * 1000); + +describe('Reindexing from Druid', () => { + let browser: playwright.Browser; + let page: playwright.Page; + + beforeAll(async () => { + await waitTillWebConsoleReady(); + browser = await createBrowser(); + }); + + beforeEach(async () => { + page = await createPage(browser); + }); + + afterAll(async () => { + await browser.close(); + }); + + it('Reindex datasource from dynamic to single dim partitions', async () => { + const testName = 'reindex-dynamic-to-single-dim-'; + const datasourceName = testName + new Date().toISOString(); + const interval = '2015-09-12/2015-09-13'; + const dataConnector = new ReindexDataConnector(page, { + datasourceName, + interval, + }); + const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false }); + const partitionConfig = new PartitionConfig({ + segmentGranularity: SegmentGranularity.DAY, + timeIntervals: interval, + forceGuaranteedRollup: true, + partitionsSpec: new SingleDimPartitionsSpec({ + partitionDimension: 'channel', + targetRowsPerSegment: 10_000, + maxRowsPerSegment: null, + }), + }); + const publishConfig = new PublishConfig({ datasourceName: datasourceName }); + + const dataLoader = new DataLoader({ + page: page, + unifiedConsoleUrl: UNIFIED_CONSOLE_URL, + connector: dataConnector, + connectValidator: validateConnectLocalData, + configureSchemaConfig: configureSchemaConfig, + partitionConfig: partitionConfig, + publishConfig: publishConfig, + }); + + loadInitialData(datasourceName); + + await saveScreenshotIfError(testName, page, async () => { + const numInitialSegment = 1; + await validateDatasourceStatus(page, datasourceName, numInitialSegment); + + await dataLoader.load(); + await validateTaskStatus(page, datasourceName); + + const numReindexedSegment = 4; // 39k rows into segments of ~10k rows + await validateDatasourceStatus(page, datasourceName, numReindexedSegment); + }); + }); +}); + +function loadInitialData(datasourceName: string) { + const ingestionSpec = path.join(DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, 'wikipedia-index.json'); + const setDatasourceName = `s/wikipedia/${datasourceName}/`; + const sedCommands = [setDatasourceName]; + runIndexTask(ingestionSpec, sedCommands); +} + +function validateConnectLocalData(preview: string) { + const lines = preview.split('\n'); + expect(lines.length).toBe(500); + const firstLine = lines[0]; + expect(firstLine).toBe( + 'Druid row: {' + + '"__time":1442018818771' + + ',"isRobot":"false"' + + ',"countryIsoCode":null' + + ',"added":"36"' + + ',"regionName":null' + + ',"channel":"#en.wikipedia"' + + ',"delta":"36"' + + ',"isUnpatrolled":"false"' + + ',"isNew":"false"' + + ',"isMinor":"false"' + + ',"isAnonymous":"false"' + + ',"deleted":"0"' + + ',"cityName":null' + + ',"metroCode":null' + + ',"namespace":"Talk"' + + ',"comment":"added project"' + + ',"countryName":null' + + ',"page":"Talk:Oswald Tilghman"' + + ',"user":"GELongstreet"' + + ',"regionIsoCode":null' + + '}', + ); + const lastLine = lines[lines.length - 1]; + expect(lastLine).toBe( + 'Druid row: {' + + '"__time":1442020314823' + + ',"isRobot":"false"' + + ',"countryIsoCode":null' + + ',"added":"1"' + + ',"regionName":null' + + ',"channel":"#en.wikipedia"' + + ',"delta":"1"' + + ',"isUnpatrolled":"false"' + + ',"isNew":"false"' + + ',"isMinor":"true"' + + ',"isAnonymous":"false"' + + ',"deleted":"0"' + + ',"cityName":null' + + ',"metroCode":null' + + ',"namespace":"Main"' + + ',"comment":"/* History */[[WP:AWB/T|Typo fixing]], [[WP:AWB/T|typo(s) fixed]]: nothern → northern using [[Project:AWB|AWB]]"' + + ',"countryName":null' + + ',"page":"Hapoel Katamon Jerusalem F.C."' + + ',"user":"The Quixotic Potato"' + + ',"regionIsoCode":null' + + '}', + ); +} + +async function validateTaskStatus(page: playwright.Page, datasourceName: string) { + const ingestionOverview = new IngestionOverview(page, UNIFIED_CONSOLE_URL); + + await retryIfJestAssertionError(async () => { + const tasks = await ingestionOverview.getTasks(); + const task = tasks.find(t => t.datasource === datasourceName); + expect(task).toBeDefined(); + expect(task!.status).toMatch('SUCCESS'); + }); +} + +async function validateDatasourceStatus( + page: playwright.Page, + datasourceName: string, + expectedNumSegment: number, +) { + const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL); + const numSegmentString = `${expectedNumSegment} segment` + (expectedNumSegment !== 1 ? 's' : ''); + + await retryIfJestAssertionError(async () => { + const datasources = await datasourcesOverview.getDatasources(); + const datasource = datasources.find(t => t.name === datasourceName); + expect(datasource).toBeDefined(); + expect(datasource!.availability).toMatch(`Fully available (${numSegmentString})`); + expect(datasource!.totalRows).toBe(39244); + }); +} diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts b/web-console/e2e-tests/tutorial-batch.spec.ts index 590893885f37..c65566323bd8 100644 --- a/web-console/e2e-tests/tutorial-batch.spec.ts +++ b/web-console/e2e-tests/tutorial-batch.spec.ts @@ -17,7 +17,6 @@ */ import * as playwright from 'playwright-core'; -import { v4 as uuid } from 'uuid'; import { DatasourcesOverview } from './component/datasources/overview'; import { IngestionOverview } from './component/ingestion/overview'; @@ -55,14 +54,19 @@ describe('Tutorial: Loading a file', () => { }); it('Loads data from local disk', async () => { - const datasourceName = uuid(); - const dataConnector = new LocalFileDataConnector( - page, - 'quickstart/tutorial/', - 'wikiticker-2015-09-12-sampled.json.gz', - ); + const testName = 'load-data-from-local-disk-'; + const datasourceName = testName + new Date().toISOString(); + const dataConnector = new LocalFileDataConnector(page, { + baseDirectory: 'quickstart/tutorial/', + fileFilter: 'wikiticker-2015-09-12-sampled.json.gz', + }); const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false }); - const partitionConfig = new PartitionConfig({ segmentGranularity: SegmentGranularity.DAY }); + const partitionConfig = new PartitionConfig({ + segmentGranularity: SegmentGranularity.DAY, + timeIntervals: null, + forceGuaranteedRollup: null, + partitionsSpec: null, + }); const publishConfig = new PublishConfig({ datasourceName: datasourceName }); const dataLoader = new DataLoader({ @@ -75,7 +79,7 @@ describe('Tutorial: Loading a file', () => { publishConfig: publishConfig, }); - await saveScreenshotIfError('load-data-from-local-disk-', page, async () => { + await saveScreenshotIfError(testName, page, async () => { await dataLoader.load(); await validateTaskStatus(page, datasourceName); await validateDatasourceStatus(page, datasourceName); diff --git a/web-console/e2e-tests/util/druid.ts b/web-console/e2e-tests/util/druid.ts index 617be859428b..87b5d4145d5a 100644 --- a/web-console/e2e-tests/util/druid.ts +++ b/web-console/e2e-tests/util/druid.ts @@ -16,6 +16,7 @@ * limitations under the License. */ +import { execSync } from 'child_process'; import path from 'path'; export const UNIFIED_CONSOLE_URL = 'http://localhost:8888/unified-console.html'; @@ -24,4 +25,24 @@ export const COORDINATOR_URL = 'http://localhost:8081'; const UTIL_DIR = __dirname; const E2E_TEST_DIR = path.dirname(UTIL_DIR); const WEB_CONSOLE_DIR = path.dirname(E2E_TEST_DIR); -export const DRUID_DIR = path.dirname(WEB_CONSOLE_DIR); +const DRUID_DIR = path.dirname(WEB_CONSOLE_DIR); +export const DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR = path.join( + DRUID_DIR, + 'examples', + 'quickstart', + 'tutorial', +); + +export function runIndexTask(ingestionSpecPath: string, sedCommands: Array) { + const postIndexTask = path.join(DRUID_DIR, 'examples', 'bin', 'post-index-task'); + const sedCommandsString = sedCommands.map(sedCommand => `-e '${sedCommand}'`).join(' '); + execSync( + `${postIndexTask} \ + --file <(sed ${sedCommandsString} ${ingestionSpecPath}) \ + --url ${COORDINATOR_URL}`, + { + shell: 'bash', + timeout: 3 * 60 * 1000, + }, + ); +} diff --git a/web-console/e2e-tests/util/playwright.ts b/web-console/e2e-tests/util/playwright.ts index 21cbd336740a..1a421a9a4f77 100644 --- a/web-console/e2e-tests/util/playwright.ts +++ b/web-console/e2e-tests/util/playwright.ts @@ -48,3 +48,69 @@ export async function createPage(browser: playwright.Browser): Promise { + return await page.$eval( + `//*[text()="${label}"]/following-sibling::div//input`, + el => (el as HTMLInputElement).value, + ); +} + +export async function setLabeledInput( + page: playwright.Page, + label: string, + value: string, +): Promise { + return setLabeledElement(page, 'input', label, value); +} + +export async function setLabeledTextarea( + page: playwright.Page, + label: string, + value: string, +): Promise { + return setLabeledElement(page, 'textarea', label, value); +} + +async function setLabeledElement( + page: playwright.Page, + type: string, + label: string, + value: string, +): Promise { + const element = await page.$(`//*[text()="${label}"]/following-sibling::div//${type}`); + await setInput(element!, value); +} + +export async function setInput( + input: playwright.ElementHandle, + value: string, +): Promise { + await input.fill(''); + await input.type(value); +} + +function buttonSelector(text: string) { + return `//button/*[contains(text(),"${text}")]`; +} + +export async function clickButton(page: playwright.Page, text: string): Promise { + await page.click(buttonSelector(text)); +} + +export async function clickLabeledButton( + page: playwright.Page, + label: string, + text: string, +): Promise { + await page.click(`//*[text()="${label}"]/following-sibling::div${buttonSelector(text)}`); +} + +export async function selectSuggestibleInput( + page: playwright.Page, + label: string, + value: string, +): Promise { + await page.click(`//*[text()="${label}"]/following-sibling::div//button`); + await page.click(`"${value}"`); +} diff --git a/web-console/package.json b/web-console/package.json index bce8dc5e79ee..6d132e983f0f 100644 --- a/web-console/package.json +++ b/web-console/package.json @@ -39,7 +39,7 @@ "test-base": "npm run tslint && npm run sasslint && npm run jest", "test": "npm run test-base -- --silent 2>&1", "test-ci": "npm run test-base -- --coverage", - "test-e2e": "jest --config jest.e2e.config.js --detectOpenHandles e2e-tests", + "test-e2e": "jest --config jest.e2e.config.js e2e-tests", "codecov": "codecov --disable=gcov -p ..", "coverage": "jest --coverage src", "update-snapshots": "jest -u", diff --git a/web-console/src/views/datasource-view/datasource-view.tsx b/web-console/src/views/datasource-view/datasource-view.tsx index 59688b04a658..9f81a03cb58b 100644 --- a/web-console/src/views/datasource-view/datasource-view.tsx +++ b/web-console/src/views/datasource-view/datasource-view.tsx @@ -1209,12 +1209,12 @@ GROUP BY 1`; width: ACTION_COLUMN_WIDTH, filterable: false, Cell: ({ value: datasource, original }) => { - const { unused, rules, compaction } = original; + const { unused, rules, compactionConfig } = original; const datasourceActions = this.getDatasourceActions( datasource, unused, rules, - compaction, + compactionConfig, ); return (