diff --git a/web-console/e2e-tests/auto-compaction.spec.ts b/web-console/e2e-tests/auto-compaction.spec.ts new file mode 100644 index 000000000000..291c61418d1e --- /dev/null +++ b/web-console/e2e-tests/auto-compaction.spec.ts @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import axios from 'axios'; +import { execSync } from 'child_process'; +import path from 'path'; +import * as playwright from 'playwright-core'; +import { v4 as uuid } from 'uuid'; + +import { CompactionConfig } from './component/datasources/compaction'; +import { Datasource } from './component/datasources/datasource'; +import { DatasourcesOverview } from './component/datasources/overview'; +import { saveScreenshotIfError } from './util/debug'; +import { COORDINATOR_URL } from './util/druid'; +import { DRUID_DIR } from './util/druid'; +import { UNIFIED_CONSOLE_URL } from './util/druid'; +import { createBrowserNormal as createBrowser } from './util/playwright'; +import { createPage } from './util/playwright'; +import { retryIfJestAssertionError } from './util/retry'; +import { waitTillWebConsoleReady } from './util/setup'; + +jest.setTimeout(5 * 60 * 1000); + +// The workflow in these tests is based on the compaction tutorial: +// https://druid.apache.org/docs/latest/tutorials/tutorial-compaction.html +describe('Auto-compaction', () => { + let browser: playwright.Browser; + let page: playwright.Page; + + beforeAll(async () => { + await waitTillWebConsoleReady(); + browser = await createBrowser(); + }); + + beforeEach(async () => { + page = await createPage(browser); + }); + + afterAll(async () => { + await browser.close(); + }); + + it('Compacts segments when max rows per segment is in tuning config', async () => { + const datasourceName = uuid(); + loadInitialData(datasourceName); + + await saveScreenshotIfError('auto-compaction-', page, async () => { + const uncompactedNumSegment = 3; + const numRow = 1412; + await validateDatasourceStatus(page, datasourceName, uncompactedNumSegment, numRow); + + const compactionConfig = new CompactionConfig({ + skipOffsetFromLatest: 'PT0S', + tuningConfig: `{ + "type" : "index_parallel", + "maxRowsInMemory" : 25000, + "partitionsSpec": { + "type": "dynamic", + "maxRowsPerSegment" : 5000000 + } + }`, + }); + await configureCompaction(page, datasourceName, compactionConfig); + + // Depending on the number of configured tasks slots, autocompaction may + // need several iterations if several time chunks need compaction + let currNumSegment = uncompactedNumSegment; + await retryIfJestAssertionError(async () => { + await triggerCompaction(); + currNumSegment = await waitForCompaction(page, datasourceName, currNumSegment); + + const compactedNumSegment = 2; + expect(currNumSegment).toBe(compactedNumSegment); + }); + }); + }); +}); + +function loadInitialData(datasourceName: string) { + const postIndexTask = path.join(DRUID_DIR, 'examples', 'bin', 'post-index-task'); + const ingestionSpec = path.join( + DRUID_DIR, + 'examples', + 'quickstart', + 'tutorial', + 'compaction-init-index.json', + ); + const setDatasourceName = `s/compaction-tutorial/${datasourceName}/`; + const setIntervals = 's|2015-09-12/2015-09-13|2015-09-12/2015-09-12T02:00|'; // shorten to reduce test duration + execSync( + `${postIndexTask} \ + --file <(sed -e '${setDatasourceName}' -e '${setIntervals}' ${ingestionSpec}) \ + --url ${COORDINATOR_URL}`, + { + shell: 'bash', + timeout: 3 * 60 * 1000, + }, + ); +} + +async function validateDatasourceStatus( + page: playwright.Page, + datasourceName: string, + expectedNumSegment: number, + expectedNumRow: number, +) { + await retryIfJestAssertionError(async () => { + const datasource = await getDatasource(page, datasourceName); + expect(datasource.availability).toMatch(`Fully available (${expectedNumSegment} segments)`); + expect(datasource.totalRows).toBe(expectedNumRow); + }); +} + +async function getDatasource(page: playwright.Page, datasourceName: string): Promise { + const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL); + const datasources = await datasourcesOverview.getDatasources(); + const datasource = datasources.find(t => t.name === datasourceName); + expect(datasource).toBeDefined(); + return datasource!; +} + +async function configureCompaction( + page: playwright.Page, + datasourceName: string, + compactionConfig: CompactionConfig, +) { + const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL); + await datasourcesOverview.setCompactionConfiguration(datasourceName, compactionConfig); +} + +async function triggerCompaction() { + const res = await axios.post(`${COORDINATOR_URL}/druid/coordinator/v1/compaction/compact`); + expect(res.status).toBe(200); +} + +async function waitForCompaction( + page: playwright.Page, + datasourceName: string, + prevNumSegment: number, +): Promise { + await retryIfJestAssertionError(async () => { + const currNumSegment = await getNumSegment(page, datasourceName); + expect(currNumSegment).toBeLessThan(prevNumSegment); + }); + + return getNumSegment(page, datasourceName); +} + +async function getNumSegment(page: playwright.Page, datasourceName: string): Promise { + const datasource = await getDatasource(page, datasourceName); + const currNumSegmentString = datasource!.availability.match(/(\d+)/)![0]; + return Number(currNumSegmentString); +} diff --git a/web-console/e2e-tests/component/datasources/compaction.ts b/web-console/e2e-tests/component/datasources/compaction.ts new file mode 100644 index 000000000000..2d7e0bf4b338 --- /dev/null +++ b/web-console/e2e-tests/component/datasources/compaction.ts @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Datasource compaction configuration + */ +export class CompactionConfig { + constructor(props: CompactionConfig) { + Object.assign(this, props); + } +} + +interface CompactionConfigProps { + readonly skipOffsetFromLatest: string; + readonly tuningConfig: string; +} + +export interface CompactionConfig extends CompactionConfigProps {} diff --git a/web-console/e2e-tests/component/datasources/overview.ts b/web-console/e2e-tests/component/datasources/overview.ts index 5b55eedeb9f1..35e3e82c643e 100644 --- a/web-console/e2e-tests/component/datasources/overview.ts +++ b/web-console/e2e-tests/component/datasources/overview.ts @@ -20,6 +20,7 @@ import * as playwright from 'playwright-core'; import { extractTable } from '../../util/table'; +import { CompactionConfig } from './compaction'; import { Datasource } from './datasource'; /** @@ -70,4 +71,55 @@ export class DatasourcesOverview { private static parseNumber(text: string): number { return Number(text.replace(/,/g, '')); } + + async setCompactionConfiguration( + datasourceName: string, + compactionConfig: CompactionConfig, + ): Promise { + await this.openEditActions(datasourceName); + + await this.page.click('"Edit compaction configuration"'); + + const skipOffsetFromLatest = await this.getInputElement('Skip offset from latest'); + await DatasourcesOverview.setInput( + skipOffsetFromLatest!, + compactionConfig.skipOffsetFromLatest, + ); + + const tuningConfig = await this.getTextareaElement('Tuning config'); + await DatasourcesOverview.setInput(tuningConfig!, compactionConfig.tuningConfig); + + await this.clickButton('Submit'); + } + + private async openEditActions(datasourceName: string): Promise { + const datasources = await this.getDatasources(); + const index = datasources.findIndex(t => t.name === datasourceName); + if (index < 0) { + throw new Error(`Could not find datasource: ${datasourceName}`); + } + + const editActions = await this.page.$$('span[icon=wrench]'); + editActions[index].click(); + await this.page.waitFor(5000); + } + + private async getInputElement(label: string): Promise | null> { + return this.page.$(`//*[text()="${label}"]/following-sibling::div//input`); + } + + private async getTextareaElement( + label: string, + ): Promise | null> { + return this.page.$(`//*[text()="${label}"]/following-sibling::div//textarea`); + } + + private static async setInput(input: playwright.ElementHandle, value: string) { + await input.fill(''); + await input.type(value); + } + + private async clickButton(text: string) { + await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any); + } } diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts b/web-console/e2e-tests/tutorial-batch.spec.ts index 52aed72c32f0..590893885f37 100644 --- a/web-console/e2e-tests/tutorial-batch.spec.ts +++ b/web-console/e2e-tests/tutorial-batch.spec.ts @@ -33,6 +33,7 @@ import { UNIFIED_CONSOLE_URL } from './util/druid'; import { createBrowserNormal as createBrowser } from './util/playwright'; import { createPage } from './util/playwright'; import { retryIfJestAssertionError } from './util/retry'; +import { waitTillWebConsoleReady } from './util/setup'; jest.setTimeout(5 * 60 * 1000); @@ -41,6 +42,7 @@ describe('Tutorial: Loading a file', () => { let page: playwright.Page; beforeAll(async () => { + await waitTillWebConsoleReady(); browser = await createBrowser(); }); diff --git a/web-console/e2e-tests/util/druid.ts b/web-console/e2e-tests/util/druid.ts index 6ad9f63b2ef0..617be859428b 100644 --- a/web-console/e2e-tests/util/druid.ts +++ b/web-console/e2e-tests/util/druid.ts @@ -16,4 +16,12 @@ * limitations under the License. */ +import path from 'path'; + export const UNIFIED_CONSOLE_URL = 'http://localhost:8888/unified-console.html'; +export const COORDINATOR_URL = 'http://localhost:8081'; + +const UTIL_DIR = __dirname; +const E2E_TEST_DIR = path.dirname(UTIL_DIR); +const WEB_CONSOLE_DIR = path.dirname(E2E_TEST_DIR); +export const DRUID_DIR = path.dirname(WEB_CONSOLE_DIR); diff --git a/web-console/e2e-tests/util/setup.ts b/web-console/e2e-tests/util/setup.ts index 8c306cc7e196..081f9ba57ca4 100644 --- a/web-console/e2e-tests/util/setup.ts +++ b/web-console/e2e-tests/util/setup.ts @@ -16,18 +16,20 @@ * limitations under the License. */ -import { createBrowserNormal } from './playwright'; +import { UNIFIED_CONSOLE_URL } from './druid'; +import { createBrowserNormal as createBrowser } from './playwright'; import { createPage } from './playwright'; -(async () => { - const browser = await createBrowserNormal(); +export async function waitTillWebConsoleReady() { + const browser = await createBrowser(); try { const page = await createPage(browser); + await page.goto(UNIFIED_CONSOLE_URL); await page.waitFor('//*[contains(text(),"console will not function at the moment")]', { visibility: 'hidden', }); } finally { await browser.close(); } -})(); +} diff --git a/web-console/jest.e2e.config.js b/web-console/jest.e2e.config.js index a2321feed8fe..b5411183dceb 100644 --- a/web-console/jest.e2e.config.js +++ b/web-console/jest.e2e.config.js @@ -21,8 +21,5 @@ const common = require('./jest.common.config'); module.exports = Object.assign(common, { "testMatch": [ "**/?(*.)+(spec).ts?(x)" - ], - "setupFilesAfterEnv": [ - "e2e-tests/util/setup.ts" - ], + ] }); diff --git a/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap b/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap index f3b66e810932..4b2d1beb27d2 100644 --- a/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap @@ -27,14 +27,6 @@ exports[`compaction dialog matches snapshot 1`] = ` "name": "skipOffsetFromLatest", "type": "string", }, - Object { - "defaultValue": 5000000, - "info":

- Determines how many rows are in each segment. -

, - "name": "maxRowsPerSegment", - "type": "number", - }, Object { "info":

>[] = [

), }, - { - name: 'maxRowsPerSegment', - type: 'number', - defaultValue: DEFAULT_MAX_ROWS_PER_SEGMENT, - info:

Determines how many rows are in each segment.

, - }, { name: 'taskContext', type: 'json',