From 9d5aa8f5defd2f3ad424804712bb17e46e9acb49 Mon Sep 17 00:00:00 2001 From: Chi Cao Minh Date: Mon, 14 Sep 2020 14:02:42 -0700 Subject: [PATCH 1/2] Remove max rows per segment from compaction dialog Configuring autocompaction with single_dim partitioning in the web console, requires specifying maxRowsPerSegment in the "Tuning config" field, rather than the "Max rows per segment" field. Specifying maxRowsPerSegment in the "Tuning config" also works for dynamic and hash partitions. To make the workflow consistent for all partitioning schemes, remove the "Max rows per segment" field so that the value is always specified in the "Tuning config" field. --- web-console/e2e-tests/auto-compaction.spec.ts | 166 ++++++++++++++++++ .../component/datasources/compaction.ts | 33 ++++ .../component/datasources/overview.ts | 52 ++++++ web-console/e2e-tests/util/druid.ts | 8 + .../compaction-dialog.spec.tsx.snap | 8 - .../compaction-dialog/compaction-dialog.tsx | 6 - 6 files changed, 259 insertions(+), 14 deletions(-) create mode 100644 web-console/e2e-tests/auto-compaction.spec.ts create mode 100644 web-console/e2e-tests/component/datasources/compaction.ts diff --git a/web-console/e2e-tests/auto-compaction.spec.ts b/web-console/e2e-tests/auto-compaction.spec.ts new file mode 100644 index 000000000000..0511ff9e1b7a --- /dev/null +++ b/web-console/e2e-tests/auto-compaction.spec.ts @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import axios from 'axios'; +import { execSync } from 'child_process'; +import path from 'path'; +import * as playwright from 'playwright-core'; +import { v4 as uuid } from 'uuid'; + +import { CompactionConfig } from './component/datasources/compaction'; +import { Datasource } from './component/datasources/datasource'; +import { DatasourcesOverview } from './component/datasources/overview'; +import { saveScreenshotIfError } from './util/debug'; +import { COORDINATOR_URL } from './util/druid'; +import { DRUID_DIR } from './util/druid'; +import { UNIFIED_CONSOLE_URL } from './util/druid'; +import { createBrowserNormal as createBrowser } from './util/playwright'; +import { createPage } from './util/playwright'; +import { retryIfJestAssertionError } from './util/retry'; + +jest.setTimeout(5 * 60 * 1000); + +// The workflow in these tests is based on the compaction tutorial: +// https://druid.apache.org/docs/latest/tutorials/tutorial-compaction.html +describe('Auto-compaction', () => { + let browser: playwright.Browser; + let page: playwright.Page; + + beforeAll(async () => { + browser = await createBrowser(); + }); + + beforeEach(async () => { + page = await createPage(browser); + }); + + afterAll(async () => { + await browser.close(); + }); + + it('Compacts segments when max rows per segment is in tuning config', async () => { + const datasourceName = uuid(); + loadInitialData(datasourceName); + + await saveScreenshotIfError('auto-compaction-', page, async () => { + const uncompactedNumSegment = 3; + const numRow = 1412; + await validateDatasourceStatus(page, datasourceName, uncompactedNumSegment, numRow); + + const compactionConfig = new CompactionConfig({ + skipOffsetFromLatest: 'PT0S', + tuningConfig: `{ + "type" : "index_parallel", + "maxRowsInMemory" : 25000, + "partitionsSpec": { + "type": "dynamic", + "maxRowsPerSegment" : 5000000 + } + }`, + }); + await configureCompaction(page, datasourceName, compactionConfig); + + // Depending on the number of configured tasks slots, autocompaction may + // need several iterations if several time chunks need compaction + let currNumSegment = uncompactedNumSegment; + await retryIfJestAssertionError(async () => { + await triggerCompaction(); + currNumSegment = await waitForCompaction(page, datasourceName, currNumSegment); + + const compactedNumSegment = 2; + expect(currNumSegment).toBe(compactedNumSegment); + }); + }); + }); +}); + +function loadInitialData(datasourceName: string) { + const postIndexTask = path.join(DRUID_DIR, 'examples', 'bin', 'post-index-task'); + const ingestionSpec = path.join( + DRUID_DIR, + 'examples', + 'quickstart', + 'tutorial', + 'compaction-init-index.json', + ); + const setDatasourceName = `s/compaction-tutorial/${datasourceName}/`; + const setIntervals = 's|2015-09-12/2015-09-13|2015-09-12/2015-09-12T02:00|'; // shorten to reduce test duration + execSync( + `${postIndexTask} \ + --file <(sed -e '${setDatasourceName}' -e '${setIntervals}' ${ingestionSpec}) \ + --url ${COORDINATOR_URL}`, + { + shell: 'bash', + timeout: 3 * 60 * 1000, + }, + ); +} + +async function validateDatasourceStatus( + page: playwright.Page, + datasourceName: string, + expectedNumSegment: number, + expectedNumRow: number, +) { + await retryIfJestAssertionError(async () => { + const datasource = await getDatasource(page, datasourceName); + expect(datasource.availability).toMatch(`Fully available (${expectedNumSegment} segments)`); + expect(datasource.totalRows).toBe(expectedNumRow); + }); +} + +async function getDatasource(page: playwright.Page, datasourceName: string): Promise { + const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL); + const datasources = await datasourcesOverview.getDatasources(); + const datasource = datasources.find(t => t.name === datasourceName); + expect(datasource).toBeDefined(); + return datasource!; +} + +async function configureCompaction( + page: playwright.Page, + datasourceName: string, + compactionConfig: CompactionConfig, +) { + const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL); + await datasourcesOverview.setCompactionConfiguration(datasourceName, compactionConfig); +} + +async function triggerCompaction() { + const res = await axios.post(`${COORDINATOR_URL}/druid/coordinator/v1/compaction/compact`); + expect(res.status).toBe(200); +} + +async function waitForCompaction( + page: playwright.Page, + datasourceName: string, + prevNumSegment: number, +): Promise { + await retryIfJestAssertionError(async () => { + const currNumSegment = await getNumSegment(page, datasourceName); + expect(currNumSegment).toBeLessThan(prevNumSegment); + }); + + return getNumSegment(page, datasourceName); +} + +async function getNumSegment(page: playwright.Page, datasourceName: string): Promise { + const datasource = await getDatasource(page, datasourceName); + const currNumSegmentString = datasource!.availability.match(/(\d+)/)![0]; + return Number(currNumSegmentString); +} diff --git a/web-console/e2e-tests/component/datasources/compaction.ts b/web-console/e2e-tests/component/datasources/compaction.ts new file mode 100644 index 000000000000..2d7e0bf4b338 --- /dev/null +++ b/web-console/e2e-tests/component/datasources/compaction.ts @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Datasource compaction configuration + */ +export class CompactionConfig { + constructor(props: CompactionConfig) { + Object.assign(this, props); + } +} + +interface CompactionConfigProps { + readonly skipOffsetFromLatest: string; + readonly tuningConfig: string; +} + +export interface CompactionConfig extends CompactionConfigProps {} diff --git a/web-console/e2e-tests/component/datasources/overview.ts b/web-console/e2e-tests/component/datasources/overview.ts index 5b55eedeb9f1..35e3e82c643e 100644 --- a/web-console/e2e-tests/component/datasources/overview.ts +++ b/web-console/e2e-tests/component/datasources/overview.ts @@ -20,6 +20,7 @@ import * as playwright from 'playwright-core'; import { extractTable } from '../../util/table'; +import { CompactionConfig } from './compaction'; import { Datasource } from './datasource'; /** @@ -70,4 +71,55 @@ export class DatasourcesOverview { private static parseNumber(text: string): number { return Number(text.replace(/,/g, '')); } + + async setCompactionConfiguration( + datasourceName: string, + compactionConfig: CompactionConfig, + ): Promise { + await this.openEditActions(datasourceName); + + await this.page.click('"Edit compaction configuration"'); + + const skipOffsetFromLatest = await this.getInputElement('Skip offset from latest'); + await DatasourcesOverview.setInput( + skipOffsetFromLatest!, + compactionConfig.skipOffsetFromLatest, + ); + + const tuningConfig = await this.getTextareaElement('Tuning config'); + await DatasourcesOverview.setInput(tuningConfig!, compactionConfig.tuningConfig); + + await this.clickButton('Submit'); + } + + private async openEditActions(datasourceName: string): Promise { + const datasources = await this.getDatasources(); + const index = datasources.findIndex(t => t.name === datasourceName); + if (index < 0) { + throw new Error(`Could not find datasource: ${datasourceName}`); + } + + const editActions = await this.page.$$('span[icon=wrench]'); + editActions[index].click(); + await this.page.waitFor(5000); + } + + private async getInputElement(label: string): Promise | null> { + return this.page.$(`//*[text()="${label}"]/following-sibling::div//input`); + } + + private async getTextareaElement( + label: string, + ): Promise | null> { + return this.page.$(`//*[text()="${label}"]/following-sibling::div//textarea`); + } + + private static async setInput(input: playwright.ElementHandle, value: string) { + await input.fill(''); + await input.type(value); + } + + private async clickButton(text: string) { + await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any); + } } diff --git a/web-console/e2e-tests/util/druid.ts b/web-console/e2e-tests/util/druid.ts index 6ad9f63b2ef0..617be859428b 100644 --- a/web-console/e2e-tests/util/druid.ts +++ b/web-console/e2e-tests/util/druid.ts @@ -16,4 +16,12 @@ * limitations under the License. */ +import path from 'path'; + export const UNIFIED_CONSOLE_URL = 'http://localhost:8888/unified-console.html'; +export const COORDINATOR_URL = 'http://localhost:8081'; + +const UTIL_DIR = __dirname; +const E2E_TEST_DIR = path.dirname(UTIL_DIR); +const WEB_CONSOLE_DIR = path.dirname(E2E_TEST_DIR); +export const DRUID_DIR = path.dirname(WEB_CONSOLE_DIR); diff --git a/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap b/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap index f3b66e810932..4b2d1beb27d2 100644 --- a/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap +++ b/web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap @@ -27,14 +27,6 @@ exports[`compaction dialog matches snapshot 1`] = ` "name": "skipOffsetFromLatest", "type": "string", }, - Object { - "defaultValue": 5000000, - "info":

- Determines how many rows are in each segment. -

, - "name": "maxRowsPerSegment", - "type": "number", - }, Object { "info":

>[] = [

), }, - { - name: 'maxRowsPerSegment', - type: 'number', - defaultValue: DEFAULT_MAX_ROWS_PER_SEGMENT, - info:

Determines how many rows are in each segment.

, - }, { name: 'taskContext', type: 'json', From 2c495a94a3b932d7b43841147b1547be8f04a3e4 Mon Sep 17 00:00:00 2001 From: Chi Cao Minh Date: Thu, 17 Sep 2020 22:15:54 -0700 Subject: [PATCH 2/2] Fix test setup --- web-console/e2e-tests/auto-compaction.spec.ts | 2 ++ web-console/e2e-tests/tutorial-batch.spec.ts | 2 ++ web-console/e2e-tests/util/setup.ts | 10 ++++++---- web-console/jest.e2e.config.js | 5 +---- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/web-console/e2e-tests/auto-compaction.spec.ts b/web-console/e2e-tests/auto-compaction.spec.ts index 0511ff9e1b7a..291c61418d1e 100644 --- a/web-console/e2e-tests/auto-compaction.spec.ts +++ b/web-console/e2e-tests/auto-compaction.spec.ts @@ -32,6 +32,7 @@ import { UNIFIED_CONSOLE_URL } from './util/druid'; import { createBrowserNormal as createBrowser } from './util/playwright'; import { createPage } from './util/playwright'; import { retryIfJestAssertionError } from './util/retry'; +import { waitTillWebConsoleReady } from './util/setup'; jest.setTimeout(5 * 60 * 1000); @@ -42,6 +43,7 @@ describe('Auto-compaction', () => { let page: playwright.Page; beforeAll(async () => { + await waitTillWebConsoleReady(); browser = await createBrowser(); }); diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts b/web-console/e2e-tests/tutorial-batch.spec.ts index 52aed72c32f0..590893885f37 100644 --- a/web-console/e2e-tests/tutorial-batch.spec.ts +++ b/web-console/e2e-tests/tutorial-batch.spec.ts @@ -33,6 +33,7 @@ import { UNIFIED_CONSOLE_URL } from './util/druid'; import { createBrowserNormal as createBrowser } from './util/playwright'; import { createPage } from './util/playwright'; import { retryIfJestAssertionError } from './util/retry'; +import { waitTillWebConsoleReady } from './util/setup'; jest.setTimeout(5 * 60 * 1000); @@ -41,6 +42,7 @@ describe('Tutorial: Loading a file', () => { let page: playwright.Page; beforeAll(async () => { + await waitTillWebConsoleReady(); browser = await createBrowser(); }); diff --git a/web-console/e2e-tests/util/setup.ts b/web-console/e2e-tests/util/setup.ts index 8c306cc7e196..081f9ba57ca4 100644 --- a/web-console/e2e-tests/util/setup.ts +++ b/web-console/e2e-tests/util/setup.ts @@ -16,18 +16,20 @@ * limitations under the License. */ -import { createBrowserNormal } from './playwright'; +import { UNIFIED_CONSOLE_URL } from './druid'; +import { createBrowserNormal as createBrowser } from './playwright'; import { createPage } from './playwright'; -(async () => { - const browser = await createBrowserNormal(); +export async function waitTillWebConsoleReady() { + const browser = await createBrowser(); try { const page = await createPage(browser); + await page.goto(UNIFIED_CONSOLE_URL); await page.waitFor('//*[contains(text(),"console will not function at the moment")]', { visibility: 'hidden', }); } finally { await browser.close(); } -})(); +} diff --git a/web-console/jest.e2e.config.js b/web-console/jest.e2e.config.js index a2321feed8fe..b5411183dceb 100644 --- a/web-console/jest.e2e.config.js +++ b/web-console/jest.e2e.config.js @@ -21,8 +21,5 @@ const common = require('./jest.common.config'); module.exports = Object.assign(common, { "testMatch": [ "**/?(*.)+(spec).ts?(x)" - ], - "setupFilesAfterEnv": [ - "e2e-tests/util/setup.ts" - ], + ] });