Remove max rows per segment from compaction dialog

web-console/e2e-tests/auto-compaction.spec.ts

@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import axios from 'axios';
+import { execSync } from 'child_process';
+import path from 'path';
+import * as playwright from 'playwright-core';
+import { v4 as uuid } from 'uuid';
+
+import { CompactionConfig } from './component/datasources/compaction';
+import { Datasource } from './component/datasources/datasource';
+import { DatasourcesOverview } from './component/datasources/overview';
+import { saveScreenshotIfError } from './util/debug';
+import { COORDINATOR_URL } from './util/druid';
+import { DRUID_DIR } from './util/druid';
+import { UNIFIED_CONSOLE_URL } from './util/druid';
+import { createBrowserNormal as createBrowser } from './util/playwright';
+import { createPage } from './util/playwright';
+import { retryIfJestAssertionError } from './util/retry';
+import { waitTillWebConsoleReady } from './util/setup';
+
+jest.setTimeout(5 * 60 * 1000);
+
+// The workflow in these tests is based on the compaction tutorial:
+// https://druid.apache.org/docs/latest/tutorials/tutorial-compaction.html
+describe('Auto-compaction', () => {
+  let browser: playwright.Browser;
+  let page: playwright.Page;
+
+  beforeAll(async () => {
+    await waitTillWebConsoleReady();
+    browser = await createBrowser();
+  });
+
+  beforeEach(async () => {
+    page = await createPage(browser);
+  });
+
+  afterAll(async () => {
+    await browser.close();
+  });
+
+  it('Compacts segments when max rows per segment is in tuning config', async () => {
+    const datasourceName = uuid();
+    loadInitialData(datasourceName);
+
+    await saveScreenshotIfError('auto-compaction-', page, async () => {
+      const uncompactedNumSegment = 3;
+      const numRow = 1412;
+      await validateDatasourceStatus(page, datasourceName, uncompactedNumSegment, numRow);
+
+      const compactionConfig = new CompactionConfig({
+        skipOffsetFromLatest: 'PT0S',
+        tuningConfig: `{
+          "type" : "index_parallel",
+          "maxRowsInMemory" : 25000,
+          "partitionsSpec": {
+            "type": "dynamic",
+            "maxRowsPerSegment" : 5000000
+          }
+        }`,
+      });
+      await configureCompaction(page, datasourceName, compactionConfig);
+
+      // Depending on the number of configured tasks slots, autocompaction may
+      // need several iterations if several time chunks need compaction
+      let currNumSegment = uncompactedNumSegment;
+      await retryIfJestAssertionError(async () => {
+        await triggerCompaction();
+        currNumSegment = await waitForCompaction(page, datasourceName, currNumSegment);
+
+        const compactedNumSegment = 2;
+        expect(currNumSegment).toBe(compactedNumSegment);
+      });
+    });
+  });
+});
+
+function loadInitialData(datasourceName: string) {
+  const postIndexTask = path.join(DRUID_DIR, 'examples', 'bin', 'post-index-task');
+  const ingestionSpec = path.join(
+    DRUID_DIR,
+    'examples',
+    'quickstart',
+    'tutorial',
+    'compaction-init-index.json',
+  );
+  const setDatasourceName = `s/compaction-tutorial/${datasourceName}/`;
+  const setIntervals = 's|2015-09-12/2015-09-13|2015-09-12/2015-09-12T02:00|'; // shorten to reduce test duration
+  execSync(
+    `${postIndexTask} \
+       --file <(sed -e '${setDatasourceName}' -e '${setIntervals}' ${ingestionSpec}) \
+       --url ${COORDINATOR_URL}`,
+    {
+      shell: 'bash',
+      timeout: 3 * 60 * 1000,
+    },
+  );
+}
+
+async function validateDatasourceStatus(
+  page: playwright.Page,
+  datasourceName: string,
+  expectedNumSegment: number,
+  expectedNumRow: number,
+) {
+  await retryIfJestAssertionError(async () => {
+    const datasource = await getDatasource(page, datasourceName);
+    expect(datasource.availability).toMatch(`Fully available (${expectedNumSegment} segments)`);
+    expect(datasource.totalRows).toBe(expectedNumRow);
+  });
+}
+
+async function getDatasource(page: playwright.Page, datasourceName: string): Promise<Datasource> {
+  const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL);
+  const datasources = await datasourcesOverview.getDatasources();
+  const datasource = datasources.find(t => t.name === datasourceName);
+  expect(datasource).toBeDefined();
+  return datasource!;
+}
+
+async function configureCompaction(
+  page: playwright.Page,
+  datasourceName: string,
+  compactionConfig: CompactionConfig,
+) {
+  const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL);
+  await datasourcesOverview.setCompactionConfiguration(datasourceName, compactionConfig);
+}
+
+async function triggerCompaction() {
+  const res = await axios.post(`${COORDINATOR_URL}/druid/coordinator/v1/compaction/compact`);
+  expect(res.status).toBe(200);
+}
+
+async function waitForCompaction(
+  page: playwright.Page,
+  datasourceName: string,
+  prevNumSegment: number,
+): Promise<number> {
+  await retryIfJestAssertionError(async () => {
+    const currNumSegment = await getNumSegment(page, datasourceName);
+    expect(currNumSegment).toBeLessThan(prevNumSegment);
+  });
+
+  return getNumSegment(page, datasourceName);
+}
+
+async function getNumSegment(page: playwright.Page, datasourceName: string): Promise<number> {
+  const datasource = await getDatasource(page, datasourceName);
+  const currNumSegmentString = datasource!.availability.match(/(\d+)/)![0];
+  return Number(currNumSegmentString);
+}

web-console/e2e-tests/component/datasources/compaction.ts

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Datasource compaction configuration
+ */
+export class CompactionConfig {
+  constructor(props: CompactionConfig) {
+    Object.assign(this, props);
+  }
+}
+
+interface CompactionConfigProps {
+  readonly skipOffsetFromLatest: string;
+  readonly tuningConfig: string;
+}
+
+export interface CompactionConfig extends CompactionConfigProps {}

web-console/e2e-tests/component/datasources/overview.ts

@@ -20,6 +20,7 @@ import * as playwright from 'playwright-core';
 
 import { extractTable } from '../../util/table';
 
+import { CompactionConfig } from './compaction';
 import { Datasource } from './datasource';
 
 /**
@@ -70,4 +71,55 @@ export class DatasourcesOverview {
   private static parseNumber(text: string): number {
     return Number(text.replace(/,/g, ''));
   }
+
+  async setCompactionConfiguration(
+    datasourceName: string,
+    compactionConfig: CompactionConfig,
+  ): Promise<void> {
+    await this.openEditActions(datasourceName);
+
+    await this.page.click('"Edit compaction configuration"');
+
+    const skipOffsetFromLatest = await this.getInputElement('Skip offset from latest');
+    await DatasourcesOverview.setInput(
+      skipOffsetFromLatest!,
+      compactionConfig.skipOffsetFromLatest,
+    );
+
+    const tuningConfig = await this.getTextareaElement('Tuning config');
+    await DatasourcesOverview.setInput(tuningConfig!, compactionConfig.tuningConfig);
+
+    await this.clickButton('Submit');
+  }
+
+  private async openEditActions(datasourceName: string): Promise<void> {
+    const datasources = await this.getDatasources();
+    const index = datasources.findIndex(t => t.name === datasourceName);
+    if (index < 0) {
+      throw new Error(`Could not find datasource: ${datasourceName}`);
+    }
+
+    const editActions = await this.page.$$('span[icon=wrench]');
+    editActions[index].click();
+    await this.page.waitFor(5000);
+  }
+
+  private async getInputElement(label: string): Promise<playwright.ElementHandle<Element> | null> {
+    return this.page.$(`//*[text()="${label}"]/following-sibling::div//input`);
+  }
+
+  private async getTextareaElement(
+    label: string,
+  ): Promise<playwright.ElementHandle<Element> | null> {
+    return this.page.$(`//*[text()="${label}"]/following-sibling::div//textarea`);
+  }
+
+  private static async setInput(input: playwright.ElementHandle<Element>, value: string) {
+    await input.fill('');
+    await input.type(value);
+  }
+
+  private async clickButton(text: string) {
+    await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any);
+  }
 }

web-console/e2e-tests/tutorial-batch.spec.ts

@@ -33,6 +33,7 @@ import { UNIFIED_CONSOLE_URL } from './util/druid';
 import { createBrowserNormal as createBrowser } from './util/playwright';
 import { createPage } from './util/playwright';
 import { retryIfJestAssertionError } from './util/retry';
+import { waitTillWebConsoleReady } from './util/setup';
 
 jest.setTimeout(5 * 60 * 1000);
 
@@ -41,6 +42,7 @@ describe('Tutorial: Loading a file', () => {
   let page: playwright.Page;
 
   beforeAll(async () => {
+    await waitTillWebConsoleReady();
     browser = await createBrowser();
   });
 

web-console/e2e-tests/util/druid.ts

@@ -16,4 +16,12 @@
  * limitations under the License.
  */
 
+import path from 'path';
+
 export const UNIFIED_CONSOLE_URL = 'http://localhost:8888/unified-console.html';
+export const COORDINATOR_URL = 'http://localhost:8081';
+
+const UTIL_DIR = __dirname;
+const E2E_TEST_DIR = path.dirname(UTIL_DIR);
+const WEB_CONSOLE_DIR = path.dirname(E2E_TEST_DIR);
+export const DRUID_DIR = path.dirname(WEB_CONSOLE_DIR);

web-console/e2e-tests/util/setup.ts

@@ -16,18 +16,20 @@
  * limitations under the License.
  */
 
-import { createBrowserNormal } from './playwright';
+import { UNIFIED_CONSOLE_URL } from './druid';
+import { createBrowserNormal as createBrowser } from './playwright';
 import { createPage } from './playwright';
 
-(async () => {
-  const browser = await createBrowserNormal();
+export async function waitTillWebConsoleReady() {
+  const browser = await createBrowser();
 
   try {
     const page = await createPage(browser);
+    await page.goto(UNIFIED_CONSOLE_URL);
     await page.waitFor('//*[contains(text(),"console will not function at the moment")]', {
       visibility: 'hidden',
     });
   } finally {
     await browser.close();
   }
-})();
+}

web-console/jest.e2e.config.js

@@ -21,8 +21,5 @@ const common = require('./jest.common.config');
 module.exports = Object.assign(common, {
   "testMatch": [
     "**/?(*.)+(spec).ts?(x)"
-  ],
-  "setupFilesAfterEnv": [
-    "<rootDir>e2e-tests/util/setup.ts"
-  ],
+  ]
 });

web-console/src/dialogs/compaction-dialog/__snapshots__/compaction-dialog.spec.tsx.snap

@@ -27,14 +27,6 @@ exports[`compaction dialog matches snapshot 1`] = `
           "name": "skipOffsetFromLatest",
           "type": "string",
         },
-        Object {
-          "defaultValue": 5000000,
-          "info": <p>
-            Determines how many rows are in each segment.
-          </p>,
-          "name": "maxRowsPerSegment",
-          "type": "number",
-        },
         Object {
           "info": <p>
             <Memo(ExternalLink)

web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx

@@ -52,12 +52,6 @@ const COMPACTION_CONFIG_FIELDS: Field<Record<string, any>>[] = [
       </p>
     ),
   },
-  {
-    name: 'maxRowsPerSegment',
-    type: 'number',
-    defaultValue: DEFAULT_MAX_ROWS_PER_SEGMENT,
-    info: <p>Determines how many rows are in each segment.</p>,
-  },
   {
     name: 'taskContext',
     type: 'json',