[Backport] Fix UI datasources view edit action compaction #10459

web-console/e2e-tests/auto-compaction.spec.ts

@@ -17,19 +17,18 @@
  */
 
 import axios from 'axios';
-import { execSync } from 'child_process';
 import path from 'path';
 import * as playwright from 'playwright-core';
-import { v4 as uuid } from 'uuid';
 
 import { CompactionConfig } from './component/datasources/compaction';
-import { CompactionHashPartitionsSpec } from './component/datasources/compaction';
 import { Datasource } from './component/datasources/datasource';
 import { DatasourcesOverview } from './component/datasources/overview';
+import { HashedPartitionsSpec } from './component/load-data/config/partition';
 import { saveScreenshotIfError } from './util/debug';
 import { COORDINATOR_URL } from './util/druid';
-import { DRUID_DIR } from './util/druid';
+import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid';
 import { UNIFIED_CONSOLE_URL } from './util/druid';
+import { runIndexTask } from './util/druid';
 import { createBrowserNormal as createBrowser } from './util/playwright';
 import { createPage } from './util/playwright';
 import { retryIfJestAssertionError } from './util/retry';
@@ -57,17 +56,18 @@ describe('Auto-compaction', () => {
   });
 
   it('Compacts segments from dynamic to hash partitions', async () => {
-    const datasourceName = uuid();
+    const testName = 'autocompaction-dynamic-to-hash-';
+    const datasourceName = testName + new Date().toISOString();
     loadInitialData(datasourceName);
 
-    await saveScreenshotIfError('auto-compaction-', page, async () => {
+    await saveScreenshotIfError(testName, page, async () => {
       const uncompactedNumSegment = 3;
       const numRow = 1412;
       await validateDatasourceStatus(page, datasourceName, uncompactedNumSegment, numRow);
 
       const compactionConfig = new CompactionConfig({
         skipOffsetFromLatest: 'PT0S',
-        partitionsSpec: new CompactionHashPartitionsSpec({
+        partitionsSpec: new HashedPartitionsSpec({
           numShards: null,
         }),
       });
@@ -88,25 +88,14 @@ describe('Auto-compaction', () => {
 });
 
 function loadInitialData(datasourceName: string) {
-  const postIndexTask = path.join(DRUID_DIR, 'examples', 'bin', 'post-index-task');
   const ingestionSpec = path.join(
-    DRUID_DIR,
-    'examples',
-    'quickstart',
-    'tutorial',
+    DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
     'compaction-init-index.json',
   );
   const setDatasourceName = `s/compaction-tutorial/${datasourceName}/`;
   const setIntervals = 's|2015-09-12/2015-09-13|2015-09-12/2015-09-12T02:00|'; // shorten to reduce test duration
-  execSync(
-    `${postIndexTask} \
-       --file <(sed -e '${setDatasourceName}' -e '${setIntervals}' ${ingestionSpec}) \
-       --url ${COORDINATOR_URL}`,
-    {
-      shell: 'bash',
-      timeout: 3 * 60 * 1000,
-    },
-  );
+  const sedCommands = [setDatasourceName, setIntervals];
+  runIndexTask(ingestionSpec, sedCommands);
 }
 
 async function validateDatasourceStatus(
@@ -137,6 +126,11 @@ async function configureCompaction(
 ) {
   const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL);
   await datasourcesOverview.setCompactionConfiguration(datasourceName, compactionConfig);
+
+  const savedCompactionConfig = await datasourcesOverview.getCompactionConfiguration(
+    datasourceName,
+  );
+  expect(savedCompactionConfig).toEqual(compactionConfig);
 }
 
 async function triggerCompaction() {

web-console/e2e-tests/component/datasources/compaction.ts

@@ -16,44 +16,7 @@
  * limitations under the License.
  */
 
-import * as playwright from 'playwright-core';
-
-/* tslint:disable max-classes-per-file */
-
-const PARTITIONING_TYPE = 'Partitioning type';
-
-interface CompactionPartitionsSpec {
-  readonly type: string;
-  apply(page: playwright.Page): Promise<void>;
-}
-
-export class CompactionHashPartitionsSpec implements CompactionPartitionsSpec {
-  readonly type: string;
-
-  constructor(props: CompactionHashPartitionsSpecProps) {
-    Object.assign(this, props);
-    this.type = 'hashed';
-  }
-
-  async apply(page: playwright.Page): Promise<void> {
-    await setInput(page, PARTITIONING_TYPE, this.type);
-    if (this.numShards != null) {
-      await setInput(page, 'Num shards', String(this.numShards));
-    }
-  }
-}
-
-async function setInput(page: playwright.Page, label: string, value: string): Promise<void> {
-  const input = await page.$(`//*[text()="${label}"]/following-sibling::div//input`);
-  await input!.fill('');
-  await input!.type(value);
-}
-
-interface CompactionHashPartitionsSpecProps {
-  readonly numShards: number | null;
-}
-
-export interface CompactionHashPartitionsSpec extends CompactionHashPartitionsSpecProps {}
+import { PartitionsSpec } from '../load-data/config/partition';
 
 /**
  * Datasource compaction configuration
@@ -66,7 +29,7 @@ export class CompactionConfig {
 
 interface CompactionConfigProps {
   readonly skipOffsetFromLatest: string;
-  readonly partitionsSpec: CompactionPartitionsSpec;
+  readonly partitionsSpec: PartitionsSpec;
 }
 
 export interface CompactionConfig extends CompactionConfigProps {}

web-console/e2e-tests/component/datasources/overview.ts

@@ -18,7 +18,11 @@
 
 import * as playwright from 'playwright-core';
 
+import { clickButton } from '../../util/playwright';
+import { getLabeledInput } from '../../util/playwright';
+import { setLabeledInput } from '../../util/playwright';
 import { extractTable } from '../../util/table';
+import { readPartitionSpec } from '../load-data/config/partition';
 
 import { CompactionConfig } from './compaction';
 import { Datasource } from './datasource';
@@ -40,6 +44,9 @@ enum DatasourceColumn {
   ACTIONS,
 }
 
+const EDIT_COMPACTION_CONFIGURATION = 'Edit compaction configuration';
+const SKIP_OFFSET_FROM_LATEST = 'Skip offset from latest';
+
 /**
  * Represents datasource overview tab.
  */
@@ -78,11 +85,26 @@ export class DatasourcesOverview {
   ): Promise<void> {
     await this.openEditActions(datasourceName);
 
-    await this.page.click('"Edit compaction configuration"');
-    await this.setInput('Skip offset from latest', compactionConfig.skipOffsetFromLatest);
+    await this.page.click(`"${EDIT_COMPACTION_CONFIGURATION}"`);
+    await setLabeledInput(
+      this.page,
+      SKIP_OFFSET_FROM_LATEST,
+      compactionConfig.skipOffsetFromLatest,
+    );
     await compactionConfig.partitionsSpec.apply(this.page);
 
-    await this.clickButton('Submit');
+    await clickButton(this.page, 'Submit');
+  }
+
+  async getCompactionConfiguration(datasourceName: string): Promise<CompactionConfig> {
+    await this.openEditActions(datasourceName);
+
+    await this.page.click(`"${EDIT_COMPACTION_CONFIGURATION}"`);
+    const skipOffsetFromLatest = await getLabeledInput(this.page, SKIP_OFFSET_FROM_LATEST);
+    const partitionsSpec = await readPartitionSpec(this.page);
+
+    await clickButton(this.page, 'Close');
+    return new CompactionConfig({ skipOffsetFromLatest, partitionsSpec: partitionsSpec! });
   }
 
   private async openEditActions(datasourceName: string): Promise<void> {
@@ -94,16 +116,6 @@ export class DatasourcesOverview {
 
     const editActions = await this.page.$$('span[icon=wrench]');
     editActions[index].click();
-    await this.page.waitFor(5000);
-  }
-
-  private async setInput(label: string, value: string) {
-    const input = await this.page.$(`//*[text()="${label}"]/following-sibling::div//input`);
-    await input!.fill('');
-    await input!.type(value);
-  }
-
-  private async clickButton(text: string) {
-    await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any);
+    await this.page.waitFor('ul.bp3-menu');
   }
 }

web-console/e2e-tests/component/load-data/config/partition.ts

@@ -16,6 +16,14 @@
  * limitations under the License.
  */
 
+import * as playwright from 'playwright-core';
+
+import { selectSuggestibleInput } from '../../../util/playwright';
+import { getLabeledInput } from '../../../util/playwright';
+import { setLabeledInput } from '../../../util/playwright';
+
+/* tslint:disable max-classes-per-file */
+
 /**
  * Possible values for partition step segment granularity.
  */
@@ -26,17 +34,144 @@ export enum SegmentGranularity {
   YEAR = 'YEAR',
 }
 
+const PARTITIONING_TYPE = 'Partitioning type';
+
+export interface PartitionsSpec {
+  readonly type: string;
+  apply(page: playwright.Page): Promise<void>;
+}
+
+export async function readPartitionSpec(page: playwright.Page): Promise<PartitionsSpec | null> {
+  const type = await getLabeledInput(page, PARTITIONING_TYPE);
+  switch (type) {
+    case HashedPartitionsSpec.TYPE:
+      return HashedPartitionsSpec.read(page);
+    case SingleDimPartitionsSpec.TYPE:
+      return SingleDimPartitionsSpec.read(page);
+  }
+  return null;
+}
+
+export class HashedPartitionsSpec implements PartitionsSpec {
+  public static TYPE = 'hashed';
+  private static NUM_SHARDS = 'Num shards';
+
+  readonly type: string;
+
+  static async read(page: playwright.Page): Promise<HashedPartitionsSpec> {
+    const numShards = await getLabeledInputAsNumber(page, HashedPartitionsSpec.NUM_SHARDS);
+    return new HashedPartitionsSpec({ numShards });
+  }
+
+  constructor(props: HashedPartitionsSpecProps) {
+    Object.assign(this, props);
+    this.type = HashedPartitionsSpec.TYPE;
+  }
+
+  async apply(page: playwright.Page): Promise<void> {
+    await setLabeledInput(page, PARTITIONING_TYPE, this.type);
+    if (this.numShards != null) {
+      await setLabeledInput(page, HashedPartitionsSpec.NUM_SHARDS, String(this.numShards));
+    }
+  }
+}
+
+async function getLabeledInputAsNumber(
+  page: playwright.Page,
+  label: string,
+): Promise<number | null> {
+  const valueString = await getLabeledInput(page, label);
+  return valueString === '' ? null : Number(valueString);
+}
+
+interface HashedPartitionsSpecProps {
+  readonly numShards: number | null;
+}
+
+export interface HashedPartitionsSpec extends HashedPartitionsSpecProps {}
+
+export class SingleDimPartitionsSpec implements PartitionsSpec {
+  public static TYPE = 'single_dim';
+  private static PARTITION_DIMENSION = 'Partition dimension';
+  private static TARGET_ROWS_PER_SEGMENT = 'Target rows per segment';
+  private static MAX_ROWS_PER_SEGMENT = 'Max rows per segment';
+
+  readonly type: string;
+
+  static async read(page: playwright.Page): Promise<SingleDimPartitionsSpec> {
+    const partitionDimension = await getLabeledInput(
+      page,
+      SingleDimPartitionsSpec.PARTITION_DIMENSION,
+    );
+    const targetRowsPerSegment = await getLabeledInputAsNumber(
+      page,
+      SingleDimPartitionsSpec.TARGET_ROWS_PER_SEGMENT,
+    );
+    const maxRowsPerSegment = await getLabeledInputAsNumber(
+      page,
+      SingleDimPartitionsSpec.MAX_ROWS_PER_SEGMENT,
+    );
+    return new SingleDimPartitionsSpec({
+      partitionDimension,
+      targetRowsPerSegment,
+      maxRowsPerSegment,
+    });
+  }
+
+  constructor(props: SingleDimPartitionsSpecProps) {
+    Object.assign(this, props);
+    this.type = SingleDimPartitionsSpec.TYPE;
+  }
+
+  async apply(page: playwright.Page): Promise<void> {
+    await selectSuggestibleInput(page, PARTITIONING_TYPE, this.type);
+    await setLabeledInput(
+      page,
+      SingleDimPartitionsSpec.PARTITION_DIMENSION,
+      this.partitionDimension,
+    );
+    if (this.targetRowsPerSegment) {
+      await setLabeledInput(
+        page,
+        SingleDimPartitionsSpec.TARGET_ROWS_PER_SEGMENT,
+        String(this.targetRowsPerSegment),
+      );
+    }
+    if (this.maxRowsPerSegment) {
+      await setLabeledInput(
+        page,
+        SingleDimPartitionsSpec.MAX_ROWS_PER_SEGMENT,
+        String(this.maxRowsPerSegment),
+      );
+    }
+  }
+}
+
+interface SingleDimPartitionsSpecProps {
+  readonly partitionDimension: string;
+  readonly targetRowsPerSegment: number | null;
+  readonly maxRowsPerSegment: number | null;
+}
+
+export interface SingleDimPartitionsSpec extends SingleDimPartitionsSpecProps {}
+
 /**
  * Data loader partition step configuration.
  */
 export class PartitionConfig {
+  readonly forceGuaranteedRollupText: string;
+
   constructor(props: PartitionConfigProps) {
     Object.assign(this, props);
+    this.forceGuaranteedRollupText = this.forceGuaranteedRollup ? 'True' : 'False';
   }
 }
 
 interface PartitionConfigProps {
   readonly segmentGranularity: SegmentGranularity;
+  readonly timeIntervals: string | null;
+  readonly forceGuaranteedRollup: boolean | null;
+  readonly partitionsSpec: PartitionsSpec | null;
 }
 
 export interface PartitionConfig extends PartitionConfigProps {}

web-console/e2e-tests/component/load-data/data-connector/data-connector.ts

@@ -16,10 +16,19 @@
  * limitations under the License.
  */
 
+import * as playwright from 'playwright-core';
+
+import { clickButton } from '../../../util/playwright';
+
 /**
  * Connector for data loader input data.
  */
 export interface DataConnector {
   readonly name: string;
+  readonly needParse: boolean;
   connect(): Promise<void>;
 }
+
+export async function clickApplyButton(page: playwright.Page): Promise<void> {
+  await clickButton(page, 'Apply');
+}