From 788bc94143677a7edc01f2a81c155625a25a852a Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Tue, 3 Nov 2020 18:01:37 -0800
Subject: [PATCH 01/13] remove unused fields

---
 .../src/druid-models/ingestion-spec.tsx       | 43 +------------------
 .../src/druid-models/timestamp-spec.tsx       |  7 ++-
 .../views/load-data-view/load-data-view.tsx   | 15 ++++++-
 3 files changed, 21 insertions(+), 44 deletions(-)

diff --git a/web-console/src/druid-models/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx
index e58c59165049..5b7abc2dd315 100644
--- a/web-console/src/druid-models/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec.tsx
@@ -983,45 +983,6 @@ export function getIoConfigTuningFormFields(
             </>
           ),
         },
-        {
-          name: 'inputSource.maxCacheCapacityBytes',
-          label: 'Max cache capacity bytes',
-          type: 'number',
-          defaultValue: 1073741824,
-          info: (
-            <>
-              <p>
-                Maximum size of the cache space in bytes. 0 means disabling cache. Cached files are
-                not removed until the ingestion task completes.
-              </p>
-            </>
-          ),
-        },
-        {
-          name: 'inputSource.maxFetchCapacityBytes',
-          label: 'Max fetch capacity bytes',
-          type: 'number',
-          defaultValue: 1073741824,
-          info: (
-            <>
-              <p>
-                Maximum size of the fetch space in bytes. 0 means disabling prefetch. Prefetched
-                files are removed immediately once they are read.
-              </p>
-            </>
-          ),
-        },
-        {
-          name: 'inputSource.prefetchTriggerBytes',
-          label: 'Prefetch trigger bytes',
-          type: 'number',
-          placeholder: 'maxFetchCapacityBytes / 2',
-          info: (
-            <>
-              <p>Threshold to trigger prefetching the objects.</p>
-            </>
-          ),
-        },
       ];
 
     case 'index_parallel:local':
@@ -2091,14 +2052,14 @@ export function updateSchemaWithSample(
   }
 
   if (rollup) {
-    newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'HOUR');
+    newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'hour');
 
     const metrics = getMetricSpecs(headerAndRows, typeHints);
     if (metrics) {
       newSpec = deepSet(newSpec, 'spec.dataSchema.metricsSpec', metrics);
     }
   } else {
-    newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'NONE');
+    newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.queryGranularity', 'none');
     newSpec = deepDelete(newSpec, 'spec.dataSchema.metricsSpec');
   }
 
diff --git a/web-console/src/druid-models/timestamp-spec.tsx b/web-console/src/druid-models/timestamp-spec.tsx
index 8e17d20dbbce..b6c595b17dab 100644
--- a/web-console/src/druid-models/timestamp-spec.tsx
+++ b/web-console/src/druid-models/timestamp-spec.tsx
@@ -123,7 +123,7 @@ export const TIMESTAMP_SPEC_FIELDS: Field<TimestampSpec>[] = [
     ],
     info: (
       <p>
-        Please specify your timestamp format by using the suggestions menu or typing in a{' '}
+        Specify your timestamp format by using the suggestions menu or typing in a{' '}
         <ExternalLink href="https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html">
           format string
         </ExternalLink>
@@ -135,7 +135,10 @@ export const TIMESTAMP_SPEC_FIELDS: Field<TimestampSpec>[] = [
     name: 'missingValue',
     type: 'string',
     placeholder: '(optional)',
-    info: <p>This value will be used if the specified column can not be found.</p>,
+    info: (
+      <p>Specify a static value for cases when the source time column is missing or is null.</p>
+    ),
+    suggestions: ['2020-01-01T00:00:00Z'],
   },
 ];
 
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index 4db8286bf532..380bc976f437 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -2419,7 +2419,20 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                     name: 'spec.dataSchema.granularitySpec.queryGranularity',
                     label: 'Query granularity',
                     type: 'string',
-                    suggestions: ['NONE', 'SECOND', 'MINUTE', 'HOUR', 'DAY'],
+                    suggestions: [
+                      'none',
+                      'second',
+                      'minute',
+                      'fifteen_minute',
+                      'thirty_minute',
+                      'hour',
+                      'day',
+                      'week',
+                      'month',
+                      'quarter',
+                      'year',
+                      'all',
+                    ],
                     info: (
                       <>
                         This granularity determines how timestamps will be truncated (not at all, to

From 6727c44f0195568fe32d95e952e7e1e31b0a9fd7 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Wed, 4 Nov 2020 18:05:49 -0800
Subject: [PATCH 02/13] keep tables live

---
 web-console/src/utils/query-manager.tsx       |   3 +
 web-console/src/utils/query-state.ts          |  15 +-
 .../views/load-data-view/load-data-view.tsx   | 305 ++++++++++--------
 .../src/views/query-view/query-view.tsx       |   5 +-
 4 files changed, 193 insertions(+), 135 deletions(-)

diff --git a/web-console/src/utils/query-manager.tsx b/web-console/src/utils/query-manager.tsx
index 906c7b6bd27f..97a239ebe35e 100644
--- a/web-console/src/utils/query-manager.tsx
+++ b/web-console/src/utils/query-manager.tsx
@@ -95,6 +95,7 @@ export class QueryManager<Q, R> {
         this.setState(
           new QueryState<R>({
             data,
+            lastData: this.state.getSomeData(),
           }),
         );
       },
@@ -107,6 +108,7 @@ export class QueryManager<Q, R> {
         this.setState(
           new QueryState<R>({
             error: e,
+            lastData: this.state.getSomeData(),
           }),
         );
       },
@@ -119,6 +121,7 @@ export class QueryManager<Q, R> {
     this.setState(
       new QueryState<R>({
         loading: true,
+        lastData: this.state.getSomeData(),
       }),
     );
 
diff --git a/web-console/src/utils/query-state.ts b/web-console/src/utils/query-state.ts
index e3eeaf2e9ce2..ecbcd020554b 100644
--- a/web-console/src/utils/query-state.ts
+++ b/web-console/src/utils/query-state.ts
@@ -18,6 +18,13 @@
 
 export type QueryStateState = 'init' | 'loading' | 'data' | 'error';
 
+export interface QueryStateOptions<T, E extends Error = Error> {
+  loading?: boolean;
+  error?: E;
+  data?: T;
+  lastData?: T;
+}
+
 export class QueryState<T, E extends Error = Error> {
   static INIT: QueryState<any> = new QueryState({});
   static LOADING: QueryState<any> = new QueryState({ loading: true });
@@ -25,8 +32,9 @@ export class QueryState<T, E extends Error = Error> {
   public state: QueryStateState = 'init';
   public error?: E;
   public data?: T;
+  public lastData?: T;
 
-  constructor(opts: { loading?: boolean; error?: E; data?: T }) {
+  constructor(opts: QueryStateOptions<T, E>) {
     const hasData = typeof opts.data !== 'undefined';
     if (typeof opts.error !== 'undefined') {
       if (hasData) {
@@ -43,6 +51,7 @@ export class QueryState<T, E extends Error = Error> {
         this.state = opts.loading ? 'loading' : 'init';
       }
     }
+    this.lastData = opts.lastData;
   }
 
   isInit(): boolean {
@@ -71,4 +80,8 @@ export class QueryState<T, E extends Error = Error> {
     const { data } = this;
     return Boolean(data && Array.isArray(data) && data.length === 0);
   }
+
+  getSomeData(): T | undefined {
+    return this.data || this.lastData;
+  }
 }
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index 380bc976f437..66bff62c5af9 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -1273,32 +1273,35 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     }
 
     if (issue) {
-      this.setState({
-        parserQueryState: initRun ? QueryState.INIT : new QueryState({ error: new Error(issue) }),
-      });
+      this.setState(({ parserQueryState }) => ({
+        parserQueryState: initRun
+          ? QueryState.INIT
+          : new QueryState({ error: new Error(issue), lastData: parserQueryState.getSomeData() }),
+      }));
       return;
     }
 
-    this.setState({
-      parserQueryState: new QueryState({ loading: true }),
-    });
+    this.setState(({ parserQueryState }) => ({
+      parserQueryState: new QueryState({ loading: true, lastData: parserQueryState.getSomeData() }),
+    }));
 
     let sampleResponse: SampleResponse;
     try {
       sampleResponse = await sampleForParser(spec, sampleStrategy);
     } catch (e) {
-      this.setState({
-        parserQueryState: new QueryState({ error: e }),
-      });
+      this.setState(({ parserQueryState }) => ({
+        parserQueryState: new QueryState({ error: e, lastData: parserQueryState.getSomeData() }),
+      }));
       return;
     }
 
-    this.setState({
+    this.setState(({ parserQueryState }) => ({
       cacheRows: getCacheRowsFromSampleResponse(sampleResponse),
       parserQueryState: new QueryState({
         data: headerAndRowsFromSampleResponse(sampleResponse, '__time', inputFormatColumns),
+        lastData: parserQueryState.getSomeData(),
       }),
-    });
+    }));
   }
 
   renderParserStep() {
@@ -1322,11 +1325,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           Please enter the parser details on the right <Icon icon={IconNames.ARROW_RIGHT} />
         </CenterMessage>
       );
-    } else if (parserQueryState.isLoading()) {
-      mainFill = <Loader />;
-    } else if (parserQueryState.error) {
-      mainFill = <CenterMessage>{`Error: ${parserQueryState.getErrorMessage()}`}</CenterMessage>;
-    } else if (parserQueryState.data) {
+    } else {
+      const data = parserQueryState.getSomeData();
       mainFill = (
         <div className="table-with-control">
           <div className="table-control">
@@ -1344,14 +1344,20 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               />
             )}
           </div>
-          <ParseDataTable
-            sampleData={parserQueryState.data}
-            columnFilter={columnFilter}
-            canFlatten={canFlatten}
-            flattenedColumnsOnly={specialColumnsOnly}
-            flattenFields={flattenFields}
-            onFlattenFieldSelect={this.onFlattenFieldSelect}
-          />
+          {data && (
+            <ParseDataTable
+              sampleData={data}
+              columnFilter={columnFilter}
+              canFlatten={canFlatten}
+              flattenedColumnsOnly={specialColumnsOnly}
+              flattenFields={flattenFields}
+              onFlattenFieldSelect={this.onFlattenFieldSelect}
+            />
+          )}
+          {parserQueryState.isLoading() && <Loader />}
+          {parserQueryState.error && (
+            <CenterMessage>{`Error: ${parserQueryState.getErrorMessage()}`}</CenterMessage>
+          )}
         </div>
       );
     }
@@ -1541,29 +1547,38 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       deepGet(spec, 'spec.ioConfig.inputFormat.columns') || EMPTY_ARRAY;
 
     if (!cacheRows) {
-      this.setState({
+      this.setState(({ timestampQueryState }) => ({
         timestampQueryState: initRun
           ? QueryState.INIT
-          : new QueryState({ error: new Error('must complete parse step') }),
-      });
+          : new QueryState({
+              error: new Error('must complete parse step'),
+              lastData: timestampQueryState.getSomeData(),
+            }),
+      }));
       return;
     }
 
-    this.setState({
-      timestampQueryState: new QueryState({ loading: true }),
-    });
+    this.setState(({ timestampQueryState }) => ({
+      timestampQueryState: new QueryState({
+        loading: true,
+        lastData: timestampQueryState.getSomeData(),
+      }),
+    }));
 
     let sampleResponse: SampleResponse;
     try {
       sampleResponse = await sampleForTimestamp(spec, cacheRows);
     } catch (e) {
-      this.setState({
-        timestampQueryState: new QueryState({ error: e }),
-      });
+      this.setState(({ timestampQueryState }) => ({
+        timestampQueryState: new QueryState({
+          error: e,
+          lastData: timestampQueryState.getSomeData(),
+        }),
+      }));
       return;
     }
 
-    this.setState({
+    this.setState(({ timestampQueryState }) => ({
       timestampQueryState: new QueryState({
         data: {
           headerAndRows: headerAndRowsFromSampleResponse(
@@ -1573,8 +1588,9 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           ),
           spec,
         },
+        lastData: timestampQueryState.getSomeData(),
       }),
-    });
+    }));
   }
 
   renderTimestampStep() {
@@ -1593,11 +1609,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           <Icon icon={IconNames.ARROW_RIGHT} />
         </CenterMessage>
       );
-    } else if (timestampQueryState.isLoading()) {
-      mainFill = <Loader />;
-    } else if (timestampQueryState.error) {
-      mainFill = <CenterMessage>{`Error: ${timestampQueryState.getErrorMessage()}`}</CenterMessage>;
-    } else if (timestampQueryState.data) {
+    } else {
+      const data = timestampQueryState.getSomeData();
       mainFill = (
         <div className="table-with-control">
           <div className="table-control">
@@ -1612,16 +1625,22 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               onChange={() => this.setState({ specialColumnsOnly: !specialColumnsOnly })}
             />
           </div>
-          <ParseTimeTable
-            sampleBundle={timestampQueryState.data}
-            columnFilter={columnFilter}
-            possibleTimestampColumnsOnly={specialColumnsOnly}
-            selectedColumnName={parseTimeTableSelectedColumnName(
-              timestampQueryState.data.headerAndRows,
-              timestampSpec,
-            )}
-            onTimestampColumnSelect={this.onTimestampColumnSelect}
-          />
+          {data && (
+            <ParseTimeTable
+              sampleBundle={data}
+              columnFilter={columnFilter}
+              possibleTimestampColumnsOnly={specialColumnsOnly}
+              selectedColumnName={parseTimeTableSelectedColumnName(
+                data.headerAndRows,
+                timestampSpec,
+              )}
+              onTimestampColumnSelect={this.onTimestampColumnSelect}
+            />
+          )}
+          {timestampQueryState.isLoading() && <Loader />}
+          {timestampQueryState.error && (
+            <CenterMessage>{`Error: ${timestampQueryState.getErrorMessage()}`}</CenterMessage>
+          )}
         </div>
       );
     }
@@ -1736,37 +1755,47 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       deepGet(spec, 'spec.ioConfig.inputFormat.columns') || EMPTY_ARRAY;
 
     if (!cacheRows) {
-      this.setState({
+      this.setState(({ transformQueryState }) => ({
         transformQueryState: initRun
           ? QueryState.INIT
-          : new QueryState({ error: new Error('must complete parse step') }),
-      });
+          : new QueryState({
+              error: new Error('must complete parse step'),
+              lastData: transformQueryState.getSomeData(),
+            }),
+      }));
       return;
     }
 
-    this.setState({
-      transformQueryState: new QueryState({ loading: true }),
-    });
+    this.setState(({ transformQueryState }) => ({
+      transformQueryState: new QueryState({
+        loading: true,
+        lastData: transformQueryState.getSomeData(),
+      }),
+    }));
 
     let sampleResponse: SampleResponse;
     try {
       sampleResponse = await sampleForTransform(spec, cacheRows);
     } catch (e) {
-      this.setState({
-        transformQueryState: new QueryState({ error: e }),
-      });
+      this.setState(({ transformQueryState }) => ({
+        transformQueryState: new QueryState({
+          error: e,
+          lastData: transformQueryState.getSomeData(),
+        }),
+      }));
       return;
     }
 
-    this.setState({
+    this.setState(({ transformQueryState }) => ({
       transformQueryState: new QueryState({
         data: headerAndRowsFromSampleResponse(
           sampleResponse,
           undefined,
           ['__time'].concat(inputFormatColumns),
         ),
+        lastData: transformQueryState.getSomeData(),
       }),
-    });
+    }));
   }
 
   renderTransformStep() {
@@ -1784,11 +1813,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     let mainFill: JSX.Element | string = '';
     if (transformQueryState.isInit()) {
       mainFill = <CenterMessage>{`Please fill in the previous steps`}</CenterMessage>;
-    } else if (transformQueryState.isLoading()) {
-      mainFill = <Loader />;
-    } else if (transformQueryState.error) {
-      mainFill = <CenterMessage>{`Error: ${transformQueryState.getErrorMessage()}`}</CenterMessage>;
-    } else if (transformQueryState.data) {
+    } else {
+      const data = transformQueryState.getSomeData();
       mainFill = (
         <div className="table-with-control">
           <div className="table-control">
@@ -1804,17 +1830,20 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               disabled={!transforms.length}
             />
           </div>
-          <TransformTable
-            sampleData={transformQueryState.data}
-            columnFilter={columnFilter}
-            transformedColumnsOnly={specialColumnsOnly}
-            transforms={transforms}
-            selectedColumnName={transformTableSelectedColumnName(
-              transformQueryState.data,
-              selectedTransform,
-            )}
-            onTransformSelect={this.onTransformSelect}
-          />
+          {data && (
+            <TransformTable
+              sampleData={data}
+              columnFilter={columnFilter}
+              transformedColumnsOnly={specialColumnsOnly}
+              transforms={transforms}
+              selectedColumnName={transformTableSelectedColumnName(data, selectedTransform)}
+              onTransformSelect={this.onTransformSelect}
+            />
+          )}
+          {transformQueryState.isLoading() && <Loader />}
+          {transformQueryState.error && (
+            <CenterMessage>{`Error: ${transformQueryState.getErrorMessage()}`}</CenterMessage>
+          )}
         </div>
       );
     }
@@ -1951,30 +1980,33 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       deepGet(spec, 'spec.ioConfig.inputFormat.columns') || EMPTY_ARRAY;
 
     if (!cacheRows) {
-      this.setState({
+      this.setState(({ filterQueryState }) => ({
         filterQueryState: initRun
           ? QueryState.INIT
-          : new QueryState({ error: new Error('must complete parse step') }),
-      });
+          : new QueryState({
+              error: new Error('must complete parse step'),
+              lastData: filterQueryState.getSomeData(),
+            }),
+      }));
       return;
     }
 
-    this.setState({
-      filterQueryState: new QueryState({ loading: true }),
-    });
+    this.setState(({ filterQueryState }) => ({
+      filterQueryState: new QueryState({ loading: true, lastData: filterQueryState.getSomeData() }),
+    }));
 
     let sampleResponse: SampleResponse;
     try {
       sampleResponse = await sampleForFilter(spec, cacheRows);
     } catch (e) {
-      this.setState({
-        filterQueryState: new QueryState({ error: e }),
-      });
+      this.setState(({ filterQueryState }) => ({
+        filterQueryState: new QueryState({ error: e, lastData: filterQueryState.getSomeData() }),
+      }));
       return;
     }
 
     if (sampleResponse.data.length) {
-      this.setState({
+      this.setState(({ filterQueryState }) => ({
         filterQueryState: new QueryState({
           data: headerAndRowsFromSampleResponse(
             sampleResponse,
@@ -1982,8 +2014,9 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
             ['__time'].concat(inputFormatColumns),
             true,
           ),
+          lastData: filterQueryState.getSomeData(),
         }),
-      });
+      }));
       return;
     }
 
@@ -1993,9 +2026,9 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       const specNoFilter = deepSet(spec, 'spec.dataSchema.transformSpec.filter', null);
       sampleResponseNoFilter = await sampleForFilter(specNoFilter, cacheRows);
     } catch (e) {
-      this.setState({
-        filterQueryState: new QueryState({ error: e }),
-      });
+      this.setState(({ filterQueryState }) => ({
+        filterQueryState: new QueryState({ error: e, lastData: filterQueryState.getSomeData() }),
+      }));
       return;
     }
 
@@ -2006,12 +2039,13 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       true,
     );
 
-    this.setState({
+    this.setState(({ filterQueryState }) => ({
       // cacheRows: sampleResponseNoFilter.cacheKey,
       filterQueryState: new QueryState({
         data: deepSet(headerAndRowsNoFilter, 'rows', []),
+        lastData: filterQueryState.getSomeData(),
       }),
-    });
+    }));
   }
 
   private getMemoizedDimensionFiltersFromSpec = memoize(spec => {
@@ -2026,11 +2060,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     let mainFill: JSX.Element | string = '';
     if (filterQueryState.isInit()) {
       mainFill = <CenterMessage>Please enter more details for the previous steps</CenterMessage>;
-    } else if (filterQueryState.isLoading()) {
-      mainFill = <Loader />;
-    } else if (filterQueryState.error) {
-      mainFill = <CenterMessage>{`Error: ${filterQueryState.getErrorMessage()}`}</CenterMessage>;
-    } else if (filterQueryState.data) {
+    } else {
+      const data = filterQueryState.getSomeData();
       mainFill = (
         <div className="table-with-control">
           <div className="table-control">
@@ -2040,17 +2071,20 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               placeholder="Search columns"
             />
           </div>
-          <FilterTable
-            sampleData={filterQueryState.data}
-            columnFilter={columnFilter}
-            dimensionFilters={dimensionFilters}
-            selectedFilterName={filterTableSelectedColumnName(
-              filterQueryState.data,
-              selectedFilter,
-            )}
-            onShowGlobalFilter={this.onShowGlobalFilter}
-            onFilterSelect={this.onFilterSelect}
-          />
+          {data && (
+            <FilterTable
+              sampleData={data}
+              columnFilter={columnFilter}
+              dimensionFilters={dimensionFilters}
+              selectedFilterName={filterTableSelectedColumnName(data, selectedFilter)}
+              onShowGlobalFilter={this.onShowGlobalFilter}
+              onFilterSelect={this.onFilterSelect}
+            />
+          )}
+          {filterQueryState.isLoading() && <Loader />}
+          {filterQueryState.error && (
+            <CenterMessage>{`Error: ${filterQueryState.getErrorMessage()}`}</CenterMessage>
+          )}
         </div>
       );
     }
@@ -2233,29 +2267,32 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       deepGet(spec, 'spec.dataSchema.dimensionsSpec') || EMPTY_OBJECT;
 
     if (!cacheRows) {
-      this.setState({
+      this.setState(({ schemaQueryState }) => ({
         schemaQueryState: initRun
           ? QueryState.INIT
-          : new QueryState({ error: new Error('must complete parse step') }),
-      });
+          : new QueryState({
+              error: new Error('must complete parse step'),
+              lastData: schemaQueryState.getSomeData(),
+            }),
+      }));
       return;
     }
 
-    this.setState({
-      schemaQueryState: new QueryState({ loading: true }),
-    });
+    this.setState(({ schemaQueryState }) => ({
+      schemaQueryState: new QueryState({ loading: true, lastData: schemaQueryState.getSomeData() }),
+    }));
 
     let sampleResponse: SampleResponse;
     try {
       sampleResponse = await sampleForSchema(spec, cacheRows);
     } catch (e) {
-      this.setState({
-        schemaQueryState: new QueryState({ error: e }),
-      });
+      this.setState(({ schemaQueryState }) => ({
+        schemaQueryState: new QueryState({ error: e, lastData: schemaQueryState.getSomeData() }),
+      }));
       return;
     }
 
-    this.setState({
+    this.setState(({ schemaQueryState }) => ({
       schemaQueryState: new QueryState({
         data: {
           headerAndRows: headerAndRowsFromSampleResponse(
@@ -2266,8 +2303,9 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           dimensionsSpec,
           metricsSpec,
         },
+        lastData: schemaQueryState.getSomeData(),
       }),
-    });
+    }));
   }
 
   renderSchemaStep() {
@@ -2287,11 +2325,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     let mainFill: JSX.Element | string = '';
     if (schemaQueryState.isInit()) {
       mainFill = <CenterMessage>Please enter more details for the previous steps</CenterMessage>;
-    } else if (schemaQueryState.isLoading()) {
-      mainFill = <Loader />;
-    } else if (schemaQueryState.error) {
-      mainFill = <CenterMessage>{`Error: ${schemaQueryState.getErrorMessage()}`}</CenterMessage>;
-    } else if (schemaQueryState.data) {
+    } else {
+      const data = schemaQueryState.getSomeData();
       mainFill = (
         <div className="table-with-control">
           <div className="table-control">
@@ -2301,13 +2336,19 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               placeholder="Search columns"
             />
           </div>
-          <SchemaTable
-            sampleBundle={schemaQueryState.data}
-            columnFilter={columnFilter}
-            selectedDimensionSpecIndex={selectedDimensionSpecIndex}
-            selectedMetricSpecIndex={selectedMetricSpecIndex}
-            onDimensionOrMetricSelect={this.onDimensionOrMetricSelect}
-          />
+          {data && (
+            <SchemaTable
+              sampleBundle={data}
+              columnFilter={columnFilter}
+              selectedDimensionSpecIndex={selectedDimensionSpecIndex}
+              selectedMetricSpecIndex={selectedMetricSpecIndex}
+              onDimensionOrMetricSelect={this.onDimensionOrMetricSelect}
+            />
+          )}
+          {schemaQueryState.isLoading() && <Loader />}
+          {schemaQueryState.error && (
+            <CenterMessage>{`Error: ${schemaQueryState.getErrorMessage()}`}</CenterMessage>
+          )}
         </div>
       );
     }
diff --git a/web-console/src/views/query-view/query-view.tsx b/web-console/src/views/query-view/query-view.tsx
index c44a3d3858b4..4cb2a2ff7c12 100644
--- a/web-console/src/views/query-view/query-view.tsx
+++ b/web-console/src/views/query-view/query-view.tsx
@@ -459,6 +459,7 @@ export class QueryView extends React.PureComponent<QueryViewProps, QueryViewStat
       }
     }
 
+    const someQueryResult = queryResultState.getSomeData();
     const runeMode = QueryView.isJsonLike(queryString);
     return (
       <SplitterLayout
@@ -501,10 +502,10 @@ export class QueryView extends React.PureComponent<QueryViewProps, QueryViewStat
           </div>
         </div>
         <div className="output-pane">
-          {queryResult && (
+          {someQueryResult && (
             <QueryOutput
               runeMode={runeMode}
-              queryResult={queryResult}
+              queryResult={someQueryResult}
               onQueryChange={this.handleQueryChange}
             />
           )}

From f8571aba080122d097858f47298c319ea139d3da Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Thu, 5 Nov 2020 17:04:06 -0800
Subject: [PATCH 03/13] advanced

---
 .../src/components/auto-form/auto-form.tsx    |  50 ++++-
 .../src/druid-models/ingestion-spec.tsx       |  50 +++++
 .../views/load-data-view/info-messages.tsx    | 196 ++++++++++++++++++
 .../views/load-data-view/load-data-view.tsx   | 173 +++++-----------
 4 files changed, 339 insertions(+), 130 deletions(-)
 create mode 100644 web-console/src/views/load-data-view/info-messages.tsx

diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx
index 6ae9e4b3b4d6..88586ac80a1d 100644
--- a/web-console/src/components/auto-form/auto-form.tsx
+++ b/web-console/src/components/auto-form/auto-form.tsx
@@ -17,6 +17,7 @@
  */
 
 import { Button, ButtonGroup, FormGroup, Intent, NumericInput } from '@blueprintjs/core';
+import { IconNames } from '@blueprintjs/icons';
 import React from 'react';
 
 import { deepDelete, deepGet, deepSet } from '../../utils';
@@ -54,6 +55,7 @@ export interface Field<M> {
   disabled?: Functor<M, boolean>;
   defined?: Functor<M, boolean>;
   required?: Functor<M, boolean>;
+  advanced?: Functor<M, boolean>;
   adjustment?: (model: M) => M;
   issueWithValue?: (value: any) => string | undefined;
 }
@@ -68,7 +70,14 @@ export interface AutoFormProps<M> {
   globalAdjustment?: (model: M) => M;
 }
 
-export class AutoForm<T extends Record<string, any>> extends React.PureComponent<AutoFormProps<T>> {
+export interface AutoFormState {
+  showAdvanced: boolean;
+}
+
+export class AutoForm<T extends Record<string, any>> extends React.PureComponent<
+  AutoFormProps<T>,
+  AutoFormState
+> {
   static REQUIRED_INTENT = Intent.PRIMARY;
 
   static makeLabelName(label: string): string {
@@ -138,7 +147,9 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
 
   constructor(props: AutoFormProps<T>) {
     super(props);
-    this.state = {};
+    this.state = {
+      showAdvanced: false,
+    };
   }
 
   private fieldChange = (field: Field<T>, newValue: any) => {
@@ -391,7 +402,6 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
   private renderField = (field: Field<T>) => {
     const { model } = this.props;
     if (!model) return;
-    if (!AutoForm.evaluateFunctor(field.defined, model, true)) return;
 
     const label = field.label || AutoForm.makeLabelName(field.name);
     return (
@@ -415,12 +425,44 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
     );
   }
 
+  renderAdvanced() {
+    const { showAdvanced } = this.state;
+
+    return (
+      <FormGroup key="advanced">
+        <Button
+          text={showAdvanced ? 'Hide advanced' : 'Show advanced'}
+          rightIcon={showAdvanced ? IconNames.CHEVRON_UP : IconNames.CHEVRON_DOWN}
+          onClick={() => {
+            this.setState(({ showAdvanced }) => ({ showAdvanced: !showAdvanced }));
+          }}
+        />
+      </FormGroup>
+    );
+  }
+
   render(): JSX.Element {
     const { fields, model, showCustom } = this.props;
+    const { showAdvanced } = this.state;
+
+    let hasAdvanced = false;
+    const shownFields = fields.filter(field => {
+      if (AutoForm.evaluateFunctor(field.defined, model, true)) {
+        if (AutoForm.evaluateFunctor(field.advanced, model, false)) {
+          hasAdvanced = true;
+          return showAdvanced;
+        }
+        return true;
+      } else {
+        return false;
+      }
+    });
+
     return (
       <div className="auto-form">
-        {model && fields.map(this.renderField)}
+        {model && shownFields.map(this.renderField)}
         {model && showCustom && showCustom(model) && this.renderCustom()}
+        {hasAdvanced && this.renderAdvanced()}
       </div>
     );
   }
diff --git a/web-console/src/druid-models/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx
index 5b7abc2dd315..9ef3c73c3944 100644
--- a/web-console/src/druid-models/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec.tsx
@@ -1571,6 +1571,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 1000,
     defined: (t: TuningConfig) => t.type === 'index_parallel',
+    advanced: true,
     info: <>Polling period in milliseconds to check running task statuses.</>,
   },
   {
@@ -1624,6 +1625,19 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
       </>
     ),
   },
+  {
+    name: 'skipSequenceNumberAvailabilityCheck',
+    type: 'boolean',
+    defaultValue: false,
+    defined: (t: TuningConfig) => t.type === 'kinesis',
+    info: (
+      <>
+        Whether to enable checking if the current sequence number is still available in a particular
+        Kinesis shard. If set to false, the indexing task will attempt to reset the current sequence
+        number (or not), depending on the value of <Code>resetOffsetAutomatically</Code>.
+      </>
+    ),
+  },
   {
     name: 'intermediatePersistPeriod',
     type: 'duration',
@@ -1647,6 +1661,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
   {
     name: 'maxPendingPersists',
     type: 'number',
+    advanced: true,
     info: (
       <>
         Maximum number of persists that can be pending but not started. If this limit would be
@@ -1659,6 +1674,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     name: 'pushTimeout',
     type: 'number',
     defaultValue: 0,
+    advanced: true,
     info: (
       <>
         Milliseconds to wait for pushing segments. It must be >= 0, where 0 means to wait forever.
@@ -1670,6 +1686,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 0,
     defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'),
+    advanced: true,
     info: <>Milliseconds to wait for segment handoff. 0 means to wait forever.</>,
   },
   {
@@ -1716,6 +1733,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'duration',
     defaultValue: 'PT10S',
     defined: (t: TuningConfig) => t.type === 'index_parallel',
+    advanced: true,
     info: <>Timeout for reporting the pushed segments in worker tasks.</>,
   },
   {
@@ -1723,6 +1741,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 5,
     defined: (t: TuningConfig) => t.type === 'index_parallel',
+    advanced: true,
     info: <>Retries for reporting the pushed segments in worker tasks.</>,
   },
   {
@@ -1739,6 +1758,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     placeholder: 'min(10, taskCount * replicas)',
     defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'),
+    advanced: true,
     info: <>The number of threads that will be used for communicating with indexing tasks.</>,
   },
   {
@@ -1746,6 +1766,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 8,
     defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'),
+    advanced: true,
     info: (
       <>
         The number of times HTTP requests to indexing tasks will be retried before considering tasks
@@ -1765,6 +1786,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'duration',
     defaultValue: 'PT80S',
     defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'),
+    advanced: true,
     info: (
       <>
         How long to wait for the supervisor to attempt a graceful shutdown of tasks before exiting.
@@ -1800,6 +1822,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 5000,
     defined: (t: TuningConfig) => t.type === 'kinesis',
+    advanced: true,
     info: (
       <>
         Length of time in milliseconds to wait for space to become available in the buffer before
@@ -1809,6 +1832,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
   },
   {
     name: 'recordBufferFullWait',
+    advanced: true,
     type: 'number',
     defaultValue: 5000,
     defined: (t: TuningConfig) => t.type === 'kinesis',
@@ -1824,6 +1848,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 60000,
     defined: (t: TuningConfig) => t.type === 'kinesis',
+    advanced: true,
     info: (
       <>
         Length of time in milliseconds to wait for Kinesis to return the earliest or latest sequence
@@ -1838,6 +1863,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     placeholder: 'max(1, {numProcessors} - 1)',
     defined: (t: TuningConfig) => t.type === 'kinesis',
+    advanced: true,
     info: (
       <>
         Size of the pool of threads fetching data from Kinesis. There is no benefit in having more
@@ -1850,6 +1876,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 100,
     defined: (t: TuningConfig) => t.type === 'kinesis',
+    advanced: true,
     info: (
       <>
         The maximum number of records/events to be fetched from buffer per poll. The actual maximum
@@ -1857,6 +1884,29 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
       </>
     ),
   },
+  {
+    name: 'repartitionTransitionDuration',
+    type: 'duration',
+    defaultValue: 'PT2M',
+    defined: (t: TuningConfig) => t.type === 'kinesis',
+    advanced: true,
+    info: (
+      <>
+        <p>
+          When shards are split or merged, the supervisor will recompute shard -> task group
+          mappings, and signal any running tasks created under the old mappings to stop early at{' '}
+          <Code>(current time + repartitionTransitionDuration)</Code>. Stopping the tasks early
+          allows Druid to begin reading from the new shards more quickly.
+        </p>
+        <p>
+          The repartition transition wait time controlled by this property gives the stream
+          additional time to write records to the new shards after the split/merge, which helps
+          avoid the issues with empty shard handling described at
+          <ExternalLink href="https://github.com/apache/druid/issues/7600">#7600</ExternalLink>.
+        </p>
+      </>
+    ),
+  },
 ];
 
 export function getTuningSpecFormFields() {
diff --git a/web-console/src/views/load-data-view/info-messages.tsx b/web-console/src/views/load-data-view/info-messages.tsx
new file mode 100644
index 000000000000..ffc7a61de603
--- /dev/null
+++ b/web-console/src/views/load-data-view/info-messages.tsx
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Callout, Code } from '@blueprintjs/core';
+import React from 'react';
+
+import { ExternalLink } from '../../components';
+import { DimensionMode, getIngestionDocLink, IngestionSpec } from '../../druid-models';
+import { getLink } from '../../links';
+
+import { LearnMore } from './learn-more/learn-more';
+
+export interface ConnectMessageProps {
+  inlineMode: boolean;
+  spec: IngestionSpec;
+}
+
+export const ConnectMessage = React.memo(function ConnectMessage(props: ConnectMessageProps) {
+  const { inlineMode, spec } = props;
+
+  return (
+    <Callout className="intro">
+      <p>
+        Druid ingests raw data and converts it into a custom,{' '}
+        <ExternalLink href={`${getLink('DOCS')}/design/segments.html`}>indexed format</ExternalLink>{' '}
+        that is optimized for analytic queries.
+      </p>
+      {inlineMode ? (
+        <>
+          <p>To get started, please paste some data in the box to the left.</p>
+          <p>Click "Apply" to verify your data with Druid.</p>
+        </>
+      ) : (
+        <p>To get started, please specify what data you want to ingest.</p>
+      )}
+      <LearnMore href={getIngestionDocLink(spec)} />
+    </Callout>
+  );
+});
+
+export interface ParserMessageProps {
+  canFlatten: boolean;
+}
+
+export const ParserMessage = React.memo(function ParserMessage(props: ParserMessageProps) {
+  const { canFlatten } = props;
+
+  return (
+    <Callout className="intro">
+      <p>
+        Druid requires flat data (non-nested, non-hierarchical). Each row should represent a
+        discrete event.
+      </p>
+      {canFlatten && (
+        <p>
+          If you have nested data, you can{' '}
+          <ExternalLink href={`${getLink('DOCS')}/ingestion/index.html#flattenspec`}>
+            flatten
+          </ExternalLink>{' '}
+          it here. If the provided flattening capabilities are not sufficient, please pre-process
+          your data before ingesting it into Druid.
+        </p>
+      )}
+      <p>Ensure that your data appears correctly in a row/column orientation.</p>
+      <LearnMore href={`${getLink('DOCS')}/ingestion/data-formats.html`} />
+    </Callout>
+  );
+});
+
+export const TimestampMessage = React.memo(function TimestampMessage() {
+  return (
+    <Callout className="intro">
+      <p>
+        Druid partitions data based on the primary time column of your data. This column is stored
+        internally in Druid as <Code>__time</Code>.
+      </p>
+      <p>Configure how to define the time column for this data.</p>
+      <p>
+        If your data does not have a time column, you can select "None" to use a placeholder value.
+        If the time information is spread across multiple columns you can combine them into one by
+        selecting "Expression" and defining a transform expression.
+      </p>
+      <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#timestampspec`} />
+    </Callout>
+  );
+});
+
+export const TransformMessage = React.memo(function TransformMessage() {
+  return (
+    <Callout className="intro">
+      <p className="optional">Optional</p>
+      <p>
+        Druid can perform per-row{' '}
+        <ExternalLink href={`${getLink('DOCS')}/ingestion/transform-spec.html#transforms`}>
+          transforms
+        </ExternalLink>{' '}
+        of column values allowing you to create new derived columns or alter existing column.
+      </p>
+      <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#transforms`} />
+    </Callout>
+  );
+});
+
+export const FilterMessage = React.memo(function FilterMessage() {
+  return (
+    <Callout className="intro">
+      <p className="optional">Optional</p>
+      <p>
+        Druid can{' '}
+        <ExternalLink href={`${getLink('DOCS')}/querying/filters.html`}>filter</ExternalLink> out
+        unwanted data by applying per-row filters.
+      </p>
+      <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#filter`} />
+    </Callout>
+  );
+});
+
+export interface SchemaMessageProps {
+  dimensionMode: DimensionMode;
+}
+
+export const SchemaMessage = React.memo(function SchemaMessage(props: SchemaMessageProps) {
+  const { dimensionMode } = props;
+
+  return (
+    <Callout className="intro">
+      <p>
+        Each column in Druid must have an assigned type (string, long, float, double, complex, etc).
+      </p>
+      {dimensionMode === 'specific' && (
+        <p>
+          Default primitive types have been automatically assigned to your columns. If you want to
+          change the type, click on the column header.
+        </p>
+      )}
+      <LearnMore href={`${getLink('DOCS')}/ingestion/schema-design.html`} />
+    </Callout>
+  );
+});
+
+export const PartitionMessage = React.memo(function PartitionMessage() {
+  return (
+    <Callout className="intro">
+      <p className="optional">Optional</p>
+      <p>Configure how Druid will partition data.</p>
+      <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#partitioning`} />
+    </Callout>
+  );
+});
+
+export const TuningMessage = React.memo(function TuningMessage() {
+  return (
+    <Callout className="intro">
+      <p className="optional">Optional</p>
+      <p>Fine tune how Druid will ingest data.</p>
+      <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#tuningconfig`} />
+    </Callout>
+  );
+});
+
+export const PublishMessage = React.memo(function PublishMessage() {
+  return (
+    <Callout className="intro">
+      <p>Configure behavior of indexed data once it reaches Druid.</p>
+    </Callout>
+  );
+});
+
+export const SpecMessage = React.memo(function SpecMessage() {
+  return (
+    <Callout className="intro">
+      <p className="optional">Optional</p>
+      <p>
+        Druid begins ingesting data once you submit a JSON ingestion spec. If you modify any values
+        in this view, the values entered in previous sections will update accordingly. If you modify
+        any values in previous sections, this spec will automatically update.
+      </p>
+      <p>Submit the spec to begin loading data into Druid.</p>
+    </Callout>
+  );
+});
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index 66bff62c5af9..e2ac1867f26e 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -85,7 +85,6 @@ import {
   FlattenField,
   getDimensionMode,
   getIngestionComboType,
-  getIngestionDocLink,
   getIngestionImage,
   getIngestionTitle,
   getIoConfigFormFields,
@@ -160,7 +159,18 @@ import {
 
 import { ExamplePicker } from './example-picker/example-picker';
 import { FilterTable, filterTableSelectedColumnName } from './filter-table/filter-table';
-import { LearnMore } from './learn-more/learn-more';
+import {
+  ConnectMessage,
+  FilterMessage,
+  ParserMessage,
+  PartitionMessage,
+  PublishMessage,
+  SchemaMessage,
+  SpecMessage,
+  TimestampMessage,
+  TransformMessage,
+  TuningMessage,
+} from './info-messages';
 import { ParseDataTable } from './parse-data-table/parse-data-table';
 import {
   ParseTimeTable,
@@ -1098,27 +1108,32 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           <Icon icon={IconNames.ARROW_RIGHT} />
         </CenterMessage>
       );
-    } else if (inputQueryState.isLoading()) {
-      mainFill = <Loader />;
-    } else if (inputQueryState.error) {
-      mainFill = <CenterMessage>{`Error: ${inputQueryState.getErrorMessage()}`}</CenterMessage>;
-    } else if (inputQueryState.data) {
-      const inputData = inputQueryState.data.data;
+    } else {
+      const data = inputQueryState.getSomeData();
+      const inputData = data ? data.data : undefined;
       mainFill = (
-        <TextArea
-          className="raw-lines"
-          readOnly
-          value={
-            inputData.length
-              ? (inputData.every(l => !l.parsed)
-                  ? inputData.map(showBlankLine)
-                  : druidSource
-                  ? inputData.map(showDruidLine)
-                  : inputData.map(showRawLine)
-                ).join('\n')
-              : 'No data returned from sampler'
-          }
-        />
+        <>
+          {inputData && (
+            <TextArea
+              className="raw-lines"
+              readOnly
+              value={
+                inputData.length
+                  ? (inputData.every(l => !l.parsed)
+                      ? inputData.map(showBlankLine)
+                      : druidSource
+                      ? inputData.map(showDruidLine)
+                      : inputData.map(showRawLine)
+                    ).join('\n')
+                  : 'No data returned from sampler'
+              }
+            />
+          )}
+          {inputQueryState.isLoading() && <Loader />}
+          {inputQueryState.error && (
+            <CenterMessage>{`Error: ${inputQueryState.getErrorMessage()}`}</CenterMessage>
+          )}
+        </>
       );
     }
 
@@ -1127,24 +1142,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       <>
         <div className="main">{mainFill}</div>
         <div className="control">
-          <Callout className="intro">
-            <p>
-              Druid ingests raw data and converts it into a custom,{' '}
-              <ExternalLink href={`${getLink('DOCS')}/design/segments.html`}>
-                indexed format
-              </ExternalLink>{' '}
-              that is optimized for analytic queries.
-            </p>
-            {inlineMode ? (
-              <>
-                <p>To get started, please paste some data in the box to the left.</p>
-                <p>Click "Apply" to verify your data with Druid.</p>
-              </>
-            ) : (
-              <p>To get started, please specify what data you want to ingest.</p>
-            )}
-            <LearnMore href={getIngestionDocLink(spec)} />
-          </Callout>
+          <ConnectMessage inlineMode={inlineMode} spec={spec} />
           {ingestionComboType ? (
             <>
               <AutoForm
@@ -1375,24 +1373,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       <>
         <div className="main">{mainFill}</div>
         <div className="control">
-          <Callout className="intro">
-            <p>
-              Druid requires flat data (non-nested, non-hierarchical). Each row should represent a
-              discrete event.
-            </p>
-            {canFlatten && (
-              <p>
-                If you have nested data, you can{' '}
-                <ExternalLink href={`${getLink('DOCS')}/ingestion/index.html#flattenspec`}>
-                  flatten
-                </ExternalLink>{' '}
-                it here. If the provided flattening capabilities are not sufficient, please
-                pre-process your data before ingesting it into Druid.
-              </p>
-            )}
-            <p>Ensure that your data appears correctly in a row/column orientation.</p>
-            <LearnMore href={`${getLink('DOCS')}/ingestion/data-formats.html`} />
-          </Callout>
+          <ParserMessage canFlatten={canFlatten} />
           {!selectedFlattenField && (
             <>
               <AutoForm
@@ -1649,19 +1630,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       <>
         <div className="main">{mainFill}</div>
         <div className="control">
-          <Callout className="intro">
-            <p>
-              Druid partitions data based on the primary time column of your data. This column is
-              stored internally in Druid as <Code>__time</Code>.
-            </p>
-            <p>Configure how to define the time column for this data.</p>
-            <p>
-              If your data does not have a time column, you can select "None" to use a placeholder
-              value. If the time information is spread across multiple columns you can combine them
-              into one by selecting "Expression" and defining a transform expression.
-            </p>
-            <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#timestampspec`} />
-          </Callout>
+          <TimestampMessage />
           <FormGroup label="Parse timestamp from">
             <ButtonGroup>
               <Button
@@ -1852,17 +1821,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       <>
         <div className="main">{mainFill}</div>
         <div className="control">
-          <Callout className="intro">
-            <p className="optional">Optional</p>
-            <p>
-              Druid can perform per-row{' '}
-              <ExternalLink href={`${getLink('DOCS')}/ingestion/transform-spec.html#transforms`}>
-                transforms
-              </ExternalLink>{' '}
-              of column values allowing you to create new derived columns or alter existing column.
-            </p>
-            <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#transforms`} />
-          </Callout>
+          <TransformMessage />
           {Boolean(transformQueryState.error && transforms.length) && (
             <FormGroup>
               <Button
@@ -2093,15 +2052,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       <>
         <div className="main">{mainFill}</div>
         <div className="control">
-          <Callout className="intro">
-            <p className="optional">Optional</p>
-            <p>
-              Druid can{' '}
-              <ExternalLink href={`${getLink('DOCS')}/querying/filters.html`}>filter</ExternalLink>{' '}
-              out unwanted data by applying per-row filters.
-            </p>
-            <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#filter`} />
-          </Callout>
+          <FilterMessage />
           {!showGlobalFilter && this.renderColumnFilterControls()}
           {!selectedFilter && this.renderGlobalFilterControls()}
         </div>
@@ -2357,19 +2308,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       <>
         <div className="main">{mainFill}</div>
         <div className="control">
-          <Callout className="intro">
-            <p>
-              Each column in Druid must have an assigned type (string, long, float, double, complex,
-              etc).
-            </p>
-            {dimensionMode === 'specific' && (
-              <p>
-                Default primitive types have been automatically assigned to your columns. If you
-                want to change the type, click on the column header.
-              </p>
-            )}
-            <LearnMore href={`${getLink('DOCS')}/ingestion/schema-design.html`} />
-          </Callout>
+          <SchemaMessage dimensionMode={dimensionMode} />
           {!somethingSelected && (
             <>
               <FormGroupWithInfo
@@ -2918,11 +2857,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           />
         </div>
         <div className="control">
-          <Callout className="intro">
-            <p className="optional">Optional</p>
-            <p>Configure how Druid will partition data.</p>
-            <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#partitioning`} />
-          </Callout>
+          <PartitionMessage />
         </div>
         {this.renderNextBar({
           disabled:
@@ -2982,11 +2917,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           />
         </div>
         <div className="control">
-          <Callout className="intro">
-            <p className="optional">Optional</p>
-            <p>Fine tune how Druid will ingest data.</p>
-            <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#tuningconfig`} />
-          </Callout>
+          <TuningMessage />
         </div>
         {this.renderNextBar({
           disabled: invalidIoConfig(ioConfig),
@@ -3087,9 +3018,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           />
         </div>
         <div className="control">
-          <Callout className="intro">
-            <p>Configure behavior of indexed data once it reaches Druid.</p>
-          </Callout>
+          <PublishMessage />
         </div>
         {this.renderNextBar({})}
       </>
@@ -3145,15 +3074,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           />
         </div>
         <div className="control">
-          <Callout className="intro">
-            <p className="optional">Optional</p>
-            <p>
-              Druid begins ingesting data once you submit a JSON ingestion spec. If you modify any
-              values in this view, the values entered in previous sections will update accordingly.
-              If you modify any values in previous sections, this spec will automatically update.
-            </p>
-            <p>Submit the spec to begin loading data into Druid.</p>
-          </Callout>
+          <SpecMessage />
         </div>
         <div className="next-bar">
           {!isEmptyIngestionSpec(spec) && (

From a5de378e4db035aa74fe9e930052163baa7506eb Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Mon, 9 Nov 2020 21:24:27 -0800
Subject: [PATCH 04/13] fix schema view

---
 .../__snapshots__/auto-form.spec.tsx.snap     |  13 +-
 .../components/auto-form/auto-form.spec.tsx   |   5 +-
 .../src/components/auto-form/auto-form.tsx    |   2 +
 .../src/druid-models/ingestion-spec.spec.ts   |   2 +-
 .../src/druid-models/ingestion-spec.tsx       |  36 +-
 web-console/src/druid-models/input-format.tsx |  35 +-
 .../src/druid-models/transform-spec.tsx       |   1 +
 web-console/src/utils/general.spec.ts         |  10 +
 web-console/src/utils/general.tsx             |  27 +-
 web-console/src/utils/sampler.ts              |  54 +-
 web-console/src/utils/utils.spec.ts           |   7 +-
 .../views/load-data-view/load-data-view.tsx   | 668 +++++++++++-------
 .../reorder-menu/reorder-menu.tsx             |  66 ++
 .../schema-table/schema-table.spec.tsx        |   8 +-
 .../schema-table/schema-table.tsx             |  65 +-
 15 files changed, 624 insertions(+), 375 deletions(-)
 create mode 100644 web-console/src/views/load-data-view/reorder-menu/reorder-menu.tsx

diff --git a/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap b/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap
index 1a4000ef00fb..aa734cd821cf 100644
--- a/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap
+++ b/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap
@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`auto-form snapshot matches snapshot 1`] = `
+exports[`AutoForm matches snapshot 1`] = `
 <div
   className="auto-form"
 >
@@ -132,5 +132,16 @@ exports[`auto-form snapshot matches snapshot 1`] = `
       placeholder=""
     />
   </Memo(FormGroupWithInfo)>
+  <Blueprint3.FormGroup
+    key="advanced"
+  >
+    <Blueprint3.Button
+      fill={true}
+      minimal={true}
+      onClick={[Function]}
+      rightIcon="chevron-down"
+      text="Show advanced"
+    />
+  </Blueprint3.FormGroup>
 </div>
 `;
diff --git a/web-console/src/components/auto-form/auto-form.spec.tsx b/web-console/src/components/auto-form/auto-form.spec.tsx
index 89bd645e9f69..9bc108b450f1 100644
--- a/web-console/src/components/auto-form/auto-form.spec.tsx
+++ b/web-console/src/components/auto-form/auto-form.spec.tsx
@@ -21,7 +21,7 @@ import React from 'react';
 
 import { AutoForm } from './auto-form';
 
-describe('auto-form snapshot', () => {
+describe('AutoForm', () => {
   it('matches snapshot', () => {
     const autoForm = shallow(
       <AutoForm
@@ -34,6 +34,9 @@ describe('auto-form snapshot', () => {
           { name: 'testFive', type: 'string-array' },
           { name: 'testSix', type: 'json' },
           { name: 'testSeven', type: 'json' },
+
+          { name: 'testNotDefined', type: 'string', defined: false },
+          { name: 'testAdvanced', type: 'string', advanced: true },
         ]}
         model={String}
         onChange={() => {}}
diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx
index 88586ac80a1d..302e89f13d27 100644
--- a/web-console/src/components/auto-form/auto-form.tsx
+++ b/web-console/src/components/auto-form/auto-form.tsx
@@ -433,6 +433,8 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
         <Button
           text={showAdvanced ? 'Hide advanced' : 'Show advanced'}
           rightIcon={showAdvanced ? IconNames.CHEVRON_UP : IconNames.CHEVRON_DOWN}
+          minimal
+          fill
           onClick={() => {
             this.setState(({ showAdvanced }) => ({ showAdvanced: !showAdvanced }));
           }}
diff --git a/web-console/src/druid-models/ingestion-spec.spec.ts b/web-console/src/druid-models/ingestion-spec.spec.ts
index adccec656f8f..4e072fa3486e 100644
--- a/web-console/src/druid-models/ingestion-spec.spec.ts
+++ b/web-console/src/druid-models/ingestion-spec.spec.ts
@@ -219,7 +219,7 @@ describe('spec utils', () => {
               ],
             },
             "granularitySpec": Object {
-              "queryGranularity": "HOUR",
+              "queryGranularity": "hour",
               "rollup": true,
               "segmentGranularity": "DAY",
               "type": "uniform",
diff --git a/web-console/src/druid-models/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx
index 9ef3c73c3944..dadc1851b330 100644
--- a/web-console/src/druid-models/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec.tsx
@@ -47,7 +47,7 @@ import {
   getMetricSpecSingleFieldName,
   MetricSpec,
 } from './metric-spec';
-import { PLACEHOLDER_TIMESTAMP_SPEC, TimestampSpec } from './timestamp-spec';
+import { TimestampSpec } from './timestamp-spec';
 import { TransformSpec } from './transform-spec';
 
 export const MAX_INLINE_DATA_LENGTH = 65536;
@@ -475,6 +475,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           label: 'Dimensions',
           type: 'string-array',
           placeholder: '(optional)',
+          advanced: true,
           info: (
             <p>
               The list of dimensions to select. If left empty, no dimensions are returned. If left
@@ -487,6 +488,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           label: 'Metrics',
           type: 'string-array',
           placeholder: '(optional)',
+          advanced: true,
           info: (
             <p>
               The list of metrics to select. If left empty, no metrics are returned. If left null or
@@ -499,6 +501,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           label: 'Filter',
           type: 'json',
           placeholder: '(optional)',
+          advanced: true,
           info: (
             <p>
               The{' '}
@@ -1943,34 +1946,6 @@ export function updateIngestionType(
 
   if (inputSourceType) {
     newSpec = deepSet(newSpec, 'spec.ioConfig.inputSource', { type: inputSourceType });
-
-    if (inputSourceType === 'local') {
-      newSpec = deepSet(newSpec, 'spec.ioConfig.inputSource.filter', '*');
-    }
-  }
-
-  if (!deepGet(spec, 'spec.dataSchema.dataSource')) {
-    newSpec = deepSet(newSpec, 'spec.dataSchema.dataSource', 'new-data-source');
-  }
-
-  if (!deepGet(spec, 'spec.dataSchema.granularitySpec')) {
-    const granularitySpec: GranularitySpec = {
-      type: 'uniform',
-      queryGranularity: 'HOUR',
-    };
-    if (ingestionType !== 'index_parallel') {
-      granularitySpec.segmentGranularity = 'HOUR';
-    }
-
-    newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec', granularitySpec);
-  }
-
-  if (!deepGet(spec, 'spec.dataSchema.timestampSpec')) {
-    newSpec = deepSet(newSpec, 'spec.dataSchema.timestampSpec', PLACEHOLDER_TIMESTAMP_SPEC);
-  }
-
-  if (!deepGet(spec, 'spec.dataSchema.dimensionsSpec')) {
-    newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec', {});
   }
 
   return newSpec;
@@ -2091,7 +2066,8 @@ export function updateSchemaWithSample(
   let newSpec = spec;
 
   if (dimensionMode === 'auto-detect') {
-    newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec.dimensions', []);
+    newSpec = deepDelete(newSpec, 'spec.dataSchema.dimensionsSpec.dimensions');
+    newSpec = deepSet(newSpec, 'spec.dataSchema.dimensionsSpec.dimensionExclusions', []);
   } else {
     newSpec = deepDelete(newSpec, 'spec.dataSchema.dimensionsSpec.dimensionExclusions');
 
diff --git a/web-console/src/druid-models/input-format.tsx b/web-console/src/druid-models/input-format.tsx
index 15cb68258d4c..7907498b3bd7 100644
--- a/web-console/src/druid-models/input-format.tsx
+++ b/web-console/src/druid-models/input-format.tsx
@@ -16,6 +16,7 @@
  * limitations under the License.
  */
 
+import { Code } from '@blueprintjs/core';
 import React from 'react';
 
 import { AutoForm, ExternalLink, Field } from '../components';
@@ -68,12 +69,6 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
     required: true,
     defined: (p: InputFormat) => p.type === 'javascript',
   },
-  {
-    name: 'findColumnsFromHeader',
-    type: 'boolean',
-    required: true,
-    defined: (p: InputFormat) => oneOf(p.type, 'csv', 'tsv'),
-  },
   {
     name: 'skipHeaderRows',
     type: 'number',
@@ -82,9 +77,22 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
     min: 0,
     info: (
       <>
-        If both skipHeaderRows and hasHeaderRow options are set, skipHeaderRows is first applied.
-        For example, if you set skipHeaderRows to 2 and hasHeaderRow to true, Druid will skip the
-        first two lines and then extract column information from the third line.
+        If this is set, skip the first <Code>skipHeaderRows</Code> rows from each file.
+      </>
+    ),
+  },
+  {
+    name: 'findColumnsFromHeader',
+    type: 'boolean',
+    required: true,
+    defined: (p: InputFormat) => oneOf(p.type, 'csv', 'tsv'),
+    info: (
+      <>
+        If this is set, find the column names from the header row. Note that
+        <Code>skipHeaderRows</Code> will be applied before finding column names from the header. For
+        example, if you set <Code>skipHeaderRows</Code> to 2 and <Code>findColumnsFromHeader</Code>{' '}
+        to true, the task will skip the first two lines and then extract column information from the
+        third line.
       </>
     ),
   },
@@ -93,7 +101,13 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
     type: 'string-array',
     required: true,
     defined: (p: InputFormat) =>
-      (oneOf(p.type, 'csv', 'tsv') && !p.findColumnsFromHeader) || p.type === 'regex',
+      (oneOf(p.type, 'csv', 'tsv') && p.findColumnsFromHeader === false) || p.type === 'regex',
+    info: (
+      <>
+        Specifies the columns of the data. The columns should be in the same order with the columns
+        of your data.
+      </>
+    ),
   },
   {
     name: 'delimiter',
@@ -106,6 +120,7 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
     name: 'listDelimiter',
     type: 'string',
     defined: (p: InputFormat) => oneOf(p.type, 'csv', 'tsv', 'regex'),
+    placeholder: '(optional, default = ctrl+A)',
     info: <>A custom delimiter for multi-value dimensions.</>,
   },
   {
diff --git a/web-console/src/druid-models/transform-spec.tsx b/web-console/src/druid-models/transform-spec.tsx
index 81e9b334e3a7..c1dc34003024 100644
--- a/web-console/src/druid-models/transform-spec.tsx
+++ b/web-console/src/druid-models/transform-spec.tsx
@@ -75,6 +75,7 @@ export function getTimestampExpressionFields(transforms: Transform[]): Field<Tra
         `timestamp_parse(concat("date", ' ', "time"))`,
         `timestamp_parse(concat("date", ' ', "time"), 'M/d/yyyy H:mm:ss')`,
         `timestamp_parse(concat("year", '-', "month", '-', "day"))`,
+        `timestamp_parse("local_time", 'yyyy-MM-dd HH:mm:ss', "timezone")`,
       ],
       info: (
         <>
diff --git a/web-console/src/utils/general.spec.ts b/web-console/src/utils/general.spec.ts
index e614b8709436..ea1078c2df75 100644
--- a/web-console/src/utils/general.spec.ts
+++ b/web-console/src/utils/general.spec.ts
@@ -24,6 +24,7 @@ import {
   formatMegabytes,
   formatMillions,
   formatPercent,
+  moveElement,
   sortWithPrefixSuffix,
   sqlQueryCustomTableFilter,
   swapElements,
@@ -96,6 +97,15 @@ describe('general', () => {
     });
   });
 
+  describe('moveElement', () => {
+    it('moves items in an array', () => {
+      expect(moveElement(['a', 'b', 'c'], 2, 0)).toEqual(['c', 'a', 'b']);
+      expect(moveElement(['a', 'b', 'c'], 1, 1)).toEqual(['a', 'b', 'c']);
+      expect(moveElement(['F', 'B', 'W', 'B'], 2, 1)).toEqual(['F', 'W', 'B', 'B']);
+      expect(moveElement([1, 2, 3], 2, 1)).toEqual([1, 3, 2]);
+    });
+  });
+
   describe('formatInteger', () => {
     it('works', () => {
       expect(formatInteger(10000)).toEqual('10,000');
diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx
index ca2a111199ec..9ce6e6186e8a 100644
--- a/web-console/src/utils/general.tsx
+++ b/web-console/src/utils/general.tsx
@@ -306,7 +306,7 @@ export function sortWithPrefixSuffix(
   things: readonly string[],
   prefix: readonly string[],
   suffix: readonly string[],
-  cmp: null | ((a: string, b: string) => number),
+  cmp?: (a: string, b: string) => number,
 ): string[] {
   const pre = uniq(prefix.filter(x => things.includes(x)));
   const mid = things.filter(x => !prefix.includes(x) && !suffix.includes(x));
@@ -356,3 +356,28 @@ export function swapElements<T>(items: readonly T[], indexA: number, indexB: num
   newItems[indexB] = t;
   return newItems;
 }
+
+export function moveElement<T>(items: readonly T[], fromIndex: number, toIndex: number): T[] {
+  const indexDiff = fromIndex - toIndex;
+  if (indexDiff > 0) {
+    // move left
+    return [
+      ...items.slice(0, toIndex),
+      items[fromIndex],
+      ...items.slice(toIndex, fromIndex),
+      ...items.slice(fromIndex + 1, items.length),
+    ];
+  } else if (indexDiff < 0) {
+    // move right
+    const targetIndex = toIndex + 1;
+    return [
+      ...items.slice(0, fromIndex),
+      ...items.slice(fromIndex + 1, targetIndex),
+      items[fromIndex],
+      ...items.slice(targetIndex, items.length),
+    ];
+  } else {
+    // do nothing
+    return items.slice();
+  }
+}
diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts
index 48db5f6c19b3..4a888600b842 100644
--- a/web-console/src/utils/sampler.ts
+++ b/web-console/src/utils/sampler.ts
@@ -139,33 +139,41 @@ export function applyCache(sampleSpec: SampleSpec, cacheRows: CacheRows) {
   return sampleSpec;
 }
 
-export function headerFromSampleResponse(
-  sampleResponse: SampleResponse,
-  ignoreColumn?: string,
-  columnOrder?: string[],
-): string[] {
+export interface HeaderFromSampleResponseOptions {
+  sampleResponse: SampleResponse;
+  ignoreTimeColumn?: boolean;
+  columnOrder?: string[];
+  suffixColumnOrder?: string[];
+}
+
+export function headerFromSampleResponse(options: HeaderFromSampleResponseOptions): string[] {
+  const { sampleResponse, ignoreTimeColumn, columnOrder, suffixColumnOrder } = options;
+
   let columns = sortWithPrefixSuffix(
     dedupe(sampleResponse.data.flatMap(s => (s.parsed ? Object.keys(s.parsed) : []))).sort(),
     columnOrder || ['__time'],
-    [],
+    suffixColumnOrder || [],
     alphanumericCompare,
   );
 
-  if (ignoreColumn) {
-    columns = columns.filter(c => c !== ignoreColumn);
+  if (ignoreTimeColumn) {
+    columns = columns.filter(c => c !== '__time');
   }
 
   return columns;
 }
 
+export interface HeaderAndRowsFromSampleResponseOptions extends HeaderFromSampleResponseOptions {
+  parsedOnly?: boolean;
+}
+
 export function headerAndRowsFromSampleResponse(
-  sampleResponse: SampleResponse,
-  ignoreColumn?: string,
-  columnOrder?: string[],
-  parsedOnly = false,
+  options: HeaderAndRowsFromSampleResponseOptions,
 ): HeaderAndRows {
+  const { sampleResponse, parsedOnly } = options;
+
   return {
-    header: headerFromSampleResponse(sampleResponse, ignoreColumn, columnOrder),
+    header: headerFromSampleResponse(options),
     rows: parsedOnly ? sampleResponse.data.filter((d: any) => d.parsed) : sampleResponse.data,
   };
 }
@@ -446,11 +454,11 @@ export async function sampleForTransform(
     );
 
     specialDimensionSpec.dimensions = dedupe(
-      headerFromSampleResponse(
-        sampleResponseHack,
-        '__time',
-        ['__time'].concat(inputFormatColumns),
-      ).concat(transforms.map(t => t.name)),
+      headerFromSampleResponse({
+        sampleResponse: sampleResponseHack,
+        ignoreTimeColumn: true,
+        columnOrder: ['__time'].concat(inputFormatColumns),
+      }).concat(transforms.map(t => t.name)),
     );
   }
 
@@ -505,11 +513,11 @@ export async function sampleForFilter(
     );
 
     specialDimensionSpec.dimensions = dedupe(
-      headerFromSampleResponse(
-        sampleResponseHack,
-        '__time',
-        ['__time'].concat(inputFormatColumns),
-      ).concat(transforms.map(t => t.name)),
+      headerFromSampleResponse({
+        sampleResponse: sampleResponseHack,
+        ignoreTimeColumn: true,
+        columnOrder: ['__time'].concat(inputFormatColumns),
+      }).concat(transforms.map(t => t.name)),
     );
   }
 
diff --git a/web-console/src/utils/utils.spec.ts b/web-console/src/utils/utils.spec.ts
index b2ddb024577a..2ff03ae1fd4d 100644
--- a/web-console/src/utils/utils.spec.ts
+++ b/web-console/src/utils/utils.spec.ts
@@ -56,8 +56,11 @@ describe('utils', () => {
   // const cacheRows: CacheRows = [{ make: 'Honda', model: 'Civic' }, { make: 'BMW', model: 'M3' }];
 
   it('spec-utils headerFromSampleResponse', () => {
-    expect(headerFromSampleResponse({ data: [{ input: { a: 1 }, parsed: { a: 1 } }] }))
-      .toMatchInlineSnapshot(`
+    expect(
+      headerFromSampleResponse({
+        sampleResponse: { data: [{ input: { a: 1 }, parsed: { a: 1 } }] },
+      }),
+    ).toMatchInlineSnapshot(`
       Array [
         "a",
       ]
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index e2ac1867f26e..adb7317a1a03 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -30,6 +30,7 @@ import {
   HTMLSelect,
   Icon,
   IconName,
+  InputGroup,
   Intent,
   Menu,
   MenuItem,
@@ -61,6 +62,8 @@ import {
   DIMENSION_SPEC_FIELDS,
   FILTER_FIELDS,
   FLATTEN_FIELD_FIELDS,
+  getDimensionSpecName,
+  getMetricSpecName,
   getTimestampExpressionFields,
   getTimestampSchema,
   INPUT_FORMAT_FIELDS,
@@ -78,7 +81,6 @@ import {
   computeFlattenPathsForData,
   DimensionMode,
   DimensionSpec,
-  DimensionsSpec,
   DruidFilter,
   fillDataSourceNameIfNeeded,
   fillInputFormat,
@@ -132,6 +134,7 @@ import {
   localStorageGet,
   LocalStorageKeys,
   localStorageSet,
+  moveElement,
   oneOf,
   parseJson,
   pluralIfNeeded,
@@ -176,6 +179,7 @@ import {
   ParseTimeTable,
   parseTimeTableSelectedColumnName,
 } from './parse-time-table/parse-time-table';
+import { ReorderMenu } from './reorder-menu/reorder-menu';
 import { SchemaTable } from './schema-table/schema-table';
 import {
   TransformTable,
@@ -338,9 +342,10 @@ export interface LoadDataViewState {
   // for schema
   schemaQueryState: QueryState<{
     headerAndRows: HeaderAndRows;
-    dimensionsSpec: DimensionsSpec;
-    metricsSpec: MetricSpec[];
+    dimensions: (string | DimensionSpec)[] | undefined;
+    metricsSpec: MetricSpec[] | undefined;
   }>;
+  selectedAutoDimension?: string;
   selectedDimensionSpecIndex: number;
   selectedDimensionSpec?: DimensionSpec;
   selectedMetricSpecIndex: number;
@@ -452,15 +457,30 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         return Boolean(!druidSource && spec.type && !issueWithIoConfig(ioConfig));
 
       case 'timestamp':
-        return Boolean(!druidSource && cacheRows);
+        return Boolean(
+          !druidSource && cacheRows && deepGet(spec, 'spec.dataSchema.timestampSpec.format'),
+        );
 
       case 'transform':
       case 'filter':
+        return Boolean(cacheRows && deepGet(spec, 'spec.dataSchema.timestampSpec.format'));
+
       case 'schema':
+        return Boolean(
+          cacheRows &&
+            deepGet(spec, 'spec.dataSchema.timestampSpec.format') &&
+            deepGet(spec, 'spec.dataSchema.dimensionsSpec'),
+        );
+
       case 'partition':
       case 'tuning':
       case 'publish':
-        return Boolean(cacheRows);
+        return Boolean(
+          cacheRows &&
+            deepGet(spec, 'spec.dataSchema.timestampSpec.format') &&
+            deepGet(spec, 'spec.dataSchema.dimensionsSpec') &&
+            deepGet(spec, 'spec.dataSchema.granularitySpec.type'),
+        );
 
       default:
         return true;
@@ -487,14 +507,14 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   };
 
   private applyPreviewSpec = () => {
-    this.setState(state => {
-      localStorageSet(LocalStorageKeys.INGESTION_SPEC, JSON.stringify(state.specPreview));
-      return { spec: Object.assign({}, state.specPreview) };
+    this.setState(({ specPreview }) => {
+      localStorageSet(LocalStorageKeys.INGESTION_SPEC, JSON.stringify(specPreview));
+      return { spec: specPreview };
     });
   };
 
   private revertPreviewSpec = () => {
-    this.setState(state => ({ specPreview: state.spec }));
+    this.setState(({ spec }) => ({ specPreview: spec }));
   };
 
   isPreviewSpecSame() {
@@ -596,7 +616,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     );
   }
 
-  renderApplyButtonBar(queryState: QueryState<unknown>) {
+  renderApplyButtonBar(queryState: QueryState<unknown>, issue: string | undefined) {
     const previewSpecSame = this.isPreviewSpecSame();
     const queryStateHasError = Boolean(queryState && queryState.error);
 
@@ -604,17 +624,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       <FormGroup className="control-buttons">
         <Button
           text="Apply"
-          disabled={previewSpecSame && !queryStateHasError}
+          disabled={(previewSpecSame && !queryStateHasError) || Boolean(issue)}
           intent={Intent.PRIMARY}
           onClick={this.applyPreviewSpec}
         />
-        {!previewSpecSame && (
-          <Button
-            text="Cancel"
-            disabled={this.isPreviewSpecSame()}
-            onClick={this.revertPreviewSpec}
-          />
-        )}
+        {!previewSpecSame && <Button text="Cancel" onClick={this.revertPreviewSpec} />}
       </FormGroup>
     );
   }
@@ -657,7 +671,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           text={`Next: ${VIEW_TITLE[nextStep]}`}
           rightIcon={IconNames.ARROW_RIGHT}
           intent={Intent.PRIMARY}
-          disabled={disabled}
+          disabled={disabled || !this.isPreviewSpecSame()}
           onClick={() => {
             if (disabled) return;
             if (onNextStep) onNextStep();
@@ -1138,15 +1152,19 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     }
 
     const ingestionComboType = getIngestionComboType(spec);
+    const ioConfigFields = ingestionComboType
+      ? getIoConfigFormFields(ingestionComboType)
+      : undefined;
+
     return (
       <>
         <div className="main">{mainFill}</div>
         <div className="control">
           <ConnectMessage inlineMode={inlineMode} spec={spec} />
-          {ingestionComboType ? (
+          {ioConfigFields ? (
             <>
               <AutoForm
-                fields={getIoConfigFormFields(ingestionComboType)}
+                fields={ioConfigFields}
                 model={ioConfig}
                 onChange={c => this.updateSpecPreview(deepSet(spec, 'spec.ioConfig', c))}
               />
@@ -1187,7 +1205,10 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               </HTMLSelect>
             </FormGroup>
           )}
-          {this.renderApplyButtonBar(inputQueryState)}
+          {this.renderApplyButtonBar(
+            inputQueryState,
+            ioConfigFields ? AutoForm.issueWithModel(ioConfig, ioConfigFields) : undefined,
+          )}
         </div>
         {this.renderNextBar({
           disabled: !inputQueryState.data,
@@ -1296,7 +1317,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     this.setState(({ parserQueryState }) => ({
       cacheRows: getCacheRowsFromSampleResponse(sampleResponse),
       parserQueryState: new QueryState({
-        data: headerAndRowsFromSampleResponse(sampleResponse, '__time', inputFormatColumns),
+        data: headerAndRowsFromSampleResponse({
+          sampleResponse,
+          ignoreTimeColumn: true,
+          columnOrder: inputFormatColumns,
+        }),
         lastData: parserQueryState.getSomeData(),
       }),
     }));
@@ -1383,7 +1408,10 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                   this.updateSpecPreview(deepSet(spec, 'spec.ioConfig.inputFormat', p))
                 }
               />
-              {this.renderApplyButtonBar(parserQueryState)}
+              {this.renderApplyButtonBar(
+                parserQueryState,
+                AutoForm.issueWithModel(inputFormat, INPUT_FORMAT_FIELDS),
+              )}
             </>
           )}
           {this.renderFlattenControls()}
@@ -1502,6 +1530,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         <FormGroup>
           <Button
             text="Add column flattening"
+            disabled={!this.isPreviewSpecSame()}
             onClick={() => {
               this.setState({
                 selectedFlattenField: { type: 'path', name: '', expr: '' },
@@ -1562,11 +1591,10 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     this.setState(({ timestampQueryState }) => ({
       timestampQueryState: new QueryState({
         data: {
-          headerAndRows: headerAndRowsFromSampleResponse(
+          headerAndRows: headerAndRowsFromSampleResponse({
             sampleResponse,
-            undefined,
-            ['__time'].concat(inputFormatColumns),
-          ),
+            columnOrder: ['__time'].concat(inputFormatColumns),
+          }),
           spec,
         },
         lastData: timestampQueryState.getSomeData(),
@@ -1702,7 +1730,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               }}
             />
           )}
-          {this.renderApplyButtonBar(timestampQueryState)}
+          {this.renderApplyButtonBar(timestampQueryState, undefined)}
         </div>
         {this.renderNextBar({
           disabled: !timestampQueryState.data,
@@ -1757,11 +1785,10 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
     this.setState(({ transformQueryState }) => ({
       transformQueryState: new QueryState({
-        data: headerAndRowsFromSampleResponse(
+        data: headerAndRowsFromSampleResponse({
           sampleResponse,
-          undefined,
-          ['__time'].concat(inputFormatColumns),
-        ),
+          columnOrder: ['__time'].concat(inputFormatColumns),
+        }),
         lastData: transformQueryState.getSomeData(),
       }),
     }));
@@ -1967,12 +1994,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     if (sampleResponse.data.length) {
       this.setState(({ filterQueryState }) => ({
         filterQueryState: new QueryState({
-          data: headerAndRowsFromSampleResponse(
+          data: headerAndRowsFromSampleResponse({
             sampleResponse,
-            undefined,
-            ['__time'].concat(inputFormatColumns),
-            true,
-          ),
+            columnOrder: ['__time'].concat(inputFormatColumns),
+            parsedOnly: true,
+          }),
           lastData: filterQueryState.getSomeData(),
         }),
       }));
@@ -1991,12 +2017,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       return;
     }
 
-    const headerAndRowsNoFilter = headerAndRowsFromSampleResponse(
-      sampleResponseNoFilter,
-      undefined,
-      ['__time'].concat(inputFormatColumns),
-      true,
-    );
+    const headerAndRowsNoFilter = headerAndRowsFromSampleResponse({
+      sampleResponse: sampleResponseNoFilter,
+      columnOrder: ['__time'].concat(inputFormatColumns),
+      parsedOnly: true,
+    });
 
     this.setState(({ filterQueryState }) => ({
       // cacheRows: sampleResponseNoFilter.cacheKey,
@@ -2211,11 +2236,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
   async queryForSchema(initRun = false) {
     const { spec, cacheRows } = this.state;
-    const inputFormatColumns: string[] =
-      deepGet(spec, 'spec.ioConfig.inputFormat.columns') || EMPTY_ARRAY;
-    const metricsSpec: MetricSpec[] = deepGet(spec, 'spec.dataSchema.metricsSpec') || EMPTY_ARRAY;
-    const dimensionsSpec: DimensionsSpec =
-      deepGet(spec, 'spec.dataSchema.dimensionsSpec') || EMPTY_OBJECT;
+    const metricsSpec: MetricSpec[] | undefined = deepGet(spec, 'spec.dataSchema.metricsSpec');
+    const dimensions: (string | DimensionSpec)[] = deepGet(
+      spec,
+      'spec.dataSchema.dimensionsSpec.dimensions',
+    );
 
     if (!cacheRows) {
       this.setState(({ schemaQueryState }) => ({
@@ -2246,12 +2271,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     this.setState(({ schemaQueryState }) => ({
       schemaQueryState: new QueryState({
         data: {
-          headerAndRows: headerAndRowsFromSampleResponse(
+          headerAndRows: headerAndRowsFromSampleResponse({
             sampleResponse,
-            undefined,
-            ['__time'].concat(inputFormatColumns),
-          ),
-          dimensionsSpec,
+            columnOrder: ['__time'].concat(dimensions ? dimensions.map(getDimensionSpecName) : []),
+            suffixColumnOrder: metricsSpec ? metricsSpec.map(getMetricSpecName) : undefined,
+          }),
+          dimensions,
           metricsSpec,
         },
         lastData: schemaQueryState.getSomeData(),
@@ -2264,13 +2289,16 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       specPreview: spec,
       columnFilter,
       schemaQueryState,
+      selectedAutoDimension,
       selectedDimensionSpec,
       selectedDimensionSpecIndex,
       selectedMetricSpec,
       selectedMetricSpecIndex,
     } = this.state;
     const rollup: boolean = Boolean(deepGet(spec, 'spec.dataSchema.granularitySpec.rollup'));
-    const somethingSelected = Boolean(selectedDimensionSpec || selectedMetricSpec);
+    const somethingSelected = Boolean(
+      selectedAutoDimension || selectedDimensionSpec || selectedMetricSpec,
+    );
     const dimensionMode = getDimensionMode(spec);
 
     let mainFill: JSX.Element | string = '';
@@ -2293,7 +2321,9 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               columnFilter={columnFilter}
               selectedDimensionSpecIndex={selectedDimensionSpecIndex}
               selectedMetricSpecIndex={selectedMetricSpecIndex}
-              onDimensionOrMetricSelect={this.onDimensionOrMetricSelect}
+              onAutoDimensionSelect={this.onAutoDimensionSelect}
+              onDimensionSelect={this.onDimensionSelect}
+              onMetricSelect={this.onMetricSelect}
             />
           )}
           {schemaQueryState.isLoading() && <Loader />}
@@ -2426,10 +2456,41 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                 onChange={s => this.updateSpecPreview(s)}
                 onFinalize={this.applyPreviewSpec}
               />
+              <FormGroup>
+                <Button
+                  text="Add dimension"
+                  disabled={dimensionMode !== 'specific'}
+                  onClick={() => {
+                    this.setState({
+                      selectedDimensionSpecIndex: -1,
+                      selectedDimensionSpec: {
+                        name: 'new_dimension',
+                        type: 'string',
+                      },
+                    });
+                  }}
+                />
+              </FormGroup>
+              <FormGroup>
+                <Button
+                  text="Add metric"
+                  onClick={() => {
+                    this.setState({
+                      selectedMetricSpecIndex: -1,
+                      selectedMetricSpec: {
+                        name: 'sum_blah',
+                        type: 'doubleSum',
+                        fieldName: '',
+                      },
+                    });
+                  }}
+                />
+              </FormGroup>
             </>
           )}
-          {!selectedMetricSpec && this.renderDimensionSpecControls()}
-          {!selectedDimensionSpec && this.renderMetricSpecControls()}
+          {this.renderAutoDimensionControls()}
+          {this.renderDimensionSpecControls()}
+          {this.renderMetricSpecControls()}
           {this.renderChangeRollupAction()}
           {this.renderChangeDimensionModeAction()}
         </div>
@@ -2444,6 +2505,14 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               newSpec = deepSet(newSpec, 'spec.tuningConfig.partitionsSpec', { type: 'dynamic' });
               newSpec = deepDelete(newSpec, 'spec.tuningConfig.forceGuaranteedRollup');
             }
+
+            if (!deepGet(newSpec, 'spec.dataSchema.granularitySpec')) {
+              newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec', {
+                type: 'uniform',
+                queryGranularity: 'hour',
+              });
+            }
+
             this.updateSpec(newSpec);
           },
         })}
@@ -2451,15 +2520,37 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     );
   }
 
-  private onDimensionOrMetricSelect = (
+  private onAutoDimensionSelect = (selectedAutoDimension: string) => {
+    this.setState({
+      selectedAutoDimension,
+      selectedDimensionSpec: undefined,
+      selectedDimensionSpecIndex: -1,
+      selectedMetricSpec: undefined,
+      selectedMetricSpecIndex: -1,
+    });
+  };
+
+  private onDimensionSelect = (
     selectedDimensionSpec: DimensionSpec | undefined,
     selectedDimensionSpecIndex: number,
-    selectedMetricSpec: MetricSpec | undefined,
-    selectedMetricSpecIndex: number,
   ) => {
     this.setState({
+      selectedAutoDimension: undefined,
       selectedDimensionSpec,
       selectedDimensionSpecIndex,
+      selectedMetricSpec: undefined,
+      selectedMetricSpecIndex: -1,
+    });
+  };
+
+  private onMetricSelect = (
+    selectedMetricSpec: MetricSpec | undefined,
+    selectedMetricSpecIndex: number,
+  ) => {
+    this.setState({
+      selectedAutoDimension: undefined,
+      selectedDimensionSpec: undefined,
+      selectedDimensionSpecIndex: -1,
       selectedMetricSpec,
       selectedMetricSpecIndex,
     });
@@ -2476,7 +2567,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           this.updateSpec(
             updateSchemaWithSample(
               spec,
-              headerAndRowsFromSampleResponse(sampleResponse),
+              headerAndRowsFromSampleResponse({ sampleResponse }),
               getDimensionMode(spec),
               newRollup,
             ),
@@ -2506,7 +2597,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           this.updateSpec(
             updateSchemaWithSample(
               spec,
-              headerAndRowsFromSampleResponse(sampleResponse),
+              headerAndRowsFromSampleResponse({ sampleResponse }),
               newDimensionMode,
               getRollup(spec),
             ),
@@ -2530,8 +2621,49 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     );
   }
 
+  renderAutoDimensionControls() {
+    const { spec, selectedAutoDimension } = this.state;
+    if (!selectedAutoDimension) return;
+
+    const close = () => {
+      this.setState({
+        selectedAutoDimension: undefined,
+      });
+    };
+
+    return (
+      <div className="edit-controls">
+        <FormGroup label="Name">
+          <InputGroup defaultValue={selectedAutoDimension} readOnly />
+        </FormGroup>
+        <FormGroup>
+          <Button
+            icon={IconNames.CROSS}
+            text="Exclude"
+            intent={Intent.DANGER}
+            onClick={() => {
+              this.updateSpec(
+                deepSet(
+                  spec,
+                  `spec.dataSchema.dimensionsSpec.dimensionExclusions.[append]`,
+                  selectedAutoDimension,
+                ),
+              );
+              close();
+            }}
+          />
+        </FormGroup>
+        <FormGroup>
+          <Button text="Close" onClick={close} />
+        </FormGroup>
+      </div>
+    );
+  }
+
   renderDimensionSpecControls() {
     const { spec, selectedDimensionSpec, selectedDimensionSpecIndex } = this.state;
+    if (!selectedDimensionSpec) return;
+    const dimensionMode = getDimensionMode(spec);
 
     const close = () => {
       this.setState({
@@ -2540,139 +2672,141 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       });
     };
 
-    if (selectedDimensionSpec) {
-      const curDimensions =
-        deepGet(spec, `spec.dataSchema.dimensionsSpec.dimensions`) || EMPTY_ARRAY;
+    const dimensions = deepGet(spec, `spec.dataSchema.dimensionsSpec.dimensions`) || EMPTY_ARRAY;
 
-      const convertToMetric = (type: string, prefix: string) => {
-        const specWithoutDimension = deepDelete(
-          spec,
-          `spec.dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
-        );
+    const moveTo = (newIndex: number) => {
+      const newDimension = moveElement(dimensions, selectedDimensionSpecIndex, newIndex);
+      const newSpec = deepSet(spec, `spec.dataSchema.dimensionsSpec.dimensions`, newDimension);
+      this.updateSpec(newSpec);
+      close();
+    };
 
-        const specWithMetric = deepSet(
-          specWithoutDimension,
-          `spec.dataSchema.metricsSpec.[append]`,
-          {
-            name: `${prefix}_${selectedDimensionSpec.name}`,
-            type,
-            fieldName: selectedDimensionSpec.name,
-          },
-        );
+    const reorderDimensionMenu = (
+      <ReorderMenu things={dimensions} selectedIndex={selectedDimensionSpecIndex} moveTo={moveTo} />
+    );
 
-        this.updateSpec(specWithMetric);
-        close();
-      };
+    const convertToMetric = (type: string, prefix: string) => {
+      const specWithoutDimension =
+        dimensionMode === 'specific'
+          ? deepDelete(
+              spec,
+              `spec.dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
+            )
+          : spec;
 
-      const convertToMetricMenu = (
-        <Menu>
-          <MenuItem
-            text="Convert to longSum metric"
-            onClick={() => convertToMetric('longSum', 'sum')}
-          />
-          <MenuItem
-            text="Convert to doubleSum metric"
-            onClick={() => convertToMetric('doubleSum', 'sum')}
-          />
-          <MenuItem
-            text="Convert to thetaSketch metric"
-            onClick={() => convertToMetric('thetaSketch', 'theta')}
-          />
-          <MenuItem
-            text="Convert to HLLSketchBuild metric"
-            onClick={() => convertToMetric('HLLSketchBuild', 'hll')}
-          />
-          <MenuItem
-            text="Convert to quantilesDoublesSketch metric"
-            onClick={() => convertToMetric('quantilesDoublesSketch', 'quantiles_doubles')}
-          />
-          <MenuItem
-            text="Convert to hyperUnique metric"
-            onClick={() => convertToMetric('hyperUnique', 'unique')}
-          />
-        </Menu>
-      );
+      const specWithMetric = deepSet(specWithoutDimension, `spec.dataSchema.metricsSpec.[append]`, {
+        name: `${prefix}_${selectedDimensionSpec.name}`,
+        type,
+        fieldName: selectedDimensionSpec.name,
+      });
 
-      return (
-        <div className="edit-controls">
-          <AutoForm
-            fields={DIMENSION_SPEC_FIELDS}
-            model={selectedDimensionSpec}
-            onChange={selectedDimensionSpec => this.setState({ selectedDimensionSpec })}
+      this.updateSpec(specWithMetric);
+      close();
+    };
+
+    const convertToMetricMenu = (
+      <Menu>
+        <MenuItem
+          text="Convert to longSum metric"
+          onClick={() => convertToMetric('longSum', 'sum')}
+        />
+        <MenuItem
+          text="Convert to doubleSum metric"
+          onClick={() => convertToMetric('doubleSum', 'sum')}
+        />
+        <MenuItem
+          text="Convert to thetaSketch metric"
+          onClick={() => convertToMetric('thetaSketch', 'theta')}
+        />
+        <MenuItem
+          text="Convert to HLLSketchBuild metric"
+          onClick={() => convertToMetric('HLLSketchBuild', 'hll')}
+        />
+        <MenuItem
+          text="Convert to quantilesDoublesSketch metric"
+          onClick={() => convertToMetric('quantilesDoublesSketch', 'quantiles_doubles')}
+        />
+        <MenuItem
+          text="Convert to hyperUnique metric"
+          onClick={() => convertToMetric('hyperUnique', 'unique')}
+        />
+      </Menu>
+    );
+
+    return (
+      <div className="edit-controls">
+        <AutoForm
+          fields={DIMENSION_SPEC_FIELDS}
+          model={selectedDimensionSpec}
+          onChange={selectedDimensionSpec => this.setState({ selectedDimensionSpec })}
+        />
+        {reorderDimensionMenu && (
+          <FormGroup>
+            <Popover content={reorderDimensionMenu}>
+              <Button
+                icon={IconNames.ARROWS_HORIZONTAL}
+                text="Reorder dimension"
+                rightIcon={IconNames.CARET_DOWN}
+              />
+            </Popover>
+          </FormGroup>
+        )}
+        {selectedDimensionSpecIndex !== -1 && deepGet(spec, 'spec.dataSchema.metricsSpec') && (
+          <FormGroup>
+            <Popover content={convertToMetricMenu}>
+              <Button
+                icon={IconNames.EXCHANGE}
+                text="Convert to metric"
+                rightIcon={IconNames.CARET_DOWN}
+                disabled={dimensions.length <= 1}
+              />
+            </Popover>
+          </FormGroup>
+        )}
+        <div className="control-buttons">
+          <Button
+            text="Apply"
+            intent={Intent.PRIMARY}
+            onClick={() => {
+              this.updateSpec(
+                deepSet(
+                  spec,
+                  `spec.dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
+                  selectedDimensionSpec,
+                ),
+              );
+              close();
+            }}
           />
-          {selectedDimensionSpecIndex !== -1 && deepGet(spec, 'spec.dataSchema.metricsSpec') && (
-            <FormGroup>
-              <Popover content={convertToMetricMenu}>
-                <Button
-                  icon={IconNames.EXCHANGE}
-                  text="Convert to metric"
-                  rightIcon={IconNames.CARET_DOWN}
-                  disabled={curDimensions.length <= 1}
-                />
-              </Popover>
-            </FormGroup>
-          )}
-          <div className="control-buttons">
+          <Button text="Cancel" onClick={close} />
+          {selectedDimensionSpecIndex !== -1 && (
             <Button
-              text="Apply"
-              intent={Intent.PRIMARY}
+              className="right"
+              icon={IconNames.TRASH}
+              intent={Intent.DANGER}
+              disabled={dimensions.length <= 1}
               onClick={() => {
+                if (dimensions.length <= 1) return; // Guard against removing the last dimension
+
                 this.updateSpec(
-                  deepSet(
+                  deepDelete(
                     spec,
                     `spec.dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
-                    selectedDimensionSpec,
                   ),
                 );
                 close();
               }}
             />
-            <Button text="Cancel" onClick={close} />
-            {selectedDimensionSpecIndex !== -1 && (
-              <Button
-                className="right"
-                icon={IconNames.TRASH}
-                intent={Intent.DANGER}
-                disabled={curDimensions.length <= 1}
-                onClick={() => {
-                  if (curDimensions.length <= 1) return; // Guard against removing the last dimension
-
-                  this.updateSpec(
-                    deepDelete(
-                      spec,
-                      `spec.dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
-                    ),
-                  );
-                  close();
-                }}
-              />
-            )}
-          </div>
+          )}
         </div>
-      );
-    } else {
-      return (
-        <FormGroup>
-          <Button
-            text="Add dimension"
-            disabled={getDimensionMode(spec) !== 'specific'}
-            onClick={() => {
-              this.setState({
-                selectedDimensionSpecIndex: -1,
-                selectedDimensionSpec: {
-                  name: 'new_dimension',
-                  type: 'string',
-                },
-              });
-            }}
-          />
-        </FormGroup>
-      );
-    }
+      </div>
+    );
   }
 
   renderMetricSpecControls() {
     const { spec, selectedMetricSpec, selectedMetricSpecIndex } = this.state;
+    if (!selectedMetricSpec) return;
+    const dimensionMode = getDimensionMode(spec);
 
     const close = () => {
       this.setState({
@@ -2681,110 +2815,108 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       });
     };
 
-    if (selectedMetricSpec) {
-      const convertToDimension = (type: string) => {
-        const specWithoutMetric = deepDelete(
-          spec,
-          `spec.dataSchema.metricsSpec.${selectedMetricSpecIndex}`,
-        );
+    const metricsSpec = deepGet(spec, `spec.dataSchema.metricsSpec`);
 
-        const specWithDimension = deepSet(
-          specWithoutMetric,
-          `spec.dataSchema.dimensionsSpec.dimensions.[append]`,
-          {
-            type,
-            name: selectedMetricSpec.fieldName,
-          },
-        );
+    const moveTo = (newIndex: number) => {
+      const newMetricsSpec = moveElement(metricsSpec, selectedMetricSpecIndex, newIndex);
+      const newSpec = deepSet(spec, `spec.dataSchema.metricsSpec`, newMetricsSpec);
+      this.updateSpec(newSpec);
+      close();
+    };
 
-        this.updateSpec(specWithDimension);
-        close();
-      };
+    const reorderMetricMenu = (
+      <ReorderMenu things={metricsSpec} selectedIndex={selectedMetricSpecIndex} moveTo={moveTo} />
+    );
 
-      const convertToDimensionMenu = (
-        <Menu>
-          <MenuItem
-            text="Convert to string dimension"
-            onClick={() => convertToDimension('string')}
-          />
-          <MenuItem text="Convert to long dimension" onClick={() => convertToDimension('long')} />
-          <MenuItem text="Convert to float dimension" onClick={() => convertToDimension('float')} />
-          <MenuItem
-            text="Convert to double dimension"
-            onClick={() => convertToDimension('double')}
-          />
-        </Menu>
+    const convertToDimension = (type: string) => {
+      const specWithoutMetric = deepDelete(
+        spec,
+        `spec.dataSchema.metricsSpec.${selectedMetricSpecIndex}`,
       );
 
-      return (
-        <div className="edit-controls">
-          <AutoForm
-            fields={METRIC_SPEC_FIELDS}
-            model={selectedMetricSpec}
-            onChange={selectedMetricSpec => this.setState({ selectedMetricSpec })}
+      const specWithDimension = deepSet(
+        specWithoutMetric,
+        `spec.dataSchema.dimensionsSpec.dimensions.[append]`,
+        {
+          type,
+          name: selectedMetricSpec.fieldName,
+        },
+      );
+
+      this.updateSpec(specWithDimension);
+      close();
+    };
+
+    const convertToDimensionMenu = (
+      <Menu>
+        <MenuItem text="Convert to string dimension" onClick={() => convertToDimension('string')} />
+        <MenuItem text="Convert to long dimension" onClick={() => convertToDimension('long')} />
+        <MenuItem text="Convert to float dimension" onClick={() => convertToDimension('float')} />
+        <MenuItem text="Convert to double dimension" onClick={() => convertToDimension('double')} />
+      </Menu>
+    );
+
+    return (
+      <div className="edit-controls">
+        <AutoForm
+          fields={METRIC_SPEC_FIELDS}
+          model={selectedMetricSpec}
+          onChange={selectedMetricSpec => this.setState({ selectedMetricSpec })}
+        />
+        {reorderMetricMenu && (
+          <FormGroup>
+            <Popover content={reorderMetricMenu}>
+              <Button
+                icon={IconNames.ARROWS_HORIZONTAL}
+                text="Reorder metric"
+                rightIcon={IconNames.CARET_DOWN}
+              />
+            </Popover>
+          </FormGroup>
+        )}
+        {selectedMetricSpecIndex !== -1 && dimensionMode === 'specific' && (
+          <FormGroup>
+            <Popover content={convertToDimensionMenu}>
+              <Button
+                icon={IconNames.EXCHANGE}
+                text="Convert to dimension"
+                rightIcon={IconNames.CARET_DOWN}
+              />
+            </Popover>
+          </FormGroup>
+        )}
+        <div className="control-buttons">
+          <Button
+            text="Apply"
+            intent={Intent.PRIMARY}
+            onClick={() => {
+              this.updateSpec(
+                deepSet(
+                  spec,
+                  `spec.dataSchema.metricsSpec.${selectedMetricSpecIndex}`,
+                  selectedMetricSpec,
+                ),
+              );
+              close();
+            }}
           />
+          <Button text="Cancel" onClick={close} />
           {selectedMetricSpecIndex !== -1 && (
-            <FormGroup>
-              <Popover content={convertToDimensionMenu}>
-                <Button
-                  icon={IconNames.EXCHANGE}
-                  text="Convert to dimension"
-                  rightIcon={IconNames.CARET_DOWN}
-                />
-              </Popover>
-            </FormGroup>
-          )}
-          <div className="control-buttons">
             <Button
-              text="Apply"
-              intent={Intent.PRIMARY}
+              className="right"
+              icon={IconNames.TRASH}
+              intent={Intent.DANGER}
               onClick={() => {
                 this.updateSpec(
-                  deepSet(
-                    spec,
-                    `spec.dataSchema.metricsSpec.${selectedMetricSpecIndex}`,
-                    selectedMetricSpec,
-                  ),
+                  deepDelete(spec, `spec.dataSchema.metricsSpec.${selectedMetricSpecIndex}`),
                 );
                 close();
               }}
             />
-            <Button text="Cancel" onClick={close} />
-            {selectedMetricSpecIndex !== -1 && (
-              <Button
-                className="right"
-                icon={IconNames.TRASH}
-                intent={Intent.DANGER}
-                onClick={() => {
-                  this.updateSpec(
-                    deepDelete(spec, `spec.dataSchema.metricsSpec.${selectedMetricSpecIndex}`),
-                  );
-                  close();
-                }}
-              />
-            )}
-          </div>
+          )}
         </div>
-      );
-    } else {
-      return (
-        <FormGroup>
-          <Button
-            text="Add metric"
-            onClick={() => {
-              this.setState({
-                selectedMetricSpecIndex: -1,
-                selectedMetricSpec: {
-                  name: 'sum_blah',
-                  type: 'doubleSum',
-                  fieldName: '',
-                },
-              });
-            }}
-          />
-        </FormGroup>
-      );
-    }
+      </div>
+    );
   }
 
   // ==================================================================
@@ -3061,6 +3193,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
   renderSpecStep() {
     const { spec, submitting } = this.state;
 
+    const fullSpec = Boolean(
+      deepGet(spec, 'spec.dataSchema.timestampSpec.format') &&
+        deepGet(spec, 'spec.dataSchema.dimensionsSpec') &&
+        deepGet(spec, 'spec.dataSchema.granularitySpec.type'),
+    );
+
     return (
       <>
         <div className="main">
@@ -3089,7 +3227,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
             text="Submit"
             rightIcon={IconNames.CLOUD_UPLOAD}
             intent={Intent.PRIMARY}
-            disabled={submitting}
+            disabled={submitting || !fullSpec}
             onClick={this.handleSubmit}
           />
         </div>
diff --git a/web-console/src/views/load-data-view/reorder-menu/reorder-menu.tsx b/web-console/src/views/load-data-view/reorder-menu/reorder-menu.tsx
new file mode 100644
index 000000000000..631b0b3df0c3
--- /dev/null
+++ b/web-console/src/views/load-data-view/reorder-menu/reorder-menu.tsx
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Menu, MenuItem } from '@blueprintjs/core';
+import { IconNames } from '@blueprintjs/icons';
+import React from 'react';
+
+export interface ReorderMenuProps {
+  things: any[] | undefined;
+  selectedIndex: number;
+  moveTo: (newIndex: number) => void;
+}
+
+export const ReorderMenu = React.memo(function ReorderMenu(props: ReorderMenuProps) {
+  const { things, selectedIndex, moveTo } = props;
+
+  if (!Array.isArray(things) || things.length <= 1 || selectedIndex === -1) return null;
+  const lastIndex = things.length - 1;
+  return (
+    <Menu>
+      {selectedIndex > 0 && (
+        <MenuItem
+          icon={IconNames.DOUBLE_CHEVRON_LEFT}
+          text="Make first"
+          onClick={() => moveTo(0)}
+        />
+      )}
+      {selectedIndex > 0 && (
+        <MenuItem
+          icon={IconNames.CHEVRON_LEFT}
+          text="Move left (earlier in the sort order)"
+          onClick={() => moveTo(selectedIndex - 1)}
+        />
+      )}
+      {selectedIndex < lastIndex && (
+        <MenuItem
+          icon={IconNames.CHEVRON_RIGHT}
+          text="Move right (later in the sort order)"
+          onClick={() => moveTo(selectedIndex + 1)}
+        />
+      )}
+      {selectedIndex < lastIndex && (
+        <MenuItem
+          icon={IconNames.DOUBLE_CHEVRON_RIGHT}
+          text="Make last"
+          onClick={() => moveTo(lastIndex)}
+        />
+      )}
+    </Menu>
+  );
+});
diff --git a/web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx b/web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx
index 2a4dad98c754..94818fc78635 100644
--- a/web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx
+++ b/web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx
@@ -21,7 +21,7 @@ import React from 'react';
 
 import { SchemaTable } from './schema-table';
 
-describe('schema table', () => {
+describe('SchemaTable', () => {
   it('matches snapshot', () => {
     const sampleData = {
       header: ['c1'],
@@ -37,13 +37,15 @@ describe('schema table', () => {
       <SchemaTable
         sampleBundle={{
           headerAndRows: sampleData,
-          dimensionsSpec: {},
+          dimensions: [],
           metricsSpec: [],
         }}
         columnFilter=""
         selectedDimensionSpecIndex={-1}
         selectedMetricSpecIndex={-1}
-        onDimensionOrMetricSelect={() => {}}
+        onAutoDimensionSelect={() => {}}
+        onDimensionSelect={() => {}}
+        onMetricSelect={() => {}}
       />
     );
 
diff --git a/web-console/src/views/load-data-view/schema-table/schema-table.tsx b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
index ecb50e5206ec..de7252be603f 100644
--- a/web-console/src/views/load-data-view/schema-table/schema-table.tsx
+++ b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
@@ -23,14 +23,13 @@ import ReactTable from 'react-table';
 import { TableCell } from '../../../components';
 import {
   DimensionSpec,
-  DimensionsSpec,
   getDimensionSpecName,
   getDimensionSpecType,
   getMetricSpecName,
   inflateDimensionSpec,
   MetricSpec,
 } from '../../../druid-models';
-import { caseInsensitiveContains, filterMap, sortWithPrefixSuffix } from '../../../utils';
+import { caseInsensitiveContains, filterMap } from '../../../utils';
 import { HeaderAndRows, SampleEntry } from '../../../utils/sampler';
 
 import './schema-table.scss';
@@ -38,15 +37,18 @@ import './schema-table.scss';
 export interface SchemaTableProps {
   sampleBundle: {
     headerAndRows: HeaderAndRows;
-    dimensionsSpec: DimensionsSpec;
-    metricsSpec: MetricSpec[];
+    dimensions: (string | DimensionSpec)[] | undefined;
+    metricsSpec: MetricSpec[] | undefined;
   };
   columnFilter: string;
   selectedDimensionSpecIndex: number;
   selectedMetricSpecIndex: number;
-  onDimensionOrMetricSelect: (
+  onAutoDimensionSelect: (dimensionName: string) => void;
+  onDimensionSelect: (
     selectedDimensionSpec: DimensionSpec | undefined,
     selectedDimensionSpecIndex: number,
+  ) => void;
+  onMetricSelect: (
     selectedMetricSpec: MetricSpec | undefined,
     selectedMetricSpecIndex: number,
   ) => void;
@@ -58,26 +60,22 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
     columnFilter,
     selectedDimensionSpecIndex,
     selectedMetricSpecIndex,
-    onDimensionOrMetricSelect,
+    onAutoDimensionSelect,
+    onDimensionSelect,
+    onMetricSelect,
   } = props;
-  const { headerAndRows, dimensionsSpec, metricsSpec } = sampleBundle;
-
-  const dimensionMetricSortedHeader = sortWithPrefixSuffix(
-    headerAndRows.header,
-    ['__time'],
-    metricsSpec.map(getMetricSpecName),
-    null,
-  );
-
+  const { headerAndRows, dimensions, metricsSpec } = sampleBundle;
   return (
     <ReactTable
       className="schema-table -striped -highlight"
       data={headerAndRows.rows}
-      columns={filterMap(dimensionMetricSortedHeader, (columnName, i) => {
+      columns={filterMap(headerAndRows.header, (columnName, i) => {
         if (!caseInsensitiveContains(columnName, columnFilter)) return;
 
-        const metricSpecIndex = metricsSpec.findIndex(m => getMetricSpecName(m) === columnName);
-        const metricSpec = metricsSpec[metricSpecIndex];
+        const metricSpecIndex = metricsSpec
+          ? metricsSpec.findIndex(m => getMetricSpecName(m) === columnName)
+          : -1;
+        const metricSpec = metricsSpec ? metricsSpec[metricSpecIndex] : undefined;
 
         if (metricSpec) {
           const columnClassName = classNames('metric', {
@@ -87,9 +85,7 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
             Header: (
               <div
                 className="clickable"
-                onClick={() =>
-                  onDimensionOrMetricSelect(undefined, -1, metricSpec, metricSpecIndex)
-                }
+                onClick={() => onMetricSelect(metricSpec, metricSpecIndex)}
               >
                 <div className="column-name">{columnName}</div>
                 <div className="column-detail">{metricSpec.type}&nbsp;</div>
@@ -103,13 +99,11 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
           };
         } else {
           const timestamp = columnName === '__time';
-          const dimensionSpecIndex = dimensionsSpec.dimensions
-            ? dimensionsSpec.dimensions.findIndex(d => getDimensionSpecName(d) === columnName)
+          const dimensionSpecIndex = dimensions
+            ? dimensions.findIndex(d => getDimensionSpecName(d) === columnName)
             : -1;
-          const dimensionSpec = dimensionsSpec.dimensions
-            ? dimensionsSpec.dimensions[dimensionSpecIndex]
-            : null;
-          const dimensionSpecType = dimensionSpec ? getDimensionSpecType(dimensionSpec) : null;
+          const dimensionSpec = dimensions ? dimensions[dimensionSpecIndex] : undefined;
+          const dimensionSpecType = dimensionSpec ? getDimensionSpecType(dimensionSpec) : undefined;
 
           const columnClassName = classNames(
             timestamp ? 'timestamp' : 'dimension',
@@ -123,18 +117,13 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
               <div
                 className="clickable"
                 onClick={() => {
-                  if (timestamp) {
-                    onDimensionOrMetricSelect(undefined, -1, undefined, -1);
-                    return;
-                  }
+                  if (timestamp) return;
 
-                  if (!dimensionSpec) return;
-                  onDimensionOrMetricSelect(
-                    inflateDimensionSpec(dimensionSpec),
-                    dimensionSpecIndex,
-                    undefined,
-                    -1,
-                  );
+                  if (dimensionSpec) {
+                    onDimensionSelect(inflateDimensionSpec(dimensionSpec), dimensionSpecIndex);
+                  } else {
+                    onAutoDimensionSelect(columnName);
+                  }
                 }}
               >
                 <div className="column-name">{columnName}</div>

From d17b8f9a8986a6d1f5e6bff3ce9c417b489168cf Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Tue, 10 Nov 2020 13:58:21 -0800
Subject: [PATCH 05/13] better indication

---
 web-console/src/views/load-data-view/load-data-view.tsx    | 3 ++-
 .../load-data-view/schema-table/schema-table.spec.tsx      | 1 +
 .../src/views/load-data-view/schema-table/schema-table.tsx | 7 ++++++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index adb7317a1a03..59824dcea690 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -2319,6 +2319,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
             <SchemaTable
               sampleBundle={data}
               columnFilter={columnFilter}
+              selectedAutoDimension={selectedAutoDimension}
               selectedDimensionSpecIndex={selectedDimensionSpecIndex}
               selectedMetricSpecIndex={selectedMetricSpecIndex}
               onAutoDimensionSelect={this.onAutoDimensionSelect}
@@ -2634,7 +2635,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     return (
       <div className="edit-controls">
         <FormGroup label="Name">
-          <InputGroup defaultValue={selectedAutoDimension} readOnly />
+          <InputGroup value={selectedAutoDimension} onChange={() => {}} readOnly />
         </FormGroup>
         <FormGroup>
           <Button
diff --git a/web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx b/web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx
index 94818fc78635..626501366645 100644
--- a/web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx
+++ b/web-console/src/views/load-data-view/schema-table/schema-table.spec.tsx
@@ -41,6 +41,7 @@ describe('SchemaTable', () => {
           metricsSpec: [],
         }}
         columnFilter=""
+        selectedAutoDimension={undefined}
         selectedDimensionSpecIndex={-1}
         selectedMetricSpecIndex={-1}
         onAutoDimensionSelect={() => {}}
diff --git a/web-console/src/views/load-data-view/schema-table/schema-table.tsx b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
index de7252be603f..f56fcd9453ce 100644
--- a/web-console/src/views/load-data-view/schema-table/schema-table.tsx
+++ b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
@@ -41,6 +41,7 @@ export interface SchemaTableProps {
     metricsSpec: MetricSpec[] | undefined;
   };
   columnFilter: string;
+  selectedAutoDimension: string | undefined;
   selectedDimensionSpecIndex: number;
   selectedMetricSpecIndex: number;
   onAutoDimensionSelect: (dimensionName: string) => void;
@@ -58,6 +59,7 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
   const {
     sampleBundle,
     columnFilter,
+    selectedAutoDimension,
     selectedDimensionSpecIndex,
     selectedMetricSpecIndex,
     onAutoDimensionSelect,
@@ -65,6 +67,7 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
     onMetricSelect,
   } = props;
   const { headerAndRows, dimensions, metricsSpec } = sampleBundle;
+
   return (
     <ReactTable
       className="schema-table -striped -highlight"
@@ -109,7 +112,9 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
             timestamp ? 'timestamp' : 'dimension',
             dimensionSpecType || 'string',
             {
-              selected: dimensionSpec && dimensionSpecIndex === selectedDimensionSpecIndex,
+              selected:
+                (dimensionSpec && dimensionSpecIndex === selectedDimensionSpecIndex) ||
+                selectedAutoDimension === columnName,
             },
           );
           return {

From 081a8bfa5dd3f21b576319450576dbb09452ac64 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Tue, 10 Nov 2020 20:41:21 -0800
Subject: [PATCH 06/13] tests pass

---
 .../component/load-data/config/partition.ts   |  8 +++----
 .../__snapshots__/ingestion-spec.spec.ts.snap |  4 ++--
 .../src/druid-models/ingestion-spec.spec.ts   | 10 ++++----
 web-console/src/utils/utils.spec.ts           |  8 +++----
 .../views/load-data-view/load-data-view.tsx   | 23 +++++++++----------
 .../__snapshots__/schema-table.spec.tsx.snap  |  2 +-
 6 files changed, 27 insertions(+), 28 deletions(-)

diff --git a/web-console/e2e-tests/component/load-data/config/partition.ts b/web-console/e2e-tests/component/load-data/config/partition.ts
index f811a531664c..375ad550745f 100644
--- a/web-console/e2e-tests/component/load-data/config/partition.ts
+++ b/web-console/e2e-tests/component/load-data/config/partition.ts
@@ -26,10 +26,10 @@ import { getLabeledInput, selectSuggestibleInput, setLabeledInput } from '../../
  * Possible values for partition step segment granularity.
  */
 export enum SegmentGranularity {
-  HOUR = 'HOUR',
-  DAY = 'DAY',
-  MONTH = 'MONTH',
-  YEAR = 'YEAR',
+  HOUR = 'hour',
+  DAY = 'day',
+  MONTH = 'month',
+  YEAR = 'year',
 }
 
 const PARTITIONING_TYPE = 'Partitioning type';
diff --git a/web-console/src/druid-models/__snapshots__/ingestion-spec.spec.ts.snap b/web-console/src/druid-models/__snapshots__/ingestion-spec.spec.ts.snap
index 900672117484..1008cb2112b3 100644
--- a/web-console/src/druid-models/__snapshots__/ingestion-spec.spec.ts.snap
+++ b/web-console/src/druid-models/__snapshots__/ingestion-spec.spec.ts.snap
@@ -13,9 +13,9 @@ Object {
         ],
       },
       "granularitySpec": Object {
-        "queryGranularity": "HOUR",
+        "queryGranularity": "hour",
         "rollup": true,
-        "segmentGranularity": "DAY",
+        "segmentGranularity": "day",
         "type": "uniform",
       },
       "metricsSpec": Array [
diff --git a/web-console/src/druid-models/ingestion-spec.spec.ts b/web-console/src/druid-models/ingestion-spec.spec.ts
index 4e072fa3486e..fb6df068351f 100644
--- a/web-console/src/druid-models/ingestion-spec.spec.ts
+++ b/web-console/src/druid-models/ingestion-spec.spec.ts
@@ -45,8 +45,8 @@ describe('ingestion-spec', () => {
         dataSource: 'wikipedia',
         granularitySpec: {
           type: 'uniform',
-          segmentGranularity: 'DAY',
-          queryGranularity: 'HOUR',
+          segmentGranularity: 'day',
+          queryGranularity: 'hour',
           rollup: true,
         },
         parser: {
@@ -183,8 +183,8 @@ describe('spec utils', () => {
         dataSource: 'wikipedia',
         granularitySpec: {
           type: 'uniform',
-          segmentGranularity: 'DAY',
-          queryGranularity: 'HOUR',
+          segmentGranularity: 'day',
+          queryGranularity: 'hour',
         },
         timestampSpec: {
           column: 'timestamp',
@@ -221,7 +221,7 @@ describe('spec utils', () => {
             "granularitySpec": Object {
               "queryGranularity": "hour",
               "rollup": true,
-              "segmentGranularity": "DAY",
+              "segmentGranularity": "day",
               "type": "uniform",
             },
             "metricsSpec": Array [
diff --git a/web-console/src/utils/utils.spec.ts b/web-console/src/utils/utils.spec.ts
index 2ff03ae1fd4d..31e2f89ab224 100644
--- a/web-console/src/utils/utils.spec.ts
+++ b/web-console/src/utils/utils.spec.ts
@@ -41,8 +41,8 @@ describe('utils', () => {
         dataSource: 'wikipedia',
         granularitySpec: {
           type: 'uniform',
-          segmentGranularity: 'DAY',
-          queryGranularity: 'HOUR',
+          segmentGranularity: 'day',
+          queryGranularity: 'hour',
         },
         timestampSpec: {
           column: 'timestamp',
@@ -89,8 +89,8 @@ describe('utils', () => {
             "dataSource": "wikipedia",
             "dimensionsSpec": Object {},
             "granularitySpec": Object {
-              "queryGranularity": "HOUR",
-              "segmentGranularity": "DAY",
+              "queryGranularity": "hour",
+              "segmentGranularity": "day",
               "type": "uniform",
             },
             "timestampSpec": Object {
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index 59824dcea690..e2bcd967d5b8 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -1870,11 +1870,17 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           disabled: !transformQueryState.data,
           onNextStep: () => {
             if (!transformQueryState.data) return;
-            if (!isDruidSource(spec)) {
-              this.updateSpec(
-                updateSchemaWithSample(spec, transformQueryState.data, 'specific', true),
-              );
+
+            let newSpec = spec;
+            if (!isDruidSource(newSpec)) {
+              newSpec = updateSchemaWithSample(newSpec, transformQueryState.data, 'specific', true);
+            }
+
+            if (!deepGet(newSpec, 'spec.dataSchema.granularitySpec.type')) {
+              newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec.type', 'uniform');
             }
+
+            this.updateSpec(newSpec);
           },
         })}
       </>
@@ -2507,13 +2513,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               newSpec = deepDelete(newSpec, 'spec.tuningConfig.forceGuaranteedRollup');
             }
 
-            if (!deepGet(newSpec, 'spec.dataSchema.granularitySpec')) {
-              newSpec = deepSet(newSpec, 'spec.dataSchema.granularitySpec', {
-                type: 'uniform',
-                queryGranularity: 'hour',
-              });
-            }
-
             this.updateSpec(newSpec);
           },
         })}
@@ -2944,7 +2943,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               {
                 name: 'segmentGranularity',
                 type: 'string',
-                suggestions: ['HOUR', 'DAY', 'WEEK', 'MONTH', 'YEAR'],
+                suggestions: ['hour', 'day', 'week', 'month', 'year'],
                 defined: (g: GranularitySpec) => g.type === 'uniform',
                 required: true,
                 info: (
diff --git a/web-console/src/views/load-data-view/schema-table/__snapshots__/schema-table.spec.tsx.snap b/web-console/src/views/load-data-view/schema-table/__snapshots__/schema-table.spec.tsx.snap
index e9d47e05787c..ef00cae858eb 100644
--- a/web-console/src/views/load-data-view/schema-table/__snapshots__/schema-table.spec.tsx.snap
+++ b/web-console/src/views/load-data-view/schema-table/__snapshots__/schema-table.spec.tsx.snap
@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`schema table matches snapshot 1`] = `
+exports[`SchemaTable matches snapshot 1`] = `
 <div
   class="ReactTable schema-table -striped -highlight"
 >

From ab5eff9e586dd2f5a86982a1a1c0b498153f1a89 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Wed, 11 Nov 2020 22:51:39 -0800
Subject: [PATCH 07/13] Show more instead of show advanced

---
 .../components/auto-form/auto-form.spec.tsx   |  2 +-
 .../src/components/auto-form/auto-form.tsx    | 28 ++++----
 .../src/druid-models/ingestion-spec.tsx       | 69 ++++++++++---------
 .../views/load-data-view/info-messages.tsx    |  2 +-
 .../views/load-data-view/load-data-view.tsx   |  7 +-
 5 files changed, 58 insertions(+), 50 deletions(-)

diff --git a/web-console/src/components/auto-form/auto-form.spec.tsx b/web-console/src/components/auto-form/auto-form.spec.tsx
index 9bc108b450f1..c2beb3d498b4 100644
--- a/web-console/src/components/auto-form/auto-form.spec.tsx
+++ b/web-console/src/components/auto-form/auto-form.spec.tsx
@@ -36,7 +36,7 @@ describe('AutoForm', () => {
           { name: 'testSeven', type: 'json' },
 
           { name: 'testNotDefined', type: 'string', defined: false },
-          { name: 'testAdvanced', type: 'string', advanced: true },
+          { name: 'testAdvanced', type: 'string', hideInMore: true },
         ]}
         model={String}
         onChange={() => {}}
diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx
index 302e89f13d27..955dbed1a219 100644
--- a/web-console/src/components/auto-form/auto-form.tsx
+++ b/web-console/src/components/auto-form/auto-form.tsx
@@ -55,7 +55,7 @@ export interface Field<M> {
   disabled?: Functor<M, boolean>;
   defined?: Functor<M, boolean>;
   required?: Functor<M, boolean>;
-  advanced?: Functor<M, boolean>;
+  hideInMore?: Functor<M, boolean>;
   adjustment?: (model: M) => M;
   issueWithValue?: (value: any) => string | undefined;
 }
@@ -71,7 +71,7 @@ export interface AutoFormProps<M> {
 }
 
 export interface AutoFormState {
-  showAdvanced: boolean;
+  showMore: boolean;
 }
 
 export class AutoForm<T extends Record<string, any>> extends React.PureComponent<
@@ -148,7 +148,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
   constructor(props: AutoFormProps<T>) {
     super(props);
     this.state = {
-      showAdvanced: false,
+      showMore: false,
     };
   }
 
@@ -425,18 +425,18 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
     );
   }
 
-  renderAdvanced() {
-    const { showAdvanced } = this.state;
+  renderMoreOrLess() {
+    const { showMore } = this.state;
 
     return (
       <FormGroup key="advanced">
         <Button
-          text={showAdvanced ? 'Hide advanced' : 'Show advanced'}
-          rightIcon={showAdvanced ? IconNames.CHEVRON_UP : IconNames.CHEVRON_DOWN}
+          text={showMore ? 'Show less' : 'Show more'}
+          rightIcon={showMore ? IconNames.CHEVRON_UP : IconNames.CHEVRON_DOWN}
           minimal
           fill
           onClick={() => {
-            this.setState(({ showAdvanced }) => ({ showAdvanced: !showAdvanced }));
+            this.setState(({ showMore }) => ({ showMore: !showMore }));
           }}
         />
       </FormGroup>
@@ -445,14 +445,14 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
 
   render(): JSX.Element {
     const { fields, model, showCustom } = this.props;
-    const { showAdvanced } = this.state;
+    const { showMore } = this.state;
 
-    let hasAdvanced = false;
+    let shouldShowMore = false;
     const shownFields = fields.filter(field => {
       if (AutoForm.evaluateFunctor(field.defined, model, true)) {
-        if (AutoForm.evaluateFunctor(field.advanced, model, false)) {
-          hasAdvanced = true;
-          return showAdvanced;
+        if (AutoForm.evaluateFunctor(field.hideInMore, model, false)) {
+          shouldShowMore = true;
+          return showMore;
         }
         return true;
       } else {
@@ -464,7 +464,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
       <div className="auto-form">
         {model && shownFields.map(this.renderField)}
         {model && showCustom && showCustom(model) && this.renderCustom()}
-        {hasAdvanced && this.renderAdvanced()}
+        {shouldShowMore && this.renderMoreOrLess()}
       </div>
     );
   }
diff --git a/web-console/src/druid-models/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx
index dadc1851b330..523c6da01ff4 100644
--- a/web-console/src/druid-models/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec.tsx
@@ -475,7 +475,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           label: 'Dimensions',
           type: 'string-array',
           placeholder: '(optional)',
-          advanced: true,
+          hideInMore: true,
           info: (
             <p>
               The list of dimensions to select. If left empty, no dimensions are returned. If left
@@ -488,7 +488,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           label: 'Metrics',
           type: 'string-array',
           placeholder: '(optional)',
-          advanced: true,
+          hideInMore: true,
           info: (
             <p>
               The list of metrics to select. If left empty, no metrics are returned. If left null or
@@ -501,7 +501,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
           label: 'Filter',
           type: 'json',
           placeholder: '(optional)',
-          advanced: true,
+          hideInMore: true,
           info: (
             <p>
               The{' '}
@@ -1567,6 +1567,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 3,
     defined: (t: TuningConfig) => t.type === 'index_parallel',
+    hideInMore: true,
     info: <>Maximum number of retries on task failures.</>,
   },
   {
@@ -1574,21 +1575,9 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 1000,
     defined: (t: TuningConfig) => t.type === 'index_parallel',
-    advanced: true,
+    hideInMore: true,
     info: <>Polling period in milliseconds to check running task statuses.</>,
   },
-  {
-    name: 'maxRowsInMemory',
-    type: 'number',
-    defaultValue: 1000000,
-    info: <>Used in determining when intermediate persists to disk should occur.</>,
-  },
-  {
-    name: 'maxBytesInMemory',
-    type: 'number',
-    placeholder: 'Default: 1/6 of max JVM memory',
-    info: <>Used in determining when intermediate persists to disk should occur.</>,
-  },
   {
     name: 'totalNumMergeTasks',
     type: 'number',
@@ -1597,7 +1586,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     defined: (t: TuningConfig) =>
       Boolean(
         t.type === 'index_parallel' &&
-          oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'hashed', 'single_dim'),
+          oneOf(deepGet(t, 'partitionsSpec.type'), 'hashed', 'single_dim'),
       ),
     info: <>Number of tasks to merge partial segments after shuffle.</>,
   },
@@ -1608,7 +1597,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     defined: (t: TuningConfig) =>
       Boolean(
         t.type === 'index_parallel' &&
-          oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'hashed', 'single_dim'),
+          oneOf(deepGet(t, 'partitionsSpec.type'), 'hashed', 'single_dim'),
       ),
     info: (
       <>
@@ -1616,6 +1605,18 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
       </>
     ),
   },
+  {
+    name: 'maxRowsInMemory',
+    type: 'number',
+    defaultValue: 1000000,
+    info: <>Used in determining when intermediate persists to disk should occur.</>,
+  },
+  {
+    name: 'maxBytesInMemory',
+    type: 'number',
+    placeholder: 'Default: 1/6 of max JVM memory',
+    info: <>Used in determining when intermediate persists to disk should occur.</>,
+  },
   {
     name: 'resetOffsetAutomatically',
     type: 'boolean',
@@ -1664,7 +1665,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
   {
     name: 'maxPendingPersists',
     type: 'number',
-    advanced: true,
+    hideInMore: true,
     info: (
       <>
         Maximum number of persists that can be pending but not started. If this limit would be
@@ -1677,7 +1678,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     name: 'pushTimeout',
     type: 'number',
     defaultValue: 0,
-    advanced: true,
+    hideInMore: true,
     info: (
       <>
         Milliseconds to wait for pushing segments. It must be >= 0, where 0 means to wait forever.
@@ -1689,7 +1690,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 0,
     defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'),
-    advanced: true,
+    hideInMore: true,
     info: <>Milliseconds to wait for segment handoff. 0 means to wait forever.</>,
   },
   {
@@ -1698,6 +1699,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'string',
     defaultValue: 'roaring',
     suggestions: ['concise', 'roaring'],
+    hideInMore: true,
     info: <>Compression format for bitmap indexes.</>,
   },
   {
@@ -1706,6 +1708,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'string',
     defaultValue: 'lz4',
     suggestions: ['lz4', 'lzf', 'uncompressed'],
+    hideInMore: true,
     info: <>Compression format for dimension columns.</>,
   },
   {
@@ -1714,6 +1717,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'string',
     defaultValue: 'lz4',
     suggestions: ['lz4', 'lzf', 'uncompressed'],
+    hideInMore: true,
     info: <>Compression format for primitive type metric columns.</>,
   },
   {
@@ -1722,6 +1726,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'string',
     defaultValue: 'longs',
     suggestions: ['longs', 'auto'],
+    hideInMore: true,
     info: (
       <>
         Encoding format for long-typed columns. Applies regardless of whether they are dimensions or
@@ -1736,7 +1741,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'duration',
     defaultValue: 'PT10S',
     defined: (t: TuningConfig) => t.type === 'index_parallel',
-    advanced: true,
+    hideInMore: true,
     info: <>Timeout for reporting the pushed segments in worker tasks.</>,
   },
   {
@@ -1744,7 +1749,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 5,
     defined: (t: TuningConfig) => t.type === 'index_parallel',
-    advanced: true,
+    hideInMore: true,
     info: <>Retries for reporting the pushed segments in worker tasks.</>,
   },
   {
@@ -1761,7 +1766,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     placeholder: 'min(10, taskCount * replicas)',
     defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'),
-    advanced: true,
+    hideInMore: true,
     info: <>The number of threads that will be used for communicating with indexing tasks.</>,
   },
   {
@@ -1769,7 +1774,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 8,
     defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'),
-    advanced: true,
+    hideInMore: true,
     info: (
       <>
         The number of times HTTP requests to indexing tasks will be retried before considering tasks
@@ -1789,7 +1794,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'duration',
     defaultValue: 'PT80S',
     defined: (t: TuningConfig) => oneOf(t.type, 'kafka', 'kinesis'),
-    advanced: true,
+    hideInMore: true,
     info: (
       <>
         How long to wait for the supervisor to attempt a graceful shutdown of tasks before exiting.
@@ -1825,7 +1830,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 5000,
     defined: (t: TuningConfig) => t.type === 'kinesis',
-    advanced: true,
+    hideInMore: true,
     info: (
       <>
         Length of time in milliseconds to wait for space to become available in the buffer before
@@ -1835,7 +1840,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
   },
   {
     name: 'recordBufferFullWait',
-    advanced: true,
+    hideInMore: true,
     type: 'number',
     defaultValue: 5000,
     defined: (t: TuningConfig) => t.type === 'kinesis',
@@ -1851,7 +1856,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 60000,
     defined: (t: TuningConfig) => t.type === 'kinesis',
-    advanced: true,
+    hideInMore: true,
     info: (
       <>
         Length of time in milliseconds to wait for Kinesis to return the earliest or latest sequence
@@ -1866,7 +1871,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     placeholder: 'max(1, {numProcessors} - 1)',
     defined: (t: TuningConfig) => t.type === 'kinesis',
-    advanced: true,
+    hideInMore: true,
     info: (
       <>
         Size of the pool of threads fetching data from Kinesis. There is no benefit in having more
@@ -1879,7 +1884,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'number',
     defaultValue: 100,
     defined: (t: TuningConfig) => t.type === 'kinesis',
-    advanced: true,
+    hideInMore: true,
     info: (
       <>
         The maximum number of records/events to be fetched from buffer per poll. The actual maximum
@@ -1892,7 +1897,7 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     type: 'duration',
     defaultValue: 'PT2M',
     defined: (t: TuningConfig) => t.type === 'kinesis',
-    advanced: true,
+    hideInMore: true,
     info: (
       <>
         <p>
diff --git a/web-console/src/views/load-data-view/info-messages.tsx b/web-console/src/views/load-data-view/info-messages.tsx
index ffc7a61de603..86f6f92f7d9c 100644
--- a/web-console/src/views/load-data-view/info-messages.tsx
+++ b/web-console/src/views/load-data-view/info-messages.tsx
@@ -184,13 +184,13 @@ export const PublishMessage = React.memo(function PublishMessage() {
 export const SpecMessage = React.memo(function SpecMessage() {
   return (
     <Callout className="intro">
-      <p className="optional">Optional</p>
       <p>
         Druid begins ingesting data once you submit a JSON ingestion spec. If you modify any values
         in this view, the values entered in previous sections will update accordingly. If you modify
         any values in previous sections, this spec will automatically update.
       </p>
       <p>Submit the spec to begin loading data into Druid.</p>
+      <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#ingestion-specs`} />
     </Callout>
   );
 });
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index e2bcd967d5b8..b820e66f95df 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -262,8 +262,11 @@ const STEPS: Step[] = [
 ];
 
 const SECTIONS: { name: string; steps: Step[] }[] = [
-  { name: 'Connect and parse raw data', steps: ['welcome', 'connect', 'parser', 'timestamp'] },
-  { name: 'Transform data and configure schema', steps: ['transform', 'filter', 'schema'] },
+  { name: 'Connect and parse raw data', steps: ['welcome', 'connect', 'parser'] },
+  {
+    name: 'Transform data and configure schema',
+    steps: ['timestamp', 'transform', 'filter', 'schema'],
+  },
   { name: 'Tune parameters', steps: ['partition', 'tuning', 'publish'] },
   { name: 'Verify and submit', steps: ['spec'] },
 ];

From ab909c6df82e5286d4e2eebb5119c22058f31719 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Fri, 13 Nov 2020 09:33:21 -0800
Subject: [PATCH 08/13] fix tests

---
 .../__snapshots__/auto-form.spec.tsx.snap     |  2 +-
 .../load-data-view.spec.tsx.snap              | 18 +++++++-------
 .../views/load-data-view/info-messages.tsx    |  9 ++-----
 .../views/load-data-view/load-data-view.tsx   | 24 -------------------
 4 files changed, 12 insertions(+), 41 deletions(-)

diff --git a/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap b/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap
index aa734cd821cf..19ea8f188c67 100644
--- a/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap
+++ b/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap
@@ -140,7 +140,7 @@ exports[`AutoForm matches snapshot 1`] = `
       minimal={true}
       onClick={[Function]}
       rightIcon="chevron-down"
-      text="Show advanced"
+      text="Show more"
     />
   </Blueprint3.FormGroup>
 </div>
diff --git a/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap b/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
index 80bbabfcab34..ef5cec674717 100644
--- a/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
+++ b/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
@@ -46,15 +46,6 @@ exports[`load data view matches snapshot 1`] = `
           onClick={[Function]}
           text="Parse data"
         />
-        <Blueprint3.Button
-          active={false}
-          className="timestamp"
-          disabled={true}
-          icon={false}
-          key="timestamp"
-          onClick={[Function]}
-          text="Parse time"
-        />
       </Blueprint3.ButtonGroup>
     </div>
     <div
@@ -69,6 +60,15 @@ exports[`load data view matches snapshot 1`] = `
       <Blueprint3.ButtonGroup
         className="step-nav-l2"
       >
+        <Blueprint3.Button
+          active={false}
+          className="timestamp"
+          disabled={true}
+          icon={false}
+          key="timestamp"
+          onClick={[Function]}
+          text="Parse time"
+        />
         <Blueprint3.Button
           active={false}
           className="transform"
diff --git a/web-console/src/views/load-data-view/info-messages.tsx b/web-console/src/views/load-data-view/info-messages.tsx
index 86f6f92f7d9c..c12352505b88 100644
--- a/web-console/src/views/load-data-view/info-messages.tsx
+++ b/web-console/src/views/load-data-view/info-messages.tsx
@@ -103,7 +103,6 @@ export const TimestampMessage = React.memo(function TimestampMessage() {
 export const TransformMessage = React.memo(function TransformMessage() {
   return (
     <Callout className="intro">
-      <p className="optional">Optional</p>
       <p>
         Druid can perform per-row{' '}
         <ExternalLink href={`${getLink('DOCS')}/ingestion/transform-spec.html#transforms`}>
@@ -119,11 +118,9 @@ export const TransformMessage = React.memo(function TransformMessage() {
 export const FilterMessage = React.memo(function FilterMessage() {
   return (
     <Callout className="intro">
-      <p className="optional">Optional</p>
       <p>
-        Druid can{' '}
-        <ExternalLink href={`${getLink('DOCS')}/querying/filters.html`}>filter</ExternalLink> out
-        unwanted data by applying per-row filters.
+        Druid can filter out unwanted data by applying per-row{' '}
+        <ExternalLink href={`${getLink('DOCS')}/querying/filters.html`}>filters</ExternalLink>.
       </p>
       <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#filter`} />
     </Callout>
@@ -156,7 +153,6 @@ export const SchemaMessage = React.memo(function SchemaMessage(props: SchemaMess
 export const PartitionMessage = React.memo(function PartitionMessage() {
   return (
     <Callout className="intro">
-      <p className="optional">Optional</p>
       <p>Configure how Druid will partition data.</p>
       <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#partitioning`} />
     </Callout>
@@ -166,7 +162,6 @@ export const PartitionMessage = React.memo(function PartitionMessage() {
 export const TuningMessage = React.memo(function TuningMessage() {
   return (
     <Callout className="intro">
-      <p className="optional">Optional</p>
       <p>Fine tune how Druid will ingest data.</p>
       <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#tuningconfig`} />
     </Callout>
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index b820e66f95df..d44d76d08d27 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -2818,19 +2818,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       });
     };
 
-    const metricsSpec = deepGet(spec, `spec.dataSchema.metricsSpec`);
-
-    const moveTo = (newIndex: number) => {
-      const newMetricsSpec = moveElement(metricsSpec, selectedMetricSpecIndex, newIndex);
-      const newSpec = deepSet(spec, `spec.dataSchema.metricsSpec`, newMetricsSpec);
-      this.updateSpec(newSpec);
-      close();
-    };
-
-    const reorderMetricMenu = (
-      <ReorderMenu things={metricsSpec} selectedIndex={selectedMetricSpecIndex} moveTo={moveTo} />
-    );
-
     const convertToDimension = (type: string) => {
       const specWithoutMetric = deepDelete(
         spec,
@@ -2866,17 +2853,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           model={selectedMetricSpec}
           onChange={selectedMetricSpec => this.setState({ selectedMetricSpec })}
         />
-        {reorderMetricMenu && (
-          <FormGroup>
-            <Popover content={reorderMetricMenu}>
-              <Button
-                icon={IconNames.ARROWS_HORIZONTAL}
-                text="Reorder metric"
-                rightIcon={IconNames.CARET_DOWN}
-              />
-            </Popover>
-          </FormGroup>
-        )}
         {selectedMetricSpecIndex !== -1 && dimensionMode === 'specific' && (
           <FormGroup>
             <Popover content={convertToDimensionMenu}>

From bd9cec4633b6f99a1d1c8090a54464952d86715e Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Fri, 13 Nov 2020 11:11:24 -0800
Subject: [PATCH 09/13] extract dynamic configs

---
 .../coordinator-dynamic-config-dialog.tsx     | 198 ++---------------
 .../overlord-dynamic-config-dialog.tsx        |  16 +-
 .../coordinator-dynamic-config.tsx            | 208 ++++++++++++++++++
 web-console/src/druid-models/index.ts         |   2 +
 .../druid-models/overlord-dynamic-config.tsx  |  35 +++
 5 files changed, 270 insertions(+), 189 deletions(-)
 create mode 100644 web-console/src/druid-models/coordinator-dynamic-config.tsx
 create mode 100644 web-console/src/druid-models/overlord-dynamic-config.tsx

diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx b/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx
index bfc197cc606d..0ec905463a16 100644
--- a/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx
+++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/coordinator-dynamic-config-dialog.tsx
@@ -16,13 +16,20 @@
  * limitations under the License.
  */
 
-import { Code, Intent } from '@blueprintjs/core';
+import { Intent } from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
 import axios from 'axios';
 import React, { useState } from 'react';
 
 import { SnitchDialog } from '..';
-import { AutoForm, ExternalLink } from '../../components';
+import {
+  AutoForm,
+  ExternalLink,
+  FormJsonSelector,
+  FormJsonTabs,
+  JsonInput,
+} from '../../components';
+import { COORDINATOR_DYNAMIC_CONFIG_FIELDS, CoordinatorDynamicConfig } from '../../druid-models';
 import { useQueryManager } from '../../hooks';
 import { getLink } from '../../links';
 import { AppToaster } from '../../singletons/toaster';
@@ -38,7 +45,8 @@ export const CoordinatorDynamicConfigDialog = React.memo(function CoordinatorDyn
   props: CoordinatorDynamicConfigDialogProps,
 ) {
   const { onClose } = props;
-  const [dynamicConfig, setDynamicConfig] = useState<Record<string, any>>({});
+  const [currentTab, setCurrentTab] = useState<FormJsonTabs>('form');
+  const [dynamicConfig, setDynamicConfig] = useState<CoordinatorDynamicConfig>({});
 
   const [historyRecordsState] = useQueryManager<null, any[]>({
     processQuery: async () => {
@@ -106,180 +114,16 @@ export const CoordinatorDynamicConfigDialog = React.memo(function CoordinatorDyn
         </ExternalLink>
         .
       </p>
-      <AutoForm
-        fields={[
-          {
-            name: 'maxSegmentsToMove',
-            type: 'number',
-            defaultValue: 5,
-            info: <>The maximum number of segments that can be moved at any given time.</>,
-          },
-          {
-            name: 'balancerComputeThreads',
-            type: 'number',
-            defaultValue: 1,
-            info: (
-              <>
-                Thread pool size for computing moving cost of segments in segment balancing.
-                Consider increasing this if you have a lot of segments and moving segments starts to
-                get stuck.
-              </>
-            ),
-          },
-          {
-            name: 'emitBalancingStats',
-            type: 'boolean',
-            defaultValue: false,
-            info: (
-              <>
-                Boolean flag for whether or not we should emit balancing stats. This is an expensive
-                operation.
-              </>
-            ),
-          },
-          {
-            name: 'killAllDataSources',
-            type: 'boolean',
-            defaultValue: false,
-            info: (
-              <>
-                Send kill tasks for ALL dataSources if property{' '}
-                <Code>druid.coordinator.kill.on</Code> is true. If this is set to true then{' '}
-                <Code>killDataSourceWhitelist</Code> must not be specified or be empty list.
-              </>
-            ),
-          },
-          {
-            name: 'killDataSourceWhitelist',
-            type: 'string-array',
-            emptyValue: [],
-            info: (
-              <>
-                List of dataSources for which kill tasks are sent if property{' '}
-                <Code>druid.coordinator.kill.on</Code> is true. This can be a list of
-                comma-separated dataSources or a JSON array.
-              </>
-            ),
-          },
-          {
-            name: 'killPendingSegmentsSkipList',
-            type: 'string-array',
-            emptyValue: [],
-            info: (
-              <>
-                List of dataSources for which pendingSegments are NOT cleaned up if property{' '}
-                <Code>druid.coordinator.kill.pendingSegments.on</Code> is true. This can be a list
-                of comma-separated dataSources or a JSON array.
-              </>
-            ),
-          },
-          {
-            name: 'maxSegmentsInNodeLoadingQueue',
-            type: 'number',
-            defaultValue: 0,
-            info: (
-              <>
-                The maximum number of segments that could be queued for loading to any given server.
-                This parameter could be used to speed up segments loading process, especially if
-                there are "slow" nodes in the cluster (with low loading speed) or if too much
-                segments scheduled to be replicated to some particular node (faster loading could be
-                preferred to better segments distribution). Desired value depends on segments
-                loading speed, acceptable replication time and number of nodes. Value 1000 could be
-                a start point for a rather big cluster. Default value is 0 (loading queue is
-                unbounded)
-              </>
-            ),
-          },
-          {
-            name: 'mergeBytesLimit',
-            type: 'size-bytes',
-            defaultValue: 524288000,
-            info: <>The maximum total uncompressed size in bytes of segments to merge.</>,
-          },
-          {
-            name: 'mergeSegmentsLimit',
-            type: 'number',
-            defaultValue: 100,
-            info: <>The maximum number of segments that can be in a single append task.</>,
-          },
-          {
-            name: 'millisToWaitBeforeDeleting',
-            type: 'number',
-            defaultValue: 900000,
-            info: (
-              <>
-                How long does the Coordinator need to be active before it can start removing
-                (marking unused) segments in metadata storage.
-              </>
-            ),
-          },
-          {
-            name: 'replicantLifetime',
-            type: 'number',
-            defaultValue: 15,
-            info: (
-              <>
-                The maximum number of Coordinator runs for a segment to be replicated before we
-                start alerting.
-              </>
-            ),
-          },
-          {
-            name: 'replicationThrottleLimit',
-            type: 'number',
-            defaultValue: 10,
-            info: <>The maximum number of segments that can be replicated at one time.</>,
-          },
-          {
-            name: 'decommissioningNodes',
-            type: 'string-array',
-            emptyValue: [],
-            info: (
-              <>
-                List of historical services to 'decommission'. Coordinator will not assign new
-                segments to 'decommissioning' services, and segments will be moved away from them to
-                be placed on non-decommissioning services at the maximum rate specified by{' '}
-                <Code>decommissioningMaxPercentOfMaxSegmentsToMove</Code>.
-              </>
-            ),
-          },
-          {
-            name: 'decommissioningMaxPercentOfMaxSegmentsToMove',
-            type: 'number',
-            defaultValue: 70,
-            info: (
-              <>
-                The maximum number of segments that may be moved away from 'decommissioning'
-                services to non-decommissioning (that is, active) services during one Coordinator
-                run. This value is relative to the total maximum segment movements allowed during
-                one run which is determined by <Code>maxSegmentsToMove</Code>. If
-                <Code>decommissioningMaxPercentOfMaxSegmentsToMove</Code> is 0, segments will
-                neither be moved from or to 'decommissioning' services, effectively putting them in
-                a sort of "maintenance" mode that will not participate in balancing or assignment by
-                load rules. Decommissioning can also become stalled if there are no available active
-                services to place the segments. By leveraging the maximum percent of decommissioning
-                segment movements, an operator can prevent active services from overload by
-                prioritizing balancing, or decrease decommissioning time instead. The value should
-                be between 0 and 100.
-              </>
-            ),
-          },
-          {
-            name: 'pauseCoordination',
-            type: 'boolean',
-            defaultValue: false,
-            info: (
-              <>
-                Boolean flag for whether or not the coordinator should execute its various duties of
-                coordinating the cluster. Setting this to true essentially pauses all coordination
-                work while allowing the API to remain up.
-              </>
-            ),
-          },
-        ]}
-        model={dynamicConfig}
-        onChange={m => setDynamicConfig(m)}
-      />
+      <FormJsonSelector tab={currentTab} onChange={setCurrentTab} />
+      {currentTab === 'form' ? (
+        <AutoForm
+          fields={COORDINATOR_DYNAMIC_CONFIG_FIELDS}
+          model={dynamicConfig}
+          onChange={setDynamicConfig}
+        />
+      ) : (
+        <JsonInput value={dynamicConfig} onChange={setDynamicConfig} height="50vh" />
+      )}
     </SnitchDialog>
   );
 });
diff --git a/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx b/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx
index 57ced4b9ac69..d54cba605213 100644
--- a/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx
+++ b/web-console/src/dialogs/overlord-dynamic-config-dialog/overlord-dynamic-config-dialog.tsx
@@ -23,6 +23,7 @@ import React, { useState } from 'react';
 
 import { SnitchDialog } from '..';
 import { AutoForm, ExternalLink } from '../../components';
+import { OVERLORD_DYNAMIC_CONFIG_FIELDS, OverlordDynamicConfig } from '../../druid-models';
 import { useQueryManager } from '../../hooks';
 import { getLink } from '../../links';
 import { AppToaster } from '../../singletons/toaster';
@@ -38,7 +39,7 @@ export const OverlordDynamicConfigDialog = React.memo(function OverlordDynamicCo
   props: OverlordDynamicConfigDialogProps,
 ) {
   const { onClose } = props;
-  const [dynamicConfig, setDynamicConfig] = useState<Record<string, any>>({});
+  const [dynamicConfig, setDynamicConfig] = useState<OverlordDynamicConfig>({});
 
   const [historyRecordsState] = useQueryManager<null, any[]>({
     processQuery: async () => {
@@ -108,18 +109,9 @@ export const OverlordDynamicConfigDialog = React.memo(function OverlordDynamicCo
         .
       </p>
       <AutoForm
-        fields={[
-          {
-            name: 'selectStrategy',
-            type: 'json',
-          },
-          {
-            name: 'autoScaler',
-            type: 'json',
-          },
-        ]}
+        fields={OVERLORD_DYNAMIC_CONFIG_FIELDS}
         model={dynamicConfig}
-        onChange={m => setDynamicConfig(m)}
+        onChange={setDynamicConfig}
       />
     </SnitchDialog>
   );
diff --git a/web-console/src/druid-models/coordinator-dynamic-config.tsx b/web-console/src/druid-models/coordinator-dynamic-config.tsx
new file mode 100644
index 000000000000..b5621ce31f8a
--- /dev/null
+++ b/web-console/src/druid-models/coordinator-dynamic-config.tsx
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Code } from '@blueprintjs/core';
+import React from 'react';
+
+import { Field } from '../components';
+
+export interface CoordinatorDynamicConfig {
+  maxSegmentsToMove?: number;
+  balancerComputeThreads?: number;
+  emitBalancingStats?: boolean;
+  killAllDataSources?: boolean;
+  killDataSourceWhitelist?: string[];
+  killPendingSegmentsSkipList?: string[];
+  maxSegmentsInNodeLoadingQueue?: number;
+  mergeBytesLimit?: number;
+  mergeSegmentsLimit?: number;
+  millisToWaitBeforeDeleting?: number;
+  replicantLifetime?: number;
+  replicationThrottleLimit?: number;
+  decommissioningNodes?: string[];
+  decommissioningMaxPercentOfMaxSegmentsToMove?: number;
+  pauseCoordination?: boolean;
+}
+
+export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field<CoordinatorDynamicConfig>[] = [
+  {
+    name: 'maxSegmentsToMove',
+    type: 'number',
+    defaultValue: 5,
+    info: <>The maximum number of segments that can be moved at any given time.</>,
+  },
+  {
+    name: 'balancerComputeThreads',
+    type: 'number',
+    defaultValue: 1,
+    info: (
+      <>
+        Thread pool size for computing moving cost of segments in segment balancing. Consider
+        increasing this if you have a lot of segments and moving segments starts to get stuck.
+      </>
+    ),
+  },
+  {
+    name: 'emitBalancingStats',
+    type: 'boolean',
+    defaultValue: false,
+    info: (
+      <>
+        Boolean flag for whether or not we should emit balancing stats. This is an expensive
+        operation.
+      </>
+    ),
+  },
+  {
+    name: 'killAllDataSources',
+    type: 'boolean',
+    defaultValue: false,
+    info: (
+      <>
+        Send kill tasks for ALL dataSources if property <Code>druid.coordinator.kill.on</Code> is
+        true. If this is set to true then <Code>killDataSourceWhitelist</Code> must not be specified
+        or be empty list.
+      </>
+    ),
+  },
+  {
+    name: 'killDataSourceWhitelist',
+    type: 'string-array',
+    emptyValue: [],
+    info: (
+      <>
+        List of dataSources for which kill tasks are sent if property{' '}
+        <Code>druid.coordinator.kill.on</Code> is true. This can be a list of comma-separated
+        dataSources or a JSON array.
+      </>
+    ),
+  },
+  {
+    name: 'killPendingSegmentsSkipList',
+    type: 'string-array',
+    emptyValue: [],
+    info: (
+      <>
+        List of dataSources for which pendingSegments are NOT cleaned up if property{' '}
+        <Code>druid.coordinator.kill.pendingSegments.on</Code> is true. This can be a list of
+        comma-separated dataSources or a JSON array.
+      </>
+    ),
+  },
+  {
+    name: 'maxSegmentsInNodeLoadingQueue',
+    type: 'number',
+    defaultValue: 0,
+    info: (
+      <>
+        The maximum number of segments that could be queued for loading to any given server. This
+        parameter could be used to speed up segments loading process, especially if there are "slow"
+        nodes in the cluster (with low loading speed) or if too much segments scheduled to be
+        replicated to some particular node (faster loading could be preferred to better segments
+        distribution). Desired value depends on segments loading speed, acceptable replication time
+        and number of nodes. Value 1000 could be a start point for a rather big cluster. Default
+        value is 0 (loading queue is unbounded)
+      </>
+    ),
+  },
+  {
+    name: 'mergeBytesLimit',
+    type: 'size-bytes',
+    defaultValue: 524288000,
+    info: <>The maximum total uncompressed size in bytes of segments to merge.</>,
+  },
+  {
+    name: 'mergeSegmentsLimit',
+    type: 'number',
+    defaultValue: 100,
+    info: <>The maximum number of segments that can be in a single append task.</>,
+  },
+  {
+    name: 'millisToWaitBeforeDeleting',
+    type: 'number',
+    defaultValue: 900000,
+    info: (
+      <>
+        How long does the Coordinator need to be active before it can start removing (marking
+        unused) segments in metadata storage.
+      </>
+    ),
+  },
+  {
+    name: 'replicantLifetime',
+    type: 'number',
+    defaultValue: 15,
+    info: (
+      <>
+        The maximum number of Coordinator runs for a segment to be replicated before we start
+        alerting.
+      </>
+    ),
+  },
+  {
+    name: 'replicationThrottleLimit',
+    type: 'number',
+    defaultValue: 10,
+    info: <>The maximum number of segments that can be replicated at one time.</>,
+  },
+  {
+    name: 'decommissioningNodes',
+    type: 'string-array',
+    emptyValue: [],
+    info: (
+      <>
+        List of historical services to 'decommission'. Coordinator will not assign new segments to
+        'decommissioning' services, and segments will be moved away from them to be placed on
+        non-decommissioning services at the maximum rate specified by{' '}
+        <Code>decommissioningMaxPercentOfMaxSegmentsToMove</Code>.
+      </>
+    ),
+  },
+  {
+    name: 'decommissioningMaxPercentOfMaxSegmentsToMove',
+    type: 'number',
+    defaultValue: 70,
+    info: (
+      <>
+        The maximum number of segments that may be moved away from 'decommissioning' services to
+        non-decommissioning (that is, active) services during one Coordinator run. This value is
+        relative to the total maximum segment movements allowed during one run which is determined
+        by <Code>maxSegmentsToMove</Code>. If
+        <Code>decommissioningMaxPercentOfMaxSegmentsToMove</Code> is 0, segments will neither be
+        moved from or to 'decommissioning' services, effectively putting them in a sort of
+        "maintenance" mode that will not participate in balancing or assignment by load rules.
+        Decommissioning can also become stalled if there are no available active services to place
+        the segments. By leveraging the maximum percent of decommissioning segment movements, an
+        operator can prevent active services from overload by prioritizing balancing, or decrease
+        decommissioning time instead. The value should be between 0 and 100.
+      </>
+    ),
+  },
+  {
+    name: 'pauseCoordination',
+    type: 'boolean',
+    defaultValue: false,
+    info: (
+      <>
+        Boolean flag for whether or not the coordinator should execute its various duties of
+        coordinating the cluster. Setting this to true essentially pauses all coordination work
+        while allowing the API to remain up.
+      </>
+    ),
+  },
+];
diff --git a/web-console/src/druid-models/index.ts b/web-console/src/druid-models/index.ts
index 5e7debe93cf2..c5186110c447 100644
--- a/web-console/src/druid-models/index.ts
+++ b/web-console/src/druid-models/index.ts
@@ -29,3 +29,5 @@ export * from './filter';
 export * from './dimension-spec';
 export * from './metric-spec';
 export * from './ingestion-spec';
+export * from './coordinator-dynamic-config';
+export * from './overlord-dynamic-config';
diff --git a/web-console/src/druid-models/overlord-dynamic-config.tsx b/web-console/src/druid-models/overlord-dynamic-config.tsx
new file mode 100644
index 000000000000..14cac8295113
--- /dev/null
+++ b/web-console/src/druid-models/overlord-dynamic-config.tsx
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Field } from '../components';
+
+export interface OverlordDynamicConfig {
+  selectStrategy?: Record<string, any>;
+  autoScaler?: Record<string, any>;
+}
+
+export const OVERLORD_DYNAMIC_CONFIG_FIELDS: Field<OverlordDynamicConfig>[] = [
+  {
+    name: 'selectStrategy',
+    type: 'json',
+  },
+  {
+    name: 'autoScaler',
+    type: 'json',
+  },
+];

From 48d7486b12d533aee1c25e66eb58591dfb66791b Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Fri, 13 Nov 2020 11:25:38 -0800
Subject: [PATCH 10/13] update snapshots

---
 .../auto-form/__snapshots__/auto-form.spec.tsx.snap    |  2 +-
 web-console/src/components/auto-form/auto-form.tsx     |  2 +-
 .../coordinator-dynamic-config-dialog.spec.tsx.snap    | 10 ++++++----
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap b/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap
index 19ea8f188c67..baf1011bdc8e 100644
--- a/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap
+++ b/web-console/src/components/auto-form/__snapshots__/auto-form.spec.tsx.snap
@@ -133,7 +133,7 @@ exports[`AutoForm matches snapshot 1`] = `
     />
   </Memo(FormGroupWithInfo)>
   <Blueprint3.FormGroup
-    key="advanced"
+    key="more-or-less"
   >
     <Blueprint3.Button
       fill={true}
diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx
index 955dbed1a219..3b2f6c0da0e5 100644
--- a/web-console/src/components/auto-form/auto-form.tsx
+++ b/web-console/src/components/auto-form/auto-form.tsx
@@ -429,7 +429,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
     const { showMore } = this.state;
 
     return (
-      <FormGroup key="advanced">
+      <FormGroup key="more-or-less">
         <Button
           text={showMore ? 'Show less' : 'Show more'}
           rightIcon={showMore ? IconNames.CHEVRON_UP : IconNames.CHEVRON_DOWN}
diff --git a/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/coordinator-dynamic-config-dialog.spec.tsx.snap b/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/coordinator-dynamic-config-dialog.spec.tsx.snap
index f061add52092..50f17d44ee65 100644
--- a/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/coordinator-dynamic-config-dialog.spec.tsx.snap
+++ b/web-console/src/dialogs/coordinator-dynamic-config-dialog/__snapshots__/coordinator-dynamic-config-dialog.spec.tsx.snap
@@ -17,6 +17,10 @@ exports[`coordinator dynamic config matches snapshot 1`] = `
     </Memo(ExternalLink)>
     .
   </p>
+  <Memo(FormJsonSelector)
+    onChange={[Function]}
+    tab="form"
+  />
   <AutoForm
     fields={
       Array [
@@ -47,13 +51,11 @@ exports[`coordinator dynamic config matches snapshot 1`] = `
         Object {
           "defaultValue": false,
           "info": <React.Fragment>
-            Send kill tasks for ALL dataSources if property
-             
+            Send kill tasks for ALL dataSources if property 
             <Unknown>
               druid.coordinator.kill.on
             </Unknown>
-             is true. If this is set to true then
-             
+             is true. If this is set to true then 
             <Unknown>
               killDataSourceWhitelist
             </Unknown>

From 34acc23fedf43c51941c370e7c3714359b5c60e0 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Mon, 16 Nov 2020 11:51:28 -0800
Subject: [PATCH 11/13] fix issues

---
 .../src/druid-models/ingestion-spec.tsx       |  49 ++++++-
 .../views/load-data-view/info-messages.tsx    |   6 +-
 .../views/load-data-view/load-data-view.tsx   | 131 ++++++++++++++----
 .../schema-table/schema-table.tsx             |  11 +-
 4 files changed, 161 insertions(+), 36 deletions(-)

diff --git a/web-console/src/druid-models/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx
index 523c6da01ff4..3149c0b4fa33 100644
--- a/web-console/src/druid-models/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec.tsx
@@ -1384,6 +1384,7 @@ export function invalidTuningConfig(tuningConfig: TuningConfig, intervals: any):
 
 export function getPartitionRelatedTuningSpecFormFields(
   specType: IngestionType,
+  dimensionSuggestions: string[] | undefined,
 ): Field<TuningConfig>[] {
   switch (specType) {
     case 'index_parallel':
@@ -1401,6 +1402,10 @@ export function getPartitionRelatedTuningSpecFormFields(
               single dimension). For best-effort rollup, you should use <Code>dynamic</Code>.
             </p>
           ),
+          adjustment: (t: TuningConfig) => {
+            if (!Array.isArray(dimensionSuggestions) || !dimensionSuggestions.length) return t;
+            return deepSet(t, 'partitionsSpec.partitionDimension', dimensionSuggestions[0]);
+          },
         },
         // partitionsSpec type: dynamic
         {
@@ -1466,6 +1471,7 @@ export function getPartitionRelatedTuningSpecFormFields(
           name: 'partitionsSpec.partitionDimensions',
           label: 'Partition dimensions',
           type: 'string-array',
+          placeholder: '(all dimensions)',
           defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'hashed',
           info: <p>The dimensions to partition on. Leave blank to select all dimensions.</p>,
         },
@@ -1476,7 +1482,19 @@ export function getPartitionRelatedTuningSpecFormFields(
           type: 'string',
           defined: (t: TuningConfig) => deepGet(t, 'partitionsSpec.type') === 'single_dim',
           required: true,
-          info: <p>The dimension to partition on.</p>,
+          suggestions: dimensionSuggestions,
+          info: (
+            <>
+              <p>The dimension to partition on.</p>
+              <p>
+                This should be the first dimension in your schema which would make it first in the
+                sort order. As{' '}
+                <ExternalLink href={`${getLink('DOCS')}/ingestion/index.html#why-partition`}>
+                  Partitioning and sorting are best friends!
+                </ExternalLink>
+              </p>
+            </>
+          ),
         },
         {
           name: 'partitionsSpec.targetRowsPerSegment',
@@ -1956,6 +1974,35 @@ export function updateIngestionType(
   return newSpec;
 }
 
+export function issueWithSampleData(sampleData: string[]): JSX.Element | undefined {
+  console.log(sampleData);
+  if (sampleData.length) {
+    const firstData = sampleData[0];
+
+    if (firstData === '{') {
+      return (
+        <>
+          This data looks like regular JSON object. For Druid to parse a text file it must have one
+          row per event. Maybe look at{' '}
+          <ExternalLink href="http://ndjson.org/">newline delimited JSON</ExternalLink> instead.
+        </>
+      );
+    }
+
+    if (oneOf(firstData, '[', '[]')) {
+      return (
+        <>
+          This data looks like a multi-line JSON array. For Druid to parse a text file it must have
+          one row per event. Maybe look at{' '}
+          <ExternalLink href="http://ndjson.org/">newline delimited JSON</ExternalLink> instead.
+        </>
+      );
+    }
+  }
+
+  return;
+}
+
 export function fillInputFormat(spec: IngestionSpec, sampleData: string[]): IngestionSpec {
   return deepSet(spec, 'spec.ioConfig.inputFormat', guessInputFormat(sampleData));
 }
diff --git a/web-console/src/views/load-data-view/info-messages.tsx b/web-console/src/views/load-data-view/info-messages.tsx
index c12352505b88..217c84200caf 100644
--- a/web-console/src/views/load-data-view/info-messages.tsx
+++ b/web-console/src/views/load-data-view/info-messages.tsx
@@ -91,9 +91,9 @@ export const TimestampMessage = React.memo(function TimestampMessage() {
       </p>
       <p>Configure how to define the time column for this data.</p>
       <p>
-        If your data does not have a time column, you can select "None" to use a placeholder value.
-        If the time information is spread across multiple columns you can combine them into one by
-        selecting "Expression" and defining a transform expression.
+        If your data does not have a time column, you can select <Code>None</Code> to use a
+        placeholder value. If the time information is spread across multiple columns you can combine
+        them into one by selecting <Code>Expression</Code> and defining a transform expression.
       </p>
       <LearnMore href={`${getLink('DOCS')}/ingestion/index.html#timestampspec`} />
     </Callout>
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index d44d76d08d27..48768a8ba799 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -67,6 +67,7 @@ import {
   getTimestampExpressionFields,
   getTimestampSchema,
   INPUT_FORMAT_FIELDS,
+  issueWithSampleData,
   METRIC_SPEC_FIELDS,
   removeTimestampTransform,
   TIMESTAMP_SPEC_FIELDS,
@@ -460,18 +461,16 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         return Boolean(!druidSource && spec.type && !issueWithIoConfig(ioConfig));
 
       case 'timestamp':
-        return Boolean(
-          !druidSource && cacheRows && deepGet(spec, 'spec.dataSchema.timestampSpec.format'),
-        );
+        return Boolean(!druidSource && cacheRows && deepGet(spec, 'spec.dataSchema.timestampSpec'));
 
       case 'transform':
       case 'filter':
-        return Boolean(cacheRows && deepGet(spec, 'spec.dataSchema.timestampSpec.format'));
+        return Boolean(cacheRows && deepGet(spec, 'spec.dataSchema.timestampSpec'));
 
       case 'schema':
         return Boolean(
           cacheRows &&
-            deepGet(spec, 'spec.dataSchema.timestampSpec.format') &&
+            deepGet(spec, 'spec.dataSchema.timestampSpec') &&
             deepGet(spec, 'spec.dataSchema.dimensionsSpec'),
         );
 
@@ -480,7 +479,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       case 'publish':
         return Boolean(
           cacheRows &&
-            deepGet(spec, 'spec.dataSchema.timestampSpec.format') &&
+            deepGet(spec, 'spec.dataSchema.timestampSpec') &&
             deepGet(spec, 'spec.dataSchema.dimensionsSpec') &&
             deepGet(spec, 'spec.dataSchema.granularitySpec.type'),
         );
@@ -663,7 +662,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     );
   }
 
-  renderNextBar(options: { nextStep?: Step; disabled?: boolean; onNextStep?: () => void }) {
+  renderNextBar(options: { nextStep?: Step; disabled?: boolean; onNextStep?: () => boolean }) {
     const { disabled, onNextStep } = options;
     const { step } = this.state;
     const nextStep = options.nextStep || STEPS[STEPS.indexOf(step) + 1] || STEPS[0];
@@ -677,7 +676,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           disabled={disabled || !this.isPreviewSpecSame()}
           onClick={() => {
             if (disabled) return;
-            if (onNextStep) onNextStep();
+            if (onNextStep && !onNextStep()) return;
 
             this.updateStep(nextStep);
           }}
@@ -1217,7 +1216,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           disabled: !inputQueryState.data,
           nextStep: druidSource ? 'transform' : 'parser',
           onNextStep: () => {
-            if (!inputQueryState.data) return;
+            if (!inputQueryState.data) return false;
             const inputData = inputQueryState.data;
 
             if (druidSource) {
@@ -1266,15 +1265,24 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
 
               this.updateSpec(fillDataSourceNameIfNeeded(newSpec));
             } else {
-              this.updateSpec(
-                fillDataSourceNameIfNeeded(
-                  fillInputFormat(
-                    spec,
-                    filterMap(inputQueryState.data.data, l => (l.input ? l.input.raw : undefined)),
-                  ),
-                ),
+              const sampleLines = filterMap(inputQueryState.data.data, l =>
+                l.input ? l.input.raw : undefined,
               );
+
+              const issue = issueWithSampleData(sampleLines);
+              if (issue) {
+                AppToaster.show({
+                  icon: IconNames.WARNING_SIGN,
+                  intent: Intent.WARNING,
+                  message: issue,
+                  timeout: 10000,
+                });
+                return false;
+              }
+
+              this.updateSpec(fillDataSourceNameIfNeeded(fillInputFormat(spec, sampleLines)));
             }
+            return true;
           },
         })}
       </>
@@ -1441,7 +1449,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         {this.renderNextBar({
           disabled: !parserQueryState.data,
           onNextStep: () => {
-            if (!parserQueryState.data) return;
+            if (!parserQueryState.data) return false;
             let possibleTimestampSpec: TimestampSpec;
             if (isDruidSource(spec)) {
               possibleTimestampSpec = {
@@ -1460,6 +1468,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
               );
               this.updateSpec(newSpec);
             }
+
+            return true;
           },
         })}
       </>
@@ -1872,11 +1882,16 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         {this.renderNextBar({
           disabled: !transformQueryState.data,
           onNextStep: () => {
-            if (!transformQueryState.data) return;
+            if (!transformQueryState.data) return false;
 
             let newSpec = spec;
             if (!isDruidSource(newSpec)) {
-              newSpec = updateSchemaWithSample(newSpec, transformQueryState.data, 'specific', true);
+              newSpec = updateSchemaWithSample(
+                newSpec,
+                transformQueryState.data,
+                'specific',
+                false,
+              );
             }
 
             if (!deepGet(newSpec, 'spec.dataSchema.granularitySpec.type')) {
@@ -1884,6 +1899,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
             }
 
             this.updateSpec(newSpec);
+            return true;
           },
         })}
       </>
@@ -2517,6 +2533,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
             }
 
             this.updateSpec(newSpec);
+            return true;
           },
         })}
       </>
@@ -2906,6 +2923,68 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     const granularitySpec: GranularitySpec =
       deepGet(spec, 'spec.dataSchema.granularitySpec') || EMPTY_OBJECT;
     const isStreaming = oneOf(spec.type, 'kafka', 'kinesis');
+    const dimensions: (string | DimensionSpec)[] | undefined = deepGet(
+      spec,
+      'spec.dataSchema.dimensionsSpec.dimensions',
+    );
+    const dimensionNames = dimensions ? dimensions.map(getDimensionSpecName) : undefined;
+
+    let nonsensicalSingleDimPartitioningMessage: JSX.Element | undefined;
+    const partitionDimension = deepGet(tuningConfig, 'partitionsSpec.partitionDimension');
+    if (
+      dimensions &&
+      Array.isArray(dimensionNames) &&
+      dimensionNames.length &&
+      deepGet(tuningConfig, 'partitionsSpec.type') === 'single_dim' &&
+      typeof partitionDimension === 'string' &&
+      partitionDimension !== dimensionNames[0]
+    ) {
+      const firstDimensionName = dimensionNames[0];
+      nonsensicalSingleDimPartitioningMessage = (
+        <Callout intent={Intent.WARNING}>
+          <p>Your partitioning and sorting configuration does not make sense.</p>
+          <p>
+            For best performance the first dimension in your schema (
+            <Code>{firstDimensionName}</Code>), which is what the data will be primarily sorted on,
+            should match the partitioning dimension (<Code>{partitionDimension}</Code>).
+          </p>
+          <p>
+            <Button
+              intent={Intent.WARNING}
+              text={`Put '${partitionDimension}' first in the dimensions list`}
+              onClick={() => {
+                this.updateSpec(
+                  deepSet(
+                    spec,
+                    'spec.dataSchema.dimensionsSpec.dimensions',
+                    moveElement(
+                      dimensions,
+                      dimensions.findIndex(d => getDimensionSpecName(d) === partitionDimension),
+                      0,
+                    ),
+                  ),
+                );
+              }}
+            />
+          </p>
+          <p>
+            <Button
+              intent={Intent.WARNING}
+              text={`Partition on '${firstDimensionName}' instead`}
+              onClick={() => {
+                this.updateSpec(
+                  deepSet(
+                    spec,
+                    'spec.tuningConfig.partitionsSpec.partitionDimension',
+                    firstDimensionName,
+                  ),
+                );
+              }}
+            />
+          </p>
+        </Callout>
+      );
+    }
 
     return (
       <>
@@ -2913,12 +2992,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           <H5>Primary partitioning (by time)</H5>
           <AutoForm
             fields={[
-              {
-                name: 'type',
-                type: 'string',
-                suggestions: ['uniform', 'arbitrary'],
-                info: <>This spec is used to generated segments with uniform intervals.</>,
-              },
               {
                 name: 'segmentGranularity',
                 type: 'string',
@@ -2961,7 +3034,10 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         <div className="other">
           <H5>Secondary partitioning</H5>
           <AutoForm
-            fields={getPartitionRelatedTuningSpecFormFields(getSpecType(spec) || 'index_parallel')}
+            fields={getPartitionRelatedTuningSpecFormFields(
+              getSpecType(spec) || 'index_parallel',
+              dimensionNames,
+            )}
             model={tuningConfig}
             globalAdjustment={adjustTuningConfig}
             onChange={t => this.updateSpec(deepSet(spec, 'spec.tuningConfig', t))}
@@ -2969,6 +3045,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
         </div>
         <div className="control">
           <PartitionMessage />
+          {nonsensicalSingleDimPartitioningMessage}
         </div>
         {this.renderNextBar({
           disabled:
@@ -3173,7 +3250,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
     const { spec, submitting } = this.state;
 
     const fullSpec = Boolean(
-      deepGet(spec, 'spec.dataSchema.timestampSpec.format') &&
+      deepGet(spec, 'spec.dataSchema.timestampSpec') &&
         deepGet(spec, 'spec.dataSchema.dimensionsSpec') &&
         deepGet(spec, 'spec.dataSchema.granularitySpec.type'),
     );
diff --git a/web-console/src/views/load-data-view/schema-table/schema-table.tsx b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
index f56fcd9453ce..6228b09af631 100644
--- a/web-console/src/views/load-data-view/schema-table/schema-table.tsx
+++ b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
@@ -101,7 +101,7 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
             Cell: ({ value }) => <TableCell value={value} />,
           };
         } else {
-          const timestamp = columnName === '__time';
+          const isTimestamp = columnName === '__time';
           const dimensionSpecIndex = dimensions
             ? dimensions.findIndex(d => getDimensionSpecName(d) === columnName)
             : -1;
@@ -109,7 +109,7 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
           const dimensionSpecType = dimensionSpec ? getDimensionSpecType(dimensionSpec) : undefined;
 
           const columnClassName = classNames(
-            timestamp ? 'timestamp' : 'dimension',
+            isTimestamp ? 'timestamp' : 'dimension',
             dimensionSpecType || 'string',
             {
               selected:
@@ -122,7 +122,7 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
               <div
                 className="clickable"
                 onClick={() => {
-                  if (timestamp) return;
+                  if (isTimestamp) return;
 
                   if (dimensionSpec) {
                     onDimensionSelect(inflateDimensionSpec(dimensionSpec), dimensionSpecIndex);
@@ -133,15 +133,16 @@ export const SchemaTable = React.memo(function SchemaTable(props: SchemaTablePro
               >
                 <div className="column-name">{columnName}</div>
                 <div className="column-detail">
-                  {timestamp ? 'long (time column)' : dimensionSpecType || 'string (auto)'}&nbsp;
+                  {isTimestamp ? 'long (time column)' : dimensionSpecType || 'string (auto)'}&nbsp;
                 </div>
               </div>
             ),
             headerClassName: columnClassName,
             className: columnClassName,
             id: String(i),
+            width: isTimestamp ? 200 : 100,
             accessor: (row: SampleEntry) => (row.parsed ? row.parsed[columnName] : null),
-            Cell: row => <TableCell value={timestamp ? new Date(row.value) : row.value} />,
+            Cell: row => <TableCell value={isTimestamp ? new Date(row.value) : row.value} />,
           };
         }
       })}

From c066b4193aa16cbfb26c8b283c5d62d4637b8ce5 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Mon, 16 Nov 2020 14:39:24 -0800
Subject: [PATCH 12/13] update snapshot

---
 .../src/druid-models/ingestion-spec.tsx       |   5 +-
 .../views/load-data-view/load-data-view.tsx   |   6 +-
 .../__snapshots__/schema-table.spec.tsx.snap  | 102 +++++++++---------
 3 files changed, 59 insertions(+), 54 deletions(-)

diff --git a/web-console/src/druid-models/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx
index 3149c0b4fa33..56201e740907 100644
--- a/web-console/src/druid-models/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec.tsx
@@ -1429,6 +1429,8 @@ export function getPartitionRelatedTuningSpecFormFields(
           name: 'partitionsSpec.targetRowsPerSegment',
           label: 'Target rows per segment',
           type: 'number',
+          zeroMeansUndefined: true,
+          defaultValue: 5000000,
           defined: (t: TuningConfig) =>
             deepGet(t, 'partitionsSpec.type') === 'hashed' &&
             !deepGet(t, 'partitionsSpec.numShards'),
@@ -1450,6 +1452,8 @@ export function getPartitionRelatedTuningSpecFormFields(
           name: 'partitionsSpec.numShards',
           label: 'Num shards',
           type: 'number',
+          zeroMeansUndefined: true,
+          hideInMore: true,
           defined: (t: TuningConfig) =>
             deepGet(t, 'partitionsSpec.type') === 'hashed' &&
             !deepGet(t, 'partitionsSpec.targetRowsPerSegment'),
@@ -1975,7 +1979,6 @@ export function updateIngestionType(
 }
 
 export function issueWithSampleData(sampleData: string[]): JSX.Element | undefined {
-  console.log(sampleData);
   if (sampleData.length) {
     const firstData = sampleData[0];
 
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index 48768a8ba799..d13e17b95d33 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -189,6 +189,8 @@ import {
 
 import './load-data-view.scss';
 
+const DEFAULT_ROLLUP_SETTING = false;
+
 function showRawLine(line: SampleEntry): string {
   if (!line.parsed) return 'No parse';
   const raw = line.parsed.raw;
@@ -1885,12 +1887,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
             if (!transformQueryState.data) return false;
 
             let newSpec = spec;
-            if (!isDruidSource(newSpec)) {
+            if (!deepGet(newSpec, 'spec.dataSchema.dimensionsSpec')) {
               newSpec = updateSchemaWithSample(
                 newSpec,
                 transformQueryState.data,
                 'specific',
-                false,
+                DEFAULT_ROLLUP_SETTING,
               );
             }
 
diff --git a/web-console/src/views/load-data-view/schema-table/__snapshots__/schema-table.spec.tsx.snap b/web-console/src/views/load-data-view/schema-table/__snapshots__/schema-table.spec.tsx.snap
index ef00cae858eb..fb54c306fce7 100644
--- a/web-console/src/views/load-data-view/schema-table/__snapshots__/schema-table.spec.tsx.snap
+++ b/web-console/src/views/load-data-view/schema-table/__snapshots__/schema-table.spec.tsx.snap
@@ -19,7 +19,7 @@ exports[`SchemaTable matches snapshot 1`] = `
         <div
           class="rt-th dimension string rt-resizable-header"
           role="columnheader"
-          style="flex: 100 0 auto; width: 100px;"
+          style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           tabindex="-1"
         >
           <div
@@ -62,7 +62,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span
               class="table-cell plain"
@@ -83,7 +83,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -102,7 +102,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -121,7 +121,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -140,7 +140,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -159,7 +159,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -178,7 +178,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -197,7 +197,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -216,7 +216,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -235,7 +235,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -254,7 +254,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -273,7 +273,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -292,7 +292,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -311,7 +311,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -330,7 +330,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -349,7 +349,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -368,7 +368,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -387,7 +387,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -406,7 +406,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -425,7 +425,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -444,7 +444,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -463,7 +463,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -482,7 +482,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -501,7 +501,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -520,7 +520,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -539,7 +539,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -558,7 +558,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -577,7 +577,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -596,7 +596,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -615,7 +615,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -634,7 +634,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -653,7 +653,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -672,7 +672,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -691,7 +691,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -710,7 +710,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -729,7 +729,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -748,7 +748,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -767,7 +767,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -786,7 +786,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -805,7 +805,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -824,7 +824,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -843,7 +843,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -862,7 +862,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -881,7 +881,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -900,7 +900,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -919,7 +919,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -938,7 +938,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -957,7 +957,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -976,7 +976,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                
@@ -995,7 +995,7 @@ exports[`SchemaTable matches snapshot 1`] = `
           <div
             class="rt-td dimension string"
             role="gridcell"
-            style="flex: 100 0 auto; width: 100px;"
+            style="flex: 100 0 auto; width: 100px; max-width: 100px;"
           >
             <span>
                

From ddeaf71fe442cc3cfedcec4acfd7e6c1df75a259 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Mon, 16 Nov 2020 15:00:14 -0800
Subject: [PATCH 13/13] reword without >

---
 web-console/src/druid-models/ingestion-spec.tsx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/web-console/src/druid-models/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx
index 56201e740907..baa8b50729ea 100644
--- a/web-console/src/druid-models/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec.tsx
@@ -1923,8 +1923,8 @@ const TUNING_CONFIG_FORM_FIELDS: Field<TuningConfig>[] = [
     info: (
       <>
         <p>
-          When shards are split or merged, the supervisor will recompute shard -> task group
-          mappings, and signal any running tasks created under the old mappings to stop early at{' '}
+          When shards are split or merged, the supervisor will recompute shard, task group mappings,
+          and signal any running tasks created under the old mappings to stop early at{' '}
           <Code>(current time + repartitionTransitionDuration)</Code>. Stopping the tasks early
           allows Druid to begin reading from the new shards more quickly.
         </p>