Skip to content

Data loader fixes #7672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
clintropolis merged 4 commits into from
May 19, 2019
Merged

Data loader fixes #7672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
dcb6f29
add ability to submit 'other' task
vogievetsky May 16, 2019
1835654
copy changes, fix sort
vogievetsky May 16, 2019
aab26b1
lets call it dialog
vogievetsky May 16, 2019
9695fd6
update name to Google Cloud Storage
vogievetsky May 16, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions web-console/src/console-application.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,

private loadDataViewSeed: LoadDataViewSeed | null;
private taskId: string | null;
private openDialog: string | null;
private datasource: string | null;
private onlyUnavailable: boolean | null;
private initSql: string | null;
Expand Down Expand Up @@ -152,6 +153,7 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,
setTimeout(() => {
this.loadDataViewSeed = null;
this.taskId = null;
this.openDialog = null;
this.datasource = null;
this.onlyUnavailable = null;
this.initSql = null;
Expand All @@ -165,8 +167,9 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,
this.resetInitialsWithDelay();
}

private goToTask = (taskId: string | null) => {
private goToTask = (taskId: string | null, openDialog: string | null = null) => {
this.taskId = taskId;
this.openDialog = openDialog;
window.location.hash = 'tasks';
this.resetInitialsWithDelay();
}
Expand Down Expand Up @@ -224,7 +227,7 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,

private wrappedTasksView = () => {
const { noSqlMode } = this.state;
return this.wrapInViewContainer('tasks', <TasksView taskId={this.taskId} goToSql={this.goToSql} goToMiddleManager={this.goToMiddleManager} goToLoadDataView={this.goToLoadDataView} noSqlMode={noSqlMode}/>, true);
return this.wrapInViewContainer('tasks', <TasksView taskId={this.taskId} openDialog={this.openDialog} goToSql={this.goToSql} goToMiddleManager={this.goToMiddleManager} goToLoadDataView={this.goToLoadDataView} noSqlMode={noSqlMode}/>, true);
}

private wrappedServersView = () => {
Expand Down
1 change: 1 addition & 0 deletions web-console/src/views/load-data-view.scss
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
.cards {
.bp3-card {
display: inline-block;
vertical-align: top;
width: 250px;
height: 140px;
margin-right: 15px;
Expand Down
41 changes: 34 additions & 7 deletions web-console/src/views/load-data-view.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ const VIEW_TITLE: Record<Stage, string> = {

export interface LoadDataViewProps extends React.Props<any> {
seed: LoadDataViewSeed | null;
goToTask: (taskId: string | null) => void;
goToTask: (taskId: string | null, openDialog?: string | null) => void;
}

export interface LoadDataViewState {
Expand Down Expand Up @@ -370,6 +370,7 @@ export class LoadDataView extends React.Component<LoadDataViewProps, LoadDataVie
}

renderInitStage() {
const { goToTask } = this.props;
const showStreaming = false;

return <>
Expand All @@ -378,8 +379,10 @@ export class LoadDataView extends React.Component<LoadDataViewProps, LoadDataVie
</div>

<Callout intent={Intent.SUCCESS} icon={IconNames.INFO_SIGN}>
Welcome to the Druid data loader.
This project is under active development and we plan to support many other sources of raw data, including stream hubs such as Apache Kafka and AWS Kinesis, in the next few releases.
Welcome to the Apache Druid graphical data loader.
This feature is under active development and currently only supports Druid's native batch ingestion.
We plan to continue building this out, including support for Druid's Apache Kafka, Apache Hadoop, and AWS Kinesis based ingestion methods, over the next few releases.
Until then, you can load from these and any other Druid ingestion source by clicking on <Code>Other</Code>.
</Callout>

{
Expand All @@ -398,10 +401,18 @@ export class LoadDataView extends React.Component<LoadDataViewProps, LoadDataVie
<div className="cards">
<Card interactive onClick={() => this.initWith({ type: 'index_parallel', firehoseType: 'http' })}>HTTP(s)</Card>
<Card interactive onClick={() => this.initWith({ type: 'index_parallel', firehoseType: 'static-s3' })}>AWS S3</Card>
<Card interactive onClick={() => this.initWith({ type: 'index_parallel', firehoseType: 'static-google-blobstore' })}>Google Blobstore</Card>
<Card interactive onClick={() => this.initWith({ type: 'index_parallel', firehoseType: 'static-google-blobstore' })}>Google Cloud Storage</Card>
<Card interactive onClick={() => this.initWith({ type: 'index_parallel', firehoseType: 'local' })}>Local disk</Card>
</div>
</div>

<div className="section">
<div className="section-title">Raw spec</div>
<div className="cards">
<Card interactive onClick={() => goToTask(null, 'supervisor')}>Other (streaming)</Card>
<Card interactive onClick={() => goToTask(null, 'task')}>Other (batch)</Card>
</div>
</div>
</>;
}

Expand Down Expand Up @@ -1179,6 +1190,22 @@ export class LoadDataView extends React.Component<LoadDataViewProps, LoadDataVie
Click "Preview" to see the result of any specified transforms.
</p>
</Callout>
{
Boolean(transformQueryState.error && transforms.length) &&
<FormGroup>
<Button
icon={IconNames.EDIT}
text="Edit last added transform"
intent={Intent.PRIMARY}
onClick={() => {
this.setState({
selectedTransformIndex: transforms.length - 1,
selectedTransform: transforms[transforms.length - 1]
});
}}
/>
</FormGroup>
}
{this.renderTransformControls()}
<Button
text="Preview"
Expand Down Expand Up @@ -1729,7 +1756,7 @@ export class LoadDataView extends React.Component<LoadDataViewProps, LoadDataVie
<Switch
checked={dimensionMode === 'specific'}
onChange={() => this.setState({ newDimensionMode: dimensionMode === 'specific' ? 'auto-detect' : 'specific' })}
label="Set dimensions and metrics"
label="Explicitly specify dimension list"
/>
<Popover
content={
Expand Down Expand Up @@ -1864,8 +1891,8 @@ export class LoadDataView extends React.Component<LoadDataViewProps, LoadDataVie
<p>
{
autoDetect ?
'Are you sure you don’t want to set the dimensions and metrics explicitly?' :
'Are you sure you want to set dimensions and metrics explicitly?'
`Are you sure you don't want to explicitly specify a dimension list?` :
`Are you sure you want to explicitly specify a dimension list?`
}
</p>
<p>
Expand Down
35 changes: 18 additions & 17 deletions web-console/src/views/tasks-view.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ const taskTableColumns: string[] = ['Task ID', 'Type', 'Datasource', 'Created ti

export interface TasksViewProps extends React.Props<any> {
taskId: string | null;
openDialog: string | null;
goToSql: (initSql: string) => void;
goToMiddleManager: (middleManager: string) => void;
goToLoadDataView: () => void;
Expand Down Expand Up @@ -117,7 +118,7 @@ export class TasksView extends React.Component<TasksViewProps, TasksViewState> {
private taskQueryManager: QueryManager<string, TaskQueryResultRow[]>;
private supervisorTableColumnSelectionHandler: TableColumnSelectionHandler;
private taskTableColumnSelectionHandler: TableColumnSelectionHandler;
static statusRanking = {RUNNING: 4, PENDING: 3, WAITING: 2, SUCCESS: 1, FAILED: 1};
static statusRanking: Record<string, number> = {RUNNING: 4, PENDING: 3, WAITING: 2, SUCCESS: 1, FAILED: 1};

constructor(props: TasksViewProps, context: any) {
super(props, context);
Expand All @@ -139,8 +140,8 @@ export class TasksView extends React.Component<TasksViewProps, TasksViewState> {

killTaskId: null,

supervisorSpecDialogOpen: false,
taskSpecDialogOpen: false,
supervisorSpecDialogOpen: props.openDialog === 'supervisor',
taskSpecDialogOpen: props.openDialog === 'task',
initSpec: null,
alertErrorMsg: null,

Expand Down Expand Up @@ -600,7 +601,7 @@ ORDER BY "rank" DESC, "created_time" DESC`);
Header: 'Status',
id: 'status',
width: 110,
accessor: (row) => { return {status: row.status, created_time: row.created_time}; },
accessor: row => ({ status: row.status, created_time: row.created_time, toString: () => row.status }),
Cell: row => {
if (row.aggregated) return '';
const { status, location } = row.original;
Expand All @@ -617,20 +618,20 @@ ORDER BY "rank" DESC, "created_time" DESC`);
{errorMsg && <a onClick={() => this.setState({ alertErrorMsg: errorMsg })} title={errorMsg}>&nbsp;?</a>}
</span>;
},
PivotValue: (opt) => {
const { subRows, value } = opt;
if (!subRows || !subRows.length) return '';
return `${subRows[0]._original['status']} (${subRows.length})`;
},
Aggregated: (opt: any) => {
const { subRows, column } = opt;
const previewValues = subRows.filter((d: any) => typeof d[column.id] !== 'undefined').map((row: any) => row._original[column.id]);
const previewCount = countBy(previewValues);
return <span>{Object.keys(previewCount).sort().map(v => `${v} (${previewCount[v]})`).join(', ')}</span>;
},
sortMethod: (d1, d2) => {
const statusRanking: any = TasksView.statusRanking;
return statusRanking[d1.status] - statusRanking[d2.status] || d1.created_time.localeCompare(d2.created_time);
const typeofD1 = typeof d1;
const typeofD2 = typeof d2;
if (typeofD1 !== typeofD2) return 0;
switch (typeofD1) {
case 'string':
return TasksView.statusRanking[d1] - TasksView.statusRanking[d2];

case 'object':
return TasksView.statusRanking[d1.status] - TasksView.statusRanking[d2.status] || d1.created_time.localeCompare(d2.created_time);

default:
return 0;
}
},
filterMethod: (filter: Filter, row: any) => {
return booleanCustomTableFilter(filter, row.status.status);
Expand Down