Skip to content

Add Azure ingestion flow to web console #9437

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vogievetsky merged 4 commits into from
Mar 3, 2020
Merged

Add Azure ingestion flow to web console #9437

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1c2aa72
add support for azure
Feb 28, 2020
1784e16
change bucket to container
Feb 28, 2020
e4763b8
add azure to input menu
Feb 28, 2020
bb92f29
remove static-azure
Mar 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added web-console/assets/azure.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
74 changes: 73 additions & 1 deletion web-console/src/utils/ingestion-spec.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ export type IngestionComboType =
| 'index_parallel:druid'
| 'index_parallel:inline'
| 'index_parallel:s3'
| 'index_parallel:azure'
| 'index_parallel:google'
| 'index_parallel:hdfs';

Expand Down Expand Up @@ -102,6 +103,7 @@ export function getIngestionComboType(spec: IngestionSpec): IngestionComboType |
case 'druid':
case 'inline':
case 's3':
case 'azure':
case 'google':
case 'hdfs':
return `${ioConfig.type}:${inputSource.type}` as IngestionComboType;
Expand All @@ -128,6 +130,9 @@ export function getIngestionTitle(ingestionType: IngestionComboTypeWithExtra): s
case 'index_parallel:s3':
return 'Amazon S3';

case 'index_parallel:azure':
return 'Azure Data Lake';

case 'index_parallel:google':
return 'Google Cloud Storage';

Expand Down Expand Up @@ -180,6 +185,9 @@ export function getRequiredModule(ingestionType: IngestionComboTypeWithExtra): s
case 'index_parallel:s3':
return 'druid-s3-extensions';

case 'index_parallel:azure':
return 'druid-azure-extensions';

case 'index_parallel:google':
return 'druid-google-extensions';

Expand Down Expand Up @@ -983,7 +991,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
name: 'inputSource.type',
label: 'Source type',
type: 'string',
suggestions: ['local', 'http', 'inline', 's3', 'google', 'hdfs'],
suggestions: ['local', 'http', 'inline', 's3', 'azure', 'google', 'hdfs'],
info: (
<p>
Druid connects to raw data through{' '}
Expand Down Expand Up @@ -1208,6 +1216,67 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
},
];

case 'index_parallel:azure':
return [
inputSourceType,
{
name: 'inputSource.uris',
label: 'Azure URIs',
type: 'string-array',
placeholder:
'azure://your-container/some-file1.ext, azure://your-container/some-file2.ext',
required: true,
defined: ioConfig =>
!deepGet(ioConfig, 'inputSource.prefixes') && !deepGet(ioConfig, 'inputSource.objects'),
info: (
<>
<p>
The full Azure URI of your file. To ingest from multiple URIs, use commas to
separate each individual URI.
</p>
<p>Either Azure URIs or prefixes or objects must be set.</p>
</>
),
},
{
name: 'inputSource.prefixes',
label: 'Azure prefixes',
type: 'string-array',
placeholder: 'azure://your-container/some-path1, azure://your-container/some-path2',
required: true,
defined: ioConfig =>
!deepGet(ioConfig, 'inputSource.uris') && !deepGet(ioConfig, 'inputSource.objects'),
info: (
<>
<p>A list of paths (with bucket) where your files are stored.</p>
<p>Either Azure URIs or prefixes or objects must be set.</p>
</>
),
},
{
name: 'inputSource.objects',
label: 'Azure objects',
type: 'json',
placeholder: '{"bucket":"your-container", "path":"some-file.ext"}',
required: true,
defined: ioConfig => deepGet(ioConfig, 'inputSource.objects'),
info: (
<>
<p>
JSON array of{' '}
<ExternalLink
href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/development/extensions-core/azure.html`}
>
S3 Objects
</ExternalLink>
.
</p>
<p>Either Azure URIs or prefixes or objects must be set.</p>
</>
),
},
];

case 'index_parallel:google':
return [
inputSourceType,
Expand Down Expand Up @@ -1423,6 +1492,7 @@ function issueWithInputSource(inputSource: InputSource | undefined): string | un
break;

case 's3':
case 'azure':
case 'google':
if (
!nonEmptyArray(inputSource.uris) &&
Expand Down Expand Up @@ -1478,6 +1548,7 @@ export function getIoConfigTuningFormFields(
switch (ingestionComboType) {
case 'index_parallel:http':
case 'index_parallel:s3':
case 'index_parallel:azure':
case 'index_parallel:google':
case 'index_parallel:hdfs':
return [
Expand Down Expand Up @@ -1815,6 +1886,7 @@ export function guessDataSourceName(spec: IngestionSpec): string | undefined {
}

case 's3':
case 'azure':
case 'google':
const actualPath = (inputSource.objects || EMPTY_ARRAY)[0];
const uriPath =
Expand Down
5 changes: 5 additions & 0 deletions web-console/src/views/load-data-view/load-data-view.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -704,6 +704,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
{this.renderIngestionCard('kafka')}
{this.renderIngestionCard('kinesis')}
{this.renderIngestionCard('index_parallel:s3')}
{this.renderIngestionCard('index_parallel:azure')}
{this.renderIngestionCard('index_parallel:google')}
{this.renderIngestionCard('index_parallel:hdfs')}
{this.renderIngestionCard('index_parallel:druid')}
Expand Down Expand Up @@ -781,6 +782,9 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
case 'index_parallel:s3':
return <p>Load text based, orc, or parquet data from Amazon S3.</p>;

case 'index_parallel:azure':
return <p>Load text based, orc, or parquet data from Azure.</p>;

case 'index_parallel:google':
return <p>Load text based, orc, or parquet data from the Google Blobstore.</p>;

Expand Down Expand Up @@ -831,6 +835,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
case 'index_parallel:druid':
case 'index_parallel:inline':
case 'index_parallel:s3':
case 'index_parallel:azure':
case 'index_parallel:google':
case 'index_parallel:hdfs':
case 'kafka':
Expand Down