From 1a0251577563f7f1f36f890b31520a0b89fe7805 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Tue, 15 Aug 2023 13:02:36 -0700 Subject: [PATCH 1/2] multi topic ux --- .../ingestion-spec/ingestion-spec.tsx | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx index 42612b120e4b..b40fab7614f8 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx @@ -38,6 +38,7 @@ import { findMap, isSimpleArray, oneOf, + oneOfKnown, parseCsvLine, typeIsKnown, } from '../../utils'; @@ -436,6 +437,7 @@ export interface IoConfig { stream?: string; endpoint?: string; useEarliestSequenceNumber?: boolean; + multiTopic?: boolean; } export function invalidIoConfig(ioConfig: IoConfig): boolean { @@ -919,12 +921,44 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F ), }, + { + name: 'multiTopic', + type: 'boolean', + defaultValue: false, + defined: typeIsKnown(KNOWN_TYPES, 'kafka'), + info: 'Set this to true if you want to ingest data from multiple Kafka topics using a single supervisor.', + }, { name: 'topic', type: 'string', required: true, - defined: typeIsKnown(KNOWN_TYPES, 'kafka'), - placeholder: 'topic_name', + defined: inputSource => + oneOfKnown(inputSource.type, KNOWN_TYPES, 'kafka') && inputSource.multiTopic !== true, + placeholder: 'your_kafka_topic', + info: 'The name of the Kafka topic to ingest from.', + }, + { + name: 'topic', + label: 'Topics (regexp)', + type: 'string', + required: true, + defined: inputSource => + oneOfKnown(inputSource.type, KNOWN_TYPES, 'kafka') && inputSource.multiTopic === true, + placeholder: 'topic1|topic2', + info: ( + <> +

+ A regular expression that represents all topics to be ingested from. For example, to + ingest data from clicks and impressions, you can set this + to clicks|impressions. To ingest from all topics starting with{' '} + metrics- set this to metrics-.*. +

+

+ If new topics are added to the cluster that match the regex, Druid will + automatically start ingesting from those new topics. +

+ + ), }, { name: 'consumerProperties', From 1c6ba035b4a627fffdb3446eac58b6e579598257 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 16 Aug 2023 11:19:15 -0700 Subject: [PATCH 2/2] updated to match new api --- .../ingestion-spec/ingestion-spec.tsx | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx index b40fab7614f8..f3865b66ae00 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx @@ -427,6 +427,7 @@ export interface IoConfig { inputFormat?: InputFormat; appendToExisting?: boolean; topic?: string; + topicPattern?: string; consumerProperties?: any; replicas?: number; taskCount?: number; @@ -437,7 +438,6 @@ export interface IoConfig { stream?: string; endpoint?: string; useEarliestSequenceNumber?: boolean; - multiTopic?: boolean; } export function invalidIoConfig(ioConfig: IoConfig): boolean { @@ -921,37 +921,28 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F ), }, - { - name: 'multiTopic', - type: 'boolean', - defaultValue: false, - defined: typeIsKnown(KNOWN_TYPES, 'kafka'), - info: 'Set this to true if you want to ingest data from multiple Kafka topics using a single supervisor.', - }, { name: 'topic', type: 'string', required: true, - defined: inputSource => - oneOfKnown(inputSource.type, KNOWN_TYPES, 'kafka') && inputSource.multiTopic !== true, + defined: ioConfig => + oneOfKnown(ioConfig.type, KNOWN_TYPES, 'kafka') && !ioConfig.topicPattern, placeholder: 'your_kafka_topic', info: 'The name of the Kafka topic to ingest from.', }, { - name: 'topic', - label: 'Topics (regexp)', + name: 'topicPattern', type: 'string', required: true, - defined: inputSource => - oneOfKnown(inputSource.type, KNOWN_TYPES, 'kafka') && inputSource.multiTopic === true, + defined: ioConfig => oneOfKnown(ioConfig.type, KNOWN_TYPES, 'kafka') && !ioConfig.topic, placeholder: 'topic1|topic2', info: ( <>

A regular expression that represents all topics to be ingested from. For example, to ingest data from clicks and impressions, you can set this - to clicks|impressions. To ingest from all topics starting with{' '} - metrics- set this to metrics-.*. + to clicks|impressions. Alternatively, to ingest from all topics + starting with metrics- set this to metrics-.*.

If new topics are added to the cluster that match the regex, Druid will @@ -1061,7 +1052,7 @@ export function issueWithIoConfig( break; case 'kafka': - if (!ioConfig.topic) return 'must have a topic'; + if (!ioConfig.topic && !ioConfig.topicPattern) return 'must have a topic or topicPattern'; break; case 'kinesis': @@ -1407,7 +1398,7 @@ export function guessDataSourceName(spec: Partial): string | unde } case 'kafka': - return ioConfig.topic; + return ioConfig.topic || ioConfig.topicPattern; case 'kinesis': return ioConfig.stream; @@ -1492,7 +1483,7 @@ export const PRIMARY_PARTITION_RELATED_FORM_FIELDS: Field[] = [ name: 'spec.dataSchema.granularitySpec.segmentGranularity', label: 'Segment granularity', type: 'string', - suggestions: ['hour', 'day', 'week', 'month', 'year', 'all'], + suggestions: ['hour', 'day', 'month', 'year', 'all'], required: true, info: ( <>