Skip to content

Delete buildV9Directly in Kafka and Kinesis Indexing Service #11351

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
clintropolis merged 5 commits into from
Jun 23, 2021
Merged

Delete buildV9Directly in Kafka and Kinesis Indexing Service #11351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1e94120
delete_buildV9Directly_in_kafka_and_kinesis_indexing_service
Jun 9, 2021
348a705
delete
Jun 10, 2021
5e59a18
delete them from server
Jun 11, 2021
b0ef1f5
delete buildV9Directly from hadoop indexing
Jun 13, 2021
5582a04
bug fixed
Jun 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions docs/ingestion/native-batch.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -605,8 +605,7 @@ An example of the result is
"logParseExceptions": false,
"maxParseExceptions": 2147483647,
"maxSavedParseExceptions": 0,
"forceGuaranteedRollup": false,
"buildV9Directly": true
"forceGuaranteedRollup": false
}
}
},
Expand Down
1 change: 0 additions & 1 deletion .../main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,6 @@ public HadoopIndexTask createTask(Interval interval, String version, List<DataSe
tuningConfig.isCombineText(),
tuningConfig.getUseCombiner(),
tuningConfig.getMaxRowsInMemory(),
tuningConfig.getBuildV9Directly(),
tuningConfig.getNumBackgroundPersistThreads(),
tuningConfig.isForceExtendableShardSpecs(),
true,
Expand Down
4 changes: 0 additions & 4 deletions ...ing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTuningConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,6 @@ public KafkaIndexTaskTuningConfig(
@JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists,
@JsonProperty("indexSpec") @Nullable IndexSpec indexSpec,
@JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists,
// This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12.
@JsonProperty("buildV9Directly") @Nullable Boolean buildV9Directly,
@Deprecated @JsonProperty("reportParseExceptions") @Nullable Boolean reportParseExceptions,
@JsonProperty("handoffConditionTimeout") @Nullable Long handoffConditionTimeout,
@JsonProperty("resetOffsetAutomatically") @Nullable Boolean resetOffsetAutomatically,
Expand All @@ -69,7 +67,6 @@ public KafkaIndexTaskTuningConfig(
maxPendingPersists,
indexSpec,
indexSpecForIntermediatePersists,
true,
reportParseExceptions,
handoffConditionTimeout,
resetOffsetAutomatically,
Expand Down Expand Up @@ -97,7 +94,6 @@ public KafkaIndexTaskTuningConfig withBasePersistDirectory(File dir)
getMaxPendingPersists(),
getIndexSpec(),
getIndexSpecForIntermediatePersists(),
true,
isReportParseExceptions(),
getHandoffConditionTimeout(),
isResetOffsetAutomatically(),
Expand Down
5 changes: 0 additions & 5 deletions ...src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTuningConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@ public static KafkaSupervisorTuningConfig defaultConfig()
null,
null,
null,
null,
null
);
}
Expand All @@ -85,8 +84,6 @@ public KafkaSupervisorTuningConfig(
@JsonProperty("maxPendingPersists") Integer maxPendingPersists,
@JsonProperty("indexSpec") IndexSpec indexSpec,
@JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists,
// This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12.
@JsonProperty("buildV9Directly") Boolean buildV9Directly,
@JsonProperty("reportParseExceptions") Boolean reportParseExceptions,
@JsonProperty("handoffConditionTimeout") Long handoffConditionTimeout,
@JsonProperty("resetOffsetAutomatically") Boolean resetOffsetAutomatically,
Expand Down Expand Up @@ -115,7 +112,6 @@ public KafkaSupervisorTuningConfig(
maxPendingPersists,
indexSpec,
indexSpecForIntermediatePersists,
true,
reportParseExceptions,
handoffConditionTimeout,
resetOffsetAutomatically,
Expand Down Expand Up @@ -237,7 +233,6 @@ public KafkaIndexTaskTuningConfig convertToTaskTuningConfig()
getMaxPendingPersists(),
getIndexSpec(),
getIndexSpecForIntermediatePersists(),
true,
isReportParseExceptions(),
getHandoffConditionTimeout(),
isResetOffsetAutomatically(),
Expand Down
1 change: 0 additions & 1 deletion ...ka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2685,7 +2685,6 @@ private KafkaIndexTask createTask(
null,
null,
null,
true,
reportParseExceptions,
handoffConditionTimeout,
resetOffsetAutomatically,
Expand Down
5 changes: 0 additions & 5 deletions ...service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTuningConfigTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,6 @@ public void testConvert()
new IndexSpec(),
new IndexSpec(),
true,
true,
5L,
null,
null,
Expand Down Expand Up @@ -178,7 +177,6 @@ public void testSerdeWithModifiedTuningConfigAddedField() throws IOException
new IndexSpec(),
new IndexSpec(),
true,
true,
5L,
null,
null,
Expand All @@ -202,7 +200,6 @@ public void testSerdeWithModifiedTuningConfigAddedField() throws IOException
Assert.assertEquals(base.getBasePersistDirectory(), deserialized.getBasePersistDirectory());
Assert.assertEquals(base.getMaxPendingPersists(), deserialized.getMaxPendingPersists());
Assert.assertEquals(base.getIndexSpec(), deserialized.getIndexSpec());
Assert.assertEquals(base.getBuildV9Directly(), deserialized.getBuildV9Directly());
Assert.assertEquals(base.isReportParseExceptions(), deserialized.isReportParseExceptions());
Assert.assertEquals(base.getHandoffConditionTimeout(), deserialized.getHandoffConditionTimeout());
Assert.assertEquals(base.isResetOffsetAutomatically(), deserialized.isResetOffsetAutomatically());
Expand All @@ -229,7 +226,6 @@ public void testSerdeWithModifiedTuningConfigRemovedField() throws IOException
new IndexSpec(),
new IndexSpec(),
true,
true,
5L,
null,
null,
Expand All @@ -253,7 +249,6 @@ public void testSerdeWithModifiedTuningConfigRemovedField() throws IOException
Assert.assertEquals(base.getBasePersistDirectory(), deserialized.getBasePersistDirectory());
Assert.assertEquals(base.getMaxPendingPersists(), deserialized.getMaxPendingPersists());
Assert.assertEquals(base.getIndexSpec(), deserialized.getIndexSpec());
Assert.assertEquals(base.getBuildV9Directly(), deserialized.getBuildV9Directly());
Assert.assertEquals(base.isReportParseExceptions(), deserialized.isReportParseExceptions());
Assert.assertEquals(base.getHandoffConditionTimeout(), deserialized.getHandoffConditionTimeout());
Assert.assertEquals(base.isResetOffsetAutomatically(), deserialized.isResetOffsetAutomatically());
Expand Down
6 changes: 0 additions & 6 deletions ...service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,6 @@ public KafkaIndexTaskClient build(
null,
null,
null,
true,
false,
null,
false,
Expand Down Expand Up @@ -468,7 +467,6 @@ public void testCreateBaseTaskContexts() throws JsonProcessingException
null,
null,
null,
null,
null
),
null
Expand Down Expand Up @@ -3280,7 +3278,6 @@ public void testIsTaskCurrent()
null,
null,
null,
true,
false,
null,
false,
Expand Down Expand Up @@ -3321,7 +3318,6 @@ public void testIsTaskCurrent()
null,
null,
null,
true,
false,
null,
null,
Expand Down Expand Up @@ -3619,7 +3615,6 @@ public KafkaIndexTaskClient build(
null,
null,
null,
true,
false,
null,
resetOffsetAutomatically,
Expand Down Expand Up @@ -3732,7 +3727,6 @@ public KafkaIndexTaskClient build(
null,
null,
null,
true,
false,
null,
false,
Expand Down
3 changes: 0 additions & 3 deletions ...est/java/org/apache/druid/indexing/kafka/test/TestModifiedKafkaIndexTaskTuningConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,6 @@ public TestModifiedKafkaIndexTaskTuningConfig(
@JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists,
@JsonProperty("indexSpec") @Nullable IndexSpec indexSpec,
@JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists,
// This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12.
@JsonProperty("buildV9Directly") @Nullable Boolean buildV9Directly,
@Deprecated @JsonProperty("reportParseExceptions") @Nullable Boolean reportParseExceptions,
@JsonProperty("handoffConditionTimeout") @Nullable Long handoffConditionTimeout,
@JsonProperty("resetOffsetAutomatically") @Nullable Boolean resetOffsetAutomatically,
Expand All @@ -74,7 +72,6 @@ public TestModifiedKafkaIndexTaskTuningConfig(
maxPendingPersists,
indexSpec,
indexSpecForIntermediatePersists,
true,
reportParseExceptions,
handoffConditionTimeout,
resetOffsetAutomatically,
Expand Down
3 changes: 0 additions & 3 deletions ...service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTuningConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ public KinesisIndexTaskTuningConfig(
@JsonProperty("maxPendingPersists") Integer maxPendingPersists,
@JsonProperty("indexSpec") IndexSpec indexSpec,
@JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists,
@JsonProperty("buildV9Directly") Boolean buildV9Directly,
@JsonProperty("reportParseExceptions") Boolean reportParseExceptions,
@JsonProperty("handoffConditionTimeout") Long handoffConditionTimeout,
@JsonProperty("resetOffsetAutomatically") Boolean resetOffsetAutomatically,
Expand Down Expand Up @@ -92,7 +91,6 @@ public KinesisIndexTaskTuningConfig(
maxPendingPersists,
indexSpec,
indexSpecForIntermediatePersists,
true,
reportParseExceptions,
handoffConditionTimeout,
resetOffsetAutomatically,
Expand Down Expand Up @@ -170,7 +168,6 @@ public KinesisIndexTaskTuningConfig withBasePersistDirectory(File dir)
getMaxPendingPersists(),
getIndexSpec(),
getIndexSpecForIntermediatePersists(),
true,
isReportParseExceptions(),
getHandoffConditionTimeout(),
isResetOffsetAutomatically(),
Expand Down
4 changes: 0 additions & 4 deletions ...main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTuningConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@ public static KinesisSupervisorTuningConfig defaultConfig()
null,
null,
null,
null,
null
);
}
Expand All @@ -94,7 +93,6 @@ public KinesisSupervisorTuningConfig(
@JsonProperty("maxPendingPersists") Integer maxPendingPersists,
@JsonProperty("indexSpec") IndexSpec indexSpec,
@JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists,
@JsonProperty("buildV9Directly") Boolean buildV9Directly,
@JsonProperty("reportParseExceptions") Boolean reportParseExceptions,
@JsonProperty("handoffConditionTimeout") Long handoffConditionTimeout,
@JsonProperty("resetOffsetAutomatically") Boolean resetOffsetAutomatically,
Expand Down Expand Up @@ -131,7 +129,6 @@ public KinesisSupervisorTuningConfig(
maxPendingPersists,
indexSpec,
indexSpecForIntermediatePersists,
buildV9Directly,
reportParseExceptions,
handoffConditionTimeout,
resetOffsetAutomatically,
Expand Down Expand Up @@ -267,7 +264,6 @@ public KinesisIndexTaskTuningConfig convertToTaskTuningConfig()
getMaxPendingPersists(),
getIndexSpec(),
getIndexSpecForIntermediatePersists(),
true,
isReportParseExceptions(),
getHandoffConditionTimeout(),
isResetOffsetAutomatically(),
Expand Down
1 change: 0 additions & 1 deletion ...ng-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskSerdeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ public class KinesisIndexTaskSerdeTest
null,
null,
null,
null,
null
);
private static final KinesisIndexTaskIOConfig IO_CONFIG = new KinesisIndexTaskIOConfig(
Expand Down
1 change: 0 additions & 1 deletion ...ndexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2766,7 +2766,6 @@ private KinesisIndexTask createTask(
null,
null,
null,
true,
reportParseExceptions,
handoffConditionTimeout,
resetOffsetAutomatically,
Expand Down
10 changes: 0 additions & 10 deletions ...ice/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTuningConfigTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ public void testSerdeWithDefaults() throws Exception
Assert.assertEquals(new Period("PT10M"), config.getIntermediatePersistPeriod());
Assert.assertEquals(0, config.getMaxPendingPersists());
Assert.assertEquals(new IndexSpec(), config.getIndexSpec());
Assert.assertTrue(config.getBuildV9Directly());
Assert.assertFalse(config.isReportParseExceptions());
Assert.assertEquals(0, config.getHandoffConditionTimeout());
Assert.assertEquals(10000, config.getRecordBufferSize());
Expand All @@ -96,7 +95,6 @@ public void testSerdeWithNonDefaults() throws Exception
+ " \"maxRowsPerSegment\": 100,\n"
+ " \"intermediatePersistPeriod\": \"PT1H\",\n"
+ " \"maxPendingPersists\": 100,\n"
+ " \"buildV9Directly\": true,\n"
+ " \"reportParseExceptions\": true,\n"
+ " \"handoffConditionTimeout\": 100,\n"
+ " \"recordBufferSize\": 1000,\n"
Expand Down Expand Up @@ -125,7 +123,6 @@ public void testSerdeWithNonDefaults() throws Exception
Assert.assertEquals(100, config.getMaxRowsPerSegment().intValue());
Assert.assertEquals(new Period("PT1H"), config.getIntermediatePersistPeriod());
Assert.assertEquals(100, config.getMaxPendingPersists());
Assert.assertTrue(config.getBuildV9Directly());
Assert.assertTrue(config.isReportParseExceptions());
Assert.assertEquals(100, config.getHandoffConditionTimeout());
Assert.assertEquals(1000, config.getRecordBufferSize());
Expand Down Expand Up @@ -153,7 +150,6 @@ public void testSerdeWithModifiedTuningConfigAddedField() throws IOException
new IndexSpec(),
new IndexSpec(),
true,
true,
5L,
true,
false,
Expand Down Expand Up @@ -184,7 +180,6 @@ public void testSerdeWithModifiedTuningConfigAddedField() throws IOException
Assert.assertEquals(base.getBasePersistDirectory(), deserialized.getBasePersistDirectory());
Assert.assertEquals(base.getMaxPendingPersists(), deserialized.getMaxPendingPersists());
Assert.assertEquals(base.getIndexSpec(), deserialized.getIndexSpec());
Assert.assertEquals(base.getBuildV9Directly(), deserialized.getBuildV9Directly());
Assert.assertEquals(base.isReportParseExceptions(), deserialized.isReportParseExceptions());
Assert.assertEquals(base.getHandoffConditionTimeout(), deserialized.getHandoffConditionTimeout());
Assert.assertEquals(base.isResetOffsetAutomatically(), deserialized.isResetOffsetAutomatically());
Expand Down Expand Up @@ -215,7 +210,6 @@ public void testSerdeWithModifiedTuningConfigRemovedField() throws IOException
new IndexSpec(),
new IndexSpec(),
true,
true,
5L,
true,
false,
Expand Down Expand Up @@ -245,7 +239,6 @@ public void testSerdeWithModifiedTuningConfigRemovedField() throws IOException
Assert.assertEquals(base.getBasePersistDirectory(), deserialized.getBasePersistDirectory());
Assert.assertEquals(base.getMaxPendingPersists(), deserialized.getMaxPendingPersists());
Assert.assertEquals(base.getIndexSpec(), deserialized.getIndexSpec());
Assert.assertEquals(base.getBuildV9Directly(), deserialized.getBuildV9Directly());
Assert.assertEquals(base.isReportParseExceptions(), deserialized.isReportParseExceptions());
Assert.assertEquals(base.getHandoffConditionTimeout(), deserialized.getHandoffConditionTimeout());
Assert.assertEquals(base.isResetOffsetAutomatically(), deserialized.isResetOffsetAutomatically());
Expand All @@ -270,7 +263,6 @@ public void testResetOffsetAndSkipSequenceNotBothTrue() throws Exception
+ " \"maxRowsPerSegment\": 100,\n"
+ " \"intermediatePersistPeriod\": \"PT1H\",\n"
+ " \"maxPendingPersists\": 100,\n"
+ " \"buildV9Directly\": true,\n"
+ " \"reportParseExceptions\": true,\n"
+ " \"handoffConditionTimeout\": 100,\n"
+ " \"recordBufferSize\": 1000,\n"
Expand Down Expand Up @@ -305,7 +297,6 @@ public void testConvert()
new IndexSpec(),
new IndexSpec(),
true,
true,
5L,
true,
false,
Expand Down Expand Up @@ -339,7 +330,6 @@ public void testConvert()
Assert.assertEquals(new File("/tmp/xxx"), copy.getBasePersistDirectory());
Assert.assertEquals(4, copy.getMaxPendingPersists());
Assert.assertEquals(new IndexSpec(), copy.getIndexSpec());
Assert.assertTrue(copy.getBuildV9Directly());
Assert.assertTrue(copy.isReportParseExceptions());
Assert.assertEquals(5L, copy.getHandoffConditionTimeout());
Assert.assertEquals(1000, copy.getRecordBufferSize());
Expand Down
3 changes: 0 additions & 3 deletions ...ice/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,6 @@ public void setupTest()
null,
null,
null,
true,
false,
null,
null,
Expand Down Expand Up @@ -3772,7 +3771,6 @@ public void testIsTaskCurrent()
null,
null,
null,
true,
false,
null,
null,
Expand Down Expand Up @@ -4827,7 +4825,6 @@ public KinesisIndexTaskClient build(
null,
null,
null,
true,
false,
null,
resetOffsetAutomatically,
Expand Down
3 changes: 0 additions & 3 deletions .../java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTuningConfigTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ public void testSerdeWithDefaults() throws Exception
Assert.assertEquals(new Period("PT10M"), config.getIntermediatePersistPeriod());
Assert.assertEquals(0, config.getMaxPendingPersists());
Assert.assertEquals(new IndexSpec(), config.getIndexSpec());
Assert.assertEquals(true, config.getBuildV9Directly());
Assert.assertEquals(false, config.isReportParseExceptions());
Assert.assertEquals(0, config.getHandoffConditionTimeout());
Assert.assertNull(config.getWorkerThreads());
Expand All @@ -86,7 +85,6 @@ public void testSerdeWithNonDefaults() throws Exception
+ " \"maxRowsPerSegment\": 100,\n"
+ " \"intermediatePersistPeriod\": \"PT1H\",\n"
+ " \"maxPendingPersists\": 100,\n"
+ " \"buildV9Directly\": false,\n"
+ " \"reportParseExceptions\": true,\n"
+ " \"handoffConditionTimeout\": 100,\n"
+ " \"workerThreads\": 12,\n"
Expand Down Expand Up @@ -114,7 +112,6 @@ public void testSerdeWithNonDefaults() throws Exception
Assert.assertEquals(100, config.getMaxRowsPerSegment().intValue());
Assert.assertEquals(new Period("PT1H"), config.getIntermediatePersistPeriod());
Assert.assertEquals(100, config.getMaxPendingPersists());
Assert.assertEquals(true, config.getBuildV9Directly());
Assert.assertEquals(true, config.isReportParseExceptions());
Assert.assertEquals(100, config.getHandoffConditionTimeout());
Assert.assertEquals(12, (int) config.getWorkerThreads());
Expand Down
Loading