diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java index eec08ff133df..7d59cb63ea5b 100644 --- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java +++ b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java @@ -184,6 +184,7 @@ public HadoopIndexTask createTask(Interval interval, String version, List= 0, "Not support persistBackgroundCount < 0"); this.useExplicitVersion = useExplicitVersion; - this.allowedHadoopPrefix = allowedHadoopPrefix == null ? ImmutableList.of() : allowedHadoopPrefix; - - this.ignoreInvalidRows = ignoreInvalidRows == null ? false : ignoreInvalidRows; - if (maxParseExceptions != null) { - this.maxParseExceptions = maxParseExceptions; - } else { - if (!this.ignoreInvalidRows) { - this.maxParseExceptions = 0; - } else { - this.maxParseExceptions = TuningConfig.DEFAULT_MAX_PARSE_EXCEPTIONS; - } - } - this.logParseExceptions = logParseExceptions == null ? TuningConfig.DEFAULT_LOG_PARSE_EXCEPTIONS : logParseExceptions; + this.allowedHadoopPrefix = Configs.valueOrDefault(allowedHadoopPrefix, Collections.emptyList()); - this.useYarnRMJobStatusFallback = useYarnRMJobStatusFallback == null ? true : useYarnRMJobStatusFallback; + this.ignoreInvalidRows = Configs.valueOrDefault(ignoreInvalidRows, false); + this.maxParseExceptions = Configs.valueOrDefault( + maxParseExceptions, + this.ignoreInvalidRows ? TuningConfig.DEFAULT_MAX_PARSE_EXCEPTIONS : 0 + ); + this.logParseExceptions = Configs.valueOrDefault(logParseExceptions, TuningConfig.DEFAULT_LOG_PARSE_EXCEPTIONS); + this.useYarnRMJobStatusFallback = Configs.valueOrDefault(useYarnRMJobStatusFallback, true); if (awaitSegmentAvailabilityTimeoutMillis == null || awaitSegmentAvailabilityTimeoutMillis < 0) { this.awaitSegmentAvailabilityTimeoutMillis = DEFAULT_AWAIT_SEGMENT_AVAILABILITY_TIMEOUT_MILLIS; @@ -263,6 +266,12 @@ public long getMaxBytesInMemory() return maxBytesInMemory; } + @JsonProperty + public boolean isUseMaxMemoryEstimates() + { + return useMaxMemoryEstimates; + } + @JsonProperty public boolean isLeaveIntermediate() { @@ -372,6 +381,7 @@ public HadoopTuningConfig withWorkingPath(String path) appendableIndexSpec, maxRowsInMemory, maxBytesInMemory, + useMaxMemoryEstimates, leaveIntermediate, cleanupOnFailure, overwriteFiles, @@ -404,6 +414,7 @@ public HadoopTuningConfig withVersion(String ver) appendableIndexSpec, maxRowsInMemory, maxBytesInMemory, + useMaxMemoryEstimates, leaveIntermediate, cleanupOnFailure, overwriteFiles, @@ -436,6 +447,7 @@ public HadoopTuningConfig withShardSpecs(Map> specs appendableIndexSpec, maxRowsInMemory, maxBytesInMemory, + useMaxMemoryEstimates, leaveIntermediate, cleanupOnFailure, overwriteFiles, diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java index 967b55665c85..c60874e7111d 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java @@ -310,6 +310,7 @@ private static IncrementalIndex makeIncrementalIndex( .setIndexSchema(indexSchema) .setMaxRowCount(tuningConfig.getMaxRowsInMemory()) .setMaxBytesInMemory(tuningConfig.getMaxBytesInMemoryOrDefault()) + .setUseMaxMemoryEstimates(tuningConfig.isUseMaxMemoryEstimates()) .build(); if (oldDimOrder != null && !indexSchema.getDimensionsSpec().hasCustomDimensions()) { diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/BatchDeltaIngestionTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/BatchDeltaIngestionTest.java index ce725230b963..ed8b8c0bb093 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/BatchDeltaIngestionTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/BatchDeltaIngestionTest.java @@ -478,6 +478,7 @@ private HadoopDruidIndexerConfig makeHadoopDruidIndexerConfig( false, false, false, + false, null, false, false, diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineHashedPartitionsJobTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineHashedPartitionsJobTest.java index fb1ff1520a28..24a8ee0ef7eb 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineHashedPartitionsJobTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineHashedPartitionsJobTest.java @@ -220,6 +220,7 @@ public DetermineHashedPartitionsJobTest( false, false, false, + false, null, false, false, diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java index 5f338936a409..a3c98f29565b 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DeterminePartitionsJobTest.java @@ -331,6 +331,7 @@ public DeterminePartitionsJobTest( false, false, false, + false, null, false, false, diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineRangePartitionsJobTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineRangePartitionsJobTest.java index f10a898d1260..e79d066ab55c 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineRangePartitionsJobTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/DetermineRangePartitionsJobTest.java @@ -382,7 +382,7 @@ public DetermineRangePartitionsJobTest( null, null, null, - false, + false, false, false, false, false, diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidIndexerConfigTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidIndexerConfigTest.java index f69ad04915c5..8aead05d625b 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidIndexerConfigTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidIndexerConfigTest.java @@ -269,6 +269,7 @@ HadoopIngestionSpec build() false, false, false, + false, null, false, false, diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopTuningConfigTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopTuningConfigTest.java index 97c764dd10b1..5857f92172f2 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopTuningConfigTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopTuningConfigTest.java @@ -48,6 +48,7 @@ public void testSerde() throws Exception null, 100, null, + false, true, true, true, diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorJobTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorJobTest.java index 2b8fc6749e31..e14ade454f4c 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorJobTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorJobTest.java @@ -532,6 +532,7 @@ public void setUp() throws Exception null, maxRowsInMemory, maxBytesInMemory, + false, true, false, false, diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/JobHelperTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/JobHelperTest.java index 3c1b80166c51..7069e9a78de3 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/JobHelperTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/JobHelperTest.java @@ -168,6 +168,7 @@ public void setup() throws Exception false, false, false, + false, //Map of job properties ImmutableMap.of( "fs.s3.impl", diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/GranularityPathSpecTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/GranularityPathSpecTest.java index d1aee3d66992..8af77ca0e4fd 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/GranularityPathSpecTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/path/GranularityPathSpecTest.java @@ -68,6 +68,7 @@ public class GranularityPathSpecTest false, false, false, + false, null, false, false, diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/AppendableIndexBuilder.java b/processing/src/main/java/org/apache/druid/segment/incremental/AppendableIndexBuilder.java index 777c4cc1fd7a..fcfca99cdfe8 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/AppendableIndexBuilder.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/AppendableIndexBuilder.java @@ -37,7 +37,7 @@ public abstract class AppendableIndexBuilder // DruidInputSource since that is the only case where we can have existing metrics. // This is currently only use by auto compaction and should not be use for anything else. protected boolean preserveExistingMetrics = false; - protected boolean useMaxMemoryEstimates = true; + protected boolean useMaxMemoryEstimates = false; protected final Logger log = new Logger(this.getClass()); diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/AppendableIndexSpec.java b/processing/src/main/java/org/apache/druid/segment/incremental/AppendableIndexSpec.java index 67cdabdf5673..44b56aaa13ad 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/AppendableIndexSpec.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/AppendableIndexSpec.java @@ -22,14 +22,18 @@ import org.apache.druid.guice.annotations.UnstableApi; /** - * AppendableIndexSpec describes the in-memory indexing method for data ingestion. + * Describes the in-memory indexing method for data ingestion. */ @UnstableApi public interface AppendableIndexSpec { - // Returns a builder of the appendable index. + /** + * Creates a new builder of the appendable index. + */ AppendableIndexBuilder builder(); - // Returns the default max bytes in memory for this index. + /** + * Returns the default max bytes in memory for this index. + */ long getDefaultMaxBytesInMemory(); }