diff --git a/api/src/main/java/io/druid/segment/loading/DataSegmentPusher.java b/api/src/main/java/io/druid/segment/loading/DataSegmentPusher.java index 9638ed48b15c..4e157cd81c9c 100644 --- a/api/src/main/java/io/druid/segment/loading/DataSegmentPusher.java +++ b/api/src/main/java/io/druid/segment/loading/DataSegmentPusher.java @@ -25,6 +25,8 @@ import java.io.File; import java.io.IOException; import java.net.URI; +import java.util.Collections; +import java.util.List; import java.util.Map; public interface DataSegmentPusher @@ -44,6 +46,15 @@ default String makeIndexPathName(DataSegment dataSegment, String indexName) { return String.format("./%s/%s", getStorageDir(dataSegment),indexName); } + /** + * Property prefixes that should be added to the "allowedHadoopPrefix" config for passing down to Hadoop jobs. These + * should be property prefixes like "druid.xxx", which means to include "druid.xxx" and "druid.xxx.*". + */ + default List getAllowedPropertyPrefixesForHadoop() + { + return Collections.emptyList(); + } + // Note: storage directory structure format = .../dataSource/interval/version/partitionNumber/ // If above format is ever changed, make sure to change it appropriately in other places // e.g. HDFSDataSegmentKiller uses this information to clean the version, interval and dataSource directories diff --git a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java index 0b9f7bea9afa..6ddaca7b64ab 100644 --- a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java +++ b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.microsoft.azure.storage.StorageException; @@ -36,6 +37,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.util.List; import java.util.Map; import java.util.concurrent.Callable; @@ -72,6 +74,12 @@ public String getPathForHadoop() return null; } + @Override + public List getAllowedPropertyPrefixesForHadoop() + { + return ImmutableList.of("druid.azure"); + } + public File createSegmentDescriptorFile(final ObjectMapper jsonMapper, final DataSegment segment) throws IOException { diff --git a/extensions-contrib/google-extensions/src/main/java/io/druid/storage/google/GoogleDataSegmentPusher.java b/extensions-contrib/google-extensions/src/main/java/io/druid/storage/google/GoogleDataSegmentPusher.java index bce847d88225..bff3949105db 100644 --- a/extensions-contrib/google-extensions/src/main/java/io/druid/storage/google/GoogleDataSegmentPusher.java +++ b/extensions-contrib/google-extensions/src/main/java/io/druid/storage/google/GoogleDataSegmentPusher.java @@ -24,6 +24,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import io.druid.java.util.common.CompressionUtils; @@ -37,6 +38,7 @@ import java.io.IOException; import java.net.URI; import java.nio.file.Files; +import java.util.List; import java.util.Map; public class GoogleDataSegmentPusher implements DataSegmentPusher @@ -74,6 +76,12 @@ public String getPathForHadoop() return String.format("gs://%s/%s", config.getBucket(), config.getPrefix()); } + @Override + public List getAllowedPropertyPrefixesForHadoop() + { + return ImmutableList.of("druid.google"); + } + public File createDescriptorFile(final ObjectMapper jsonMapper, final DataSegment segment) throws IOException { diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java index f5020ed297c4..8c79c1039fe6 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.metamx.emitter.EmittingLogger; @@ -37,6 +38,7 @@ import java.io.IOException; import java.net.URI; import java.nio.file.Files; +import java.util.List; import java.util.Map; import java.util.concurrent.Callable; @@ -78,6 +80,12 @@ public String getPathForHadoop(String dataSource) return getPathForHadoop(); } + @Override + public List getAllowedPropertyPrefixesForHadoop() + { + return ImmutableList.of("druid.s3"); + } + @Override public DataSegment push(final File indexFilesDir, final DataSegment inSegment) throws IOException { diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java index d035536cba0b..39057573c6cd 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java @@ -73,6 +73,7 @@ import java.io.Reader; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -260,7 +261,13 @@ public ShardSpec apply(HadoopyShardSpec input) } this.rollupGran = spec.getDataSchema().getGranularitySpec().getQueryGranularity(); - this.allowedHadoopPrefix = spec.getTuningConfig().getAllowedHadoopPrefix(); + + // User-specified list plus our additional bonus list. + this.allowedHadoopPrefix = new ArrayList<>(); + this.allowedHadoopPrefix.add("druid.storage"); + this.allowedHadoopPrefix.add("druid.javascript"); + this.allowedHadoopPrefix.addAll(DATA_SEGMENT_PUSHER.getAllowedPropertyPrefixesForHadoop()); + this.allowedHadoopPrefix.addAll(spec.getTuningConfig().getUserAllowedHadoopPrefix()); } @JsonProperty(value = "spec") diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopTuningConfig.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopTuningConfig.java index b131a5ed5d7d..44e7c1a65b04 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopTuningConfig.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopTuningConfig.java @@ -139,9 +139,7 @@ public HadoopTuningConfig( this.forceExtendableShardSpecs = forceExtendableShardSpecs; Preconditions.checkArgument(this.numBackgroundPersistThreads >= 0, "Not support persistBackgroundCount < 0"); this.useExplicitVersion = useExplicitVersion; - this.allowedHadoopPrefix = allowedHadoopPrefix == null - ? ImmutableList.of("druid.storage.", "druid.javascript.") - : allowedHadoopPrefix; + this.allowedHadoopPrefix = allowedHadoopPrefix == null ? ImmutableList.of() : allowedHadoopPrefix; } @JsonProperty @@ -321,9 +319,10 @@ public HadoopTuningConfig withShardSpecs(Map> specs ); } - @JsonProperty - public List getAllowedHadoopPrefix() + @JsonProperty("allowedHadoopPrefix") + public List getUserAllowedHadoopPrefix() { + // Just the user-specified list. More are added in HadoopDruidIndexerConfig. return allowedHadoopPrefix; } } diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java index 34b19245138f..34cc92e65e95 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java @@ -317,7 +317,7 @@ public static void injectDruidProperties(Configuration configuration, List