From 2ea78315c8a62273b8fe7410539075109f179aae Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Fri, 9 Jan 2015 23:39:47 -0800 Subject: [PATCH 1/2] Walk around a typical case that 'maven-assembly' causes bug about 'No FileSystem for scheme: hdfs' while loading hadoop-hdfs dependency --- .../java/io/druid/storage/hdfs/HdfsStorageDruidModule.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java b/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java index 10e86973f22f..ab6f3dd4a3b5 100644 --- a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java +++ b/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java @@ -30,6 +30,7 @@ import io.druid.storage.hdfs.tasklog.HdfsTaskLogs; import io.druid.storage.hdfs.tasklog.HdfsTaskLogsConfig; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DistributedFileSystem; import java.util.List; import java.util.Properties; @@ -60,6 +61,10 @@ public void configure(Binder binder) Binders.dataSegmentKillerBinder(binder).addBinding("hdfs").to(HdfsDataSegmentKiller.class).in(LazySingleton.class); final Configuration conf = new Configuration(); + + // Walk around a typical case that "maven-assembly" causes bug about "No FileSystem for scheme: hdfs" while loading hadoop-hdfs dependency + conf.set("fs.hdfs.impl", DistributedFileSystem.class.getName()); + if (props != null) { for (String propName : System.getProperties().stringPropertyNames()) { if (propName.startsWith("hadoop.")) { From c5a713db362996ad729b83412285e3fe9ddd64a1 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Sun, 11 Jan 2015 18:05:35 -0800 Subject: [PATCH 2/2] Set fs.file.impl configure --- .../main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java b/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java index ab6f3dd4a3b5..51d761fca971 100644 --- a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java +++ b/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java @@ -30,6 +30,7 @@ import io.druid.storage.hdfs.tasklog.HdfsTaskLogs; import io.druid.storage.hdfs.tasklog.HdfsTaskLogsConfig; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.hdfs.DistributedFileSystem; import java.util.List; @@ -64,6 +65,7 @@ public void configure(Binder binder) // Walk around a typical case that "maven-assembly" causes bug about "No FileSystem for scheme: hdfs" while loading hadoop-hdfs dependency conf.set("fs.hdfs.impl", DistributedFileSystem.class.getName()); + conf.set("fs.file.impl", LocalFileSystem.class.getName()); if (props != null) { for (String propName : System.getProperties().stringPropertyNames()) {