From 1931491c9f13f7e1b5dbeacab15ece3d2cd910f7 Mon Sep 17 00:00:00 2001 From: Hao Xia Date: Fri, 19 Jun 2015 17:22:20 -0700 Subject: [PATCH] A couple of hdfs related fixes * Class loading issue with hdfs-storage extension * Exception when using hdfs with non-fully qualified segment path --- .../java/io/druid/storage/hdfs/HdfsStorageDruidModule.java | 4 ++++ indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java b/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java index 6707cdf97f5e..52eadd3a308e 100644 --- a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java +++ b/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java @@ -81,6 +81,10 @@ public void configure(Binder binder) Binders.dataSegmentKillerBinder(binder).addBinding(SCHEME).to(HdfsDataSegmentKiller.class).in(LazySingleton.class); final Configuration conf = new Configuration(); + + // Set explicit CL. Otherwise it'll try to use thread context CL, which may not have all of our dependencies. + conf.setClassLoader(getClass().getClassLoader()); + if (props != null) { for (String propName : System.getProperties().stringPropertyNames()) { if (propName.startsWith("hadoop.")) { diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java index 22f1f605e4cf..19c743b82409 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java @@ -518,7 +518,7 @@ public Boolean call() throws Exception public static Path prependFSIfNullScheme(FileSystem fs, Path path) { if (path.toUri().getScheme() == null) { - path = new Path(fs.getUri().toString(), String.format("./%s", path)); + path = fs.makeQualified(path); } return path; }