From a7c3dac554815386886ca2a14a0fe695a8274c2f Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 24 Jan 2018 14:24:06 -0600 Subject: [PATCH 1/2] use reflection to call hadoop fs.rename to workaround different hadoop jar version in main and hdfs-storage extension class loader --- .../org/apache/hadoop/fs/HadoopFsWrapper.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java b/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java index 4604241dd76a..be7610c49943 100644 --- a/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java +++ b/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java @@ -19,9 +19,12 @@ package org.apache.hadoop.fs; +import com.google.common.base.Throwables; import io.druid.java.util.common.logger.Logger; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; /** * This wrapper class is created to be able to access some of the the "protected" methods inside Hadoop's @@ -50,12 +53,23 @@ private HadoopFsWrapper() {} public static boolean rename(FileSystem fs, Path from, Path to, boolean replaceExisting) throws IOException { try { - fs.rename(from, to, replaceExisting ? Options.Rename.OVERWRITE : Options.Rename.NONE); + // Note: Using reflection instead of simpler + // fs.rename(from, to, replaceExisting ? Options.Rename.OVERWRITE : Options.Rename.NONE); + // due to the issues discussed in https://github.com/druid-io/druid/pull/3787 + Method renameMethod = fs.getClass().getMethod("rename", Path.class, Path.class, Options.Rename[].class); + renameMethod.invoke(fs, from, to, new Options.Rename[]{Options.Rename.NONE}); return true; } - catch (FileAlreadyExistsException ex) { - log.info(ex, "Destination exists while renaming [%s] to [%s]", from, to); - return false; + catch (InvocationTargetException ex) { + if (ex.getTargetException() instanceof FileAlreadyExistsException) { + log.info(ex, "Destination exists while renaming [%s] to [%s]", from, to); + return false; + } else { + throw Throwables.propagate(ex); + } + } + catch (NoSuchMethodException | IllegalAccessException ex) { + throw Throwables.propagate(ex); } } } From 15e738393d0c170f8d24bc0f42d75e662e8532f9 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Thu, 25 Jan 2018 11:58:26 -0600 Subject: [PATCH 2/2] find rename method recursively --- .../org/apache/hadoop/fs/HadoopFsWrapper.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java b/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java index be7610c49943..eb0b8c6a62e2 100644 --- a/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java +++ b/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java @@ -56,7 +56,7 @@ public static boolean rename(FileSystem fs, Path from, Path to, boolean replaceE // Note: Using reflection instead of simpler // fs.rename(from, to, replaceExisting ? Options.Rename.OVERWRITE : Options.Rename.NONE); // due to the issues discussed in https://github.com/druid-io/druid/pull/3787 - Method renameMethod = fs.getClass().getMethod("rename", Path.class, Path.class, Options.Rename[].class); + Method renameMethod = findRenameMethodRecursively(fs.getClass()); renameMethod.invoke(fs, from, to, new Options.Rename[]{Options.Rename.NONE}); return true; } @@ -69,7 +69,33 @@ public static boolean rename(FileSystem fs, Path from, Path to, boolean replaceE } } catch (NoSuchMethodException | IllegalAccessException ex) { + + for (Method method : fs.getClass().getDeclaredMethods()) { + log.error(method.toGenericString()); + } throw Throwables.propagate(ex); } } + + /** + * Finds "rename" method recursively through the FileSystem class hierarchy. This is required because + * clazz.getMethod(..) only returns PUBLIC methods in clazz hierarchy. + * and clazz.getDeclaredMethod(..) only returns all methods declared in clazz but not inherited ones. + */ + private static Method findRenameMethodRecursively(Class clazz) throws NoSuchMethodException + { + try { + Method renameMethod = clazz.getDeclaredMethod("rename", Path.class, Path.class, Options.Rename[].class); + renameMethod.setAccessible(true); + return renameMethod; + } + catch (NoSuchMethodException ex) { + Class superClazz = clazz.getSuperclass(); + if (superClazz == null) { + throw ex; + } else { + return findRenameMethodRecursively(superClazz); + } + } + } }