diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java index 2dc1c0d6e486fc..a268916d33d411 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java @@ -79,7 +79,7 @@ public class HMSExternalTable extends ExternalTable { private static final Logger LOG = LogManager.getLogger(HMSExternalTable.class); - private static final Set SUPPORTED_HIVE_FILE_FORMATS; + public static final Set SUPPORTED_HIVE_FILE_FORMATS; private static final Set SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS; private static final String TBL_PROP_TXN_PROPERTIES = "transactional_properties"; @@ -712,13 +712,13 @@ public long getDataSize(boolean singleReplica) { @Override public boolean isDistributionColumn(String columnName) { return getRemoteTable().getSd().getBucketCols().stream().map(String::toLowerCase) - .collect(Collectors.toSet()).contains(columnName.toLowerCase()); + .collect(Collectors.toSet()).contains(columnName.toLowerCase()); } @Override public Set getDistributionColumnNames() { return getRemoteTable().getSd().getBucketCols().stream().map(String::toLowerCase) - .collect(Collectors.toSet()); + .collect(Collectors.toSet()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java b/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java index deb048b59439ec..f0f013fa1f0cb1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java @@ -21,16 +21,14 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import org.apache.doris.catalog.external.HMSExternalTable; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; -import org.apache.doris.fs.FileSystemFactory; import org.apache.doris.fs.remote.BrokerFileSystem; import org.apache.doris.fs.remote.RemoteFileSystem; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat; import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; @@ -46,10 +44,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.util.List; @@ -169,7 +164,7 @@ private static Type convertHiveTypeToiveDoris(TypeInfo hiveTypeInfo) { return Type.DECIMALV2; default: throw new UnsupportedOperationException("Unsupported type: " - + primitiveTypeInfo.getPrimitiveCategory()); + + primitiveTypeInfo.getPrimitiveCategory()); } } case LIST: @@ -191,38 +186,13 @@ private static Type convertHiveTypeToiveDoris(TypeInfo hiveTypeInfo) { } public static boolean isSplittable(RemoteFileSystem remoteFileSystem, InputFormat inputFormat, - String location, JobConf jobConf) throws UserException { + String location, JobConf jobConf) throws UserException { if (remoteFileSystem instanceof BrokerFileSystem) { return ((BrokerFileSystem) remoteFileSystem) .isSplittable(location, inputFormat.getClass().getCanonicalName()); } - // ORC uses a custom InputFormat but is always splittable - if (inputFormat.getClass().getSimpleName().equals("OrcInputFormat")) { - return true; - } - // use reflection to get isSplitable method on FileInputFormat - // ATTN: the method name is actually "isSplitable", but the right spell is "isSplittable" - Method method = null; - for (Class clazz = inputFormat.getClass(); clazz != null; clazz = clazz.getSuperclass()) { - try { - method = clazz.getDeclaredMethod("isSplitable", FileSystem.class, Path.class); - break; - } catch (NoSuchMethodException ignored) { - LOG.debug("Class {} doesn't contain isSplitable method.", clazz); - } - } - - if (method == null) { - return false; - } - Path path = new Path(location); - try { - method.setAccessible(true); - return (boolean) method.invoke(inputFormat, FileSystemFactory.getNativeByPath(path, jobConf), path); - } catch (InvocationTargetException | IllegalAccessException | IOException e) { - throw new RuntimeException(e); - } + return HMSExternalTable.SUPPORTED_HIVE_FILE_FORMATS.contains(inputFormat); } public static String getHivePartitionValue(String part) {