Skip to content

[opt](hive) simplify the isSplitable method to avoid too many hadoop metrics #33242

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 1 commit into from
Apr 4, 2024
Merged

[opt](hive) simplify the isSplitable method to avoid too many hadoop metrics #33242

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@
public class HMSExternalTable extends ExternalTable {
private static final Logger LOG = LogManager.getLogger(HMSExternalTable.class);

private static final Set<String> SUPPORTED_HIVE_FILE_FORMATS;
public static final Set<String> SUPPORTED_HIVE_FILE_FORMATS;
private static final Set<String> SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS;

private static final String TBL_PROP_TXN_PROPERTIES = "transactional_properties";
Expand Down Expand Up @@ -712,13 +712,13 @@ public long getDataSize(boolean singleReplica) {
@Override
public boolean isDistributionColumn(String columnName) {
return getRemoteTable().getSd().getBucketCols().stream().map(String::toLowerCase)
.collect(Collectors.toSet()).contains(columnName.toLowerCase());
.collect(Collectors.toSet()).contains(columnName.toLowerCase());
}

@Override
public Set<String> getDistributionColumnNames() {
return getRemoteTable().getSd().getBucketCols().stream().map(String::toLowerCase)
.collect(Collectors.toSet());
.collect(Collectors.toSet());
}
}

Expand Down
38 changes: 4 additions & 34 deletions fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,14 @@
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.fs.remote.BrokerFileSystem;
import org.apache.doris.fs.remote.RemoteFileSystem;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat;
import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat;
Expand All @@ -46,10 +44,7 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.List;
Expand Down Expand Up @@ -169,7 +164,7 @@ private static Type convertHiveTypeToiveDoris(TypeInfo hiveTypeInfo) {
return Type.DECIMALV2;
default:
throw new UnsupportedOperationException("Unsupported type: "
+ primitiveTypeInfo.getPrimitiveCategory());
+ primitiveTypeInfo.getPrimitiveCategory());
}
}
case LIST:
Expand All @@ -191,38 +186,13 @@ private static Type convertHiveTypeToiveDoris(TypeInfo hiveTypeInfo) {
}

public static boolean isSplittable(RemoteFileSystem remoteFileSystem, InputFormat<?, ?> inputFormat,
String location, JobConf jobConf) throws UserException {
String location, JobConf jobConf) throws UserException {
if (remoteFileSystem instanceof BrokerFileSystem) {
return ((BrokerFileSystem) remoteFileSystem)
.isSplittable(location, inputFormat.getClass().getCanonicalName());
}

// ORC uses a custom InputFormat but is always splittable
if (inputFormat.getClass().getSimpleName().equals("OrcInputFormat")) {
return true;
}
// use reflection to get isSplitable method on FileInputFormat
// ATTN: the method name is actually "isSplitable", but the right spell is "isSplittable"
Method method = null;
for (Class<?> clazz = inputFormat.getClass(); clazz != null; clazz = clazz.getSuperclass()) {
try {
method = clazz.getDeclaredMethod("isSplitable", FileSystem.class, Path.class);
break;
} catch (NoSuchMethodException ignored) {
LOG.debug("Class {} doesn't contain isSplitable method.", clazz);
}
}

if (method == null) {
return false;
}
Path path = new Path(location);
try {
method.setAccessible(true);
return (boolean) method.invoke(inputFormat, FileSystemFactory.getNativeByPath(path, jobConf), path);
} catch (InvocationTargetException | IllegalAccessException | IOException e) {
throw new RuntimeException(e);
}
return HMSExternalTable.SUPPORTED_HIVE_FILE_FORMATS.contains(inputFormat);
}

public static String getHivePartitionValue(String part) {
Expand Down