diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index c7ad269844b316..2d847bbd3f216d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -1306,10 +1306,6 @@ public void notifyPropertiesUpdated(Map updatedProps) { } } - public ThreadPoolExecutor getThreadPoolExecutor() { - return threadPoolWithPreAuth; - } - public CatalogProperty getCatalogProperty() { return catalogProperty; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 1fcc7083258910..a1516ae530eff2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -582,7 +582,7 @@ private Namespace getNamespace() { } public ThreadPoolExecutor getThreadPoolWithPreAuth() { - return dorisCatalog.getThreadPoolExecutor(); + return dorisCatalog.getThreadPoolWithPreAuth(); } private void performDropView(String remoteDbName, String remoteViewName) throws DdlException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java index 18af0fc3c117b8..021a3d3905f69d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/CsvFileFormatProperties.java @@ -50,6 +50,8 @@ public class CsvFileFormatProperties extends FileFormatProperties { public static final String PROP_ENCLOSE = "enclose"; public static final String PROP_ESCAPE = "escape"; + public static final String PROP_ENABLE_TEXT_VALIDATE_UTF8 = "enable_text_validate_utf8"; + private String headerType = ""; private String columnSeparator = DEFAULT_COLUMN_SEPARATOR; private String lineDelimiter = DEFAULT_LINE_DELIMITER; @@ -57,6 +59,7 @@ public class CsvFileFormatProperties extends FileFormatProperties { private int skipLines; private byte enclose; private byte escape; + private boolean enableTextValidateUTF8 = true; public CsvFileFormatProperties(String formatName) { super(TFileFormatType.FORMAT_CSV_PLAIN, formatName); @@ -118,6 +121,18 @@ public void analyzeFileFormatProperties(Map formatProperties, bo PROP_COMPRESS_TYPE, "UNKNOWN", isRemoveOriginProperty); compressionType = Util.getFileCompressType(compressTypeStr); + // get ENABLE_TEXT_VALIDATE_UTF8 from properties map first, + // if not exist, try getting from session variable, + // if connection context is null, use "true" as default value. + String validateUtf8 = getOrDefault(formatProperties, PROP_ENABLE_TEXT_VALIDATE_UTF8, "", + isRemoveOriginProperty); + if (Strings.isNullOrEmpty(validateUtf8)) { + enableTextValidateUTF8 = ConnectContext.get() == null ? true + : ConnectContext.get().getSessionVariable().enableTextValidateUtf8; + } else { + enableTextValidateUTF8 = Boolean.parseBoolean(validateUtf8); + } + } catch (org.apache.doris.common.AnalysisException e) { throw new AnalysisException(e.getMessage()); } @@ -142,8 +157,7 @@ public TFileAttributes toTFileAttributes() { fileAttributes.setHeaderType(headerType); fileAttributes.setTrimDoubleQuotes(trimDoubleQuotes); fileAttributes.setSkipLines(skipLines); - fileAttributes.setEnableTextValidateUtf8( - ConnectContext.get().getSessionVariable().enableTextValidateUtf8); + fileAttributes.setEnableTextValidateUtf8(enableTextValidateUTF8); return fileAttributes; }