[cherry-pick](branch-2.1) Make doris read hive text table parameters and behavior consistent with hive

docker/thirdparties/docker-compose/hive/scripts/data/regression/serde_prop/some_serde_table.hql

@@ -30,5 +30,62 @@ TBLPROPERTIES (
    'field.delim'='|'
 );
 
+CREATE TABLE `serde_test3`(
+  `id` int, 
+  `name` string)
+ROW FORMAT SERDE 
+  'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' 
+WITH SERDEPROPERTIES (  
+  'serialization.format'='g') 
+STORED AS INPUTFORMAT 
+  'org.apache.hadoop.mapred.TextInputFormat' 
+OUTPUTFORMAT 
+  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+
+CREATE TABLE `serde_test4`(
+  `id` int, 
+  `name` string)
+ROW FORMAT SERDE 
+  'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' 
+WITH SERDEPROPERTIES (
+  'field.delim' = 'gg',
+  "line.delim" = "hh")
+STORED AS INPUTFORMAT 
+  'org.apache.hadoop.mapred.TextInputFormat' 
+OUTPUTFORMAT 
+  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+CREATE TABLE `serde_test5`(
+  `id` int, 
+  `name` string)
+ROW FORMAT SERDE 
+  'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' 
+WITH SERDEPROPERTIES (
+  'field.delim' = '16',
+  "line.delim" = "21")
+STORED AS INPUTFORMAT 
+  'org.apache.hadoop.mapred.TextInputFormat' 
+OUTPUTFORMAT 
+  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+CREATE TABLE `serde_test6`(
+  `id` int, 
+  `name` string)
+ROW FORMAT SERDE 
+  'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' 
+WITH SERDEPROPERTIES (
+  'field.delim' = '\16',
+  "line.delim" = "\21")
+STORED AS INPUTFORMAT 
+  'org.apache.hadoop.mapred.TextInputFormat' 
+OUTPUTFORMAT 
+  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
+
 insert into serde_test1 values(1, "abc"),(2, "def");
 insert into serde_test2 values(1, "abc"),(2, "def");
+insert into serde_test3 values(1, "abc"),(2, "def");
+insert into serde_test4 values(1, "abc"),(2, "def");
+insert into serde_test5 values(1, "abc"),(2, "def");
+insert into serde_test6 values(1, "abc"),(2, "def");

fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java

@@ -873,4 +873,21 @@ public static String firstPresentOrDefault(String defaultValue, Optional<String>
         }
         return defaultValue;
     }
+
+    /**
+     * Return the byte value of the number string.
+     *
+     * @param altValue
+     *                 The string containing a number.
+     */
+    public static String getByte(String altValue) {
+        if (altValue != null && altValue.length() > 0) {
+            try {
+                return String.valueOf((char) ((Byte.parseByte(altValue) + 256) % 256));
+            } catch (NumberFormatException e) {
+                return altValue.substring(0, 1);
+            }
+        }
+        return null;
+    }
 }

fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java

@@ -86,6 +86,7 @@ public class HiveScanNode extends FileQueryScanNode {
     public static final String DEFAULT_LINE_DELIMITER = "\n";
     public static final String PROP_SEPARATOR_CHAR = "separatorChar";
     public static final String PROP_QUOTE_CHAR = "quoteChar";
+    public static final String PROP_SERIALIZATION_FORMAT = "serialization.format";
 
     public static final String PROP_COLLECTION_DELIMITER_HIVE2 = "colelction.delim";
     public static final String PROP_COLLECTION_DELIMITER_HIVE3 = "collection.delim";
@@ -447,29 +448,32 @@ protected TFileAttributes getFileAttributes() throws UserException {
         TFileTextScanRangeParams textParams = new TFileTextScanRangeParams();
 
         // 1. set column separator
-        Optional<String> fieldDelim =
-                HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(), PROP_FIELD_DELIMITER);
-        Optional<String> columnSeparator =
-                HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(), PROP_SEPARATOR_CHAR);
-        textParams.setColumnSeparator(HiveMetaStoreClientHelper.firstPresentOrDefault(
-                DEFAULT_FIELD_DELIMITER, fieldDelim, columnSeparator));
+        Optional<String> fieldDelim = HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(),
+                PROP_FIELD_DELIMITER);
+        Optional<String> serFormat = HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(),
+                PROP_SERIALIZATION_FORMAT);
+        Optional<String> columnSeparator = HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(),
+                PROP_SEPARATOR_CHAR);
+        textParams.setColumnSeparator(HiveMetaStoreClientHelper.getByte(HiveMetaStoreClientHelper.firstPresentOrDefault(
+                DEFAULT_FIELD_DELIMITER, fieldDelim, columnSeparator, serFormat)));
         // 2. set line delimiter
-        Optional<String> lineDelim =
-                HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(), PROP_LINE_DELIMITER);
-        textParams.setLineDelimiter(HiveMetaStoreClientHelper.firstPresentOrDefault(
-                DEFAULT_LINE_DELIMITER, lineDelim));
+        Optional<String> lineDelim = HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(),
+                PROP_LINE_DELIMITER);
+        textParams.setLineDelimiter(HiveMetaStoreClientHelper.getByte(HiveMetaStoreClientHelper.firstPresentOrDefault(
+                DEFAULT_LINE_DELIMITER, lineDelim)));
         // 3. set mapkv delimiter
-        Optional<String> mapkvDelim =
-                HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(), PROP_MAP_KV_DELIMITER);
-        textParams.setMapkvDelimiter(HiveMetaStoreClientHelper.firstPresentOrDefault(
-                DEFAULT_MAP_KV_DELIMITER, mapkvDelim));
+        Optional<String> mapkvDelim = HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(),
+                PROP_MAP_KV_DELIMITER);
+        textParams.setMapkvDelimiter(HiveMetaStoreClientHelper.getByte(HiveMetaStoreClientHelper.firstPresentOrDefault(
+                DEFAULT_MAP_KV_DELIMITER, mapkvDelim)));
         // 4. set collection delimiter
-        Optional<String> collectionDelimHive2 =
-                HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(), PROP_COLLECTION_DELIMITER_HIVE2);
-        Optional<String> collectionDelimHive3 =
-                HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(), PROP_COLLECTION_DELIMITER_HIVE3);
-        textParams.setCollectionDelimiter(HiveMetaStoreClientHelper.firstPresentOrDefault(
-                DEFAULT_COLLECTION_DELIMITER, collectionDelimHive2, collectionDelimHive3));
+        Optional<String> collectionDelimHive2 = HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(),
+                PROP_COLLECTION_DELIMITER_HIVE2);
+        Optional<String> collectionDelimHive3 = HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(),
+                PROP_COLLECTION_DELIMITER_HIVE3);
+        textParams.setCollectionDelimiter(
+                HiveMetaStoreClientHelper.getByte(HiveMetaStoreClientHelper.firstPresentOrDefault(
+                        DEFAULT_COLLECTION_DELIMITER, collectionDelimHive2, collectionDelimHive3)));
         // 5. set quote char
         Map<String, String> serdeParams = hmsTable.getRemoteTable().getSd().getSerdeInfo().getParameters();
         if (serdeParams.containsKey(PROP_QUOTE_CHAR)) {

regression-test/data/external_table_p0/hive/test_hive_serde_prop.out

@@ -7,7 +7,23 @@ b	2.2
 1	abc
 2	def
 
--- !2 --
+-- !3 --
+1	abc
+2	def
+
+-- !4 --
+1	abc
+2	def
+
+-- !5 --
+1	abc
+2	def
+
+-- !6 --
+1	abc
+2	def
+
+-- !7 --
 1	abc
 2	def
 
@@ -19,7 +35,23 @@ b	2.2
 1	abc
 2	def
 
--- !2 --
+-- !3 --
+1	abc
+2	def
+
+-- !4 --
+1	abc
+2	def
+
+-- !5 --
+1	abc
+2	def
+
+-- !6 --
+1	abc
+2	def
+
+-- !7 --
 1	abc
 2	def
 

regression-test/suites/external_table_p0/hive/test_hive_serde_prop.groovy

@@ -40,7 +40,11 @@ suite("test_hive_serde_prop", "external_docker,hive,external_docker_hive,p0,exte
 
 
         qt_2 """select * from ${catalog_name}.regression.serde_test1 order by id;"""
-        qt_2 """select * from ${catalog_name}.regression.serde_test2 order by id;"""
+        qt_3 """select * from ${catalog_name}.regression.serde_test2 order by id;"""
+        qt_4 """select * from ${catalog_name}.regression.serde_test3 order by id;"""
+        qt_5 """select * from ${catalog_name}.regression.serde_test4 order by id;"""
+        qt_6 """select * from ${catalog_name}.regression.serde_test5 order by id;"""
+        qt_7 """select * from ${catalog_name}.regression.serde_test6 order by id;"""
     }
 }