From 637ebe99d64551e3b420baa94388e168fe94339e Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Mon, 1 Apr 2024 17:40:45 +0800
Subject: [PATCH 01/26] add hive3 docker compose

---
 .../docker-compose/hive/hive-3x.yaml.tpl      | 105 ++++++++++++++++++
 .../thirdparties/run-thirdparties-docker.sh   |  39 ++++++-
 2 files changed, 141 insertions(+), 3 deletions(-)
 create mode 100644 docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl

diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
new file mode 100644
index 00000000000000..ee47231eb2188c
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -0,0 +1,105 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+version: "3.8"
+
+services:
+  doris--namenode:
+    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
+    environment:
+      - CLUSTER_NAME=test
+    env_file:
+      - ./hadoop-hive.env
+    container_name: doris--namenode
+    expose:
+      - "9870"
+      - "8020"
+      - "9000"
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:9870/" ]
+      interval: 5s
+      timeout: 120s
+      retries: 120
+    network_mode: "host"
+
+  doris--datanode:
+    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
+    env_file:
+      - ./hadoop-hive.env
+    environment:
+      SERVICE_PRECONDITION: "externalEnvIp:9870"
+    container_name: doris--datanode
+    expose:
+      - "9864"
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:9864" ]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    network_mode: "host"
+
+  doris--hive-server:
+    image: lishizhen/hive:3.1.2-postgresql-metastore
+    env_file:
+      - ./hadoop-hive.env
+    environment:
+      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://externalEnvIp:5432/metastore"
+      SERVICE_PRECONDITION: "externalEnvIp:9083"
+    container_name: doris--hive-server
+    expose:
+      - "10000"
+    depends_on:
+      - doris--datanode
+      - doris--namenode
+    healthcheck:
+      test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;"
+      interval: 10s
+      timeout: 120s
+      retries: 120
+    network_mode: "host"
+
+
+  doris--hive-metastore:
+    image: lishizhen/hive:3.1.2-postgresql-metastore
+    env_file:
+      - ./hadoop-hive.env
+    command: /bin/bash /mnt/scripts/hive-metastore.sh
+    # command: /opt/hive/bin/hive --service metastore
+    environment:
+      SERVICE_PRECONDITION: "externalEnvIp:9870 externalEnvIp:9864 externalEnvIp:5432"
+    container_name: doris--hive-metastore
+    expose:
+      - "9083"
+    volumes:
+      - ./scripts:/mnt/scripts
+    depends_on:
+      - doris--hive-metastore-postgresql
+    network_mode: "host"
+
+  doris--hive-metastore-postgresql:
+    image: bde2020/hive-metastore-postgresql:3.1.0
+    restart: always
+    container_name: doris--hive-metastore-postgresql
+    expose:
+      - "5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    network_mode: "host"
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index 0e565c2af3b875..ef69a618b3dd7f 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -37,7 +37,7 @@ Usage: $0 <options>
      --stop             stop the specified components
 
   All valid components:
-    mysql,pg,oracle,sqlserver,clickhouse,es,hive,iceberg,hudi,trino,kafka,mariadb,db2
+    mysql,pg,oracle,sqlserver,clickhouse,es,hive,hive3,iceberg,hudi,trino,kafka,mariadb,db2
   "
     exit 1
 }
@@ -60,7 +60,7 @@ STOP=0
 
 if [[ "$#" == 1 ]]; then
     # default
-    COMPONENTS="mysql,es,hive,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2"
+    COMPONENTS="mysql,es,hive,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2"
 else
     while true; do
         case "$1" in
@@ -92,7 +92,7 @@ else
     done
     if [[ "${COMPONENTS}"x == ""x ]]; then
         if [[ "${STOP}" -eq 1 ]]; then
-            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino,kafka,mariadb,db2"
+            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,hive3,iceberg,hudi,trino,kafka,mariadb,db2"
         fi
     fi
 fi
@@ -126,6 +126,7 @@ RUN_ORACLE=0
 RUN_SQLSERVER=0
 RUN_CLICKHOUSE=0
 RUN_HIVE=0
+RUN_HIVE3=0;
 RUN_ES=0
 RUN_ICEBERG=0
 RUN_HUDI=0
@@ -150,6 +151,8 @@ for element in "${COMPONENTS_ARR[@]}"; do
         RUN_ES=1
     elif [[ "${element}"x == "hive"x ]]; then
         RUN_HIVE=1
+    elif [[ "${element}"x == "hive3"x ]]; then
+        RUN_HIVE3=1
     elif [[ "${element}"x == "kafka"x ]]; then
         RUN_KAFKA=1
     elif [[ "${element}"x == "iceberg"x ]]; then
@@ -328,6 +331,36 @@ if [[ "${RUN_HIVE}" -eq 1 ]]; then
     fi
 fi
 
+if [[ "${RUN_HIVE3}" -eq 1 ]]; then
+    # hive3
+    # If the doris cluster you need to test is single-node, you can use the default values; If the doris cluster you need to test is composed of multiple nodes, then you need to set the IP_HOST according to the actual situation of your machine
+    #default value
+    IP_HOST="127.0.0.1"
+    eth0_num=$(ifconfig -a|grep flags=|grep -n ^eth0|awk -F ':' '{print $1}')
+    IP_HOST=$(ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d "addr:"|tail -n +${eth0_num}|head -n 1)
+    if [ "_${IP_HOST}" == "_" ];then
+        echo "please set IP_HOST according to your actual situation"
+        exit -1
+    fi
+    # before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/"
+    cp "${ROOT}"/docker-compose/hive/gen_env.sh.tpl "${ROOT}"/docker-compose/hive/gen_env.sh
+    sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/gen_env.sh
+    cp "${ROOT}"/docker-compose/hive/hive-3x.yaml.tpl "${ROOT}"/docker-compose/hive/hive-3x.yaml
+    cp "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl.tpl "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
+    sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/hive-3x.yaml
+    sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
+    sed -i "s/externalEnvIp/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/hive-3x.yaml
+    sed -i "s/externalEnvIp/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
+    sed -i "s/\${externalEnvIp}/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/gen_env.sh
+    sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
+    sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
+    sudo bash "${ROOT}"/docker-compose/hive/gen_env.sh
+    sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
+    if [[ "${STOP}" -ne 1 ]]; then
+        sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d
+    fi
+fi
+
 if [[ "${RUN_SPARK}" -eq 1 ]]; then
     sudo docker compose -f "${ROOT}"/docker-compose/spark/spark.yaml down
     if [[ "${STOP}" -ne 1 ]]; then

From 91ee08583e349fae2e7e107cf0b8f60e6ab6c261 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Tue, 2 Apr 2024 18:03:11 +0800
Subject: [PATCH 02/26] use brigde mode not host

---
 .../docker-compose/hive/hive-3x.yaml.tpl      | 30 ++++++++-----------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index ee47231eb2188c..ba4fef419e2bbe 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -26,16 +26,14 @@ services:
     env_file:
       - ./hadoop-hive.env
     container_name: doris--namenode
-    expose:
-      - "9870"
-      - "8020"
-      - "9000"
+    ports:
+      - "9870:9870"
+      - "9820:9820"
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:9870/" ]
       interval: 5s
       timeout: 120s
       retries: 120
-    network_mode: "host"
 
   doris--datanode:
     image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
@@ -44,14 +42,13 @@ services:
     environment:
       SERVICE_PRECONDITION: "externalEnvIp:9870"
     container_name: doris--datanode
-    expose:
-      - "9864"
+    ports:
+      - "9864:9864"
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:9864" ]
       interval: 5s
       timeout: 60s
       retries: 120
-    network_mode: "host"
 
   doris--hive-server:
     image: lishizhen/hive:3.1.2-postgresql-metastore
@@ -61,8 +58,8 @@ services:
       HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://externalEnvIp:5432/metastore"
       SERVICE_PRECONDITION: "externalEnvIp:9083"
     container_name: doris--hive-server
-    expose:
-      - "10000"
+    ports:
+      - "10000:10000"
     depends_on:
       - doris--datanode
       - doris--namenode
@@ -71,7 +68,6 @@ services:
       interval: 10s
       timeout: 120s
       retries: 120
-    network_mode: "host"
 
 
   doris--hive-metastore:
@@ -83,23 +79,21 @@ services:
     environment:
       SERVICE_PRECONDITION: "externalEnvIp:9870 externalEnvIp:9864 externalEnvIp:5432"
     container_name: doris--hive-metastore
-    expose:
-      - "9083"
+    ports:
+      - "9083:9083"
     volumes:
       - ./scripts:/mnt/scripts
     depends_on:
       - doris--hive-metastore-postgresql
-    network_mode: "host"
 
   doris--hive-metastore-postgresql:
     image: bde2020/hive-metastore-postgresql:3.1.0
     restart: always
     container_name: doris--hive-metastore-postgresql
-    expose:
-      - "5432"
+    ports:
+      - "5432:5432"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s
       timeout: 60s
-      retries: 120
-    network_mode: "host"
+      retries: 120
\ No newline at end of file

From eb733bb25d37253f6fbf7bd9bd6343a064575466 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Sun, 7 Apr 2024 11:22:57 +0800
Subject: [PATCH 03/26] create_preinstalled_table for hive3

---
 .../hive/scripts/create_preinstalled_table.hql           | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
index 730c8bda04d755..ded0132a6d64d8 100644
--- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
+++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
@@ -601,6 +601,8 @@ CREATE TABLE `unsupported_type_table`(
   k6 int
 );
 
+set hive.stats.column.autogather=false;
+
 CREATE TABLE `schema_evo_test_text`(
   id int,
   name string
@@ -2440,3 +2442,10 @@ PARTITIONED BY (
   `varchar_col` varchar(50))
 stored as orc
 TBLPROPERTIES("orc.compress"="ZLIB");
+
+ create table struct_test(id int,sf struct<f1: int, f2: string>) stored as parquet;
+ insert into struct_test values
+  (1, named_struct('f1', 1, 'f2', 's1')),
+  (2, named_struct('f1', 2, 'f2', 's2')),
+  (3, named_struct('f1', 3, 'f2', 's3'));
+alter table struct_test change sf sf struct<f1:int, f3:string>;
\ No newline at end of file

From c03c36fd10b174d0faee243d4f7578fdd6a97d1f Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Sun, 7 Apr 2024 14:11:00 +0800
Subject: [PATCH 04/26] change hive3 to bridge

---
 .../docker-compose/hive/hive-3x.yaml.tpl      | 29 +++++++++++--------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index ba4fef419e2bbe..48422c9e005578 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -26,14 +26,15 @@ services:
     env_file:
       - ./hadoop-hive.env
     container_name: doris--namenode
-    ports:
-      - "9870:9870"
-      - "9820:9820"
+    expose:
+      - "9870"
+      - "8020"
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:9870/" ]
       interval: 5s
       timeout: 120s
       retries: 120
+    network_mode: "host"
 
   doris--datanode:
     image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
@@ -42,13 +43,14 @@ services:
     environment:
       SERVICE_PRECONDITION: "externalEnvIp:9870"
     container_name: doris--datanode
-    ports:
-      - "9864:9864"
+    expose:
+      - "9864"
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:9864" ]
       interval: 5s
       timeout: 60s
       retries: 120
+    network_mode: "host"
 
   doris--hive-server:
     image: lishizhen/hive:3.1.2-postgresql-metastore
@@ -58,8 +60,8 @@ services:
       HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://externalEnvIp:5432/metastore"
       SERVICE_PRECONDITION: "externalEnvIp:9083"
     container_name: doris--hive-server
-    ports:
-      - "10000:10000"
+    expose:
+      - "10000"
     depends_on:
       - doris--datanode
       - doris--namenode
@@ -68,6 +70,7 @@ services:
       interval: 10s
       timeout: 120s
       retries: 120
+    network_mode: "host"
 
 
   doris--hive-metastore:
@@ -79,21 +82,23 @@ services:
     environment:
       SERVICE_PRECONDITION: "externalEnvIp:9870 externalEnvIp:9864 externalEnvIp:5432"
     container_name: doris--hive-metastore
-    ports:
-      - "9083:9083"
+    expose:
+      - "9083"
     volumes:
       - ./scripts:/mnt/scripts
     depends_on:
       - doris--hive-metastore-postgresql
+    network_mode: "host"
 
   doris--hive-metastore-postgresql:
     image: bde2020/hive-metastore-postgresql:3.1.0
     restart: always
     container_name: doris--hive-metastore-postgresql
-    ports:
-      - "5432:5432"
+    expose:
+      - "5432"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s
       timeout: 60s
-      retries: 120
\ No newline at end of file
+      retries: 120
+    network_mode: "host"
\ No newline at end of file

From 4469fc077998678ff76fed5fcc0bac4d606d4ef6 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Sun, 7 Apr 2024 16:26:26 +0800
Subject: [PATCH 05/26] add env path in custom_settings.env for hive

---
 .../docker-compose/hive/custom_settings.env   | 10 ++++
 .../docker-compose/hive/gen_env.sh.tpl        | 41 -----------------
 ...p-hive.env.tpl.tpl => hadoop-hive.env.tpl} | 17 +++++--
 .../docker-compose/hive/hive-3x.yaml.tpl      | 46 +++++++++----------
 .../thirdparties/run-thirdparties-docker.sh   | 34 ++++++--------
 5 files changed, 59 insertions(+), 89 deletions(-)
 create mode 100644 docker/thirdparties/docker-compose/hive/custom_settings.env
 delete mode 100755 docker/thirdparties/docker-compose/hive/gen_env.sh.tpl
 rename docker/thirdparties/docker-compose/hive/{hadoop-hive.env.tpl.tpl => hadoop-hive.env.tpl} (77%)

diff --git a/docker/thirdparties/docker-compose/hive/custom_settings.env b/docker/thirdparties/docker-compose/hive/custom_settings.env
new file mode 100644
index 00000000000000..e403c914aed4e0
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/custom_settings.env
@@ -0,0 +1,10 @@
+export FS_PORT=8020 #should be same in regression-conf.groovy
+export HMS_PORT=9083 #should be same in regression-conf.groovy
+export NAMENODE_HTTP_PORT=50070
+export NAMENODE_HTTPS_PORT=50470
+export NAMENODE_SECONDARY_HTTP_PORT=50090
+export NAMENODE_SECONDARY_HTTPS_PORT=50091
+export DATANODE_PORT=50010
+export DATANODE_HTTP_PORT=50075
+export DATANODE_HTTPS_PORT=50475
+export DATANODE_IPC_PORT=50020
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/hive/gen_env.sh.tpl b/docker/thirdparties/docker-compose/hive/gen_env.sh.tpl
deleted file mode 100755
index a2c708841daf1e..00000000000000
--- a/docker/thirdparties/docker-compose/hive/gen_env.sh.tpl
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-####################################################################
-# This script will generate hadoop-hive.env from hadoop-hive.env.tpl
-####################################################################
-
-set -eo pipefail
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
-FS_PORT=8020
-HMS_PORT=9083
-
-cp "${ROOT}"/hadoop-hive.env.tpl "${ROOT}"/hadoop-hive.env
-# Need to set hostname of container to same as host machine's.
-# Otherwise, the doris process can not connect to namenode directly.
-HOST_NAME="doris--"
-
-{
-    echo "FS_PORT=${FS_PORT}"
-    echo "HMS_PORT=${HMS_PORT}"
-    echo "CORE_CONF_fs_defaultFS=hdfs://${externalEnvIp}:${FS_PORT}"
-    echo "HOST_NAME=${HOST_NAME}"
-    echo "externalEnvIp=${externalEnvIp}"
-
-} >>"${ROOT}"/hadoop-hive.env
diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl.tpl b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
similarity index 77%
rename from docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl.tpl
rename to docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
index 7fb85d72fa837c..02cebc090eac0d 100644
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl.tpl
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
@@ -15,13 +15,12 @@
 # limitations under the License.
 #
 
-HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://externalEnvIp:5432/metastore
+HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://${externalEnvIp}:5432/metastore
 HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver
 HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive
 HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive
 HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
-HIVE_SITE_CONF_hive_metastore_uris=thrift://externalEnvIp:9083
-HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
+HIVE_SITE_CONF_hive_metastore_uris=thrift://${externalEnvIp}:${HMS_PORT}
 HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
 HIVE_SITE_CONF_hive_server2_thrift_port=10000
 HIVE_SITE_CONF_hive_compactor_initiator_on=true
@@ -31,9 +30,19 @@ CORE_CONF_hadoop_http_staticuser_user=root
 CORE_CONF_hadoop_proxyuser_hue_hosts=*
 CORE_CONF_hadoop_proxyuser_hue_groups=*
 CORE_CONF_hadoop_proxyuser_hive_hosts=*
+CORE_CONF_fs_defaultFS=hdfs://${externalEnvIp}:${FS_PORT}
 
 HDFS_CONF_dfs_webhdfs_enabled=true
 HDFS_CONF_dfs_permissions_enabled=false
+HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
+HDFS_CONF_dfs_namenode_http___address=0.0.0.0:${NAMENODE_HTTP_PORT}
+HDFS_CONF_dfs_namenode_https___address=0.0.0.0:${NAMENODE_HTTPS_PORT}
+HDFS_CONF_dfs_namenode_secondary_http___address=0.0.0.0:${NAMENODE_SECONDARY_HTTP_PORT}
+HDFS_CONF_dfs_namenode_secondary_https___address=0.0.0.0:${NAMENODE_SECONDARY_HTTPS_PORT}
+HDFS_CONF_dfs_datanode_address=0.0.0.0:${DATANODE_PORT}
+HDFS_CONF_dfs_datanode_http_address=0.0.0.0:${DATANODE_HTTP_PORT}
+HDFS_CONF_dfs_datanode_https_address=0.0.0.0:${DATANODE_HTTPS_PORT}
+HDFS_CONF_dfs_datanode_ipc_address=0.0.0.0:${DATANODE_IPC_PORT}
 
 YARN_CONF_yarn_log___aggregation___enable=true
 YARN_CONF_yarn_resourcemanager_recovery_enabled=true
@@ -48,4 +57,4 @@ YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
 YARN_CONF_yarn_timeline___service_hostname=historyserver
 YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032
 YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030
-YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031
+YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index 48422c9e005578..5b81a39e29fe07 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -19,52 +19,52 @@
 version: "3.8"
 
 services:
-  doris--namenode:
+  ${CONTAINER_UID}namenode:
     image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
     environment:
       - CLUSTER_NAME=test
     env_file:
       - ./hadoop-hive.env
-    container_name: doris--namenode
+    container_name: ${CONTAINER_UID}namenode
     expose:
-      - "9870"
-      - "8020"
+      - "${NAMENODE_HTTP_PORT}"
+      - "${FS_PORT}"
     healthcheck:
-      test: [ "CMD", "curl", "http://localhost:9870/" ]
+      test: [ "CMD", "curl", "http://localhost:${NAMENODE_HTTP_PORT}/" ]
       interval: 5s
       timeout: 120s
       retries: 120
     network_mode: "host"
 
-  doris--datanode:
+  ${CONTAINER_UID}datanode:
     image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
     env_file:
       - ./hadoop-hive.env
     environment:
-      SERVICE_PRECONDITION: "externalEnvIp:9870"
-    container_name: doris--datanode
+      SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT}"
+    container_name: ${CONTAINER_UID}datanode
     expose:
-      - "9864"
+      - "${DATANODE_HTTP_PORT}"
     healthcheck:
-      test: [ "CMD", "curl", "http://localhost:9864" ]
+      test: [ "CMD", "curl", "http://localhost:${DATANODE_HTTP_PORT}" ]
       interval: 5s
       timeout: 60s
       retries: 120
     network_mode: "host"
 
-  doris--hive-server:
+  ${CONTAINER_UID}hive-server:
     image: lishizhen/hive:3.1.2-postgresql-metastore
     env_file:
       - ./hadoop-hive.env
     environment:
-      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://externalEnvIp:5432/metastore"
-      SERVICE_PRECONDITION: "externalEnvIp:9083"
-    container_name: doris--hive-server
+      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://${externalEnvIp}:5432/metastore"
+      SERVICE_PRECONDITION: "${externalEnvIp}:${HMS_PORT}"
+    container_name: ${CONTAINER_UID}hive-server
     expose:
       - "10000"
     depends_on:
-      - doris--datanode
-      - doris--namenode
+      - ${CONTAINER_UID}datanode
+      - ${CONTAINER_UID}namenode
     healthcheck:
       test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;"
       interval: 10s
@@ -73,27 +73,27 @@ services:
     network_mode: "host"
 
 
-  doris--hive-metastore:
+  ${CONTAINER_UID}hive-metastore:
     image: lishizhen/hive:3.1.2-postgresql-metastore
     env_file:
       - ./hadoop-hive.env
     command: /bin/bash /mnt/scripts/hive-metastore.sh
     # command: /opt/hive/bin/hive --service metastore
     environment:
-      SERVICE_PRECONDITION: "externalEnvIp:9870 externalEnvIp:9864 externalEnvIp:5432"
-    container_name: doris--hive-metastore
+      SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT} ${externalEnvIp}:${DATANODE_HTTP_PORT} ${externalEnvIp}:5432"
+    container_name: ${CONTAINER_UID}hive-metastore
     expose:
-      - "9083"
+      - "${HMS_PORT}"
     volumes:
       - ./scripts:/mnt/scripts
     depends_on:
-      - doris--hive-metastore-postgresql
+      - ${CONTAINER_UID}hive-metastore-postgresql
     network_mode: "host"
 
-  doris--hive-metastore-postgresql:
+  ${CONTAINER_UID}hive-metastore-postgresql:
     image: bde2020/hive-metastore-postgresql:3.1.0
     restart: always
-    container_name: doris--hive-metastore-postgresql
+    container_name: ${CONTAINER_UID}hive-metastore-postgresql
     expose:
       - "5432"
     healthcheck:
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index ef69a618b3dd7f..b99dcce8e59441 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -302,7 +302,7 @@ if [[ "${RUN_KAFKA}" -eq 1 ]]; then
 fi
 
 if [[ "${RUN_HIVE}" -eq 1 ]]; then
-    # hive
+    # hive2
     # If the doris cluster you need to test is single-node, you can use the default values; If the doris cluster you need to test is composed of multiple nodes, then you need to set the IP_HOST according to the actual situation of your machine
     #default value
     IP_HOST="127.0.0.1"
@@ -313,18 +313,14 @@ if [[ "${RUN_HIVE}" -eq 1 ]]; then
         exit -1
     fi
     # before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/"
-    cp "${ROOT}"/docker-compose/hive/gen_env.sh.tpl "${ROOT}"/docker-compose/hive/gen_env.sh
-    sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/gen_env.sh
-    cp "${ROOT}"/docker-compose/hive/hive-2x.yaml.tpl "${ROOT}"/docker-compose/hive/hive-2x.yaml
-    cp "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl.tpl "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
-    sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/hive-2x.yaml
-    sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
-    sed -i "s/externalEnvIp/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/hive-2x.yaml
-    sed -i "s/externalEnvIp/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
-    sed -i "s/\${externalEnvIp}/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/gen_env.sh
     sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
     sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
-    sudo bash "${ROOT}"/docker-compose/hive/gen_env.sh
+    # generate hive-3x.yaml and hadoop-hive.env
+    export externalEnvIp=${IP_HOST}
+    export CONTAINER_UID=${CONTAINER_UID}
+    . "${ROOT}"/docker-compose/hive/custom_settings.env
+    envsubst < "${ROOT}"/docker-compose/hive/hive-2x.yaml.tpl > "${ROOT}"/docker-compose/hive/hive-2x.yaml
+    envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl > "${ROOT}"/docker-compose/hive/hadoop-hive.env
     sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
     if [[ "${STOP}" -ne 1 ]]; then
         sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d
@@ -343,18 +339,14 @@ if [[ "${RUN_HIVE3}" -eq 1 ]]; then
         exit -1
     fi
     # before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/"
-    cp "${ROOT}"/docker-compose/hive/gen_env.sh.tpl "${ROOT}"/docker-compose/hive/gen_env.sh
-    sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/gen_env.sh
-    cp "${ROOT}"/docker-compose/hive/hive-3x.yaml.tpl "${ROOT}"/docker-compose/hive/hive-3x.yaml
-    cp "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl.tpl "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
-    sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/hive-3x.yaml
-    sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
-    sed -i "s/externalEnvIp/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/hive-3x.yaml
-    sed -i "s/externalEnvIp/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl
-    sed -i "s/\${externalEnvIp}/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/gen_env.sh
     sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
     sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
-    sudo bash "${ROOT}"/docker-compose/hive/gen_env.sh
+    # generate hive-3x.yaml and hadoop-hive.env
+    export externalEnvIp=${IP_HOST}
+    export CONTAINER_UID=${CONTAINER_UID}
+    . "${ROOT}"/docker-compose/hive/custom_settings.env
+    envsubst < "${ROOT}"/docker-compose/hive/hive-3x.yaml.tpl > "${ROOT}"/docker-compose/hive/hive-3x.yaml
+    envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl > "${ROOT}"/docker-compose/hive/hadoop-hive.env
     sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
     if [[ "${STOP}" -ne 1 ]]; then
         sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d

From 71174206a1d4b19573ae6b9b437fe5957b9a3817 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Sun, 7 Apr 2024 16:28:53 +0800
Subject: [PATCH 06/26] change hive-2x.yaml.tpl to support custom_sttings.env

---
 .../docker-compose/hive/hive-2x.yaml.tpl      | 49 +++++++++----------
 1 file changed, 24 insertions(+), 25 deletions(-)

diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
index 32b99a0b0cc849..72ad2e268800d0 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
@@ -19,53 +19,52 @@
 version: "3.8"
 
 services:
-  doris--namenode:
+  ${CONTAINER_UID}namenode:
     image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8
     environment:
       - CLUSTER_NAME=test
     env_file:
       - ./hadoop-hive.env
-    container_name: doris--namenode
+    container_name: ${CONTAINER_UID}namenode
     expose:
-      - "50070"
-      - "8020"
-      - "9000"
+      - "${NAMENODE_HTTP_PORT}"
+      - "${FS_PORT}"
     healthcheck:
-      test: [ "CMD", "curl", "http://localhost:50070/" ]
+      test: [ "CMD", "curl", "http://localhost:${NAMENODE_HTTP_PORT}/" ]
       interval: 5s
       timeout: 120s
       retries: 120
     network_mode: "host"
 
-  doris--datanode:
+  ${CONTAINER_UID}datanode:
     image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
     env_file:
       - ./hadoop-hive.env
     environment:
-      SERVICE_PRECONDITION: "externalEnvIp:50070"
-    container_name: doris--datanode
+      SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT}"
+    container_name: ${CONTAINER_UID}datanode
     expose:
-      - "50075"
+      - "${DATANODE_HTTP_PORT}"
     healthcheck:
-      test: [ "CMD", "curl", "http://localhost:50075" ]
+      test: [ "CMD", "curl", "http://localhost:${DATANODE_HTTP_PORT}" ]
       interval: 5s
       timeout: 60s
       retries: 120
     network_mode: "host"
 
-  doris--hive-server:
+  ${CONTAINER_UID}hive-server:
     image: bde2020/hive:2.3.2-postgresql-metastore
     env_file:
       - ./hadoop-hive.env
     environment:
-      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://externalEnvIp:5432/metastore"
-      SERVICE_PRECONDITION: "externalEnvIp:9083"
-    container_name: doris--hive-server
+      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://${externalEnvIp}:5432/metastore"
+      SERVICE_PRECONDITION: "${externalEnvIp}:${HMS_PORT}"
+    container_name: ${CONTAINER_UID}hive-server
     expose:
       - "10000"
     depends_on:
-      - doris--datanode
-      - doris--namenode
+      - ${CONTAINER_UID}datanode
+      - ${CONTAINER_UID}namenode
     healthcheck:
       test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;"
       interval: 10s
@@ -74,27 +73,27 @@ services:
     network_mode: "host"
 
 
-  doris--hive-metastore:
+  ${CONTAINER_UID}hive-metastore:
     image: bde2020/hive:2.3.2-postgresql-metastore
     env_file:
       - ./hadoop-hive.env
     command: /bin/bash /mnt/scripts/hive-metastore.sh
     # command: /opt/hive/bin/hive --service metastore
     environment:
-      SERVICE_PRECONDITION: "externalEnvIp:50070 externalEnvIp:50075 externalEnvIp:5432"
-    container_name: doris--hive-metastore
+      SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT} ${externalEnvIp}:${DATANODE_HTTP_PORT} ${externalEnvIp}:5432"
+    container_name: ${CONTAINER_UID}hive-metastore
     expose:
-      - "9083"
+      - "${HMS_PORT}"
     volumes:
       - ./scripts:/mnt/scripts
     depends_on:
-      - doris--hive-metastore-postgresql
+      - ${CONTAINER_UID}hive-metastore-postgresql
     network_mode: "host"
 
-  doris--hive-metastore-postgresql:
+  ${CONTAINER_UID}hive-metastore-postgresql:
     image: bde2020/hive-metastore-postgresql:2.3.0
     restart: always
-    container_name: doris--hive-metastore-postgresql
+    container_name: ${CONTAINER_UID}hive-metastore-postgresql
     expose:
       - "5432"
     healthcheck:
@@ -102,4 +101,4 @@ services:
       interval: 5s
       timeout: 60s
       retries: 120
-    network_mode: "host"
+    network_mode: "host"
\ No newline at end of file

From 77048dda68273063a4f378e1c589b1fd0d61bcef Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Sun, 7 Apr 2024 16:32:09 +0800
Subject: [PATCH 07/26] fix

---
 .../hive/scripts/create_preinstalled_table.hql           | 9 +--------
 regression-test/conf/regression-conf.groovy              | 4 ++--
 2 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
index ded0132a6d64d8..eb16c1239cae66 100644
--- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
+++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
@@ -2441,11 +2441,4 @@ PARTITIONED BY (
   `char_col` char(50),
   `varchar_col` varchar(50))
 stored as orc
-TBLPROPERTIES("orc.compress"="ZLIB");
-
- create table struct_test(id int,sf struct<f1: int, f2: string>) stored as parquet;
- insert into struct_test values
-  (1, named_struct('f1', 1, 'f2', 's1')),
-  (2, named_struct('f1', 2, 'f2', 's2')),
-  (3, named_struct('f1', 3, 'f2', 's3'));
-alter table struct_test change sf sf struct<f1:int, f3:string>;
\ No newline at end of file
+TBLPROPERTIES("orc.compress"="ZLIB");
\ No newline at end of file
diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy
index 02a947b739c0e0..2bb5ca5f6f4256 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -124,8 +124,8 @@ db2_11_port=50000
 // See `docker/thirdparties/start-thirdparties-docker.sh`
 enableHiveTest=false
 enablePaimonTest=false
-hms_port=9183
-hdfs_port=8120
+hms_port=9083
+hdfs_port=8020
 hiveServerPort=10000
 hive_pg_port=5432
 

From 384762ee944cd6590b57b924432794381d2bfce5 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Sun, 7 Apr 2024 16:46:49 +0800
Subject: [PATCH 08/26] add HS_PORT in custom_settings add license for
 custom_settings

---
 .../docker-compose/hive/custom_settings.env   | 25 +++++++++++++++++++
 .../docker-compose/hive/hadoop-hive.env.tpl   |  2 +-
 .../docker-compose/hive/hive-2x.yaml.tpl      |  4 +--
 .../docker-compose/hive/hive-3x.yaml.tpl      |  4 +--
 4 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/docker/thirdparties/docker-compose/hive/custom_settings.env b/docker/thirdparties/docker-compose/hive/custom_settings.env
index e403c914aed4e0..6c3008d76986fc 100644
--- a/docker/thirdparties/docker-compose/hive/custom_settings.env
+++ b/docker/thirdparties/docker-compose/hive/custom_settings.env
@@ -1,5 +1,30 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Change this to a specific string.
+# Do not use "_" or other sepcial characters, only number and alphabeta.
+# NOTICE: change this uid will modify hadoop-hive.env and hive-*.yaml
+
 export FS_PORT=8020 #should be same in regression-conf.groovy
 export HMS_PORT=9083 #should be same in regression-conf.groovy
+export HS_PORT=10000 #should be same in regression-conf.groovy
+export PG_PORT=5432 #TODO: this port is not supported to change yet
+
 export NAMENODE_HTTP_PORT=50070
 export NAMENODE_HTTPS_PORT=50470
 export NAMENODE_SECONDARY_HTTP_PORT=50090
diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
index 02cebc090eac0d..ea31af143dd25f 100644
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
@@ -22,7 +22,7 @@ HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive
 HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
 HIVE_SITE_CONF_hive_metastore_uris=thrift://${externalEnvIp}:${HMS_PORT}
 HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
-HIVE_SITE_CONF_hive_server2_thrift_port=10000
+HIVE_SITE_CONF_hive_server2_thrift_port=${HS_PORT}
 HIVE_SITE_CONF_hive_compactor_initiator_on=true
 HIVE_SITE_CONF_hive_compactor_worker_threads=2
 
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
index 72ad2e268800d0..9c5cb8d0289f6b 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
@@ -61,12 +61,12 @@ services:
       SERVICE_PRECONDITION: "${externalEnvIp}:${HMS_PORT}"
     container_name: ${CONTAINER_UID}hive-server
     expose:
-      - "10000"
+      - "${HS_PORT}"
     depends_on:
       - ${CONTAINER_UID}datanode
       - ${CONTAINER_UID}namenode
     healthcheck:
-      test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;"
+      test: beeline -u "jdbc:hive2://127.0.0.1:${HS_PORT}/default" -n health_check -e "show databases;"
       interval: 10s
       timeout: 120s
       retries: 120
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index 5b81a39e29fe07..5630ca22d2c456 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -61,12 +61,12 @@ services:
       SERVICE_PRECONDITION: "${externalEnvIp}:${HMS_PORT}"
     container_name: ${CONTAINER_UID}hive-server
     expose:
-      - "10000"
+      - "${HS_PORT}"
     depends_on:
       - ${CONTAINER_UID}datanode
       - ${CONTAINER_UID}namenode
     healthcheck:
-      test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;"
+      test: beeline -u "jdbc:hive2://127.0.0.1:${HS_PORT}/default" -n health_check -e "show databases;"
       interval: 10s
       timeout: 120s
       retries: 120

From 8de0e295ab3bcd10e96e8f459548cfc3b755797f Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Sun, 7 Apr 2024 16:50:24 +0800
Subject: [PATCH 09/26] fix

---
 docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl     | 2 +-
 docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl        | 2 +-
 docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl        | 2 +-
 .../docker-compose/hive/scripts/create_preinstalled_table.hql   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
index ea31af143dd25f..e9a89e8f852fb5 100644
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
@@ -57,4 +57,4 @@ YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
 YARN_CONF_yarn_timeline___service_hostname=historyserver
 YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032
 YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030
-YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031
\ No newline at end of file
+YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
index 9c5cb8d0289f6b..3bb039958d9ad2 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
@@ -101,4 +101,4 @@ services:
       interval: 5s
       timeout: 60s
       retries: 120
-    network_mode: "host"
\ No newline at end of file
+    network_mode: "host"
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index 5630ca22d2c456..45a84df826d32b 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -101,4 +101,4 @@ services:
       interval: 5s
       timeout: 60s
       retries: 120
-    network_mode: "host"
\ No newline at end of file
+    network_mode: "host"
diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
index eb16c1239cae66..d832981ae99a75 100644
--- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
+++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
@@ -2441,4 +2441,4 @@ PARTITIONED BY (
   `char_col` char(50),
   `varchar_col` varchar(50))
 stored as orc
-TBLPROPERTIES("orc.compress"="ZLIB");
\ No newline at end of file
+TBLPROPERTIES("orc.compress"="ZLIB");

From e73d4f07c4fa90c04b6b1ee33a317c216ee54ce7 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Sun, 7 Apr 2024 17:00:56 +0800
Subject: [PATCH 10/26] fix

---
 docker/thirdparties/docker-compose/hive/custom_settings.env | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/thirdparties/docker-compose/hive/custom_settings.env b/docker/thirdparties/docker-compose/hive/custom_settings.env
index 6c3008d76986fc..7ae393ae1c9760 100644
--- a/docker/thirdparties/docker-compose/hive/custom_settings.env
+++ b/docker/thirdparties/docker-compose/hive/custom_settings.env
@@ -32,4 +32,4 @@ export NAMENODE_SECONDARY_HTTPS_PORT=50091
 export DATANODE_PORT=50010
 export DATANODE_HTTP_PORT=50075
 export DATANODE_HTTPS_PORT=50475
-export DATANODE_IPC_PORT=50020
\ No newline at end of file
+export DATANODE_IPC_PORT=50020

From 9d65464e27d71f8a34c916a3b81f155c6ae50957 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Sun, 7 Apr 2024 17:01:44 +0800
Subject: [PATCH 11/26] remove PG_PORT from custom_settings.env

---
 docker/thirdparties/docker-compose/hive/custom_settings.env | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/thirdparties/docker-compose/hive/custom_settings.env b/docker/thirdparties/docker-compose/hive/custom_settings.env
index 7ae393ae1c9760..6ae6b136b246ba 100644
--- a/docker/thirdparties/docker-compose/hive/custom_settings.env
+++ b/docker/thirdparties/docker-compose/hive/custom_settings.env
@@ -23,7 +23,6 @@
 export FS_PORT=8020 #should be same in regression-conf.groovy
 export HMS_PORT=9083 #should be same in regression-conf.groovy
 export HS_PORT=10000 #should be same in regression-conf.groovy
-export PG_PORT=5432 #TODO: this port is not supported to change yet
 
 export NAMENODE_HTTP_PORT=50070
 export NAMENODE_HTTPS_PORT=50470

From abdb74b15ae0bf1537cc4a332f51c5bc94fb5ad3 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Tue, 9 Apr 2024 00:19:04 +0800
Subject: [PATCH 12/26] set autogather

---
 .../docker-compose/hive/scripts/create_preinstalled_table.hql   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
index d832981ae99a75..dbeeab972f6b0d 100644
--- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
+++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
@@ -630,6 +630,8 @@ insert into `schema_evo_test_orc` select 1, "kaka";
 alter table `schema_evo_test_orc` ADD COLUMNS (`ts` timestamp);
 insert into `schema_evo_test_orc` select 2, "messi", from_unixtime(to_unix_timestamp('20230101 13:01:03','yyyyMMdd HH:mm:ss'));
 
+set hive.stats.column.autogather=true;
+
 -- Currently docker is hive 2.x version. Hive 2.x versioned full-acid tables need to run major compaction.
 SET hive.support.concurrency=true;
 SET hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;

From c3ae65df38b6809764d5f24dae579a7eed27a3fc Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Tue, 9 Apr 2024 00:20:53 +0800
Subject: [PATCH 13/26] not to start hive2 default

---
 docker/thirdparties/run-thirdparties-docker.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index b99dcce8e59441..fc50c2675012de 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -37,7 +37,7 @@ Usage: $0 <options>
      --stop             stop the specified components
 
   All valid components:
-    mysql,pg,oracle,sqlserver,clickhouse,es,hive,hive3,iceberg,hudi,trino,kafka,mariadb,db2
+    mysql,pg,oracle,sqlserver,clickhouse,es,hive/hive3,iceberg,hudi,trino,kafka,mariadb,db2
   "
     exit 1
 }
@@ -60,7 +60,7 @@ STOP=0
 
 if [[ "$#" == 1 ]]; then
     # default
-    COMPONENTS="mysql,es,hive,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2"
+    COMPONENTS="mysql,es,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2"
 else
     while true; do
         case "$1" in
@@ -92,7 +92,7 @@ else
     done
     if [[ "${COMPONENTS}"x == ""x ]]; then
         if [[ "${STOP}" -eq 1 ]]; then
-            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,hive3,iceberg,hudi,trino,kafka,mariadb,db2"
+            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive3,iceberg,hudi,trino,kafka,mariadb,db2"
         fi
     fi
 fi

From de1f299bceae665972186c3991ac2a7143e24b1b Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Tue, 9 Apr 2024 00:24:29 +0800
Subject: [PATCH 14/26] can not both run hive2 and hive3

---
 docker/thirdparties/run-thirdparties-docker.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index fc50c2675012de..9ab66064e5a583 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -301,6 +301,11 @@ if [[ "${RUN_KAFKA}" -eq 1 ]]; then
     fi
 fi
 
+if [[ $RUN_HIVE -eq 1 && $RUN_HIVE3 -eq 1 ]]; then
+    echo "can not both run hive2 and hive3"
+    exit 1
+fi
+
 if [[ "${RUN_HIVE}" -eq 1 ]]; then
     # hive2
     # If the doris cluster you need to test is single-node, you can use the default values; If the doris cluster you need to test is composed of multiple nodes, then you need to set the IP_HOST according to the actual situation of your machine

From 796f8b94e7cacca4d300d925e35c9b8c688d64fb Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Tue, 9 Apr 2024 17:19:53 +0800
Subject: [PATCH 15/26] support to run both hive and hive3 support to run multi
 hive docker compose by CONTAINER_UID

---
 .../docker-compose/hive/hadoop-hive.env.tpl   |  3 +-
 .../docker-compose/hive/hive-2x.yaml.tpl      | 10 +++---
 ...stom_settings.env => hive-2x_settings.env} | 23 ++++++------
 .../docker-compose/hive/hive-3x.yaml.tpl      | 34 +++++++++---------
 .../docker-compose/hive/hive-3x_settings.env  | 35 +++++++++++++++++++
 .../thirdparties/run-thirdparties-docker.sh   | 23 +++++-------
 6 files changed, 80 insertions(+), 48 deletions(-)
 rename docker/thirdparties/docker-compose/hive/{custom_settings.env => hive-2x_settings.env} (65%)
 create mode 100644 docker/thirdparties/docker-compose/hive/hive-3x_settings.env

diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
index e9a89e8f852fb5..51041e15341508 100644
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
@@ -15,12 +15,13 @@
 # limitations under the License.
 #
 
-HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://${externalEnvIp}:5432/metastore
+HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://${externalEnvIp}:${PG_PORT}/metastore
 HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver
 HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive
 HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive
 HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
 HIVE_SITE_CONF_hive_metastore_uris=thrift://${externalEnvIp}:${HMS_PORT}
+HIVE_SITE_CONF_hive_metastore_port=${HMS_PORT}
 HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
 HIVE_SITE_CONF_hive_server2_thrift_port=${HS_PORT}
 HIVE_SITE_CONF_hive_compactor_initiator_on=true
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
index 3bb039958d9ad2..256dd2ba509375 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
@@ -57,7 +57,7 @@ services:
     env_file:
       - ./hadoop-hive.env
     environment:
-      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://${externalEnvIp}:5432/metastore"
+      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://${externalEnvIp}:${PG_PORT}/metastore"
       SERVICE_PRECONDITION: "${externalEnvIp}:${HMS_PORT}"
     container_name: ${CONTAINER_UID}hive-server
     expose:
@@ -80,7 +80,7 @@ services:
     command: /bin/bash /mnt/scripts/hive-metastore.sh
     # command: /opt/hive/bin/hive --service metastore
     environment:
-      SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT} ${externalEnvIp}:${DATANODE_HTTP_PORT} ${externalEnvIp}:5432"
+      SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT} ${externalEnvIp}:${DATANODE_HTTP_PORT} ${externalEnvIp}:${PG_PORT}"
     container_name: ${CONTAINER_UID}hive-metastore
     expose:
       - "${HMS_PORT}"
@@ -94,11 +94,11 @@ services:
     image: bde2020/hive-metastore-postgresql:2.3.0
     restart: always
     container_name: ${CONTAINER_UID}hive-metastore-postgresql
-    expose:
-      - "5432"
+    ports:
+      - ${PG_PORT}:5432
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s
       timeout: 60s
       retries: 120
-    network_mode: "host"
+    network_mode: "bridge"
diff --git a/docker/thirdparties/docker-compose/hive/custom_settings.env b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
similarity index 65%
rename from docker/thirdparties/docker-compose/hive/custom_settings.env
rename to docker/thirdparties/docker-compose/hive/hive-2x_settings.env
index 6ae6b136b246ba..347142e2cddfa2 100644
--- a/docker/thirdparties/docker-compose/hive/custom_settings.env
+++ b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
@@ -20,15 +20,16 @@
 # Do not use "_" or other sepcial characters, only number and alphabeta.
 # NOTICE: change this uid will modify hadoop-hive.env and hive-*.yaml
 
-export FS_PORT=8020 #should be same in regression-conf.groovy
-export HMS_PORT=9083 #should be same in regression-conf.groovy
-export HS_PORT=10000 #should be same in regression-conf.groovy
+export FS_PORT=8220 #should be same in regression-conf.groovy
+export HMS_PORT=9283 #should be same in regression-conf.groovy
+export HS_PORT=12000 #should be same in regression-conf.groovy
+export PG_PORT=5632 #should be same in regression-conf.groovy
 
-export NAMENODE_HTTP_PORT=50070
-export NAMENODE_HTTPS_PORT=50470
-export NAMENODE_SECONDARY_HTTP_PORT=50090
-export NAMENODE_SECONDARY_HTTPS_PORT=50091
-export DATANODE_PORT=50010
-export DATANODE_HTTP_PORT=50075
-export DATANODE_HTTPS_PORT=50475
-export DATANODE_IPC_PORT=50020
+export NAMENODE_HTTP_PORT=52070
+export NAMENODE_HTTPS_PORT=52470
+export NAMENODE_SECONDARY_HTTP_PORT=52090
+export NAMENODE_SECONDARY_HTTPS_PORT=52091
+export DATANODE_PORT=52010
+export DATANODE_HTTP_PORT=52075
+export DATANODE_HTTPS_PORT=52475
+export DATANODE_IPC_PORT=52020
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index 45a84df826d32b..150be0a9d8955a 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -19,13 +19,13 @@
 version: "3.8"
 
 services:
-  ${CONTAINER_UID}namenode:
+  ${CONTAINER_UID}hadoop3-namenode:
     image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
     environment:
       - CLUSTER_NAME=test
     env_file:
       - ./hadoop-hive.env
-    container_name: ${CONTAINER_UID}namenode
+    container_name: ${CONTAINER_UID}hadoop3-namenode
     expose:
       - "${NAMENODE_HTTP_PORT}"
       - "${FS_PORT}"
@@ -36,13 +36,13 @@ services:
       retries: 120
     network_mode: "host"
 
-  ${CONTAINER_UID}datanode:
+  ${CONTAINER_UID}hadoop3-datanode:
     image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
     env_file:
       - ./hadoop-hive.env
     environment:
       SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT}"
-    container_name: ${CONTAINER_UID}datanode
+    container_name: ${CONTAINER_UID}hadoop3-datanode
     expose:
       - "${DATANODE_HTTP_PORT}"
     healthcheck:
@@ -57,14 +57,14 @@ services:
     env_file:
       - ./hadoop-hive.env
     environment:
-      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://${externalEnvIp}:5432/metastore"
+      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://${externalEnvIp}:${PG_PORT}/metastore"
       SERVICE_PRECONDITION: "${externalEnvIp}:${HMS_PORT}"
-    container_name: ${CONTAINER_UID}hive-server
+    container_name: ${CONTAINER_UID}hive3-server
     expose:
       - "${HS_PORT}"
     depends_on:
-      - ${CONTAINER_UID}datanode
-      - ${CONTAINER_UID}namenode
+      - ${CONTAINER_UID}hadoop3-datanode
+      - ${CONTAINER_UID}hadoop3-namenode
     healthcheck:
       test: beeline -u "jdbc:hive2://127.0.0.1:${HS_PORT}/default" -n health_check -e "show databases;"
       interval: 10s
@@ -73,32 +73,32 @@ services:
     network_mode: "host"
 
 
-  ${CONTAINER_UID}hive-metastore:
+  ${CONTAINER_UID}hive3-metastore:
     image: lishizhen/hive:3.1.2-postgresql-metastore
     env_file:
       - ./hadoop-hive.env
     command: /bin/bash /mnt/scripts/hive-metastore.sh
     # command: /opt/hive/bin/hive --service metastore
     environment:
-      SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT} ${externalEnvIp}:${DATANODE_HTTP_PORT} ${externalEnvIp}:5432"
-    container_name: ${CONTAINER_UID}hive-metastore
+      SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT} ${externalEnvIp}:${DATANODE_HTTP_PORT} ${externalEnvIp}:${PG_PORT}"
+    container_name: ${CONTAINER_UID}hive3-metastore
     expose:
       - "${HMS_PORT}"
     volumes:
       - ./scripts:/mnt/scripts
     depends_on:
-      - ${CONTAINER_UID}hive-metastore-postgresql
+      - ${CONTAINER_UID}hive3-metastore-postgresql
     network_mode: "host"
 
-  ${CONTAINER_UID}hive-metastore-postgresql:
+  ${CONTAINER_UID}hive3-metastore-postgresql:
     image: bde2020/hive-metastore-postgresql:3.1.0
     restart: always
-    container_name: ${CONTAINER_UID}hive-metastore-postgresql
-    expose:
-      - "5432"
+    container_name: ${CONTAINER_UID}hive3-metastore-postgresql
+    ports:
+      - ${PG_PORT}:5432
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s
       timeout: 60s
       retries: 120
-    network_mode: "host"
+    network_mode: "bridge"
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
new file mode 100644
index 00000000000000..dbf403f41e98f3
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Change this to a specific string.
+# Do not use "_" or other sepcial characters, only number and alphabeta.
+# NOTICE: change this uid will modify hadoop-hive.env and hive-*.yaml
+
+export FS_PORT=8320 #should be same in regression-conf.groovy
+export HMS_PORT=9383 #should be same in regression-conf.groovy
+export HS_PORT=13000 #should be same in regression-conf.groovy
+export PG_PORT=5732 #should be same in regression-conf.groovy
+
+export NAMENODE_HTTP_PORT=53070
+export NAMENODE_HTTPS_PORT=53470
+export NAMENODE_SECONDARY_HTTP_PORT=53090
+export NAMENODE_SECONDARY_HTTPS_PORT=53091
+export DATANODE_PORT=53010
+export DATANODE_HTTP_PORT=53075
+export DATANODE_HTTPS_PORT=53475
+export DATANODE_IPC_PORT=53020
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index 9ab66064e5a583..c83b8dadf415ca 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -37,7 +37,7 @@ Usage: $0 <options>
      --stop             stop the specified components
 
   All valid components:
-    mysql,pg,oracle,sqlserver,clickhouse,es,hive/hive3,iceberg,hudi,trino,kafka,mariadb,db2
+    mysql,pg,oracle,sqlserver,clickhouse,es,hive,hive3,iceberg,hudi,trino,kafka,mariadb,db2
   "
     exit 1
 }
@@ -60,7 +60,7 @@ STOP=0
 
 if [[ "$#" == 1 ]]; then
     # default
-    COMPONENTS="mysql,es,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2"
+    COMPONENTS="mysql,es,hive,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2"
 else
     while true; do
         case "$1" in
@@ -92,7 +92,7 @@ else
     done
     if [[ "${COMPONENTS}"x == ""x ]]; then
         if [[ "${STOP}" -eq 1 ]]; then
-            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive3,iceberg,hudi,trino,kafka,mariadb,db2"
+            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,hive3,iceberg,hudi,trino,kafka,mariadb,db2"
         fi
     fi
 fi
@@ -301,11 +301,6 @@ if [[ "${RUN_KAFKA}" -eq 1 ]]; then
     fi
 fi
 
-if [[ $RUN_HIVE -eq 1 && $RUN_HIVE3 -eq 1 ]]; then
-    echo "can not both run hive2 and hive3"
-    exit 1
-fi
-
 if [[ "${RUN_HIVE}" -eq 1 ]]; then
     # hive2
     # If the doris cluster you need to test is single-node, you can use the default values; If the doris cluster you need to test is composed of multiple nodes, then you need to set the IP_HOST according to the actual situation of your machine
@@ -323,12 +318,12 @@ if [[ "${RUN_HIVE}" -eq 1 ]]; then
     # generate hive-3x.yaml and hadoop-hive.env
     export externalEnvIp=${IP_HOST}
     export CONTAINER_UID=${CONTAINER_UID}
-    . "${ROOT}"/docker-compose/hive/custom_settings.env
+    . "${ROOT}"/docker-compose/hive/hive-2x_settings.env
     envsubst < "${ROOT}"/docker-compose/hive/hive-2x.yaml.tpl > "${ROOT}"/docker-compose/hive/hive-2x.yaml
     envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl > "${ROOT}"/docker-compose/hive/hadoop-hive.env
-    sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
+    sudo docker compose -p ${CONTAINER_UID}hive2 -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
     if [[ "${STOP}" -ne 1 ]]; then
-        sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d
+        sudo docker compose -p ${CONTAINER_UID}hive2 -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d
     fi
 fi
 
@@ -349,12 +344,12 @@ if [[ "${RUN_HIVE3}" -eq 1 ]]; then
     # generate hive-3x.yaml and hadoop-hive.env
     export externalEnvIp=${IP_HOST}
     export CONTAINER_UID=${CONTAINER_UID}
-    . "${ROOT}"/docker-compose/hive/custom_settings.env
+    . "${ROOT}"/docker-compose/hive/hive-3x_settings.env
     envsubst < "${ROOT}"/docker-compose/hive/hive-3x.yaml.tpl > "${ROOT}"/docker-compose/hive/hive-3x.yaml
     envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl > "${ROOT}"/docker-compose/hive/hadoop-hive.env
-    sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
+    sudo docker compose -p ${CONTAINER_UID}hive3 -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
     if [[ "${STOP}" -ne 1 ]]; then
-        sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d
+        sudo docker compose -p ${CONTAINER_UID}hive3 -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d
     fi
 fi
 

From 2aceabd9929e73cbe211bd9d3e71d1095a53353e Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Wed, 10 Apr 2024 12:08:00 +0800
Subject: [PATCH 16/26] fix hive3 read schema error

---
 docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
index 51041e15341508..e97e71b444eda2 100644
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
@@ -26,6 +26,7 @@ HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
 HIVE_SITE_CONF_hive_server2_thrift_port=${HS_PORT}
 HIVE_SITE_CONF_hive_compactor_initiator_on=true
 HIVE_SITE_CONF_hive_compactor_worker_threads=2
+HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader
 
 CORE_CONF_hadoop_http_staticuser_user=root
 CORE_CONF_hadoop_proxyuser_hue_hosts=*

From 4c4f647c182494c9dabc7acaeec89f6e9a1bd55b Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Wed, 10 Apr 2024 13:10:38 +0800
Subject: [PATCH 17/26] change hive docker to bridge network

---
 .../{hadoop-hive.env.tpl => hadoop-hive.env}  | 22 ++----
 .../docker-compose/hive/hive-2x.yaml.tpl      | 69 +++++++++----------
 .../docker-compose/hive/hive-2x_settings.env  | 19 ++---
 .../docker-compose/hive/hive-3x.yaml.tpl      | 60 ++++++++--------
 .../docker-compose/hive/hive-3x_settings.env  | 19 ++---
 .../thirdparties/run-thirdparties-docker.sh   |  8 +--
 6 files changed, 79 insertions(+), 118 deletions(-)
 rename docker/thirdparties/docker-compose/hive/{hadoop-hive.env.tpl => hadoop-hive.env} (71%)

diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl b/docker/thirdparties/docker-compose/hive/hadoop-hive.env
similarity index 71%
rename from docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
rename to docker/thirdparties/docker-compose/hive/hadoop-hive.env
index e97e71b444eda2..5f24a742831e1b 100644
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive.env
@@ -15,36 +15,24 @@
 # limitations under the License.
 #
 
-HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://${externalEnvIp}:${PG_PORT}/metastore
+HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore-postgresql/metastore
 HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver
 HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive
 HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive
 HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
-HIVE_SITE_CONF_hive_metastore_uris=thrift://${externalEnvIp}:${HMS_PORT}
-HIVE_SITE_CONF_hive_metastore_port=${HMS_PORT}
+HIVE_SITE_CONF_hive_metastore_uris=thrift://hive-metastore:9083
 HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
-HIVE_SITE_CONF_hive_server2_thrift_port=${HS_PORT}
-HIVE_SITE_CONF_hive_compactor_initiator_on=true
-HIVE_SITE_CONF_hive_compactor_worker_threads=2
+HIVE_SITE_CONF_hive_server2_thrift_port=10000
 HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader
 
+CORE_CONF_fs_defaultFS=hdfs://namenode:8020
 CORE_CONF_hadoop_http_staticuser_user=root
 CORE_CONF_hadoop_proxyuser_hue_hosts=*
 CORE_CONF_hadoop_proxyuser_hue_groups=*
-CORE_CONF_hadoop_proxyuser_hive_hosts=*
-CORE_CONF_fs_defaultFS=hdfs://${externalEnvIp}:${FS_PORT}
 
 HDFS_CONF_dfs_webhdfs_enabled=true
 HDFS_CONF_dfs_permissions_enabled=false
 HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
-HDFS_CONF_dfs_namenode_http___address=0.0.0.0:${NAMENODE_HTTP_PORT}
-HDFS_CONF_dfs_namenode_https___address=0.0.0.0:${NAMENODE_HTTPS_PORT}
-HDFS_CONF_dfs_namenode_secondary_http___address=0.0.0.0:${NAMENODE_SECONDARY_HTTP_PORT}
-HDFS_CONF_dfs_namenode_secondary_https___address=0.0.0.0:${NAMENODE_SECONDARY_HTTPS_PORT}
-HDFS_CONF_dfs_datanode_address=0.0.0.0:${DATANODE_PORT}
-HDFS_CONF_dfs_datanode_http_address=0.0.0.0:${DATANODE_HTTP_PORT}
-HDFS_CONF_dfs_datanode_https_address=0.0.0.0:${DATANODE_HTTPS_PORT}
-HDFS_CONF_dfs_datanode_ipc_address=0.0.0.0:${DATANODE_IPC_PORT}
 
 YARN_CONF_yarn_log___aggregation___enable=true
 YARN_CONF_yarn_resourcemanager_recovery_enabled=true
@@ -59,4 +47,4 @@ YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
 YARN_CONF_yarn_timeline___service_hostname=historyserver
 YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032
 YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030
-YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031
+YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
index 256dd2ba509375..7af6ea3fbde52a 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
@@ -19,86 +19,83 @@
 version: "3.8"
 
 services:
-  ${CONTAINER_UID}namenode:
+  namenode:
     image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8
     environment:
       - CLUSTER_NAME=test
     env_file:
       - ./hadoop-hive.env
-    container_name: ${CONTAINER_UID}namenode
-    expose:
-      - "${NAMENODE_HTTP_PORT}"
-      - "${FS_PORT}"
+    container_name: ${CONTAINER_UID}hadoop2-namenode
+    ports:
+      - "${FS_PORT}:8020"
     healthcheck:
-      test: [ "CMD", "curl", "http://localhost:${NAMENODE_HTTP_PORT}/" ]
+      test: [ "CMD", "curl", "http://localhost:50070/" ]
       interval: 5s
       timeout: 120s
       retries: 120
-    network_mode: "host"
 
-  ${CONTAINER_UID}datanode:
+  datanode:
     image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
     env_file:
       - ./hadoop-hive.env
     environment:
-      SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT}"
-    container_name: ${CONTAINER_UID}datanode
-    expose:
-      - "${DATANODE_HTTP_PORT}"
+      SERVICE_PRECONDITION: "namenode:50070"
+    container_name: ${CONTAINER_UID}hadoop2-datanode
     healthcheck:
-      test: [ "CMD", "curl", "http://localhost:${DATANODE_HTTP_PORT}" ]
+      test: [ "CMD", "curl", "http://localhost:50075" ]
       interval: 5s
       timeout: 60s
       retries: 120
-    network_mode: "host"
 
-  ${CONTAINER_UID}hive-server:
+  hive-server:
     image: bde2020/hive:2.3.2-postgresql-metastore
     env_file:
       - ./hadoop-hive.env
     environment:
-      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://${externalEnvIp}:${PG_PORT}/metastore"
-      SERVICE_PRECONDITION: "${externalEnvIp}:${HMS_PORT}"
-    container_name: ${CONTAINER_UID}hive-server
-    expose:
-      - "${HS_PORT}"
+      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
+      SERVICE_PRECONDITION: "hive-metastore:9083"
+    container_name: ${CONTAINER_UID}hive2-server
+    ports:
+      - "${HS_PORT}:10000"
     depends_on:
-      - ${CONTAINER_UID}datanode
-      - ${CONTAINER_UID}namenode
+      - datanode
+      - namenode
     healthcheck:
-      test: beeline -u "jdbc:hive2://127.0.0.1:${HS_PORT}/default" -n health_check -e "show databases;"
+      test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;"
       interval: 10s
       timeout: 120s
       retries: 120
-    network_mode: "host"
 
 
-  ${CONTAINER_UID}hive-metastore:
+  hive-metastore:
     image: bde2020/hive:2.3.2-postgresql-metastore
     env_file:
       - ./hadoop-hive.env
     command: /bin/bash /mnt/scripts/hive-metastore.sh
     # command: /opt/hive/bin/hive --service metastore
     environment:
-      SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT} ${externalEnvIp}:${DATANODE_HTTP_PORT} ${externalEnvIp}:${PG_PORT}"
-    container_name: ${CONTAINER_UID}hive-metastore
-    expose:
-      - "${HMS_PORT}"
+      SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432"
+    container_name: ${CONTAINER_UID}hive2-metastore
+    ports:
+      - "${HMS_PORT}:9083"
     volumes:
       - ./scripts:/mnt/scripts
     depends_on:
-      - ${CONTAINER_UID}hive-metastore-postgresql
-    network_mode: "host"
+      - hive-metastore-postgresql
 
-  ${CONTAINER_UID}hive-metastore-postgresql:
+  hive-metastore-postgresql:
     image: bde2020/hive-metastore-postgresql:2.3.0
-    restart: always
-    container_name: ${CONTAINER_UID}hive-metastore-postgresql
+    container_name: ${CONTAINER_UID}hive2-metastore-postgresql
     ports:
-      - ${PG_PORT}:5432
+      - "${PG_PORT}:5432"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s
       timeout: 60s
       retries: 120
-    network_mode: "bridge"
+
+# solve HiveServer2 connect error:
+# java.net.URISyntaxException Illegal character in hostname :thrift://${CONTAINER_UID}hive2_default:9083
+networks:
+  default:
+    name: ${CONTAINER_UID}hive2-default
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x_settings.env b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
index 347142e2cddfa2..8baad99cbc81b7 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
+++ b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
@@ -18,18 +18,9 @@
 
 # Change this to a specific string.
 # Do not use "_" or other sepcial characters, only number and alphabeta.
-# NOTICE: change this uid will modify hadoop-hive.env and hive-*.yaml
+# NOTICE: change this uid will modify hive-*.yaml
 
-export FS_PORT=8220 #should be same in regression-conf.groovy
-export HMS_PORT=9283 #should be same in regression-conf.groovy
-export HS_PORT=12000 #should be same in regression-conf.groovy
-export PG_PORT=5632 #should be same in regression-conf.groovy
-
-export NAMENODE_HTTP_PORT=52070
-export NAMENODE_HTTPS_PORT=52470
-export NAMENODE_SECONDARY_HTTP_PORT=52090
-export NAMENODE_SECONDARY_HTTPS_PORT=52091
-export DATANODE_PORT=52010
-export DATANODE_HTTP_PORT=52075
-export DATANODE_HTTPS_PORT=52475
-export DATANODE_IPC_PORT=52020
+export FS_PORT=8120 #should be same in regression-conf.groovy
+export HMS_PORT=9183 #should be same in regression-conf.groovy
+export HS_PORT=11000 #should be same in regression-conf.groovy
+export PG_PORT=5532 #should be same in regression-conf.groovy
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index 150be0a9d8955a..67ce71fd1aee56 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -19,86 +19,84 @@
 version: "3.8"
 
 services:
-  ${CONTAINER_UID}hadoop3-namenode:
+  namenode:
     image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
     environment:
       - CLUSTER_NAME=test
     env_file:
       - ./hadoop-hive.env
     container_name: ${CONTAINER_UID}hadoop3-namenode
-    expose:
-      - "${NAMENODE_HTTP_PORT}"
-      - "${FS_PORT}"
+    ports:
+      - "${FS_PORT}:8020"
     healthcheck:
-      test: [ "CMD", "curl", "http://localhost:${NAMENODE_HTTP_PORT}/" ]
+      test: [ "CMD", "curl", "http://localhost:9870/" ]
       interval: 5s
       timeout: 120s
       retries: 120
-    network_mode: "host"
 
-  ${CONTAINER_UID}hadoop3-datanode:
+  datanode:
     image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
     env_file:
       - ./hadoop-hive.env
     environment:
-      SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT}"
+      SERVICE_PRECONDITION: "namenode:9870"
     container_name: ${CONTAINER_UID}hadoop3-datanode
-    expose:
-      - "${DATANODE_HTTP_PORT}"
     healthcheck:
-      test: [ "CMD", "curl", "http://localhost:${DATANODE_HTTP_PORT}" ]
+      test: [ "CMD", "curl", "http://localhost:9864" ]
       interval: 5s
       timeout: 60s
       retries: 120
-    network_mode: "host"
 
-  ${CONTAINER_UID}hive-server:
+  hive-server:
     image: lishizhen/hive:3.1.2-postgresql-metastore
     env_file:
       - ./hadoop-hive.env
     environment:
-      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://${externalEnvIp}:${PG_PORT}/metastore"
-      SERVICE_PRECONDITION: "${externalEnvIp}:${HMS_PORT}"
+      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
+      SERVICE_PRECONDITION: "hive-metastore:9083"
     container_name: ${CONTAINER_UID}hive3-server
-    expose:
-      - "${HS_PORT}"
+    ports:
+      - "${HS_PORT}:10000"
     depends_on:
-      - ${CONTAINER_UID}hadoop3-datanode
-      - ${CONTAINER_UID}hadoop3-namenode
+      - datanode
+      - namenode
     healthcheck:
-      test: beeline -u "jdbc:hive2://127.0.0.1:${HS_PORT}/default" -n health_check -e "show databases;"
+      test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;"
       interval: 10s
       timeout: 120s
       retries: 120
-    network_mode: "host"
 
 
-  ${CONTAINER_UID}hive3-metastore:
+  hive-metastore:
     image: lishizhen/hive:3.1.2-postgresql-metastore
     env_file:
       - ./hadoop-hive.env
     command: /bin/bash /mnt/scripts/hive-metastore.sh
     # command: /opt/hive/bin/hive --service metastore
     environment:
-      SERVICE_PRECONDITION: "${externalEnvIp}:${NAMENODE_HTTP_PORT} ${externalEnvIp}:${DATANODE_HTTP_PORT} ${externalEnvIp}:${PG_PORT}"
+      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 hive-metastore-postgresql:5432"
     container_name: ${CONTAINER_UID}hive3-metastore
-    expose:
-      - "${HMS_PORT}"
+    ports:
+      - "${HMS_PORT}:9083"
     volumes:
       - ./scripts:/mnt/scripts
     depends_on:
-      - ${CONTAINER_UID}hive3-metastore-postgresql
-    network_mode: "host"
+      - hive-metastore-postgresql
 
-  ${CONTAINER_UID}hive3-metastore-postgresql:
+  hive-metastore-postgresql:
     image: bde2020/hive-metastore-postgresql:3.1.0
-    restart: always
     container_name: ${CONTAINER_UID}hive3-metastore-postgresql
     ports:
-      - ${PG_PORT}:5432
+      - "${PG_PORT}:5432"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s
       timeout: 60s
       retries: 120
-    network_mode: "bridge"
+
+# solve HiveServer2 connect error:
+# java.net.URISyntaxException Illegal character in hostname :thrift://${CONTAINER_UID}hive3_default:9083
+
+networks:
+  default:
+    name: ${CONTAINER_UID}hive3-default
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
index dbf403f41e98f3..82486a17f0aa3b 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
+++ b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
@@ -18,18 +18,9 @@
 
 # Change this to a specific string.
 # Do not use "_" or other sepcial characters, only number and alphabeta.
-# NOTICE: change this uid will modify hadoop-hive.env and hive-*.yaml
+# NOTICE: change this uid will modify hive-*.yaml
 
-export FS_PORT=8320 #should be same in regression-conf.groovy
-export HMS_PORT=9383 #should be same in regression-conf.groovy
-export HS_PORT=13000 #should be same in regression-conf.groovy
-export PG_PORT=5732 #should be same in regression-conf.groovy
-
-export NAMENODE_HTTP_PORT=53070
-export NAMENODE_HTTPS_PORT=53470
-export NAMENODE_SECONDARY_HTTP_PORT=53090
-export NAMENODE_SECONDARY_HTTPS_PORT=53091
-export DATANODE_PORT=53010
-export DATANODE_HTTP_PORT=53075
-export DATANODE_HTTPS_PORT=53475
-export DATANODE_IPC_PORT=53020
+export FS_PORT=8020 #should be same in regression-conf.groovy
+export HMS_PORT=9083 #should be same in regression-conf.groovy
+export HS_PORT=10000 #should be same in regression-conf.groovy
+export PG_PORT=5432 #should be same in regression-conf.groovy
\ No newline at end of file
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index c83b8dadf415ca..4f7f1d24627ada 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -315,12 +315,10 @@ if [[ "${RUN_HIVE}" -eq 1 ]]; then
     # before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/"
     sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
     sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
-    # generate hive-3x.yaml and hadoop-hive.env
-    export externalEnvIp=${IP_HOST}
+    # generate hive-3x.yaml
     export CONTAINER_UID=${CONTAINER_UID}
     . "${ROOT}"/docker-compose/hive/hive-2x_settings.env
     envsubst < "${ROOT}"/docker-compose/hive/hive-2x.yaml.tpl > "${ROOT}"/docker-compose/hive/hive-2x.yaml
-    envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl > "${ROOT}"/docker-compose/hive/hadoop-hive.env
     sudo docker compose -p ${CONTAINER_UID}hive2 -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
     if [[ "${STOP}" -ne 1 ]]; then
         sudo docker compose -p ${CONTAINER_UID}hive2 -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d
@@ -341,12 +339,10 @@ if [[ "${RUN_HIVE3}" -eq 1 ]]; then
     # before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/"
     sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
     sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
-    # generate hive-3x.yaml and hadoop-hive.env
-    export externalEnvIp=${IP_HOST}
+    # generate hive-3x.yaml
     export CONTAINER_UID=${CONTAINER_UID}
     . "${ROOT}"/docker-compose/hive/hive-3x_settings.env
     envsubst < "${ROOT}"/docker-compose/hive/hive-3x.yaml.tpl > "${ROOT}"/docker-compose/hive/hive-3x.yaml
-    envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl > "${ROOT}"/docker-compose/hive/hadoop-hive.env
     sudo docker compose -p ${CONTAINER_UID}hive3 -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
     if [[ "${STOP}" -ne 1 ]]; then
         sudo docker compose -p ${CONTAINER_UID}hive3 -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d

From 13510bf0f12453e2e8d89fbd5256901ab22d2709 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Thu, 11 Apr 2024 00:59:34 +0800
Subject: [PATCH 18/26] fix

---
 docker/thirdparties/docker-compose/hive/hadoop-hive.env | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive.env b/docker/thirdparties/docker-compose/hive/hadoop-hive.env
index 5f24a742831e1b..0e092766dc334d 100644
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive.env
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive.env
@@ -47,4 +47,4 @@ YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
 YARN_CONF_yarn_timeline___service_hostname=historyserver
 YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032
 YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030
-YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031
\ No newline at end of file
+YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031

From 28e45309340c9ea8dbbb659d1a514d4f1669b440 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Thu, 11 Apr 2024 02:58:38 +0800
Subject: [PATCH 19/26] fix UnknownHostException: namenode when doris read hdfs
 url from hive metastore service

---
 .../hive/hadoop-hive-metastore.env.tpl        | 50 +++++++++++++++++++
 .../docker-compose/hive/hive-2x.yaml.tpl      |  2 +-
 .../docker-compose/hive/hive-3x.yaml.tpl      |  2 +-
 .../thirdparties/run-thirdparties-docker.sh   |  6 ++-
 4 files changed, 57 insertions(+), 3 deletions(-)
 create mode 100644 docker/thirdparties/docker-compose/hive/hadoop-hive-metastore.env.tpl

diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive-metastore.env.tpl b/docker/thirdparties/docker-compose/hive/hadoop-hive-metastore.env.tpl
new file mode 100644
index 00000000000000..828a194e2b3524
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive-metastore.env.tpl
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore-postgresql/metastore
+HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver
+HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive
+HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive
+HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
+HIVE_SITE_CONF_hive_metastore_uris=thrift://hive-metastore:9083
+HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
+HIVE_SITE_CONF_hive_server2_thrift_port=10000
+HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader
+
+CORE_CONF_fs_defaultFS=hdfs://${IP_HOST}:${FS_PORT}
+CORE_CONF_hadoop_http_staticuser_user=root
+CORE_CONF_hadoop_proxyuser_hue_hosts=*
+CORE_CONF_hadoop_proxyuser_hue_groups=*
+
+HDFS_CONF_dfs_webhdfs_enabled=true
+HDFS_CONF_dfs_permissions_enabled=false
+HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
+
+YARN_CONF_yarn_log___aggregation___enable=true
+YARN_CONF_yarn_resourcemanager_recovery_enabled=true
+YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
+YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate
+YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
+YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/
+YARN_CONF_yarn_timeline___service_enabled=true
+YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true
+YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true
+YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
+YARN_CONF_yarn_timeline___service_hostname=historyserver
+YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032
+YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030
+YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
index 7af6ea3fbde52a..cf912640160b82 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
@@ -70,7 +70,7 @@ services:
   hive-metastore:
     image: bde2020/hive:2.3.2-postgresql-metastore
     env_file:
-      - ./hadoop-hive.env
+      - ./hadoop-hive-metastore.env
     command: /bin/bash /mnt/scripts/hive-metastore.sh
     # command: /opt/hive/bin/hive --service metastore
     environment:
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index 67ce71fd1aee56..dc5c37b0dcdd69 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -70,7 +70,7 @@ services:
   hive-metastore:
     image: lishizhen/hive:3.1.2-postgresql-metastore
     env_file:
-      - ./hadoop-hive.env
+      - ./hadoop-hive-metastore.env
     command: /bin/bash /mnt/scripts/hive-metastore.sh
     # command: /opt/hive/bin/hive --service metastore
     environment:
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index 4f7f1d24627ada..47f6c8915f2923 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -315,10 +315,12 @@ if [[ "${RUN_HIVE}" -eq 1 ]]; then
     # before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/"
     sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
     sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
-    # generate hive-3x.yaml
+    # generate hive-2x.yaml
+    export IP_HOST=${IP_HOST}
     export CONTAINER_UID=${CONTAINER_UID}
     . "${ROOT}"/docker-compose/hive/hive-2x_settings.env
     envsubst < "${ROOT}"/docker-compose/hive/hive-2x.yaml.tpl > "${ROOT}"/docker-compose/hive/hive-2x.yaml
+    envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive-metastore.env.tpl > "${ROOT}"/docker-compose/hive/hadoop-hive-metastore.env
     sudo docker compose -p ${CONTAINER_UID}hive2 -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
     if [[ "${STOP}" -ne 1 ]]; then
         sudo docker compose -p ${CONTAINER_UID}hive2 -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d
@@ -340,9 +342,11 @@ if [[ "${RUN_HIVE3}" -eq 1 ]]; then
     sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
     sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh
     # generate hive-3x.yaml
+    export IP_HOST=${IP_HOST}
     export CONTAINER_UID=${CONTAINER_UID}
     . "${ROOT}"/docker-compose/hive/hive-3x_settings.env
     envsubst < "${ROOT}"/docker-compose/hive/hive-3x.yaml.tpl > "${ROOT}"/docker-compose/hive/hive-3x.yaml
+    envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive-metastore.env.tpl > "${ROOT}"/docker-compose/hive/hadoop-hive-metastore.env
     sudo docker compose -p ${CONTAINER_UID}hive3 -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down
     if [[ "${STOP}" -ne 1 ]]; then
         sudo docker compose -p ${CONTAINER_UID}hive3 -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d

From edff305fed1d4a2abd128ab13dff16e9a27fec96 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Thu, 11 Apr 2024 16:03:03 +0800
Subject: [PATCH 20/26] fix UnknownHostException:namenode

---
 docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl  | 2 +-
 .../thirdparties/docker-compose/hive/hive-2x_settings.env | 8 ++++----
 docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl  | 2 +-
 .../thirdparties/docker-compose/hive/hive-3x_settings.env | 8 ++++----
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
index cf912640160b82..9b201a5416e44f 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
@@ -50,7 +50,7 @@ services:
   hive-server:
     image: bde2020/hive:2.3.2-postgresql-metastore
     env_file:
-      - ./hadoop-hive.env
+      - ./hadoop-hive-metastore.env
     environment:
       HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
       SERVICE_PRECONDITION: "hive-metastore:9083"
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x_settings.env b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
index 8baad99cbc81b7..82486a17f0aa3b 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
+++ b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
@@ -20,7 +20,7 @@
 # Do not use "_" or other sepcial characters, only number and alphabeta.
 # NOTICE: change this uid will modify hive-*.yaml
 
-export FS_PORT=8120 #should be same in regression-conf.groovy
-export HMS_PORT=9183 #should be same in regression-conf.groovy
-export HS_PORT=11000 #should be same in regression-conf.groovy
-export PG_PORT=5532 #should be same in regression-conf.groovy
\ No newline at end of file
+export FS_PORT=8020 #should be same in regression-conf.groovy
+export HMS_PORT=9083 #should be same in regression-conf.groovy
+export HS_PORT=10000 #should be same in regression-conf.groovy
+export PG_PORT=5432 #should be same in regression-conf.groovy
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index dc5c37b0dcdd69..5ce81774c69ce2 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -50,7 +50,7 @@ services:
   hive-server:
     image: lishizhen/hive:3.1.2-postgresql-metastore
     env_file:
-      - ./hadoop-hive.env
+      - ./hadoop-hive-metastore.env
     environment:
       HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
       SERVICE_PRECONDITION: "hive-metastore:9083"
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
index 82486a17f0aa3b..dc21bce0243c2a 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
+++ b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
@@ -20,7 +20,7 @@
 # Do not use "_" or other sepcial characters, only number and alphabeta.
 # NOTICE: change this uid will modify hive-*.yaml
 
-export FS_PORT=8020 #should be same in regression-conf.groovy
-export HMS_PORT=9083 #should be same in regression-conf.groovy
-export HS_PORT=10000 #should be same in regression-conf.groovy
-export PG_PORT=5432 #should be same in regression-conf.groovy
\ No newline at end of file
+export FS_PORT=8320 #should be same in regression-conf.groovy
+export HMS_PORT=9383 #should be same in regression-conf.groovy
+export HS_PORT=13000 #should be same in regression-conf.groovy
+export PG_PORT=5732 #should be same in regression-conf.groovy
\ No newline at end of file

From be01e1a8b231ab41504fa43c50a4213d846ad31c Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Thu, 11 Apr 2024 21:37:03 +0800
Subject: [PATCH 21/26] test hive3

---
 .../thirdparties/docker-compose/hive/hive-2x_settings.env | 8 ++++----
 .../thirdparties/docker-compose/hive/hive-3x_settings.env | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/docker/thirdparties/docker-compose/hive/hive-2x_settings.env b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
index 82486a17f0aa3b..1d021ba20c925c 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
+++ b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
@@ -20,7 +20,7 @@
 # Do not use "_" or other sepcial characters, only number and alphabeta.
 # NOTICE: change this uid will modify hive-*.yaml
 
-export FS_PORT=8020 #should be same in regression-conf.groovy
-export HMS_PORT=9083 #should be same in regression-conf.groovy
-export HS_PORT=10000 #should be same in regression-conf.groovy
-export PG_PORT=5432 #should be same in regression-conf.groovy
\ No newline at end of file
+export FS_PORT=8220 #should be same in regression-conf.groovy
+export HMS_PORT=9283 #should be same in regression-conf.groovy
+export HS_PORT=12000 #should be same in regression-conf.groovy
+export PG_PORT=5632 #should be same in regression-conf.groovy
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
index dc21bce0243c2a..82486a17f0aa3b 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
+++ b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
@@ -20,7 +20,7 @@
 # Do not use "_" or other sepcial characters, only number and alphabeta.
 # NOTICE: change this uid will modify hive-*.yaml
 
-export FS_PORT=8320 #should be same in regression-conf.groovy
-export HMS_PORT=9383 #should be same in regression-conf.groovy
-export HS_PORT=13000 #should be same in regression-conf.groovy
-export PG_PORT=5732 #should be same in regression-conf.groovy
\ No newline at end of file
+export FS_PORT=8020 #should be same in regression-conf.groovy
+export HMS_PORT=9083 #should be same in regression-conf.groovy
+export HS_PORT=10000 #should be same in regression-conf.groovy
+export PG_PORT=5432 #should be same in regression-conf.groovy
\ No newline at end of file

From 4bde2bad58ba62ea34952ef58edd29d0977a8f5c Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Fri, 12 Apr 2024 02:13:07 +0800
Subject: [PATCH 22/26] change unreasonable tests

---
 .../export/hive_read/orc/test_hive_read_orc.out    |  3 ---
 .../orc/test_hive_read_orc_complex_type.out        |  2 +-
 .../hive_read/parquet/test_hive_read_parquet.out   |  3 ---
 .../export/hive_read/orc/test_hive_read_orc.groovy | 14 +++-----------
 .../orc/test_hive_read_orc_complex_type.groovy     |  8 ++++----
 .../parquet/test_hive_read_parquet.groovy          | 14 +++-----------
 .../test_hive_read_parquet_comlex_type.groovy      |  8 ++++----
 7 files changed, 15 insertions(+), 37 deletions(-)

diff --git a/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc.out b/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc.out
index 26799d9a5320dd..690889873a6fe5 100644
--- a/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc.out
+++ b/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc.out
@@ -33,17 +33,14 @@
 1	2023-04-20	2023-04-20	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	Beijing	Haidian	1	1	true	1	1	1	1.1	1.1	char1	1	1	1	0.1	1.00000000	1.0000000000	1	1.0000000000000000000000000000000000000	0.10000000000000000000000000000000000000
 2	9999-12-31	9999-12-31	9999-12-31T23:59:59	9999-12-31T23:59:59	2023-04-20T00:00:00.120	2023-04-20T00:00:00.334400		Haidian	-32768	-128	true	-2147483648	-9223372036854775808	-170141183460469231731687303715884105728	1.4E-45	4.9E-324	char2	100000000	100000000	4	0.1	0.99999999	9999999999.9999999999	99999999999999999999999999999999999999	9.9999999999999999999999999999999999999	0.99999999999999999999999999999999999999
 3	2023-04-21	2023-04-21	2023-04-20T12:34:56	2023-04-20T00:00	2023-04-20T00:00:00.123	2023-04-20T00:00:00.123456	Beijing		32767	127	true	2147483647	9223372036854775807	170141183460469231731687303715884105727	3.4028235e+38	1.7976931348623157E308	char3	999999999	999999999	9	0.9	9.99999999	1234567890.0123456789	12345678901234567890123456789012345678	1.2345678901234567890123456789012345678	0.12345678901234567890123456789012345678
-4	0000-01-01	0000-01-01	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	Beijing	Haidian	4	4	true	4	4	4	4.4	4.4	char4	4	4	4	0.4	4.00000000	4.0000000000	4	4.0000000000000000000000000000000000000	0.40000000000000000000000000000000000000
 
 -- !select_tvf2 --
 1	2023-04-20	2023-04-20	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	Beijing	Haidian	1	1	true	1	1	1	1.1	1.1	char1	1	1	1	0.1	1.00000000	1.0000000000	1	1.0000000000000000000000000000000000000	0.10000000000000000000000000000000000000
 2	9999-12-31	9999-12-31	9999-12-31T23:59:59	9999-12-31T23:59:59	2023-04-20T00:00:00.120	2023-04-20T00:00:00.334400		Haidian	-32768	-128	true	-2147483648	-9223372036854775808	-170141183460469231731687303715884105728	1.4E-45	4.9E-324	char2	100000000	100000000	4	0.1	0.99999999	9999999999.9999999999	99999999999999999999999999999999999999	9.9999999999999999999999999999999999999	0.99999999999999999999999999999999999999
 3	2023-04-21	2023-04-21	2023-04-20T12:34:56	2023-04-20T00:00	2023-04-20T00:00:00.123	2023-04-20T00:00:00.123456	Beijing		32767	127	true	2147483647	9223372036854775807	170141183460469231731687303715884105727	3.4028235e+38	1.7976931348623157E308	char3	999999999	999999999	9	0.9	9.99999999	1234567890.0123456789	12345678901234567890123456789012345678	1.2345678901234567890123456789012345678	0.12345678901234567890123456789012345678
-4	0000-01-01	0000-01-01	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	Beijing	Haidian	4	4	true	4	4	4	4.4	4.4	char4	4	4	4	0.4	4.00000000	4.0000000000	4	4.0000000000000000000000000000000000000	0.40000000000000000000000000000000000000
 
 -- !hive_docker_02 --
 1	2023-04-20	2023-04-20	2023-04-20 00:00:00.0	2023-04-20 00:00:00.0	2023-04-20 00:00:00.0	2023-04-20 00:00:00.0	Beijing	Haidian	1	1	true	1	1	1	1.1	1.1	char1	1	1	1	0.1	1.00000000	1.0000000000	1	1.0000000000000000000000000000000000000	0.10000000000000000000000000000000000000
 2	9999-12-31	9999-12-31	9999-12-31 23:59:59.0	9999-12-31 23:59:59.0	2023-04-20 00:00:00.12	2023-04-20 00:00:00.3344		Haidian	-32768	-128	true	-2147483648	-9223372036854775808	-170141183460469231731687303715884105728	1.4E-45	4.9E-324	char2	100000000	100000000	4	0.1	0.99999999	9999999999.9999999999	99999999999999999999999999999999999999	9.9999999999999999999999999999999999999	0.99999999999999999999999999999999999999
 3	2023-04-21	2023-04-21	2023-04-20 12:34:56.0	2023-04-20 00:00:00.0	2023-04-20 00:00:00.123	2023-04-20 00:00:00.123456	Beijing		32767	127	true	2147483647	9223372036854775807	170141183460469231731687303715884105727	3.4028235E38	1.7976931348623157E308	char3	999999999	999999999	9	0.9	9.99999999	1234567890.0123456789	12345678901234567890123456789012345678	1.2345678901234567890123456789012345678	0.12345678901234567890123456789012345678
-4	0001-01-04	0001-01-04	2023-04-20 00:00:00.0	2023-04-20 00:00:00.0	2023-04-20 00:00:00.0	2023-04-20 00:00:00.0	Beijing	Haidian	4	4	true	4	4	4	4.4	4.4	char4	4	4	4	0.4	4.00000000	4.0000000000	4	4.0000000000000000000000000000000000000	0.40000000000000000000000000000000000000
 
diff --git a/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.out b/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.out
index d13a47d516f295..3f4935a2d295b0 100644
--- a/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.out
+++ b/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.out
@@ -51,7 +51,7 @@
 1	doris1	{"a":"100","b":"111"}
 2	doris2	{"a":"200","b":"222"}
 3	doris3	{"a":null,"b":"333","c":"399","d":"399999999999999"}
-4	doris4	{}
+4	doris4	{null:null}
 5	doris5	{"b":null}
 6	\N	\N
 7	doris7	\N
diff --git a/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out b/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out
index 90a5cf00ee32ec..9002c1c4490109 100644
--- a/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out
+++ b/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out
@@ -33,17 +33,14 @@
 1	2023-04-20	2023-04-20	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	Beijing	Haidian	1	1	true	1	1	1	1.1	1.1	char1	1	1	1	0.1	1.00000000	1.0000000000	1	1.0000000000000000000000000000000000000	0.10000000000000000000000000000000000000
 2	9999-12-31	9999-12-31	9999-12-31T23:59:59	9999-12-31T23:59:59	2023-04-20T00:00:00.120	2023-04-20T00:00:00.334400		Haidian	-32768	-128	true	-2147483648	-9223372036854775808	-170141183460469231731687303715884105728	1.4E-45	4.9E-324	char2	100000000	100000000	4	0.1	0.99999999	9999999999.9999999999	99999999999999999999999999999999999999	9.9999999999999999999999999999999999999	0.99999999999999999999999999999999999999
 3	2023-04-21	2023-04-21	2023-04-20T12:34:56	2023-04-20T00:00	2023-04-20T00:00:00.123	2023-04-20T00:00:00.123456	Beijing		32767	127	true	2147483647	9223372036854775807	170141183460469231731687303715884105727	3.4028235e+38	1.7976931348623157E308	char3	999999999	999999999	9	0.9	9.99999999	1234567890.0123456789	12345678901234567890123456789012345678	1.2345678901234567890123456789012345678	0.12345678901234567890123456789012345678
-4	0000-01-01	0000-01-01	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	Beijing	Haidian	4	4	true	4	4	4	4.4	4.4	char4	4	4	4	0.4	4.00000000	4.0000000000	4	4.0000000000000000000000000000000000000	0.40000000000000000000000000000000000000
 
 -- !select_tvf2 --
 1	2023-04-20	2023-04-20	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	Beijing	Haidian	1	1	true	1	1	1	1.1	1.1	char1	1	1	1	0.1	1.00000000	1.0000000000	1	1.0000000000000000000000000000000000000	0.10000000000000000000000000000000000000
 2	9999-12-31	9999-12-31	9999-12-31T23:59:59	9999-12-31T23:59:59	2023-04-20T00:00:00.120	2023-04-20T00:00:00.334400		Haidian	-32768	-128	true	-2147483648	-9223372036854775808	-170141183460469231731687303715884105728	1.4E-45	4.9E-324	char2	100000000	100000000	4	0.1	0.99999999	9999999999.9999999999	99999999999999999999999999999999999999	9.9999999999999999999999999999999999999	0.99999999999999999999999999999999999999
 3	2023-04-21	2023-04-21	2023-04-20T12:34:56	2023-04-20T00:00	2023-04-20T00:00:00.123	2023-04-20T00:00:00.123456	Beijing		32767	127	true	2147483647	9223372036854775807	170141183460469231731687303715884105727	3.4028235e+38	1.7976931348623157E308	char3	999999999	999999999	9	0.9	9.99999999	1234567890.0123456789	12345678901234567890123456789012345678	1.2345678901234567890123456789012345678	0.12345678901234567890123456789012345678
-4	0000-01-01	0000-01-01	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	2023-04-20T00:00	Beijing	Haidian	4	4	true	4	4	4	4.4	4.4	char4	4	4	4	0.4	4.00000000	4.0000000000	4	4.0000000000000000000000000000000000000	0.40000000000000000000000000000000000000
 
 -- !hive_docker_02 --
 1	2023-04-20	2023-04-20	2023-04-19 16:00:00.0	2023-04-19 16:00:00.0	2023-04-19 16:00:00.0	2023-04-19 16:00:00.0	Beijing	Haidian	1	1	true	1	1	1	1.1	1.1	char1	1	1	1	0.1	1.00000000	1.0000000000	1	1.0000000000000000000000000000000000000	0.10000000000000000000000000000000000000
 2	9999-12-31	9999-12-31	9999-12-31 15:59:59.0	9999-12-31 15:59:59.0	2023-04-19 16:00:00.12	2023-04-19 16:00:00.3344		Haidian	-32768	-128	true	-2147483648	-9223372036854775808	-170141183460469231731687303715884105728	1.4E-45	4.9E-324	char2	100000000	100000000	4	0.1	0.99999999	9999999999.9999999999	99999999999999999999999999999999999999	9.9999999999999999999999999999999999999	0.99999999999999999999999999999999999999
 3	2023-04-21	2023-04-21	2023-04-20 04:34:56.0	2023-04-19 16:00:00.0	2023-04-19 16:00:00.123	2023-04-19 16:00:00.123456	Beijing		32767	127	true	2147483647	9223372036854775807	170141183460469231731687303715884105727	3.4028235E38	1.7976931348623157E308	char3	999999999	999999999	9	0.9	9.99999999	1234567890.0123456789	12345678901234567890123456789012345678	1.2345678901234567890123456789012345678	0.12345678901234567890123456789012345678
-4	0001-01-04	0001-01-04	2023-04-19 16:00:00.0	2023-04-19 16:00:00.0	2023-04-19 16:00:00.0	2023-04-19 16:00:00.0	Beijing	Haidian	4	4	true	4	4	4	4.4	4.4	char4	4	4	4	0.4	4.00000000	4.0000000000	4	4.0000000000000000000000000000000000000	0.40000000000000000000000000000000000000
 
diff --git a/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc.groovy b/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc.groovy
index b897263ea86f4c..0fead618e91bb9 100644
--- a/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc.groovy
+++ b/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc.groovy
@@ -144,7 +144,7 @@ suite("test_hive_read_orc", "external,hive,external_docker") {
                     "format" = "${format}");
                     """
 
-        qt_hive_docker_01 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;"""
+        qt_hive_docker_01 """ SELECT * FROM ${hive_database}.${hive_table};"""
 
     } finally {
     }
@@ -235,16 +235,8 @@ suite("test_hive_read_orc", "external,hive,external_docker") {
                 (${++i}, '2023-04-21', '2023-04-21', '2023-04-20 12:34:56', '2023-04-20 00:00:00', '2023-04-20 00:00:00.123', '2023-04-20 00:00:00.123456',
                 'Beijing', '', 
                 ${Short.MAX_VALUE}, ${Byte.MAX_VALUE}, true, ${Integer.MAX_VALUE}, ${Long.MAX_VALUE}, 170141183460469231731687303715884105727, ${Float.MAX_VALUE}, ${Double.MAX_VALUE}, 'char${i}',
-                999999999, 999999999, 9, 0.9, 9.99999999, 1234567890.0123456789, 12345678901234567890123456789012345678, 1.2345678901234567890123456789012345678, 0.12345678901234567890123456789012345678),
+                999999999, 999999999, 9, 0.9, 9.99999999, 1234567890.0123456789, 12345678901234567890123456789012345678, 1.2345678901234567890123456789012345678, 0.12345678901234567890123456789012345678)
             """)
-
-        sb.append("""
-                (${++i}, '0000-01-01', '0000-01-01', '2023-04-20 00:00:00', '2023-04-20 00:00:00', '2023-04-20 00:00:00', '2023-04-20 00:00:00',
-                'Beijing', 'Haidian',
-                ${i}, ${i % 128}, true, ${i}, ${i}, ${i}, ${i}.${i}, ${i}.${i}, 'char${i}',
-                ${i}, ${i}, ${i}, 0.${i}, ${i}, ${i}, ${i}, ${i}, 0.${i})
-            """)
-
         
         sql """ INSERT INTO ${export_table_name} VALUES
                 ${sb.toString()}
@@ -265,7 +257,7 @@ suite("test_hive_read_orc", "external,hive,external_docker") {
                     "format" = "${format}");
                     """
 
-        qt_hive_docker_02""" SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;"""
+        qt_hive_docker_02""" SELECT * FROM ${hive_database}.${hive_table};"""
 
     } finally {
     }
diff --git a/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.groovy b/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.groovy
index 9edac8229a12f7..c5577baa60ed59 100644
--- a/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.groovy
+++ b/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.groovy
@@ -143,7 +143,7 @@ suite("test_hive_read_orc_complex_type", "external,hive,external_docker") {
                         "format" = "${format}");
                         """
 
-        qt_hive_docker_01 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;"""
+        qt_hive_docker_01 """ SELECT * FROM ${hive_database}.${hive_table};"""
 
     } finally {
     }
@@ -181,7 +181,7 @@ suite("test_hive_read_orc_complex_type", "external,hive,external_docker") {
                         "format" = "${format}");
                         """
 
-        qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;"""
+        qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table};"""
             
     } finally {
     }
@@ -221,7 +221,7 @@ suite("test_hive_read_orc_complex_type", "external,hive,external_docker") {
                         "format" = "${format}");
                         """
 
-        qt_hive_docker_03 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;"""
+        qt_hive_docker_03 """ SELECT * FROM ${hive_database}.${hive_table};"""
 
     } finally {
     }
@@ -272,7 +272,7 @@ suite("test_hive_read_orc_complex_type", "external,hive,external_docker") {
                         "format" = "${format}");
                         """
 
-        qt_hive_docker_04 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;"""
+        qt_hive_docker_04 """ SELECT * FROM ${hive_database}.${hive_table};"""
 
     } finally {
     }
diff --git a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy
index 1b69c917835584..479183daa939d4 100644
--- a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy
+++ b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy
@@ -144,7 +144,7 @@ suite("test_hive_read_parquet", "external,hive,external_docker") {
                     "format" = "${format}");
                     """
 
-        qt_hive_docker_01 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;"""
+        qt_hive_docker_01 """ SELECT * FROM ${hive_database}.${hive_table};"""
 
     } finally {
     }
@@ -235,16 +235,8 @@ suite("test_hive_read_parquet", "external,hive,external_docker") {
                 (${++i}, '2023-04-21', '2023-04-21', '2023-04-20 12:34:56', '2023-04-20 00:00:00', '2023-04-20 00:00:00.123', '2023-04-20 00:00:00.123456',
                 'Beijing', '', 
                 ${Short.MAX_VALUE}, ${Byte.MAX_VALUE}, true, ${Integer.MAX_VALUE}, ${Long.MAX_VALUE}, 170141183460469231731687303715884105727, ${Float.MAX_VALUE}, ${Double.MAX_VALUE}, 'char${i}',
-                999999999, 999999999, 9, 0.9, 9.99999999, 1234567890.0123456789, 12345678901234567890123456789012345678, 1.2345678901234567890123456789012345678, 0.12345678901234567890123456789012345678),
+                999999999, 999999999, 9, 0.9, 9.99999999, 1234567890.0123456789, 12345678901234567890123456789012345678, 1.2345678901234567890123456789012345678, 0.12345678901234567890123456789012345678)
             """)
-
-        sb.append("""
-                (${++i}, '0000-01-01', '0000-01-01', '2023-04-20 00:00:00', '2023-04-20 00:00:00', '2023-04-20 00:00:00', '2023-04-20 00:00:00',
-                'Beijing', 'Haidian',
-                ${i}, ${i % 128}, true, ${i}, ${i}, ${i}, ${i}.${i}, ${i}.${i}, 'char${i}',
-                ${i}, ${i}, ${i}, 0.${i}, ${i}, ${i}, ${i}, ${i}, 0.${i})
-            """)
-
         
         sql """ INSERT INTO ${export_table_name} VALUES
                 ${sb.toString()}
@@ -265,7 +257,7 @@ suite("test_hive_read_parquet", "external,hive,external_docker") {
                     "format" = "${format}");
                     """
 
-        qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;"""
+        qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table};"""
         
     } finally {
     }
diff --git a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.groovy b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.groovy
index e4e80d74b57155..eb741e1e5db004 100644
--- a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.groovy
+++ b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.groovy
@@ -152,7 +152,7 @@ suite("test_hive_read_parquet_complex_type", "external,hive,external_docker") {
                         "format" = "${format}");
                         """
 
-        qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;"""
+        qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table};"""
             
     } finally {
     }
@@ -191,7 +191,7 @@ suite("test_hive_read_parquet_complex_type", "external,hive,external_docker") {
                         "format" = "${format}");
                         """
 
-        qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;"""
+        qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table};"""
 
     } finally {
     }
@@ -231,7 +231,7 @@ suite("test_hive_read_parquet_complex_type", "external,hive,external_docker") {
                         "format" = "${format}");
                         """
 
-        qt_hive_docker_03 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;"""
+        qt_hive_docker_03 """ SELECT * FROM ${hive_database}.${hive_table};"""
 
     } finally {
     }
@@ -276,7 +276,7 @@ suite("test_hive_read_parquet_complex_type", "external,hive,external_docker") {
                         "format" = "${format}");
                         """
 
-        qt_hive_docker_04 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;"""
+        qt_hive_docker_04 """ SELECT * FROM ${hive_database}.${hive_table};"""
 
     } finally {
     }

From 2272bd8fa35c52ff92353416685e65dfe1ae7052 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Fri, 12 Apr 2024 16:21:05 +0800
Subject: [PATCH 23/26] fix test for hive3 about hive.stats.column.autogather

---
 .../suites/mtmv_p0/test_hive_multi_partition_mtmv.groovy  | 6 ++++++
 .../suites/mtmv_p0/test_hive_refresh_mtmv.groovy          | 8 ++++++++
 .../mv/external_table/mv_contain_external_table.groovy    | 4 ++++
 3 files changed, 18 insertions(+)

diff --git a/regression-test/suites/mtmv_p0/test_hive_multi_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_multi_partition_mtmv.groovy
index c8e5830dcacc60..269bf95a34d38a 100644
--- a/regression-test/suites/mtmv_p0/test_hive_multi_partition_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_hive_multi_partition_mtmv.groovy
@@ -25,6 +25,8 @@ suite("test_hive_multi_partition_mtmv", "p0,external,hive,external_docker,extern
     def hive_database = "test_hive_multi_partition_mtmv_db"
     def hive_table = "partition2"
 
+    def autogather_off_str = """ set hive.stats.column.autogather = false; """
+    def autogather_on_str = """ set hive.stats.column.autogather = true; """
     def drop_table_str = """ drop table if exists ${hive_database}.${hive_table} """
     def drop_database_str = """ drop database if exists ${hive_database}"""
     def create_database_str = """ create database ${hive_database}"""
@@ -51,6 +53,8 @@ suite("test_hive_multi_partition_mtmv", "p0,external,hive,external_docker,extern
     def insert_str5 = """insert into ${hive_database}.${hive_table} PARTITION(year=2022,region="bj") values(5)"""
     def insert_str6 = """insert into ${hive_database}.${hive_table} PARTITION(year=2022,region="sh") values(6)"""
 
+    logger.info("hive sql: " + autogather_off_str)
+    hive_docker """ ${autogather_off_str} """
     logger.info("hive sql: " + drop_table_str)
     hive_docker """ ${drop_table_str} """
     logger.info("hive sql: " + drop_database_str)
@@ -247,6 +251,8 @@ suite("test_hive_multi_partition_mtmv", "p0,external,hive,external_docker,extern
        assertTrue(showPartitionsResult.toString().contains("p_sh"))
        assertFalse(showPartitionsResult.toString().contains("p_tj"))
 
+       logger.info("hive sql: " + autogather_on_str)
+       hive_docker """ ${autogather_on_str} """
        sql """drop materialized view if exists ${mvName};"""
        sql """drop catalog if exists ${catalog_name}"""
 }
diff --git a/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy
index 616ff77ca9a852..23135ea62a09cc 100644
--- a/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy
@@ -25,6 +25,8 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke
     def hive_database = "mtmv_test_db"
     def hive_table = "test_hive_refresh_mtmv_t1"
 
+    def autogather_off_str = """ set hive.stats.column.autogather = false; """
+    def autogather_on_str = """ set hive.stats.column.autogather = true; """
     def drop_table_str = """ drop table if exists ${hive_database}.${hive_table} """
     def drop_database_str = """ drop database if exists ${hive_database}"""
     def create_database_str = """ create database ${hive_database}"""
@@ -40,6 +42,9 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke
                                 partition(year=2020);
                             """
     def insert_str = """ insert into ${hive_database}.${hive_table} PARTITION(year=2020) values(1,1)"""
+
+    logger.info("hive sql: " + autogather_off_str)
+    hive_docker """ ${autogather_off_str} """
     logger.info("hive sql: " + drop_table_str)
     hive_docker """ ${drop_table_str} """
     logger.info("hive sql: " + drop_database_str)
@@ -173,6 +178,9 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke
        waitingMTMVTaskFinishedNotNeedSuccess(jobName)
        order_qt_task_recover "select Status from tasks('type'='mv') where JobName = '${jobName}' order by CreateTime DESC limit 1"
 
+       logger.info("hive sql: " + autogather_on_str)
+       hive_docker """ ${autogather_on_str} """
+
        sql """drop materialized view if exists ${mvName};"""
 
        sql """drop catalog if exists ${catalog_name}"""
diff --git a/regression-test/suites/nereids_rules_p0/mv/external_table/mv_contain_external_table.groovy b/regression-test/suites/nereids_rules_p0/mv/external_table/mv_contain_external_table.groovy
index 392997a82b9192..fc630a0e4ebf55 100644
--- a/regression-test/suites/nereids_rules_p0/mv/external_table/mv_contain_external_table.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/external_table/mv_contain_external_table.groovy
@@ -25,6 +25,8 @@ suite("mv_contain_external_table", "p0,external,hive,external_docker,external_do
     def hive_database = "test_mv_contain_external_table_rewrite_db"
     def hive_table = "orders"
 
+    def autogather_off_str = """ set hive.stats.column.autogather = false; """
+    def autogather_on_str = """ set hive.stats.column.autogather = true; """
     def drop_table_str = """ drop table if exists ${hive_database}.${hive_table} """
     def drop_database_str = """ drop database if exists ${hive_database}"""
     def create_database_str = """ create database ${hive_database}"""
@@ -60,6 +62,7 @@ suite("mv_contain_external_table", "p0,external,hive,external_docker,external_do
     def insert_str3 = """ insert into ${hive_database}.${hive_table} 
     PARTITION(o_orderdate='2023-10-19') values(3, 3, 'ok', 99.5, 'a', 'b', 1, 'yy')"""
 
+    hive_docker """ ${autogather_off_str} """
     hive_docker """ ${drop_table_str} """
     hive_docker """ ${drop_database_str} """
     hive_docker """ ${create_database_str}"""
@@ -211,6 +214,7 @@ suite("mv_contain_external_table", "p0,external,hive,external_docker,external_do
     }
     order_qt_query_rewritten_with_new_data """ ${query_sql}"""
 
+    hive_docker """ ${autogather_on_str} """
     sql """drop materialized view if exists ${mv_name};"""
     sql """drop catalog if exists ${catalog_name}"""
 }

From a3aacf865c5686496b47658b83c47b965c37b643 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Sun, 14 Apr 2024 23:46:33 +0800
Subject: [PATCH 24/26] fix test_hive_ddl for hive3

---
 .../suites/external_table_p0/hive/ddl/test_hive_ddl.groovy      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy
index b0543931371625..5169145da8e3c4 100644
--- a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy
+++ b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy
@@ -73,7 +73,7 @@ suite("test_hive_ddl", "p0,external,hive,external_docker,external_docker_hive")
                         properties('location'='tt://${externalEnvIp}:${hdfs_port}/exist_check')
                     """
             } catch (Exception e) {
-                assertTrue(e.getMessage().contains("No FileSystem for scheme: tt"))
+                assertTrue(e.getMessage().contains("No FileSystem for scheme"))
             }
             try {
                 sql """ create database if not exists `test_hive_loc_no_exist`

From 520e0334104c24250c286f3380e8731597d887c3 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Mon, 15 Apr 2024 17:58:42 +0800
Subject: [PATCH 25/26] only run hive3 default

---
 docker/thirdparties/run-thirdparties-docker.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index 47f6c8915f2923..6ca328433b0bee 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -60,7 +60,7 @@ STOP=0
 
 if [[ "$#" == 1 ]]; then
     # default
-    COMPONENTS="mysql,es,hive,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2"
+    COMPONENTS="mysql,es,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2"
 else
     while true; do
         case "$1" in
@@ -92,7 +92,7 @@ else
     done
     if [[ "${COMPONENTS}"x == ""x ]]; then
         if [[ "${STOP}" -eq 1 ]]; then
-            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,hive3,iceberg,hudi,trino,kafka,mariadb,db2"
+            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive3,iceberg,hudi,trino,kafka,mariadb,db2"
         fi
     fi
 fi

From 0ca926f33740f3a16d5f7a8c8ba1e8679b608060 Mon Sep 17 00:00:00 2001
From: suxiaogang223 <suxiaogang223@icloud.com>
Date: Tue, 16 Apr 2024 13:06:33 +0800
Subject: [PATCH 26/26] Revert "only run hive3 default"

This reverts commit 520e0334104c24250c286f3380e8731597d887c3.
---
 docker/thirdparties/run-thirdparties-docker.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index 6ca328433b0bee..47f6c8915f2923 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -60,7 +60,7 @@ STOP=0
 
 if [[ "$#" == 1 ]]; then
     # default
-    COMPONENTS="mysql,es,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2"
+    COMPONENTS="mysql,es,hive,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2"
 else
     while true; do
         case "$1" in
@@ -92,7 +92,7 @@ else
     done
     if [[ "${COMPONENTS}"x == ""x ]]; then
         if [[ "${STOP}" -eq 1 ]]; then
-            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive3,iceberg,hudi,trino,kafka,mariadb,db2"
+            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,hive3,iceberg,hudi,trino,kafka,mariadb,db2"
         fi
     fi
 fi