From 295fb03a5e1fe4503f9b9b115a7ea62587b8e0c1 Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Mon, 2 Sep 2024 13:11:58 +0800 Subject: [PATCH 1/5] [chore](conf) Specify UTF8 as the default charset. (#39521) According the [JEP 400](https://openjdk.org/jeps/400), UTF-8 was be used as the default charset of the JavaSE 18. But when you use the version below 18, the default charset depends on your locale. Usually, it can work well in many sence, but Apache Doris only supports UTF-8 as its charset, which may occur some decoding incorrectly. So it is necessary to set UTF8 as the default JDK charset. (cherry picked from commit 48991df34f0cf0f6889cf8887ee1a716afdeb70c) --- conf/be.conf | 10 +++++----- conf/fe.conf | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conf/be.conf b/conf/be.conf index c40af9b9e4b5e8..25fc2af9b8a5d6 100644 --- a/conf/be.conf +++ b/conf/be.conf @@ -19,10 +19,10 @@ CUR_DATE=`date +%Y%m%d-%H%M%S` PPROF_TMPDIR="$DORIS_HOME/log/" -JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.java.command=DorisBE -XX:-CriticalJNINatives" +JAVA_OPTS="-Dfile.encoding=UTF-8 -Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.java.command=DorisBE -XX:-CriticalJNINatives" # For jdk 9+, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.java.command=DorisBE -XX:-CriticalJNINatives" +JAVA_OPTS_FOR_JDK_9="-Dfile.encoding=UTF-8 -Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.java.command=DorisBE -XX:-CriticalJNINatives" # since 1.2, the JAVA_HOME need to be set to run BE process. # JAVA_HOME=/path/to/jdk/ @@ -51,7 +51,7 @@ ssl_private_key_path = "$DORIS_HOME/conf/key.pem" # enable auth check enable_auth = false -# Choose one if there are more than one ip except loopback address. +# Choose one if there are more than one ip except loopback address. # Note that there should at most one ip match this list. # If no ip match this rule, will choose one randomly. # use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1 @@ -82,7 +82,7 @@ enable_auth = false # sys_log_roll_num = 10 # sys_log_verbose_modules = * # log_buffer_level = -1 -# palo_cgroups +# palo_cgroups # aws sdk log level # Off = 0, @@ -95,4 +95,4 @@ enable_auth = false # Default to turn off aws sdk log, because aws sdk errors that need to be cared will be output through Doris logs aws_log_level=0 ## If you are not running in aws cloud, you can disable EC2 metadata -AWS_EC2_METADATA_DISABLED=true \ No newline at end of file +AWS_EC2_METADATA_DISABLED=true diff --git a/conf/fe.conf b/conf/fe.conf index 7272e1298a3d24..b7d6ba5db083e3 100644 --- a/conf/fe.conf +++ b/conf/fe.conf @@ -27,11 +27,11 @@ CUR_DATE=`date +%Y%m%d-%H%M%S` LOG_DIR = ${DORIS_HOME}/log #JAVA_OPTS="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$CUR_DATE" -JAVA_OPTS="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UnlockExperimentalVMOptions -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:$LOG_DIR/log/fe.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Dlog4j2.formatMsgNoLookups=true" +JAVA_OPTS="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UnlockExperimentalVMOptions -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:$LOG_DIR/log/fe.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Dlog4j2.formatMsgNoLookups=true" # For jdk 9+, this JAVA_OPTS will be used as default JVM options #JAVA_OPTS_FOR_JDK_9="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time" -JAVA_OPTS_FOR_JDK_9="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:$LOG_DIR/fe.gc.log.$CUR_DATE:time -Dlog4j2.formatMsgNoLookups=true" +JAVA_OPTS_FOR_JDK_9="-Dfile.encoding=UTF-8 -Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:$LOG_DIR/fe.gc.log.$CUR_DATE:time -Dlog4j2.formatMsgNoLookups=true" ## ## the lowercase properties are read by main program. From 8e3eb7e625c084baf003f75088f5fbb739b9e064 Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Wed, 11 Sep 2024 16:12:17 +0800 Subject: [PATCH 2/5] Update be.conf --- conf/be.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/be.conf b/conf/be.conf index 25fc2af9b8a5d6..01657e313d709c 100644 --- a/conf/be.conf +++ b/conf/be.conf @@ -51,7 +51,7 @@ ssl_private_key_path = "$DORIS_HOME/conf/key.pem" # enable auth check enable_auth = false -# Choose one if there are more than one ip except loopback address. +# Choose one if there are more than one ip except loopback address. # Note that there should at most one ip match this list. # If no ip match this rule, will choose one randomly. # use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1 @@ -82,7 +82,7 @@ enable_auth = false # sys_log_roll_num = 10 # sys_log_verbose_modules = * # log_buffer_level = -1 -# palo_cgroups +# palo_cgroups # aws sdk log level # Off = 0, From 98a8089016f21054f01f9e281f5528a15cffaa1d Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Wed, 11 Sep 2024 16:12:44 +0800 Subject: [PATCH 3/5] Update be.conf --- conf/be.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/be.conf b/conf/be.conf index 01657e313d709c..9fa75b17b3030c 100644 --- a/conf/be.conf +++ b/conf/be.conf @@ -94,5 +94,7 @@ enable_auth = false # Trace = 6 # Default to turn off aws sdk log, because aws sdk errors that need to be cared will be output through Doris logs aws_log_level=0 + + ## If you are not running in aws cloud, you can disable EC2 metadata AWS_EC2_METADATA_DISABLED=true From ea377df70f66bcd8731453382e5bedd71976aac5 Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Wed, 11 Sep 2024 16:13:23 +0800 Subject: [PATCH 4/5] Update be.conf --- conf/be.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/be.conf b/conf/be.conf index 9fa75b17b3030c..605d62d7cc25c4 100644 --- a/conf/be.conf +++ b/conf/be.conf @@ -94,7 +94,6 @@ enable_auth = false # Trace = 6 # Default to turn off aws sdk log, because aws sdk errors that need to be cared will be output through Doris logs aws_log_level=0 - - ## If you are not running in aws cloud, you can disable EC2 metadata AWS_EC2_METADATA_DISABLED=true + From cdbc052b870c787a4a5e82f3b07798aa77995512 Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Wed, 11 Sep 2024 16:13:50 +0800 Subject: [PATCH 5/5] Update be.conf --- conf/be.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/be.conf b/conf/be.conf index 605d62d7cc25c4..01657e313d709c 100644 --- a/conf/be.conf +++ b/conf/be.conf @@ -96,4 +96,3 @@ enable_auth = false aws_log_level=0 ## If you are not running in aws cloud, you can disable EC2 metadata AWS_EC2_METADATA_DISABLED=true -