From bf903d4bb1855fd8bcc9c0c3c6c87541948e1327 Mon Sep 17 00:00:00 2001 From: Yijia Su Date: Sat, 8 Oct 2022 12:56:06 +0800 Subject: [PATCH 01/10] fix doc --- .../Show-Statements/SHOW-ALTER-TABLE-MATERIALIZED-VIEW.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-ALTER-TABLE-MATERIALIZED-VIEW.md b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-ALTER-TABLE-MATERIALIZED-VIEW.md index 62ee04bd969e68..ba1f6590527a5f 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-ALTER-TABLE-MATERIALIZED-VIEW.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-ALTER-TABLE-MATERIALIZED-VIEW.md @@ -32,7 +32,7 @@ SHOW ALTER TABLE MATERIALIZED VIEW ### Description -该命令用于查看通过 [CREATE-MATERIALIZED-VIEW](../../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-MATERIALIZED-VIEW.md) 语句提交的创建物化视图作业的执行情况。 +该命令用于查看通过 [CREATE-MATERIALIZED-VIEW](../../sql-reference/Data-Definition-Statements/Create/CREATE-MATERIALIZED-VIEW) 语句提交的创建物化视图作业的执行情况。 > 该语句等同于 `SHOW ALTER TABLE ROLLUP`; From 527726201c002d58d55e33b3328ab4b9466022ff Mon Sep 17 00:00:00 2001 From: Yijia Su Date: Mon, 21 Nov 2022 18:04:49 +0800 Subject: [PATCH 02/10] change docs --- docker/runtime/be/Dockerfile | 20 ++++++++++---------- docker/runtime/fe/Dockerfile | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docker/runtime/be/Dockerfile b/docker/runtime/be/Dockerfile index d4b60eec65082b..75b64176ddf5c5 100644 --- a/docker/runtime/be/Dockerfile +++ b/docker/runtime/be/Dockerfile @@ -21,20 +21,20 @@ FROM openjdk:8u342-jdk # set environment variables ENV JAVA_HOME="/usr/local/openjdk-8/" \ - PATH="/opt/apache-doris/be/bin:${PATH}" + PATH="/opt/apache-doris/be/bin:${PATH}" # download the software to the mirror and replace it as needed -ADD ./resource/apache-doris-be-x.x.x-bin-x86_64.tar.gz /opt/ +ADD ./resource/apache-doris-be-"${*.*.*}"-bin-x86_64.tar.gz /opt/ # deploy software RUN apt-get update && \ - apt-get install -y default-mysql-client && \ - apt-get clean && \ - mkdir /opt/apache-doris && \ - cd /opt && \ - mv apache-doris-be-x.x.x-bin-x86_64 /opt/apache-doris/be + apt-get install -y default-mysql-client && \ + apt-get clean && \ + mkdir /opt/apache-doris && \ + cd /opt && \ + mv apache-doris-be-"${*.*.*}"-bin-x86_64 /opt/apache-doris/be -ADD resource/init_be.sh /opt/apache-doris/be/bin -RUN chmod 755 /opt/apache-doris/be/bin/init_be.sh +ADD ./resource/run_be.sh /opt/apache-doris/be/bin +RUN chmod 755 /opt/apache-doris/be/bin/run_be.sh -ENTRYPOINT ["/opt/apache-doris/be/bin/init_be.sh"] +ENTRYPOINT ["/opt/apache-doris/be/bin/run_be.sh"] diff --git a/docker/runtime/fe/Dockerfile b/docker/runtime/fe/Dockerfile index 44796e1933774b..fd24aa0019b5e5 100644 --- a/docker/runtime/fe/Dockerfile +++ b/docker/runtime/fe/Dockerfile @@ -21,20 +21,20 @@ FROM openjdk:8u342-jdk # set environment variables ENV JAVA_HOME="/usr/local/openjdk-8/" \ - PATH="/opt/apache-doris/fe/bin:${PATH}" + PATH="/opt/apache-doris/fe/bin:${PATH}" # download the software to the mirror and replace it as needed -ADD ./resource/apache-doris-fe-x.x.x-bin.tar.gz /opt/ +ADD ./resource/apache-doris-fe-"${*.*.*}"-bin.tar.gz /opt/ # deploy software RUN apt-get update && \ - apt-get install -y default-mysql-client && \ - apt-get clean && \ - mkdir /opt/apache-doris && \ - cd /opt && \ - mv apache-doris-fe-x.x.x-bin /opt/apache-doris/fe + apt-get install -y default-mysql-client && \ + apt-get clean && \ + mkdir /opt/apache-doris && \ + cd /opt && \ + mv apache-doris-fe-"${*.*.*}"-bin /opt/apache-doris/fe -ADD resource/init_fe.sh /opt/apache-doris/fe/bin -RUN chmod 755 /opt/apache-doris/fe/bin/init_fe.sh +ADD ./resource/run_fe.sh /opt/apache-doris/fe/bin +RUN chmod 755 /opt/apache-doris/fe/bin/run_fe.sh -ENTRYPOINT ["/opt/apache-doris/fe/bin/init_fe.sh"] +ENTRYPOINT ["/opt/apache-doris/fe/bin/run_fe.sh"] From 34d88d26056cba2bd2158be5924f5d3edb9236d0 Mon Sep 17 00:00:00 2001 From: Yijia Su Date: Mon, 21 Nov 2022 18:07:39 +0800 Subject: [PATCH 03/10] change docs --- docker/runtime/be/Dockerfile | 4 ++-- docker/runtime/fe/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/runtime/be/Dockerfile b/docker/runtime/be/Dockerfile index 75b64176ddf5c5..3cba0fa36768d1 100644 --- a/docker/runtime/be/Dockerfile +++ b/docker/runtime/be/Dockerfile @@ -24,7 +24,7 @@ ENV JAVA_HOME="/usr/local/openjdk-8/" \ PATH="/opt/apache-doris/be/bin:${PATH}" # download the software to the mirror and replace it as needed -ADD ./resource/apache-doris-be-"${*.*.*}"-bin-x86_64.tar.gz /opt/ +ADD ./resource/apache-doris-be-x.x.x-bin-x86_64.tar.gz /opt/ # deploy software RUN apt-get update && \ @@ -32,7 +32,7 @@ RUN apt-get update && \ apt-get clean && \ mkdir /opt/apache-doris && \ cd /opt && \ - mv apache-doris-be-"${*.*.*}"-bin-x86_64 /opt/apache-doris/be + mv apache-doris-be-x.x.x-bin-x86_64 /opt/apache-doris/be ADD ./resource/run_be.sh /opt/apache-doris/be/bin RUN chmod 755 /opt/apache-doris/be/bin/run_be.sh diff --git a/docker/runtime/fe/Dockerfile b/docker/runtime/fe/Dockerfile index fd24aa0019b5e5..ad2b73989bdbc1 100644 --- a/docker/runtime/fe/Dockerfile +++ b/docker/runtime/fe/Dockerfile @@ -24,7 +24,7 @@ ENV JAVA_HOME="/usr/local/openjdk-8/" \ PATH="/opt/apache-doris/fe/bin:${PATH}" # download the software to the mirror and replace it as needed -ADD ./resource/apache-doris-fe-"${*.*.*}"-bin.tar.gz /opt/ +ADD ./resource/apache-doris-fe-x.x.x-bin.tar.gz /opt/ # deploy software RUN apt-get update && \ @@ -32,7 +32,7 @@ RUN apt-get update && \ apt-get clean && \ mkdir /opt/apache-doris && \ cd /opt && \ - mv apache-doris-fe-"${*.*.*}"-bin /opt/apache-doris/fe + mv apache-doris-fe-x.x.x-bin /opt/apache-doris/fe ADD ./resource/run_fe.sh /opt/apache-doris/fe/bin RUN chmod 755 /opt/apache-doris/fe/bin/run_fe.sh From 2345b204f1a60134f68d653710676eabd6c14c8a Mon Sep 17 00:00:00 2001 From: Yijia Su Date: Mon, 21 Nov 2022 18:11:09 +0800 Subject: [PATCH 04/10] change docs --- docker/runtime/be/Dockerfile | 14 +++++++------- docker/runtime/fe/Dockerfile | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docker/runtime/be/Dockerfile b/docker/runtime/be/Dockerfile index 3cba0fa36768d1..dddea76d0e2782 100644 --- a/docker/runtime/be/Dockerfile +++ b/docker/runtime/be/Dockerfile @@ -21,18 +21,18 @@ FROM openjdk:8u342-jdk # set environment variables ENV JAVA_HOME="/usr/local/openjdk-8/" \ - PATH="/opt/apache-doris/be/bin:${PATH}" + PATH="/opt/apache-doris/be/bin:${PATH}" # download the software to the mirror and replace it as needed ADD ./resource/apache-doris-be-x.x.x-bin-x86_64.tar.gz /opt/ # deploy software -RUN apt-get update && \ - apt-get install -y default-mysql-client && \ - apt-get clean && \ - mkdir /opt/apache-doris && \ - cd /opt && \ - mv apache-doris-be-x.x.x-bin-x86_64 /opt/apache-doris/be +RUN apt-get update && + apt-get install -y default-mysql-client && + apt-get clean && + mkdir /opt/apache-doris && + cd /opt && + mv apache-doris-be-x.x.x-bin-x86_64 /opt/apache-doris/be ADD ./resource/run_be.sh /opt/apache-doris/be/bin RUN chmod 755 /opt/apache-doris/be/bin/run_be.sh diff --git a/docker/runtime/fe/Dockerfile b/docker/runtime/fe/Dockerfile index ad2b73989bdbc1..0ca2474d7e45ec 100644 --- a/docker/runtime/fe/Dockerfile +++ b/docker/runtime/fe/Dockerfile @@ -21,18 +21,18 @@ FROM openjdk:8u342-jdk # set environment variables ENV JAVA_HOME="/usr/local/openjdk-8/" \ - PATH="/opt/apache-doris/fe/bin:${PATH}" + PATH="/opt/apache-doris/fe/bin:${PATH}" # download the software to the mirror and replace it as needed ADD ./resource/apache-doris-fe-x.x.x-bin.tar.gz /opt/ # deploy software -RUN apt-get update && \ - apt-get install -y default-mysql-client && \ - apt-get clean && \ - mkdir /opt/apache-doris && \ - cd /opt && \ - mv apache-doris-fe-x.x.x-bin /opt/apache-doris/fe +RUN apt-get update && + apt-get install -y default-mysql-client && + apt-get clean && + mkdir /opt/apache-doris && + cd /opt && + mv apache-doris-fe-x.x.x-bin /opt/apache-doris/fe ADD ./resource/run_fe.sh /opt/apache-doris/fe/bin RUN chmod 755 /opt/apache-doris/fe/bin/run_fe.sh From 0967a619d647f66b84ba2ceef67706d9931989ad Mon Sep 17 00:00:00 2001 From: Yijia Su Date: Mon, 21 Nov 2022 18:18:42 +0800 Subject: [PATCH 05/10] change docs --- docker/runtime/be/Dockerfile | 10 +++++----- docker/runtime/fe/Dockerfile | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/runtime/be/Dockerfile b/docker/runtime/be/Dockerfile index dddea76d0e2782..19a1868b86ce84 100644 --- a/docker/runtime/be/Dockerfile +++ b/docker/runtime/be/Dockerfile @@ -27,11 +27,11 @@ ENV JAVA_HOME="/usr/local/openjdk-8/" \ ADD ./resource/apache-doris-be-x.x.x-bin-x86_64.tar.gz /opt/ # deploy software -RUN apt-get update && - apt-get install -y default-mysql-client && - apt-get clean && - mkdir /opt/apache-doris && - cd /opt && +RUN apt-get update && \ + apt-get install -y default-mysql-client && \ + apt-get clean && \ + mkdir /opt/apache-doris && \ + cd /opt && \ mv apache-doris-be-x.x.x-bin-x86_64 /opt/apache-doris/be ADD ./resource/run_be.sh /opt/apache-doris/be/bin diff --git a/docker/runtime/fe/Dockerfile b/docker/runtime/fe/Dockerfile index 0ca2474d7e45ec..caac91d72ed3cc 100644 --- a/docker/runtime/fe/Dockerfile +++ b/docker/runtime/fe/Dockerfile @@ -27,11 +27,11 @@ ENV JAVA_HOME="/usr/local/openjdk-8/" \ ADD ./resource/apache-doris-fe-x.x.x-bin.tar.gz /opt/ # deploy software -RUN apt-get update && - apt-get install -y default-mysql-client && - apt-get clean && - mkdir /opt/apache-doris && - cd /opt && +RUN apt-get update && \ + apt-get install -y default-mysql-client && \ + apt-get clean && \ + mkdir /opt/apache-doris && \ + cd /opt && \ mv apache-doris-fe-x.x.x-bin /opt/apache-doris/fe ADD ./resource/run_fe.sh /opt/apache-doris/fe/bin From b24fcafeef82ec4fd6e9d9366c7c960ae19c9787 Mon Sep 17 00:00:00 2001 From: Yijia Su Date: Tue, 22 Nov 2022 17:33:38 +0800 Subject: [PATCH 06/10] change sh_checker_exclude --- docker/runtime/be/Dockerfile | 6 +++--- docker/runtime/fe/Dockerfile | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/runtime/be/Dockerfile b/docker/runtime/be/Dockerfile index 19a1868b86ce84..d4b60eec65082b 100644 --- a/docker/runtime/be/Dockerfile +++ b/docker/runtime/be/Dockerfile @@ -34,7 +34,7 @@ RUN apt-get update && \ cd /opt && \ mv apache-doris-be-x.x.x-bin-x86_64 /opt/apache-doris/be -ADD ./resource/run_be.sh /opt/apache-doris/be/bin -RUN chmod 755 /opt/apache-doris/be/bin/run_be.sh +ADD resource/init_be.sh /opt/apache-doris/be/bin +RUN chmod 755 /opt/apache-doris/be/bin/init_be.sh -ENTRYPOINT ["/opt/apache-doris/be/bin/run_be.sh"] +ENTRYPOINT ["/opt/apache-doris/be/bin/init_be.sh"] diff --git a/docker/runtime/fe/Dockerfile b/docker/runtime/fe/Dockerfile index caac91d72ed3cc..44796e1933774b 100644 --- a/docker/runtime/fe/Dockerfile +++ b/docker/runtime/fe/Dockerfile @@ -34,7 +34,7 @@ RUN apt-get update && \ cd /opt && \ mv apache-doris-fe-x.x.x-bin /opt/apache-doris/fe -ADD ./resource/run_fe.sh /opt/apache-doris/fe/bin -RUN chmod 755 /opt/apache-doris/fe/bin/run_fe.sh +ADD resource/init_fe.sh /opt/apache-doris/fe/bin +RUN chmod 755 /opt/apache-doris/fe/bin/init_fe.sh -ENTRYPOINT ["/opt/apache-doris/fe/bin/run_fe.sh"] +ENTRYPOINT ["/opt/apache-doris/fe/bin/init_fe.sh"] From efa085baa8a09e985df6a37785c70aa38b13b963 Mon Sep 17 00:00:00 2001 From: Yijia Su Date: Sun, 27 Nov 2022 17:54:13 +0800 Subject: [PATCH 07/10] add shell --- docker/runtime/be/resource/init_be.sh | 13 +- docker/runtime/broker/Dockerfile | 21 ++++ docker/runtime/broker/resource/init_broker.sh | 114 ++++++++++++++++++ docker/runtime/fe/resource/init_fe.sh | 3 - 4 files changed, 141 insertions(+), 10 deletions(-) create mode 100644 docker/runtime/broker/Dockerfile create mode 100644 docker/runtime/broker/resource/init_broker.sh diff --git a/docker/runtime/be/resource/init_be.sh b/docker/runtime/be/resource/init_be.sh index 2b329c80cae899..e4de6b7c7e1f2f 100644 --- a/docker/runtime/be/resource/init_be.sh +++ b/docker/runtime/be/resource/init_be.sh @@ -57,11 +57,9 @@ feServerArray=(${FE_SERVERS}) for i in "${!feServerArray[@]}"; do val=${feServerArray[i]} val=${val// /} - #echo "DEBUG >>>>>>>>> $i => 【$val】" tmpFeId=$(echo "${val}" | awk -F ':' '{ sub(/fe/, ""); sub(/ /, ""); print$1}') tmpFeIp=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$2}') tmpFeEditLogPort=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$3}') - #echo "DEBUG >>>>>>>>> tmpFeId = $tmpFeIdi, tmpFeIp = $tmpFeIp, tmpFeEditLogPort = $tmpFeEditLogPort" feIpArray[tmpFeId]=${tmpFeIp} feEditLogPortArray[tmpFeId]=${tmpFeEditLogPort} done @@ -79,20 +77,18 @@ echo "DEBUG >>>>>> Append the configuration [priority_networks = ${priority_netw echo "priority_networks = ${priority_networks}" >>/opt/apache-doris/be/conf/be.conf registerMySQL="mysql -uroot -P9030 -h${feIpArray[1]} -e \"alter system add backend '${be_ip}:${be_heartbeat_port}'\"" -registerShell="/opt/apache-doris/be/bin/start_be.sh" - echo "DEBUG >>>>>> registerMySQL = ${registerMySQL}" + +registerShell="/opt/apache-doris/be/bin/start_be.sh &" echo "DEBUG >>>>>> registerShell = ${registerShell}" for ((i = 0; i <= 20; i++)); do - echo "DEBUG >>>>>> The " "${i}" "time to register BE node, be_join_status=${be_join_status}" - ## check be register status echo "mysql -uroot -P9030 -h${feIpArray[1]} -e \"show backends\" | grep \" ${be_ip} \" | grep \" ${be_heartbeat_port} \"" mysql -uroot -P9030 -h"${feIpArray[1]}" -e "show backends" | grep "[[:space:]]${be_ip}[[:space:]]" | grep "[[:space:]]${be_heartbeat_port}[[:space:]]" be_join_status=$? - echo "DEBUG >>>>>> be_join_status = ${be_join_status}" + echo "DEBUG >>>>>> The " "${i}" "time to register BE node, be_join_status=${be_join_status}" if [[ "${be_join_status}" == 0 ]]; then ## be registe successfully echo "DEBUG >>>>>> run command ${registerShell}" @@ -101,6 +97,9 @@ for ((i = 0; i <= 20; i++)); do ## be doesn't registe echo "DEBUG >>>>>> run commnad ${registerMySQL}" eval "${registerMySQL}" + if [[ "${i}" == 20 ]]; then + echo "DEBUG >>>>>> BE Start Or Register FAILED!" + fi sleep 5 fi done diff --git a/docker/runtime/broker/Dockerfile b/docker/runtime/broker/Dockerfile new file mode 100644 index 00000000000000..ee8ba0dc638a46 --- /dev/null +++ b/docker/runtime/broker/Dockerfile @@ -0,0 +1,21 @@ +# 选择基础镜像 +FROM openjdk:8u342-jdk + +# 设置环境变量 +ENV JAVA_HOME="/usr/local/openjdk-8/" \ + PATH="/opt/apache-doris/broker/bin:$PATH" + +# 下载软件至镜像内，此处 broker 目录被同步压缩至 FE 的二进制包，需要自行解压重新打包，可根据需要替换 +ADD ./resource/apache_hdfs_broker.tar.gz /opt/ + +RUN apt-get update && \ + apt-get install -y default-mysql-client && \ + apt-get clean && \ + mkdir /opt/apache-doris && \ + cd /opt && \ + mv apache_hdfs_broker /opt/apache-doris/broker + +ADD ./resource/init_be.sh /opt/apache-doris/broker/bin +RUN chmod 755 /opt/apache-doris/broker/bin/init_broker.sh + +ENTRYPOINT ["/opt/apache-doris/broker/bin/init_broker.sh"] diff --git a/docker/runtime/broker/resource/init_broker.sh b/docker/runtime/broker/resource/init_broker.sh new file mode 100644 index 00000000000000..5123cc9116281b --- /dev/null +++ b/docker/runtime/broker/resource/init_broker.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FE_SERVERS="" +BROKER_ADDR="" + +ARGS=$(getopt -o -h: --long fe_servers:,broker_addr: -n "$0" -- "$@") + +eval set -- "${ARGS}" + +while [[ -n "$1" ]]; do + case "$1" in + --fe_servers) + FE_SERVERS=$2 + shift + ;; + --broker_addr) + BROKER_ADDR=$2 + shift + ;; + --) ;; + + *) + echo "Error option $1" + break + ;; + esac + shift +done + +#echo FE_SERVERS = $FE_SERVERS +echo "DEBUG >>>>>> FE_SERVERS=[${FE_SERVERS}]" +echo "DEBUG >>>>>> BROKER_ADDR=[${BROKER_ADDR}]" + +feIpArray=() +feEditLogPortArray=() + +IFS="," +# shellcheck disable=SC2206 +feServerArray=(${FE_SERVERS}) + +for i in "${!feServerArray[@]}"; do + val=${feServerArray[i]} + val=${val// /} + tmpFeId=$(echo "${val}" | awk -F ':' '{ sub(/fe/, ""); sub(/ /, ""); print$1}') + tmpFeIp=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$2}') + tmpFeEditLogPort=$(echo "${val}" | awk -F ':' '{ sub(/ /, ""); print$3}') + feIpArray[tmpFeId]=${tmpFeIp} + feEditLogPortArray[tmpFeId]=${tmpFeEditLogPort} +done + +broker_name=$(echo "${BROKER_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$1}') +broker_ip=$(echo "${BROKER_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$2}') +broker_ipc_port=$(echo "${BROKER_ADDR}" | awk -F ':' '{ sub(/ /, ""); print$3}') + +echo "DEBUG >>>>>> feIpArray = ${feIpArray[*]}" +echo "DEBUG >>>>>> feEditLogPortArray = ${feEditLogPortArray[*]}" +echo "DEBUG >>>>>> masterFe = ${feIpArray[1]}:${feEditLogPortArray[1]}" +echo "DEBUG >>>>>> broker_addr = ${broker_ip}:${broker_ipc_port}" + +dropMySQL="/usr/bin/mysql -uroot -P9030 -h${feIpArray[1]} -e \"alter system drop broker ${broker_name} '${broker_ip}:${broker_ipc_port}'\"" +echo "DEBUG >>>>>> dropMySQL = ${dropMySQL}" +eval "${dropMySQL}" && echo "DEBUG >>>>>> drop history registe SUCCESS!" || echo "DEBUG >>>>>> drop history registe FAILED!" + +# register broker to FE through mysql +registerMySQL="/usr/bin/mysql -uroot -P9030 -h${feIpArray[1]} -e \"alter system add broker ${broker_name} '${broker_ip}:${broker_ipc_port}'\"" +echo "DEBUG >>>>>> registerMySQL = ${registerMySQL}" +eval "${registerMySQL}" && echo "DEBUG >>>>>> mysql register is SUCCESS!" || echo "DEBUG >>>>>> mysql register is FAILED!" + +# start broker +registerShell="/opt/apache-doris/broker/bin/start_broker.sh &" +echo "DEBUG >>>>>> registerShell = ${registerShell}" +eval "${registerShell}" && echo "DEBUG >>>>>> start_broker SUCCESS！" || echo "DEBUG >>>>>> start_broker FAILED!" + +for ((i = 0; i <= 20; i++)); do + sleep 10 + ## check broker register status + echo "DEBUG >>>>>> run commnad mysql -uroot -P9030 -h${feIpArray[1]} -e \"show proc '/brokers'\" | grep \" ${broker_ip} \" | grep \" ${broker_ipc_port} \" | grep \" true \"" + mysql -uroot -P9030 -h"${feIpArray[1]}" -e "show proc '/brokers'" | grep "[[:space:]]${broker_ip}[[:space:]]" | grep "[[:space:]]${broker_ipc_port}[[:space:]]" | grep "[[:space:]]true[[:space:]]" + broker_join_status=$? + echo "DEBUG >>>>>> The " "${i}" "time to register Broker node, broker_join_status=${broker_join_status}" + if [[ "${broker_join_status}" == 0 ]]; then + ## broker registe successfully + echo "BROKER START SUCCESS!!!" + "${is_success}"=1 + break + else + ## broker doesn't registe + echo "DEBUG >>>>>> run commnad ${registerMySQL}" + eval "${registerMySQL}" + if [[ "${i}" == 20 ]]; then + echo "DEBUG >>>>>> Broker Start Or Register FAILED!" + fi + sleep 3 + fi +done + + + diff --git a/docker/runtime/fe/resource/init_fe.sh b/docker/runtime/fe/resource/init_fe.sh index 1753d391e753bc..64e97a08dc46f7 100644 --- a/docker/runtime/fe/resource/init_fe.sh +++ b/docker/runtime/fe/resource/init_fe.sh @@ -116,14 +116,11 @@ if [[ "${FE_ID}" != 1 ]]; then echo "The resutl of run registerShell command, [ res = $? ]" fi sleep 5 - done else - registerShell="/opt/apache-doris/fe/bin/start_fe.sh" eval "${registerShell}" echo "DEBUG >>>>>> FE is master, fe_id = ${FE_ID}" echo "DEBUG >>>>>> registerShell = ${registerShell}" - fi From 44aab66a0ae921dd64e6679fda5d71736e6d2ec4 Mon Sep 17 00:00:00 2001 From: Yijia Su Date: Sun, 27 Nov 2022 18:41:00 +0800 Subject: [PATCH 08/10] add broker Dockerfile and init_broker.sh --- docker/runtime/broker/Dockerfile | 26 ++++++++++++++++--- docker/runtime/broker/resource/init_broker.sh | 1 - 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docker/runtime/broker/Dockerfile b/docker/runtime/broker/Dockerfile index ee8ba0dc638a46..e20d189f71019e 100644 --- a/docker/runtime/broker/Dockerfile +++ b/docker/runtime/broker/Dockerfile @@ -1,13 +1,33 @@ -# 选择基础镜像 +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# choose a base image FROM openjdk:8u342-jdk -# 设置环境变量 +# set environment variables ENV JAVA_HOME="/usr/local/openjdk-8/" \ PATH="/opt/apache-doris/broker/bin:$PATH" -# 下载软件至镜像内，此处 broker 目录被同步压缩至 FE 的二进制包，需要自行解压重新打包，可根据需要替换 +# Download the software to the mirror, where the broker directory is synchronously compressed to the binary package of FE, +# which needs to be decompressed and repackaged by itself, and can be replaced as needed ADD ./resource/apache_hdfs_broker.tar.gz /opt/ +# deploy software RUN apt-get update && \ apt-get install -y default-mysql-client && \ apt-get clean && \ diff --git a/docker/runtime/broker/resource/init_broker.sh b/docker/runtime/broker/resource/init_broker.sh index 5123cc9116281b..8cf069cd81985e 100644 --- a/docker/runtime/broker/resource/init_broker.sh +++ b/docker/runtime/broker/resource/init_broker.sh @@ -97,7 +97,6 @@ for ((i = 0; i <= 20; i++)); do if [[ "${broker_join_status}" == 0 ]]; then ## broker registe successfully echo "BROKER START SUCCESS!!!" - "${is_success}"=1 break else ## broker doesn't registe From 1b4b90e0d8992bdcff177b8c6417a7755d0ff9ae Mon Sep 17 00:00:00 2001 From: Yijia Su Date: Sun, 27 Nov 2022 21:29:36 +0800 Subject: [PATCH 09/10] add docker docs --- docker/runtime/broker/resource/init_broker.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/runtime/broker/resource/init_broker.sh b/docker/runtime/broker/resource/init_broker.sh index 8cf069cd81985e..1d7c4fba78092f 100644 --- a/docker/runtime/broker/resource/init_broker.sh +++ b/docker/runtime/broker/resource/init_broker.sh @@ -108,6 +108,3 @@ for ((i = 0; i <= 20; i++)); do sleep 3 fi done - - - From 6e5856bd6107faf4235d085063684b698a8a3836 Mon Sep 17 00:00:00 2001 From: Yijia Su Date: Tue, 29 Nov 2022 18:52:31 +0800 Subject: [PATCH 10/10] Adjust the field naming rules when creating tables --- .../org/apache/doris/common/FeNameFormat.java | 2 +- regression-test/suites/ddl_p0/test_fnf.groovy | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/ddl_p0/test_fnf.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java index 65ac16a1b93543..4048d9f6479547 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java @@ -28,7 +28,7 @@ public class FeNameFormat { private static final String LABEL_REGEX = "^[-_A-Za-z0-9]{1,128}$"; private static final String COMMON_NAME_REGEX = "^[a-zA-Z][a-zA-Z0-9_]{0,63}$"; private static final String COMMON_TABLE_NAME_REGEX = "^[a-zA-Z][a-zA-Z0-9_]*$"; - private static final String COLUMN_NAME_REGEX = "^[_a-zA-Z@][a-zA-Z0-9_]{0,63}$"; + private static final String COLUMN_NAME_REGEX = "^[_a-zA-Z@0-9][a-zA-Z0-9_]{0,255}$"; public static final String FORBIDDEN_PARTITION_NAME = "placeholder_"; diff --git a/regression-test/suites/ddl_p0/test_fnf.groovy b/regression-test/suites/ddl_p0/test_fnf.groovy new file mode 100644 index 00000000000000..e2c3697716bb51 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_fnf.groovy @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_fnf") { + try { + sql """ + CREATE TABLE IF NOT EXISTS `test_fnf` ( + `test_varchar` varchar(1) NULL, + `0test_varchar` varchar(1) NULL, + `@test_varchar` varchar(1) NULL, + `_test_varchar` varchar(1) NULL, + `_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_var` varchar(1) NULL, + `@test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_var` varchar(1) NULL, + `0test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_var` varchar(1) NULL + ) ENGINE=OLAP + UNIQUE KEY(`test_varchar`) + DISTRIBUTED BY HASH(`test_varchar`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ) + """ + } catch (java.sql.SQLException t){ + assertTrue(false) + } + + try { + sql """ + CREATE TABLE IF NOT EXISTS `test_fnf_error_1` ( + `test@_varchar` varchar(1) NULL + ) ENGINE=OLAP + UNIQUE KEY(`test_varchar`) + DISTRIBUTED BY HASH(`test_varchar`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ) + """ + } catch (java.sql.SQLException t){ + assertTrue(true) + } + + try { + sql """ + CREATE TABLE IF NOT EXISTS `test_fnf_error_2` ( + `test_varchar` varchar(1) NULL, + `_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varchar_test_varc` varchar(1) NULL + ) ENGINE=OLAP + UNIQUE KEY(`test_varchar`) + DISTRIBUTED BY HASH(`test_varchar`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ) + """ + } catch (java.sql.SQLException t){ + assertTrue(true) + } finally { + sql """ DROP TABLE IF EXISTS test_fnf """ + + sql """ DROP TABLE IF EXISTS test_fnf_error_1 """ + + sql """ DROP TABLE IF EXISTS test_fnf_error_2 """ + } + +} +