Pick "[chore](ColdHeatSeparation) Reduce the frequency of log following read failure #32706"

.asf.yaml

@@ -52,12 +52,12 @@ github:
           - Clang Formatter
           - CheckStyle
           - P0 Regression (Doris Regression)
+          - P1 Regression (Doris Regression)
           - FE UT (Doris FE UT)
           - BE UT (Doris BE UT)
           - Build Broker
           - Build Documents
           - BE UT (Clang)
-          - ShellCheck
           - clickbench (benchmark)
           - BE UT (macOS)
           - Build Third Party Libraries (Linux)
@@ -78,6 +78,27 @@ github:
       required_pull_request_reviews:
         dismiss_stale_reviews: true
         required_approving_review_count: 1
+
+    branch-2.0:
+      required_status_checks:
+      # if strict is true, means "Require branches to be up to date before merging".
+        strict: false
+        contexts:
+          - License Check
+          - Clang Formatter
+          - CheckStyle
+          - Build Documents
+          - ShellCheck
+          - Need_2_Approval
+          - COMPILE (DORIS_COMPILE)
+          - P0 Regression (Doris Regression)
+          - FE UT (Doris FE UT)
+          - BE UT (Doris BE UT)
+          - clickbench-new (clickbench)
+        required_pull_request_reviews:
+        dismiss_stale_reviews: true
+        required_approving_review_count: 1
+
   collaborators:
     - LemonLiTree
     - zy-kkk

.dockerignore

@@ -0,0 +1,7 @@
+be*
+fs_brokers*
+thirdparty*
+fe*
+.git*
+fs_brokers*
+

.github/workflows/auto_trigger_teamcity.yml

@@ -95,6 +95,7 @@ jobs:
 
           for pipeline in ${trigger_pipelines}
           do
+            same_build_sign="false"
             echo "-----------------------------------------------------------------"
             running_builds_command="curl -s -X GET ${teamcity_url}/app/rest/builds\?locator\=buildType\:${pipeline}\,branch:pull/${pull_request_num}\,running:true"
             echo "${running_builds_command}" 

.github/workflows/be-ut-mac.yml

@@ -29,7 +29,7 @@ concurrency:
 jobs:
   run-ut:
     name: BE UT (macOS)
-    runs-on: macos-12
+    runs-on: macos-13
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -81,7 +81,7 @@ jobs:
             'node'
             'llvm@16'
           )
-          brew install "${cellars[@]}"
+          brew install "${cellars[@]}" || true
 
           pushd thirdparty
           branch="${{ github.base_ref }}"

.github/workflows/build-extension.yml

@@ -100,6 +100,7 @@ jobs:
         run: |
           git clone https://github.com/apache/doris-website.git website
           cd website
+          echo "[\"current\"]" > versions.json 
           mkdir -p docs
           cp -R ../docs/en/docs/* docs/
           cp -R ../docs/sidebars.json sidebars.json

.github/workflows/clang-format.yml

@@ -30,7 +30,21 @@ jobs:
         uses: actions/checkout@v3
         with:
           persist-credentials: false
-          submodules: recursive
+
+      - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
+        if: ${{ github.event_name == 'pull_request_target' }}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Checkout paths-filter
+        run: |
+          rm -rf ./.github/actions/paths-filter
+          git clone https://github.com/dorny/paths-filter .github/actions/paths-filter
+
+          pushd .github/actions/paths-filter &>/dev/null
+          git checkout 4512585405083f25c027a35db413c2b3b9006d50
+          popd &>/dev/null
 
       - name: Paths filter
         uses: ./.github/actions/paths-filter
@@ -41,6 +55,15 @@ jobs:
               - 'be/src/**'
               - 'be/test/**'
 
+      - name: Checkout clang-format-lint-action
+        run: |
+          rm -rf ./.github/actions/clang-format-lint-action
+          git clone https://github.com/DoozyX/clang-format-lint-action .github/actions/clang-format-lint-action
+
+          pushd .github/actions/clang-format-lint-action &>/dev/null
+          git checkout 6adbe14579e5b8e19eb3e31e5ff2479f3bd302c7
+          popd &>/dev/null
+
       - name: "Format it!"
         if: ${{ steps.filter.outputs.be_changes == 'true' }}
         uses: ./.github/actions/clang-format-lint-action

.github/workflows/code-checks.yml

@@ -27,40 +27,52 @@ jobs:
       - name: Checkout ${{ github.ref }} ( ${{ github.sha }} )
         if: ${{ github.event_name != 'pull_request_target' }}
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
       - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
         if: ${{ github.event_name == 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
-      - name: Patch
+      - name: Checkout action-sh-checker
         run: |
-          pushd .github/actions/action-sh-checker >/dev/null
-          git apply ../patches/action-sh-checker.patch
-          popd >/dev/null
+          rm -rf ./.github/actions/action-sh-checker
+          git clone https://github.com/luizm/action-sh-checker .github/actions/action-sh-checker
+
+          pushd .github/actions/action-sh-checker &>/dev/null
+          git checkout 76ab0b22e1f194e4a582edc7969df6485c4e9246
+          sed -i 's/\[ "$GITHUB_EVENT_NAME" == "pull_request" \]/\[\[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "pull_request_target" \]\]/' entrypoint.sh
+          popd &>/dev/null
 
       - name: Run ShellCheck
         uses: ./.github/actions/action-sh-checker
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           sh_checker_comment: true
-          sh_checker_exclude: .git .github ^docker ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$ ^be/src/apache-orc ^be/src/clucene
+          sh_checker_exclude: .git .github ^docker ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$ ^be/src/apache-orc ^be/src/clucene ^pytest
 
-  clang-tidy:
-    name: "Clang Tidy"
+  preparation:
+    name: "Clang Tidy Preparation"
     if: ${{ github.event_name == 'pull_request_target' }}
     runs-on: ubuntu-22.04
+    permissions: read-all
+    outputs:
+      should_check: ${{ steps.generate.outputs.should_check }}
     steps:
       - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
+
+      - name: Checkout paths-filter
+        run: |
+          rm -rf ./.github/actions/paths-filter
+          git clone https://github.com/dorny/paths-filter .github/actions/paths-filter
+
+          pushd .github/actions/paths-filter &>/dev/null
+          git checkout 4512585405083f25c027a35db413c2b3b9006d50
+          popd &>/dev/null
 
       - name: Paths Filter
         uses: ./.github/actions/paths-filter
@@ -73,28 +85,64 @@ jobs:
               - 'gensrc/thrift/**'
 
       - name: Generate compile_commands.json
-        if: ${{ steps.filter.outputs.be_changes == 'true' }}
+        id: generate
         run: |
-          export DEFAULT_DIR='/opt/doris'
+          if [[ "${{ steps.filter.outputs.be_changes }}" == 'true' ]]; then
+            export DEFAULT_DIR='/opt/doris'
 
-          mkdir "${DEFAULT_DIR}"
-          wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.18/ldb_toolchain_gen.sh \
-            -q -O /tmp/ldb_toolchain_gen.sh
-          bash /tmp/ldb_toolchain_gen.sh "${DEFAULT_DIR}/ldb-toolchain"
+            mkdir "${DEFAULT_DIR}"
+            wget https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.18/ldb_toolchain_gen.sh \
+              -q -O /tmp/ldb_toolchain_gen.sh
+            bash /tmp/ldb_toolchain_gen.sh "${DEFAULT_DIR}/ldb-toolchain"
 
-          sudo DEBIAN_FRONTEND=noninteractive apt install --yes tzdata byacc
+            sudo DEBIAN_FRONTEND=noninteractive apt install --yes tzdata byacc
 
-          pushd thirdparty
-          curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-prebuilt-linux-x86_64.tar.xz \
-            -o doris-thirdparty-prebuilt-linux-x86_64.tar.xz
-          tar -xvf doris-thirdparty-prebuilt-linux-x86_64.tar.xz
-          popd
+            pushd thirdparty
+            curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-prebuilt-linux-x86_64.tar.xz \
+              -o doris-thirdparty-prebuilt-linux-x86_64.tar.xz
+            tar -xvf doris-thirdparty-prebuilt-linux-x86_64.tar.xz
+            popd
 
-          export PATH="${DEFAULT_DIR}/ldb-toolchain/bin/:$(pwd)/thirdparty/installed/bin/:${PATH}"
-          DISABLE_JAVA_UDF=ON DORIS_TOOLCHAIN=clang OUTPUT_BE_BINARY=0 ./build.sh --be
+            export PATH="${DEFAULT_DIR}/ldb-toolchain/bin/:$(pwd)/thirdparty/installed/bin/:${PATH}"
+            DISABLE_JAVA_UDF=ON DORIS_TOOLCHAIN=clang ENABLE_PCH=OFF OUTPUT_BE_BINARY=0 ./build.sh --be
+          fi
 
-      - name: Run clang-tidy review
+          echo "should_check=${{ steps.filter.outputs.be_changes }}" >>${GITHUB_OUTPUT}
+
+      - name: Upload
+        uses: actions/upload-artifact@v3
         if: ${{ steps.filter.outputs.be_changes == 'true' }}
+        with:
+          name: compile_commands
+          path: ./be/build_Release/compile_commands.json
+
+  clang-tidy:
+    name: "Clang Tidy"
+    needs: preparation
+    if: ${{ needs.preparation.outputs.should_check == 'true' }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Download
+        uses: actions/download-artifact@v3
+        with:
+          name: compile_commands
+          path: ./be/build_Release
+
+      - name: Checkout clang-tidy review
+        run: |
+          rm -rf ./.github/actions/clang-tidy-review
+          git clone https://github.com/ZedThree/clang-tidy-review .github/actions/clang-tidy-review
+
+          pushd .github/actions/clang-tidy-review &>/dev/null
+          git checkout 2c55ef8cfc9acb3715d433e58aea086dcec9b206
+          popd &>/dev/null
+
+      - name: Run clang-tidy review
         uses: ./.github/actions/clang-tidy-review
         id: review
         with:

.github/workflows/sonarcloud.yml

@@ -62,4 +62,4 @@ jobs:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
           cd fe
-          mvn --batch-mode verify sonar:sonar -DskipTests -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_incubator-doris -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
+          mvn --batch-mode verify sonar:sonar -DskipTests -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_incubator-doris -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Xmx4g -XX:MaxPermSize=512m

.gitignore

@@ -1,4 +1,5 @@
 # ignore file with wildcard anywhere
+*.swo
 *.swp
 *.pyc
 *.class
@@ -13,13 +14,14 @@ core.*
 .DS_Store
 .classpath
 nohup.out
-custom_env.sh
-custom_env_mac.sh
+/custom_env.sh
+/custom_env_mac.sh
 derby.log
 dependency-reduced-pom.xml
 yarn.lock
 ut_dir
 package-lock.json
+__pycache__
 
 # ignore all sub `/bin` and `/build` directory
 **/build/
@@ -42,6 +44,7 @@ docs/.temp
 
 # output, thirdparty, extension
 output/
+output.bak/
 rpc_data/
 metastore_db/
 thirdparty/src*
@@ -72,6 +75,7 @@ samples/doris-demo/remote-udf-cpp-demo/function_server_demo
 
 # FE
 fe/mocked/
+fe/gen/
 fe/fe-core/gen/
 fe/fe-core/src/main/resources/static/
 fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.tokens
@@ -90,6 +94,7 @@ be/tags
 be/test/olap/test_data/tablet_meta_test.hdr
 be/.devcontainer/
 be/src/apache-orc/
+zoneinfo/
 
 ## tools
 tools/ssb-tools/ssb-data/

.gitmodules

@@ -4,9 +4,6 @@
 [submodule ".github/actions/get-workflow-origin"]
 	path = .github/actions/get-workflow-origin
 	url = https://github.com/potiuk/get-workflow-origin.git
-[submodule ".github/actions/clang-format-lint-action"]
-	path = .github/actions/clang-format-lint-action
-	url = https://github.com/DoozyX/clang-format-lint-action.git
 [submodule ".github/actions/setup-maven"]
 	path = .github/actions/setup-maven
 	url = https://github.com/stCarolas/setup-maven.git
@@ -19,17 +16,11 @@
 [submodule ".github/actions/ccache-action"]
 	path = .github/actions/ccache-action
 	url = https://github.com/hendrikmuhs/ccache-action
-[submodule ".github/actions/action-sh-checker"]
-	path = .github/actions/action-sh-checker
-	url = https://github.com/luizm/action-sh-checker
-[submodule ".github/actions/clang-tidy-review"]
-	path = .github/actions/clang-tidy-review
-	url = https://github.com/ZedThree/clang-tidy-review.git
 [submodule "be/src/apache-orc"]
 	path = be/src/apache-orc
 	url = https://github.com/apache/doris-thirdparty.git
 	branch = orc
 [submodule "be/src/clucene"]
 	path = be/src/clucene
 	url = https://github.com/apache/doris-thirdparty.git
-	branch = clucene
+	branch = clucene-2.0

.licenserc.yaml

@@ -9,7 +9,7 @@ header:
     - "**/*.md"
     - "LICENSE"
     - "NOTICE"
-    - "DISCLAIMER"
+    - "doap_Doris.rdf"
     - ".clang-format"
     - ".clang-format-ignore"
     - ".clang-tidy"
@@ -18,6 +18,8 @@ header:
     - ".gitmodules"
     - ".licenserc.yaml"
     - ".rat-excludes"
+    - ".github/**"
+    - ".dockerignore"
     - "be/src/common/status.cpp"
     - "be/src/common/status.h"
     - "be/src/env/env.h"
@@ -86,7 +88,11 @@ header:
     - "conf/mysql_ssl_default_certificate/client_certificate/client-cert.pem"
     - "conf/mysql_ssl_default_certificate/client_certificate/client-key.pem"
     - "regression-test/ssl_default_certificate/*"
+    - "regression-test/pipeline/performance/**"
     - "extension/beats/go.mod"
     - "extension/beats/go.sum"
-
+    - "pytest/hdfs"
+    - "pytest/qe"
+    - "pytest/sys/data"
+    - "pytest/deploy/*.conf"
   comment: on-failure