From 1fad66446a3fa346eb9726b36a23ac67cd3133ef Mon Sep 17 00:00:00 2001 From: andylokandy Date: Wed, 16 Feb 2022 18:40:38 +0800 Subject: [PATCH 01/14] test Signed-off-by: andylokandy --- .ci/build.groovy | 63 ----------- .ci/build.sh | 6 -- .ci/integration_test.groovy | 117 --------------------- .ci/test.sh | 8 -- {config => .github/config}/pd.toml | 2 +- {config => .github/config}/tikv_rawkv.toml | 6 ++ {config => .github/config}/tikv_txnkv.toml | 6 ++ .github/workflows/ci.yml | 52 +++++++++ 8 files changed, 65 insertions(+), 195 deletions(-) delete mode 100644 .ci/build.groovy delete mode 100755 .ci/build.sh delete mode 100644 .ci/integration_test.groovy delete mode 100755 .ci/test.sh rename {config => .github/config}/pd.toml (79%) rename {config => .github/config}/tikv_rawkv.toml (66%) rename {config => .github/config}/tikv_txnkv.toml (60%) create mode 100644 .github/workflows/ci.yml diff --git a/.ci/build.groovy b/.ci/build.groovy deleted file mode 100644 index e8ddbdd00eb..00000000000 --- a/.ci/build.groovy +++ /dev/null @@ -1,63 +0,0 @@ -def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPullDescription, credentialsId) { - - catchError { - node ('build') { - container("java") { - stage('Prepare') { - dir("/home/jenkins/agent/git/client-java") { - sh """ - rm -rf /maven/.m2/repository/* - rm -rf /maven/.m2/settings.xml - rm -rf ~/.m2/settings.xml - archive_url=http://fileserver.pingcap.net/download/builds/pingcap/client-java/cache/tikv-client-java-m2-cache-latest.tar.gz - if [ ! "\$(ls -A /maven/.m2/repository)" ]; then curl -sL \$archive_url | tar -zx -C /maven || true; fi - """ - if (sh(returnStatus: true, script: '[ -d .git ] && [ -f Makefile ] && git rev-parse --git-dir > /dev/null 2>&1') != 0) { - deleteDir() - } - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'master']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: credentialsId, refspec: '+refs/pull/*:refs/remotes/origin/pr/*', url: 'git@github.com:tikv/client-java.git']]] - sh "git checkout -f ${ghprbActualCommit}" - } - } - - stage('Format') { - dir("/home/jenkins/agent/git/client-java") { - sh """ - ./dev/javafmt - git diff --quiet - formatted="\$?" - if [[ "\${formatted}" -eq 1 ]] - then - echo "code format error, please run the following commands:" - echo " mvn com.coveo:fmt-maven-plugin:format" - exit 1 - fi - """ - } - } - - stage('Build') { - dir("/home/jenkins/agent/git/client-java") { - timeout(30) { - sh ".ci/build.sh" - } - } - } - } - } - currentBuild.result = "SUCCESS" - } - - stage('Summary') { - def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) - def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" + - "${ghprbPullLink}" + "\n" + - "${ghprbPullDescription}" + "\n" + - "Build Result: `${currentBuild.result}`" + "\n" + - "Elapsed Time: `${duration} mins` " + "\n" + - "${env.RUN_DISPLAY_URL}" - print msg - } -} - -return this diff --git a/.ci/build.sh b/.ci/build.sh deleted file mode 100755 index 9be12fe6a53..00000000000 --- a/.ci/build.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -x -set -euo pipefail - -mvn clean compile \ No newline at end of file diff --git a/.ci/integration_test.groovy b/.ci/integration_test.groovy deleted file mode 100644 index fac1bd92468..00000000000 --- a/.ci/integration_test.groovy +++ /dev/null @@ -1,117 +0,0 @@ -def call(ghprbActualCommit, ghprbPullId, ghprbPullTitle, ghprbPullLink, ghprbPullDescription, credentialsId) { - - def TIKV_BRANCH = "master" - def PD_BRANCH = "master" - - // parse pd branch - def m2 = ghprbCommentBody =~ /pd\s*=\s*([^\s\\]+)(\s|\\|$)/ - if (m2) { - PD_BRANCH = "${m2[0][1]}" - } - m2 = null - println "PD_BRANCH=${PD_BRANCH}" - - // parse tikv branch - def m3 = ghprbCommentBody =~ /tikv\s*=\s*([^\s\\]+)(\s|\\|$)/ - if (m3) { - TIKV_BRANCH = "${m3[0][1]}" - } - m3 = null - println "TIKV_BRANCH=${TIKV_BRANCH}" - - catchError { - node ('build') { - container("java") { - stage('Prepare') { - dir("/home/jenkins/agent/git/client-java") { - sh """ - rm -rf /maven/.m2/repository/* - rm -rf /maven/.m2/settings.xml - rm -rf ~/.m2/settings.xml - archive_url=http://fileserver.pingcap.net/download/builds/pingcap/client-java/cache/tikv-client-java-m2-cache-latest.tar.gz - if [ ! "\$(ls -A /maven/.m2/repository)" ]; then curl -sL \$archive_url | tar -zx -C /maven || true; fi - """ - if (sh(returnStatus: true, script: '[ -d .git ] && [ -f Makefile ] && git rev-parse --git-dir > /dev/null 2>&1') != 0) { - deleteDir() - } - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: 'master']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'PruneStaleBranch'], [$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: credentialsId, refspec: '+refs/pull/*:refs/remotes/origin/pr/*', url: 'git@github.com:tikv/client-java.git']]] - sh "git checkout -f ${ghprbActualCommit}" - } - - dir("/home/jenkins/agent/git/client-java/_run") { - // tikv - def tikv_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/tikv/${TIKV_BRANCH}/sha1").trim() - sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/tikv/${tikv_sha1}/centos7/tikv-server.tar.gz | tar xz" - // pd - def pd_sha1 = sh(returnStdout: true, script: "curl ${FILE_SERVER_URL}/download/refs/pingcap/pd/${PD_BRANCH}/sha1").trim() - sh "curl ${FILE_SERVER_URL}/download/builds/pingcap/pd/${pd_sha1}/centos7/pd-server.tar.gz | tar xz" - - sh """ - killall -9 tikv-server || true - killall -9 pd-server || true - killall -9 java || true - sleep 10 - """ - - sh """ - echo "start TiKV for RawKV test" - bin/pd-server --name=pd_rawkv --data-dir=pd_rawkv --client-urls="http://0.0.0.0:2379" --advertise-client-urls="http://127.0.0.1:2379" --peer-urls="http://0.0.0.0:2380" --advertise-peer-urls="http://127.0.0.1:2380" --config=../config/pd.toml &>pd_rawkv.log & - sleep 10 - bin/tikv-server --pd 127.0.0.1:2379 --data-dir tikv_rawkv --addr 0.0.0.0:20160 --advertise-addr 127.0.0.1:20160 --status-addr 0.0.0.0:20180 --config ../config/tikv_rawkv.toml &>tikv_rawkv.log & - sleep 10 - ps aux | grep '-server' || true - curl -s 127.0.0.1:2379/pd/api/v1/status || true - """ - - sh """ - echo "start TiKV for TxnKV test" - bin/pd-server --name=pd_txnkv --data-dir=pd_txnkv --client-urls="http://0.0.0.0:3379" --advertise-client-urls="http://127.0.0.1:3379" --peer-urls="http://0.0.0.0:3380" --advertise-peer-urls="http://127.0.0.1:3380" --config=../config/pd.toml &>pd_txnkv.log & - sleep 10 - bin/tikv-server --pd 127.0.0.1:3379 --data-dir tikv_txnkv --addr 0.0.0.0:21160 --advertise-addr 127.0.0.1:21160 --status-addr 0.0.0.0:21180 --config ../config/tikv_txnkv.toml &>tikv_txnkv.log & - sleep 10 - ps aux | grep '-server' || true - curl -s 127.0.0.1:3379/pd/api/v1/status || true - """ - - sh "sleep 30" - } - } - - stage('Test') { - dir("/home/jenkins/agent/git/client-java") { - try { - timeout(30) { - sh ".ci/test.sh" - } - } catch (err) { - sh """ - ps aux | grep '-server' || true - curl -s 127.0.0.1:2379/pd/api/v1/status || true - """ - sh "cat _run/pd_rawkv.log" - sh "cat _run/tikv_rawkv.log" - sh "cat _run/pd_txnkv.log" - sh "cat _run/tikv_txnkv.log" - throw err - } - } - } - } - } - currentBuild.result = "SUCCESS" - } - - stage('Summary') { - def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) - def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" + - "${ghprbPullLink}" + "\n" + - "${ghprbPullDescription}" + "\n" + - "Integration Common Test Result: `${currentBuild.result}`" + "\n" + - "Elapsed Time: `${duration} mins` " + "\n" + - "${env.RUN_DISPLAY_URL}" - - print msg - } -} - -return this diff --git a/.ci/test.sh b/.ci/test.sh deleted file mode 100755 index 6ba95963458..00000000000 --- a/.ci/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -x -set -euo pipefail - -export RAWKV_PD_ADDRESSES=127.0.0.1:2379 -export TXNKV_PD_ADDRESSES=127.0.0.1:3379 -mvn clean test \ No newline at end of file diff --git a/config/pd.toml b/.github/config/pd.toml similarity index 79% rename from config/pd.toml rename to .github/config/pd.toml index f2795de343f..526fa3fa745 100644 --- a/config/pd.toml +++ b/.github/config/pd.toml @@ -1,4 +1,4 @@ # PD Configuration. [replication] enable-placement-rules = true -max-replicas = 1 \ No newline at end of file +max-replicas = 1 diff --git a/config/tikv_rawkv.toml b/.github/config/tikv_rawkv.toml similarity index 66% rename from config/tikv_rawkv.toml rename to .github/config/tikv_rawkv.toml index f525fa753c5..48dbc4d0882 100644 --- a/config/tikv_rawkv.toml +++ b/.github/config/tikv_rawkv.toml @@ -6,3 +6,9 @@ capacity = "8G" [storage] enable-ttl = true + +[rocksdb] +max-open-files = 10000 + +[raftdb] +max-open-files = 10000 diff --git a/config/tikv_txnkv.toml b/.github/config/tikv_txnkv.toml similarity index 60% rename from config/tikv_txnkv.toml rename to .github/config/tikv_txnkv.toml index 287abc69e41..57cb2bceeca 100644 --- a/config/tikv_txnkv.toml +++ b/.github/config/tikv_txnkv.toml @@ -3,3 +3,9 @@ [raftstore] # set store capacity, if no set, use disk capacity. capacity = "8G" + +[rocksdb] +max-open-files = 10000 + +[raftdb] +max-open-files = 10000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..fdc46cfe556 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,52 @@ +on: + pull_request: + push: + branches: + - master + +name: CI + +jobs: + format: + name: Check Format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + java-version: '8.0' + distribution: 'adopt' + - run: | + ./dev/javafmt + if [[ $(git diff) != "" ]] + then + echo "code format error, please run the following commands:" + echo " mvn com.coveo:fmt-maven-plugin:format" + exit 1 + fi + + integration-test: + name: Integration Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + java-version: '8.0' + distribution: 'adopt' + - name: Install TiUP + run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh + - name: Start TiUP Playground + run: | + /home/runner/.tiup/bin/tiup playground nightly --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> raw.out & + /home/runner/.tiup/bin/tiup playground nightly --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> txn.out & + sleep 2m + echo "RAWKV_PD_ADDRESSES=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV + echo "TXNKV_PD_ADDRESSES=$(cat txn.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV + echo $GITHUB_ENV + cat raw.out | echo + cat txn.out | echo + - name: Run Integration Test + run: mvn clean test From 9fbc2d480c645b05e58c88218a5c3e2d05a192f3 Mon Sep 17 00:00:00 2001 From: andylokandy Date: Wed, 16 Feb 2022 19:14:07 +0800 Subject: [PATCH 02/14] test Signed-off-by: andylokandy --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdc46cfe556..c39002bbb15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,8 +45,8 @@ jobs: sleep 2m echo "RAWKV_PD_ADDRESSES=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV echo "TXNKV_PD_ADDRESSES=$(cat txn.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV - echo $GITHUB_ENV - cat raw.out | echo - cat txn.out | echo - - name: Run Integration Test - run: mvn clean test + echo "::set-output name=GITHUB_ENV::$(echo $GITHUB_ENV)" + echo "::set-output name=rawout::$(echo $(cat raw.out))" + echo "::set-output name=txnout::$(echo $(cat txn.out))" + # - name: Run Integration Test + # run: mvn clean test From 32833735de392ca059e89990e0c3f5f826baf753 Mon Sep 17 00:00:00 2001 From: andylokandy Date: Wed, 16 Feb 2022 19:30:02 +0800 Subject: [PATCH 03/14] test Signed-off-by: andylokandy --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c39002bbb15..c5815129fdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,13 +40,13 @@ jobs: run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh - name: Start TiUP Playground run: | - /home/runner/.tiup/bin/tiup playground nightly --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> raw.out & - /home/runner/.tiup/bin/tiup playground nightly --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> txn.out & + /home/runner/.tiup/bin/tiup playground nightly --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> raw.out 2>&1 & + /home/runner/.tiup/bin/tiup playground nightly --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> txn.out 2>&1 & sleep 2m echo "RAWKV_PD_ADDRESSES=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV echo "TXNKV_PD_ADDRESSES=$(cat txn.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV - echo "::set-output name=GITHUB_ENV::$(echo $GITHUB_ENV)" - echo "::set-output name=rawout::$(echo $(cat raw.out))" - echo "::set-output name=txnout::$(echo $(cat txn.out))" + echo "$GITHUB_ENV" + echo "$(cat raw.out)" + echo "$(cat txn.out)" # - name: Run Integration Test # run: mvn clean test From 415c24703ced9649cf54000fc5435e6715ee210f Mon Sep 17 00:00:00 2001 From: andylokandy Date: Wed, 16 Feb 2022 19:35:46 +0800 Subject: [PATCH 04/14] test Signed-off-by: andylokandy --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5815129fdd..e997f14f280 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,8 +45,8 @@ jobs: sleep 2m echo "RAWKV_PD_ADDRESSES=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV echo "TXNKV_PD_ADDRESSES=$(cat txn.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV - echo "$GITHUB_ENV" - echo "$(cat raw.out)" - echo "$(cat txn.out)" + echo "$GITHUB_ENV" >&2 + echo "$(cat raw.out)" >&2 + echo "$(cat txn.out)" >&2 # - name: Run Integration Test # run: mvn clean test From 16c46f6015c88c09a3fd027323f0e19f72a4296c Mon Sep 17 00:00:00 2001 From: andylokandy Date: Wed, 16 Feb 2022 19:41:02 +0800 Subject: [PATCH 05/14] test Signed-off-by: andylokandy --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e997f14f280..63ad5f57f68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,8 +41,9 @@ jobs: - name: Start TiUP Playground run: | /home/runner/.tiup/bin/tiup playground nightly --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> raw.out 2>&1 & + sleep 1m 30s /home/runner/.tiup/bin/tiup playground nightly --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> txn.out 2>&1 & - sleep 2m + sleep 30s echo "RAWKV_PD_ADDRESSES=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV echo "TXNKV_PD_ADDRESSES=$(cat txn.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV echo "$GITHUB_ENV" >&2 From 466c363b0c282c52934e40dd592c379cd33a3bcf Mon Sep 17 00:00:00 2001 From: andylokandy Date: Wed, 16 Feb 2022 19:45:07 +0800 Subject: [PATCH 06/14] test Signed-off-by: andylokandy --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63ad5f57f68..a0048eec3f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,5 +49,5 @@ jobs: echo "$GITHUB_ENV" >&2 echo "$(cat raw.out)" >&2 echo "$(cat txn.out)" >&2 - # - name: Run Integration Test - # run: mvn clean test + - name: Run Integration Test + run: mvn clean test From 0fc0c1866648a1aac95e39c28adae4756ebc883a Mon Sep 17 00:00:00 2001 From: andylokandy Date: Wed, 16 Feb 2022 20:40:15 +0800 Subject: [PATCH 07/14] Add coverage Signed-off-by: andylokandy --- .github/workflows/ci.yml | 6 +++++- pom.xml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0048eec3f8..2406c852a39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,8 +46,12 @@ jobs: sleep 30s echo "RAWKV_PD_ADDRESSES=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV echo "TXNKV_PD_ADDRESSES=$(cat txn.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV - echo "$GITHUB_ENV" >&2 echo "$(cat raw.out)" >&2 echo "$(cat txn.out)" >&2 - name: Run Integration Test run: mvn clean test + - name: Comment coverage to PR + uses: madrapps/jacoco-report@v1.2 + with: + paths: ${{ github.workspace }}/target/site/jacoco/jacoco.xml + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/pom.xml b/pom.xml index a452e2891fd..eaf7276f387 100644 --- a/pom.xml +++ b/pom.xml @@ -495,7 +495,7 @@ jacoco-site - package + test report From 568cd77d0730a575cc9ccc2fd8f7faeb19eb327a Mon Sep 17 00:00:00 2001 From: andylokandy Date: Wed, 16 Feb 2022 21:20:18 +0800 Subject: [PATCH 08/14] Elevate permission Signed-off-by: andylokandy --- .github/config/tikv_rawkv.toml | 4 ++++ .github/config/tikv_txnkv.toml | 3 +++ .github/workflows/ci.yml | 13 ++++++++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/config/tikv_rawkv.toml b/.github/config/tikv_rawkv.toml index 48dbc4d0882..5ee1bfe07ed 100644 --- a/.github/config/tikv_rawkv.toml +++ b/.github/config/tikv_rawkv.toml @@ -3,6 +3,9 @@ [raftstore] # set store capacity, if no set, use disk capacity. capacity = "8G" +pd-heartbeat-tick-interval = "2s" +pd-store-heartbeat-tick-interval = "5s" +split-region-check-tick-interval = "1s" [storage] enable-ttl = true @@ -12,3 +15,4 @@ max-open-files = 10000 [raftdb] max-open-files = 10000 + diff --git a/.github/config/tikv_txnkv.toml b/.github/config/tikv_txnkv.toml index 57cb2bceeca..c083cfa31b0 100644 --- a/.github/config/tikv_txnkv.toml +++ b/.github/config/tikv_txnkv.toml @@ -3,6 +3,9 @@ [raftstore] # set store capacity, if no set, use disk capacity. capacity = "8G" +pd-heartbeat-tick-interval = "2s" +pd-store-heartbeat-tick-interval = "5s" +split-region-check-tick-interval = "1s" [rocksdb] max-open-files = 10000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2406c852a39..74f053b9245 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,8 @@ jobs: integration-test: name: Integration Test + permissions: + pull-requests: write runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -50,8 +52,17 @@ jobs: echo "$(cat txn.out)" >&2 - name: Run Integration Test run: mvn clean test - - name: Comment coverage to PR + - name: Upload coverage + uses: actions/upload-artifact@v2 + with: + name: jacoco + path: | + ${{ github.workspace }}/target/jacoco.exec + ${{ github.workspace }}/target/site/jacoco + - name: Comment coverage uses: madrapps/jacoco-report@v1.2 with: paths: ${{ github.workspace }}/target/site/jacoco/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} + min-coverage-overall: 0 + min-coverage-changed-files: 0 From 778650140444e6463566ea748962596339246121 Mon Sep 17 00:00:00 2001 From: andylokandy Date: Wed, 16 Feb 2022 22:24:10 +0800 Subject: [PATCH 09/14] use codecov Signed-off-by: andylokandy --- .github/workflows/ci.yml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74f053b9245..04a1f94363f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,11 +1,8 @@ -on: - pull_request: - push: - branches: - - master - name: CI +on: +- pull_request_target + jobs: format: name: Check Format @@ -28,8 +25,6 @@ jobs: integration-test: name: Integration Test - permissions: - pull-requests: write runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -59,10 +54,8 @@ jobs: path: | ${{ github.workspace }}/target/jacoco.exec ${{ github.workspace }}/target/site/jacoco - - name: Comment coverage - uses: madrapps/jacoco-report@v1.2 + - uses: codecov/codecov-action@v2 with: - paths: ${{ github.workspace }}/target/site/jacoco/jacoco.xml - token: ${{ secrets.GITHUB_TOKEN }} - min-coverage-overall: 0 - min-coverage-changed-files: 0 + files: ${{ github.workspace }}/target/site/jacoco/jacoco.xml + fail_ci_if_error: true + verbose: true From fd69ab9301a4c71cd876a27c4a5a6ddc034d051d Mon Sep 17 00:00:00 2001 From: andylokandy Date: Wed, 16 Feb 2022 22:30:26 +0800 Subject: [PATCH 10/14] fix Signed-off-by: andylokandy --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04a1f94363f..fb35a6e01e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,10 @@ name: CI on: -- pull_request_target + pull_request: + push: + branches: + - master jobs: format: From 9891d2404036fd9d43829553ea7d2b1101caa9c1 Mon Sep 17 00:00:00 2001 From: andylokandy Date: Wed, 16 Feb 2022 22:56:01 +0800 Subject: [PATCH 11/14] add baget Signed-off-by: andylokandy --- .github/workflows/ci.yml | 8 +------- README.md | 1 + 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb35a6e01e9..8d364db3c34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,13 +51,7 @@ jobs: - name: Run Integration Test run: mvn clean test - name: Upload coverage - uses: actions/upload-artifact@v2 - with: - name: jacoco - path: | - ${{ github.workspace }}/target/jacoco.exec - ${{ github.workspace }}/target/site/jacoco - - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v2 with: files: ${{ github.workspace }}/target/site/jacoco/jacoco.xml fail_ci_if_error: true diff --git a/README.md b/README.md index 065a5869bd1..0acd816deb1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Maven Central](https://img.shields.io/maven-central/v/org.tikv/tikv-client-java.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:%22org.tikv%22%20AND%20a:%22tikv-client-java%22) [![Slack](https://img.shields.io/badge/chat-on%20Slack-brightgreen.svg)](https://slack.tidb.io/invite?team=tikv-wg&channel=client) +[![codecov](https://codecov.io/gh/tikv/client-java/branch/master/graph/badge.svg?token=nSAjGaN0EH)](https://codecov.io/gh/tikv/client-java) ## TiKV JAVA Client From a7dedd2a0e6bb69ab4b095af8497d270ffb15228 Mon Sep 17 00:00:00 2001 From: andylokandy Date: Thu, 17 Feb 2022 15:55:40 +0800 Subject: [PATCH 12/14] address comment Signed-off-by: andylokandy --- .github/workflows/ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d364db3c34..f10d5e417a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,13 +22,16 @@ jobs: if [[ $(git diff) != "" ]] then echo "code format error, please run the following commands:" - echo " mvn com.coveo:fmt-maven-plugin:format" + echo " ./dev/javafmt" exit 1 fi integration-test: - name: Integration Test + name: Integration Test - ${{ matrix.tikv_version }} runs-on: ubuntu-latest + strategy: + matrix: + tikv_version: [nightly, v5.0.4] steps: - uses: actions/checkout@v2 - name: Set up JDK 8 @@ -40,9 +43,9 @@ jobs: run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh - name: Start TiUP Playground run: | - /home/runner/.tiup/bin/tiup playground nightly --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> raw.out 2>&1 & + /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> raw.out 2>&1 & sleep 1m 30s - /home/runner/.tiup/bin/tiup playground nightly --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> txn.out 2>&1 & + /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> txn.out 2>&1 & sleep 30s echo "RAWKV_PD_ADDRESSES=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV echo "TXNKV_PD_ADDRESSES=$(cat txn.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV From 6d539bf93687544f98a123faa7d4ddd736efbae0 Mon Sep 17 00:00:00 2001 From: andylokandy Date: Thu, 17 Feb 2022 15:56:22 +0800 Subject: [PATCH 13/14] address comment Signed-off-by: andylokandy --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f10d5e417a3..b5dda4b1210 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - tikv_version: [nightly, v5.0.4] + tikv_version: [nightly, v5.0.4, v5.3.0, v5.4.0] steps: - uses: actions/checkout@v2 - name: Set up JDK 8 From 2cefaba55e5b7b280dba0af4f9505a44d637e496 Mon Sep 17 00:00:00 2001 From: andylokandy Date: Thu, 17 Feb 2022 16:08:45 +0800 Subject: [PATCH 14/14] add comment Signed-off-by: andylokandy --- .github/workflows/ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5dda4b1210..c2fbf68aa88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,12 +43,22 @@ jobs: run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh - name: Start TiUP Playground run: | + # Start TiKV in APIV1TTL /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> raw.out 2>&1 & + + # The first run of `tiup` has to download all components so it'll take longer. sleep 1m 30s + + # Start TiKV in APIV1 /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> txn.out 2>&1 & + sleep 30s + + # Parse PD address from `tiup` output echo "RAWKV_PD_ADDRESSES=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV echo "TXNKV_PD_ADDRESSES=$(cat txn.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV + + # Log the output echo "$(cat raw.out)" >&2 echo "$(cat txn.out)" >&2 - name: Run Integration Test