From 715cebafa77ab0712e9afba787c20390a0a914f7 Mon Sep 17 00:00:00 2001 From: linlin-s <84819822+linlin-s@users.noreply.github.com> Date: Thu, 22 Jul 2021 12:51:20 -0700 Subject: [PATCH 1/4] Add ion-java performance regression detector to GitHub workflow (#2) --- ...n-java-performance-regression-detector.yml | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/ion-java-performance-regression-detector.yml diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml new file mode 100644 index 0000000000..bfef7bfc82 --- /dev/null +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -0,0 +1,72 @@ +# This workflow will build a Java project with Maven +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven + +name: Ion Jave performance regression detector + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + detetct-regression: + name: Detetct Regression + + runs-on: ubuntu-latest + + steps: + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: Checkout ion-java repository from the current commit + uses: actions/checkout@v2 + with: + fetch-depth: 2 + submodules: recursive + - run: git checkout HEAD^ + + - name: Build ion-java from the current commit + run: mvn clean install + + - name: Build ion-java-benchmark-cli based on the current ion-java + uses: actions/checkout@v2 + with: + repository: amzn/ion-java-benchmark-cli + ref: master + - run: mvn clean install + + - name: Check the version of ion-java + run: java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version + + - name: Generate test Ion Data + run: java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 500 -T string -f ion_text testWorkflow.ion + + - name: Test read preformance of the ion-java from the current commit + run: java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar read testWorkflow.ion -o readPerformanceCurrent.ion -r ion + + - name: Clean maven dependency repository + run : rm -r /home/runner/.m2 + + - name: Checkout ion-java repository from the new commit + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Build ion-java from the new commit + run: mvn clean install + + - name: Build ion-java-benchmark-cli based on the new ion-java + uses: actions/checkout@v2 + with: + repository: amzn/ion-java-benchmark-cli + ref: master + - run: mvn clean install + + - name: Check the version of ion-java + run: java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version + + - name: Test read preformance of ion-java from the new commit + run: java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar read testWorkflow.ion -o readPerformanceNew.ion -r ion \ No newline at end of file From acbd0e2c4d8f3659fac6f0ebb616515992449773 Mon Sep 17 00:00:00 2001 From: linlin-s <84819822+linlin-s@users.noreply.github.com> Date: Mon, 9 Aug 2021 17:42:39 -0700 Subject: [PATCH 2/4] Adds the ability to detect ion-java performance regression in GitHub workflow (#3) --- ...n-java-performance-regression-detector.yml | 115 +++++++++++++----- 1 file changed, 85 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml index bfef7bfc82..6853f4d498 100644 --- a/.github/workflows/ion-java-performance-regression-detector.yml +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -1,7 +1,7 @@ # This workflow will build a Java project with Maven # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven -name: Ion Jave performance regression detector +name: Ion Java performance regression detector on: push: @@ -10,8 +10,8 @@ on: branches: [ master ] jobs: - detetct-regression: - name: Detetct Regression + detect-regression: + name: Detect Regression runs-on: ubuntu-latest @@ -21,52 +21,107 @@ jobs: with: java-version: 1.8 - - name: Checkout ion-java repository from the current commit + - name: Checkout ion-java from the new commit. uses: actions/checkout@v2 with: fetch-depth: 2 submodules: recursive - - run: git checkout HEAD^ + path: ion-java - - name: Build ion-java from the current commit - run: mvn clean install + - name: Build ion-java from the new commit + run: cd ion-java && mvn clean install - - name: Build ion-java-benchmark-cli based on the current ion-java + - name: Checkout ion-java-benchmark-cli uses: actions/checkout@v2 with: repository: amzn/ion-java-benchmark-cli ref: master - - run: mvn clean install + path: ion-java-benchmark-cli - - name: Check the version of ion-java - run: java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version + - name: Build ion-java-benchmark-cli + run: cd ion-java-benchmark-cli && mvn clean install + + - name: Check the version of ion-java. + run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version - name: Generate test Ion Data - run: java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 500 -T string -f ion_text testWorkflow.ion + run: | + mkdir -p testData + java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/benchmark/testStruct.isl testData/testStruct.10n + java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/benchmark/testList.isl testData/testList.10n + java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/benchmark/testNestedStruct.isl testData/testNestedStruct.10n + + - name: Upload test Ion Data to artifacts + uses: actions/upload-artifact@v2 + with: + name: test Ion Data + path: testData + + - name: Benchmark ion-java from the new commit + run: | + mkdir -p benchmarkResults + cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults - - name: Test read preformance of the ion-java from the current commit - run: java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar read testWorkflow.ion -o readPerformanceCurrent.ion -r ion + - name: Upload benchmark results to artifacts + uses: actions/upload-artifact@v2 + with: + name: Benchmark result + path: benchmarkResults - - name: Clean maven dependency repository + - name: Clean maven dependencies repository run : rm -r /home/runner/.m2 - - name: Checkout ion-java repository from the new commit - uses: actions/checkout@v2 - with: - submodules: recursive + - name: Build ion-java from the previous commit + run: cd ion-java && git checkout HEAD^ && mvn clean install - - name: Build ion-java from the new commit - run: mvn clean install + - name: Build ion-java-benchmark-cli + run: cd ion-java-benchmark-cli && mvn clean install - - name: Build ion-java-benchmark-cli based on the new ion-java - uses: actions/checkout@v2 + - name: Check the version of ion-java + run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version + + - name: Create directories for test data and benchmark results + run: | + mkdir -p benchmarkResults + mkdir -p testData + + - name: Download test Ion Data from artifacts + uses: actions/download-artifact@v2 with: - repository: amzn/ion-java-benchmark-cli - ref: master - - run: mvn clean install + name: test Ion Data + path: testData - - name: Check the version of ion-java - run: java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version + - name: Download benchmark results of ion-java from the new commit from artifacts + uses: actions/download-artifact@v2 + with: + name: Benchmark result + path: benchmarkResults + + - name: Benchmark ion-java from the previous commit and add the generated benchmark results to the existing directories + run: cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults - - name: Test read preformance of ion-java from the new commit - run: java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar read testWorkflow.ion -o readPerformanceNew.ion -r ion \ No newline at end of file + - name: Upload new benchmark results directory to artifacts + uses: actions/upload-artifact@v2 + with: + name: Benchmark result + path: benchmarkResults + + - name: Detect performance regression + id: regression_result + run: | + result=true + cd benchmarkResults && for FILE in *; do message=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion | tee /dev/stderr) && if [ "$message" != "" ]; then result=false; fi; done + echo "::set-output name=regression-result::$result" + echo $result + + - name: Upload comparison reports to the benchmark results directory + uses: actions/upload-artifact@v2 + with: + name: Benchmark result + path: benchmarkResults + + - name: Fail the workflow if regression happened + env: + regression_detect: ${{steps.regression_result.outputs.regression-result}} + if: ${{ env.regression_detect == 'false' }} + run: exit 1 From c3af9983e4bf9e5c6adfec25e552dad750837fa3 Mon Sep 17 00:00:00 2001 From: Sun Date: Tue, 10 Aug 2021 13:16:29 -0700 Subject: [PATCH 3/4] Delete redundant 'push:branch:[master]' --- .../workflows/ion-java-performance-regression-detector.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml index 6853f4d498..e1d256dfc3 100644 --- a/.github/workflows/ion-java-performance-regression-detector.yml +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -4,10 +4,8 @@ name: Ion Java performance regression detector on: - push: - branches: [ master ] pull_request: - branches: [ master ] + branches: [master] jobs: detect-regression: From de595d5d790ed830a013bd0525d0ca816e42a714 Mon Sep 17 00:00:00 2001 From: Sun Date: Tue, 10 Aug 2021 14:16:23 -0700 Subject: [PATCH 4/4] Fix the bug that block the workflow --- .../workflows/ion-java-performance-regression-detector.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml index e1d256dfc3..6bf0cd43c5 100644 --- a/.github/workflows/ion-java-performance-regression-detector.yml +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -3,9 +3,7 @@ name: Ion Java performance regression detector -on: - pull_request: - branches: [master] +on: [pull_request] jobs: detect-regression: