From 233fc2251bb2850be6af18ad86aec36fe575dafc Mon Sep 17 00:00:00 2001 From: Jenn Magder Date: Fri, 3 Mar 2023 10:54:01 -0800 Subject: [PATCH 01/10] Upload xcresults to LUCI cloud storage upload scenario app test result fix script --- testing/run_tests.py | 23 ++++++++++++++- testing/scenario_app/run_ios_tests.sh | 42 +++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/testing/run_tests.py b/testing/run_tests.py index f800a474544a6..aa9ebf5a224a1 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -17,6 +17,7 @@ import multiprocessing import os import re +import shutil import subprocess import sys import tempfile @@ -752,6 +753,12 @@ def run_objc_tests(ios_variant='ios_debug_sim_unopt', test_filter=None): ios_unit_test_dir = os.path.join( BUILDROOT_DIR, 'flutter', 'testing', 'ios', 'IosUnitTests' ) + + result_bundle_temp = tempfile.TemporaryDirectory( + suffix='ios_embedding_xcresult' + ).name + result_bundle_path = os.path.join(result_bundle_temp, 'ios_embedding') + # Avoid using xcpretty unless the following can be addressed: # - Make sure all relevant failure output is printed on a failure. # - Make sure that a failing exit code is set for CI. @@ -760,13 +767,27 @@ def run_objc_tests(ios_variant='ios_debug_sim_unopt', test_filter=None): 'xcodebuild ' '-sdk iphonesimulator ' '-scheme IosUnitTests ' - "-destination name='" + new_simulator_name + "' " + '-resultBundlePath ' + result_bundle_path + " -destination name='" + + new_simulator_name + "' " 'test ' 'FLUTTER_ENGINE=' + ios_variant ] if test_filter is not None: test_command[0] = test_command[0] + ' -only-testing:%s' % test_filter run_cmd(test_command, cwd=ios_unit_test_dir, shell=True) + + # except: + # The LUCI environment may provide a variable containing a directory path + # for additional output files that will be uploaded to cloud storage. + # Upload the xcresult when the tests fail. + luci_test_outputs_path = os.environ.get('FLUTTER_TEST_OUTPUTS_DIR') + xcresult_bundle = os.path.join(result_bundle_temp, 'ios_embedding.xcresult') + print(xcresult_bundle) + if luci_test_outputs_path and os.path.exists(xcresult_bundle): + dump_path = os.path.join(luci_test_outputs_path, 'ios_embedding.xcresult') + # xcresults contain many little files. Archive the bundle before upload. + shutil.make_archive(dump_path, 'zip', root_dir=xcresult_bundle) + finally: delete_simulator(new_simulator_name) diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index fd84de214150f..d8a54146d6f16 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -41,25 +41,57 @@ fi # Can also be set via Simulator app Device > Rotate Device Automatically defaults write com.apple.iphonesimulator RotateWindowWhenSignaledByGuest -int 1 -cd $SRC_DIR/out/$FLUTTER_ENGINE/scenario_app/Scenarios +SCENARIO_PATH=$SRC_DIR/out/$FLUTTER_ENGINE/scenario_app/Scenarios +cd $SCENARIO_PATH + +RESULT_BUNDLE_FOLDER="ios_scenario_xcresult" +RESULT_BUNDLE_PATH="${SCENARIO_PATH}/${RESULT_BUNDLE_FOLDER}" + +# Zip and upload xcresult to luci. +# First parameter ($1) is the zip output name. +ZIP_AND_UPLOAD_XCRESULT_TO_LUCI () { + # Using RESULT_BUNDLE_PATH causes the zip containing all the sub directories. + # So use relative directory instead. + echo $1 + zip -q -r $1 "./$RESULT_BUNDLE_FOLDER" + if ( -z "$FLUTTER_TEST_OUTPUTS_DIR") then + mv -f $1 $FLUTTER_TEST_OUTPUTS_DIR + fi + exit 1 +} echo "Running simulator tests with Skia" echo "" -set -o pipefail && xcodebuild -sdk iphonesimulator \ +mktemp -d $RESULT_BUNDLE_PATH +trap 'rm -rf $RESULT_BUNDLE_PATH' EXIT + +if set -o pipefail && xcodebuild -sdk iphonesimulator \ -scheme Scenarios \ + -resultBundlePath "$RESULT_BUNDLE_PATH/ios_scenario.xcresult" \ -destination 'platform=iOS Simulator,OS=16.2,name=iPhone SE (3rd generation)' \ clean test \ - FLUTTER_ENGINE="$FLUTTER_ENGINE" + FLUTTER_ENGINE="$FLUTTER_ENGINE"; then + echo "test success." +else + echo "test failed." + ZIP_AND_UPLOAD_XCRESULT_TO_LUCI "ios_scenario_xcresult.zip" +fi echo "Running simulator tests with Impeller" echo "" # Skip testFontRenderingWhenSuppliedWithBogusFont: https://github.com/flutter/flutter/issues/113250 -set -o pipefail && xcodebuild -sdk iphonesimulator \ +if set -o pipefail && xcodebuild -sdk iphonesimulator \ -scheme Scenarios \ + -resultBundlePath "$RESULT_BUNDLE_PATH/ios_scenario.xcresult" \ -destination 'platform=iOS Simulator,OS=16.2,name=iPhone SE (3rd generation)' \ clean test \ FLUTTER_ENGINE="$FLUTTER_ENGINE" \ -skip-testing "ScenariosUITests/BogusFontTextTest/testFontRenderingWhenSuppliedWithBogusFont" \ - INFOPLIST_FILE="Scenarios/Info_Impeller.plist" # Plist with FLTEnableImpeller=YES + INFOPLIST_FILE="Scenarios/Info_Impeller.plist"; then # Plist with FLTEnableImpeller=YES + echo "test success." +else + echo "test failed." + ZIP_AND_UPLOAD_XCRESULT_TO_LUCI "ios_scenario_impeller_xcresult.zip" +fi From 244d39ed0a838d059d2468cf35232b197aacacd5 Mon Sep 17 00:00:00 2001 From: Chris Yang Date: Thu, 11 May 2023 09:45:15 -0700 Subject: [PATCH 02/10] fix --- testing/scenario_app/run_ios_tests.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index d8a54146d6f16..7c397a345f7af 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -50,13 +50,11 @@ RESULT_BUNDLE_PATH="${SCENARIO_PATH}/${RESULT_BUNDLE_FOLDER}" # Zip and upload xcresult to luci. # First parameter ($1) is the zip output name. ZIP_AND_UPLOAD_XCRESULT_TO_LUCI () { - # Using RESULT_BUNDLE_PATH causes the zip containing all the sub directories. - # So use relative directory instead. + # Using absolute directory causes the zip containing all the sub directories. + # So use relative directory (./$RESULT_BUNDLE_FOLDER) instead. echo $1 zip -q -r $1 "./$RESULT_BUNDLE_FOLDER" - if ( -z "$FLUTTER_TEST_OUTPUTS_DIR") then - mv -f $1 $FLUTTER_TEST_OUTPUTS_DIR - fi + mv -f $1 $FLUTTER_TEST_OUTPUTS_DIR exit 1 } @@ -95,3 +93,4 @@ else echo "test failed." ZIP_AND_UPLOAD_XCRESULT_TO_LUCI "ios_scenario_impeller_xcresult.zip" fi + From df346c9fedaa0143ff139e4fa9ef663f8a575c3b Mon Sep 17 00:00:00 2001 From: Chris Yang Date: Thu, 11 May 2023 13:24:25 -0700 Subject: [PATCH 03/10] remove result bundle after zipped and moved --- testing/scenario_app/run_ios_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index 7c397a345f7af..5581b28824204 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -55,6 +55,7 @@ ZIP_AND_UPLOAD_XCRESULT_TO_LUCI () { echo $1 zip -q -r $1 "./$RESULT_BUNDLE_FOLDER" mv -f $1 $FLUTTER_TEST_OUTPUTS_DIR + rm -rf $RESULT_BUNDLE_PATH exit 1 } @@ -62,7 +63,6 @@ echo "Running simulator tests with Skia" echo "" mktemp -d $RESULT_BUNDLE_PATH -trap 'rm -rf $RESULT_BUNDLE_PATH' EXIT if set -o pipefail && xcodebuild -sdk iphonesimulator \ -scheme Scenarios \ From d6c732fbcbf0b77f8a36eccc0f6e01e1e3ea6136 Mon Sep 17 00:00:00 2001 From: Chris Yang Date: Thu, 11 May 2023 14:13:43 -0700 Subject: [PATCH 04/10] remove --- testing/scenario_app/run_ios_tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index 5581b28824204..f88dcb01bd5c5 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -55,7 +55,6 @@ ZIP_AND_UPLOAD_XCRESULT_TO_LUCI () { echo $1 zip -q -r $1 "./$RESULT_BUNDLE_FOLDER" mv -f $1 $FLUTTER_TEST_OUTPUTS_DIR - rm -rf $RESULT_BUNDLE_PATH exit 1 } @@ -75,6 +74,7 @@ else echo "test failed." ZIP_AND_UPLOAD_XCRESULT_TO_LUCI "ios_scenario_xcresult.zip" fi +rm -rf $RESULT_BUNDLE_PATH echo "Running simulator tests with Impeller" echo "" @@ -93,4 +93,5 @@ else echo "test failed." ZIP_AND_UPLOAD_XCRESULT_TO_LUCI "ios_scenario_impeller_xcresult.zip" fi +rm -rf $RESULT_BUNDLE_PATH From 348a709edb1ec816e00ec3ca1b0c40520b1504ab Mon Sep 17 00:00:00 2001 From: Chris Yang Date: Fri, 12 May 2023 09:51:38 -0700 Subject: [PATCH 05/10] fix py --- testing/run_tests.py | 63 ++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/testing/run_tests.py b/testing/run_tests.py index aa9ebf5a224a1..0deac86a23310 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -754,39 +754,38 @@ def run_objc_tests(ios_variant='ios_debug_sim_unopt', test_filter=None): BUILDROOT_DIR, 'flutter', 'testing', 'ios', 'IosUnitTests' ) - result_bundle_temp = tempfile.TemporaryDirectory( + with tempfile.TemporaryDirectory( suffix='ios_embedding_xcresult' - ).name - result_bundle_path = os.path.join(result_bundle_temp, 'ios_embedding') - - # Avoid using xcpretty unless the following can be addressed: - # - Make sure all relevant failure output is printed on a failure. - # - Make sure that a failing exit code is set for CI. - # See https://github.com/flutter/flutter/issues/63742 - test_command = [ - 'xcodebuild ' - '-sdk iphonesimulator ' - '-scheme IosUnitTests ' - '-resultBundlePath ' + result_bundle_path + " -destination name='" + - new_simulator_name + "' " - 'test ' - 'FLUTTER_ENGINE=' + ios_variant - ] - if test_filter is not None: - test_command[0] = test_command[0] + ' -only-testing:%s' % test_filter - run_cmd(test_command, cwd=ios_unit_test_dir, shell=True) - - # except: - # The LUCI environment may provide a variable containing a directory path - # for additional output files that will be uploaded to cloud storage. - # Upload the xcresult when the tests fail. - luci_test_outputs_path = os.environ.get('FLUTTER_TEST_OUTPUTS_DIR') - xcresult_bundle = os.path.join(result_bundle_temp, 'ios_embedding.xcresult') - print(xcresult_bundle) - if luci_test_outputs_path and os.path.exists(xcresult_bundle): - dump_path = os.path.join(luci_test_outputs_path, 'ios_embedding.xcresult') - # xcresults contain many little files. Archive the bundle before upload. - shutil.make_archive(dump_path, 'zip', root_dir=xcresult_bundle) + ) as result_bundle_temp: + result_bundle_path = os.path.join(result_bundle_temp, 'ios_embedding') + + # Avoid using xcpretty unless the following can be addressed: + # - Make sure all relevant failure output is printed on a failure. + # - Make sure that a failing exit code is set for CI. + # See https://github.com/flutter/flutter/issues/63742 + test_command = [ + 'xcodebuild ' + '-sdk iphonesimulator ' + '-scheme IosUnitTests ' + '-resultBundlePath ' + result_bundle_path + ' ' + '-destination name=' + new_simulator_name + ' ' + 'test ' + 'FLUTTER_ENGINE=' + ios_variant + ] + if test_filter is not None: + test_command[0] = test_command[0] + ' -only-testing:%s' % test_filter + run_cmd(test_command, cwd=ios_unit_test_dir, shell=True) + + # except: + # The LUCI environment may provide a variable containing a directory path + # for additional output files that will be uploaded to cloud storage. + # Upload the xcresult when the tests fail. + luci_test_outputs_path = os.environ.get('FLUTTER_TEST_OUTPUTS_DIR') + xcresult_bundle = os.path.join(result_bundle_temp, 'ios_embedding.xcresult') + if luci_test_outputs_path and os.path.exists(xcresult_bundle): + dump_path = os.path.join(luci_test_outputs_path, 'ios_embedding.xcresult') + # xcresults contain many little files. Archive the bundle before upload. + shutil.make_archive(dump_path, 'zip', root_dir=xcresult_bundle) finally: delete_simulator(new_simulator_name) From 6f7a7ca645f03b0878feb0f2a5fac7346604e728 Mon Sep 17 00:00:00 2001 From: Chris Yang Date: Fri, 12 May 2023 09:58:36 -0700 Subject: [PATCH 06/10] some bash fixes --- testing/scenario_app/run_ios_tests.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index f88dcb01bd5c5..f477d4f1229b7 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -49,11 +49,10 @@ RESULT_BUNDLE_PATH="${SCENARIO_PATH}/${RESULT_BUNDLE_FOLDER}" # Zip and upload xcresult to luci. # First parameter ($1) is the zip output name. -ZIP_AND_UPLOAD_XCRESULT_TO_LUCI () { +zip_and_upload_xcresult_to_luci () { # Using absolute directory causes the zip containing all the sub directories. # So use relative directory (./$RESULT_BUNDLE_FOLDER) instead. - echo $1 - zip -q -r $1 "./$RESULT_BUNDLE_FOLDER" + zip -q -r $1 $RESULT_BUNDLE_PATH mv -f $1 $FLUTTER_TEST_OUTPUTS_DIR exit 1 } @@ -72,7 +71,7 @@ if set -o pipefail && xcodebuild -sdk iphonesimulator \ echo "test success." else echo "test failed." - ZIP_AND_UPLOAD_XCRESULT_TO_LUCI "ios_scenario_xcresult.zip" + zip_and_upload_xcresult_to_luci "ios_scenario_xcresult.zip" fi rm -rf $RESULT_BUNDLE_PATH @@ -91,7 +90,7 @@ if set -o pipefail && xcodebuild -sdk iphonesimulator \ echo "test success." else echo "test failed." - ZIP_AND_UPLOAD_XCRESULT_TO_LUCI "ios_scenario_impeller_xcresult.zip" + zip_and_upload_xcresult_to_luci "ios_scenario_impeller_xcresult.zip" fi rm -rf $RESULT_BUNDLE_PATH From 619332071cf4b253298b13d5f8a73bc906309f9a Mon Sep 17 00:00:00 2001 From: Chris Yang Date: Fri, 12 May 2023 09:59:18 -0700 Subject: [PATCH 07/10] format --- testing/scenario_app/run_ios_tests.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index f477d4f1229b7..439082e808d4c 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -93,4 +93,3 @@ else zip_and_upload_xcresult_to_luci "ios_scenario_impeller_xcresult.zip" fi rm -rf $RESULT_BUNDLE_PATH - From 0493106f182b042435bf15db94fc97731fccc07b Mon Sep 17 00:00:00 2001 From: Chris Yang Date: Fri, 12 May 2023 10:10:11 -0700 Subject: [PATCH 08/10] zip comment --- testing/scenario_app/run_ios_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index 439082e808d4c..f0518a776ac62 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -50,9 +50,9 @@ RESULT_BUNDLE_PATH="${SCENARIO_PATH}/${RESULT_BUNDLE_FOLDER}" # Zip and upload xcresult to luci. # First parameter ($1) is the zip output name. zip_and_upload_xcresult_to_luci () { - # Using absolute directory causes the zip containing all the sub directories. - # So use relative directory (./$RESULT_BUNDLE_FOLDER) instead. - zip -q -r $1 $RESULT_BUNDLE_PATH + # We don't want the zip to contain the abusolute path, + # so use relative path (./$RESULT_BUNDLE_FOLDER) instead. + zip -q -r $1 "./$RESULT_BUNDLE_FOLDER" mv -f $1 $FLUTTER_TEST_OUTPUTS_DIR exit 1 } From c94b62b98a8b4152a1f25970dac0009963eb52a4 Mon Sep 17 00:00:00 2001 From: Chris Yang Date: Fri, 12 May 2023 10:12:29 -0700 Subject: [PATCH 09/10] format python --- testing/run_tests.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/testing/run_tests.py b/testing/run_tests.py index 0deac86a23310..0d7962f9dc98a 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -754,9 +754,8 @@ def run_objc_tests(ios_variant='ios_debug_sim_unopt', test_filter=None): BUILDROOT_DIR, 'flutter', 'testing', 'ios', 'IosUnitTests' ) - with tempfile.TemporaryDirectory( - suffix='ios_embedding_xcresult' - ) as result_bundle_temp: + with tempfile.TemporaryDirectory(suffix='ios_embedding_xcresult' + ) as result_bundle_temp: result_bundle_path = os.path.join(result_bundle_temp, 'ios_embedding') # Avoid using xcpretty unless the following can be addressed: @@ -781,9 +780,13 @@ def run_objc_tests(ios_variant='ios_debug_sim_unopt', test_filter=None): # for additional output files that will be uploaded to cloud storage. # Upload the xcresult when the tests fail. luci_test_outputs_path = os.environ.get('FLUTTER_TEST_OUTPUTS_DIR') - xcresult_bundle = os.path.join(result_bundle_temp, 'ios_embedding.xcresult') + xcresult_bundle = os.path.join( + result_bundle_temp, 'ios_embedding.xcresult' + ) if luci_test_outputs_path and os.path.exists(xcresult_bundle): - dump_path = os.path.join(luci_test_outputs_path, 'ios_embedding.xcresult') + dump_path = os.path.join( + luci_test_outputs_path, 'ios_embedding.xcresult' + ) # xcresults contain many little files. Archive the bundle before upload. shutil.make_archive(dump_path, 'zip', root_dir=xcresult_bundle) From 6fddd21db3330462e8a270919d923302618bb7a9 Mon Sep 17 00:00:00 2001 From: Chris Yang Date: Fri, 12 May 2023 10:15:21 -0700 Subject: [PATCH 10/10] push and pop --- testing/scenario_app/run_ios_tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testing/scenario_app/run_ios_tests.sh b/testing/scenario_app/run_ios_tests.sh index f0518a776ac62..0034a42e4899a 100755 --- a/testing/scenario_app/run_ios_tests.sh +++ b/testing/scenario_app/run_ios_tests.sh @@ -42,6 +42,7 @@ fi defaults write com.apple.iphonesimulator RotateWindowWhenSignaledByGuest -int 1 SCENARIO_PATH=$SRC_DIR/out/$FLUTTER_ENGINE/scenario_app/Scenarios +pushd . cd $SCENARIO_PATH RESULT_BUNDLE_FOLDER="ios_scenario_xcresult" @@ -93,3 +94,5 @@ else zip_and_upload_xcresult_to_luci "ios_scenario_impeller_xcresult.zip" fi rm -rf $RESULT_BUNDLE_PATH + +popd