From 252b536372fc09c6d7e6f9ff8bc94d361a508f62 Mon Sep 17 00:00:00 2001 From: Ricardo Amador Date: Wed, 25 Jan 2023 10:11:44 -0800 Subject: [PATCH 01/10] Compatibility with python2 and python3 --- ci/firebase_testlab.py | 12 +++++++++--- ci/scan_flattened_deps.py | 11 ++++++++--- testing/run_tests.py | 4 +++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/ci/firebase_testlab.py b/ci/firebase_testlab.py index 92069a7c1f584..a9614db43a208 100755 --- a/ci/firebase_testlab.py +++ b/ci/firebase_testlab.py @@ -21,6 +21,8 @@ sys.exit(1) PROJECT = os.environ['GCP_PROJECT'] +ENCODING = 'UTF-8' + script_dir = os.path.dirname(os.path.realpath(__file__)) buildroot_dir = os.path.abspath(os.path.join(script_dir, '..', '..')) out_dir = os.path.join(buildroot_dir, 'out') @@ -67,6 +69,7 @@ def check_logcat(results_dir): 'gsutil', 'cat', '%s/%s/*/logcat' % (BUCKET, results_dir) ]) + logcat = logcat if isinstance(logcat, str) else logcat.decode(ENCODING) if not logcat: sys.exit(1) @@ -82,7 +85,9 @@ def check_timeline(results_dir): 'gsutil', 'du', '%s/%s/*/game_loop_results/results_scenario_0.json' % (BUCKET, results_dir) - ]).strip() + ]) + gsutil_du = gsutil_du if isinstance(gsutil_du, str) else gsutil_du.decode(ENCODING) + gsutil_du = gsutil_du.strip() if gsutil_du == '0': print('Failed to produce a timeline.') sys.exit(1) @@ -113,8 +118,9 @@ def main(): return 1 git_revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], - cwd=script_dir).strip() - + cwd=script_dir) + git_revision = git_revision if isinstance(git_revision, str) else git_revision.decode(ENCODING) + git_revision = git_revision.strip() results = [] apk = None for apk in apks: diff --git a/ci/scan_flattened_deps.py b/ci/scan_flattened_deps.py index 2f51bdf09b683..7d24a090e009e 100644 --- a/ci/scan_flattened_deps.py +++ b/ci/scan_flattened_deps.py @@ -26,6 +26,7 @@ OSV_VULN_DB_URL = 'https://osv.dev/vulnerability/' SECONDS_PER_YEAR = 31556952 UPSTREAM_PREFIX = 'upstream_' +ENCODING = 'UTF-8' failed_deps = [] # deps which fail to be cloned or git-merge based @@ -206,7 +207,9 @@ def get_common_ancestor_commit(dep, deps_list): 'git --git-dir ' + temp_dep_dir + '/.git remote show upstream ' + "| sed -n \'/HEAD branch/s/.*: //p\'", shell=True - ).decode().strip() + ) + default_branch = default_branch if isinstance(default_branch, str) else default_branch.decode(ENCODING) + default_branch = default_branch.strip() print( 'default_branch found: {default_branch}'.format( default_branch=default_branch @@ -223,7 +226,8 @@ def get_common_ancestor_commit(dep, deps_list): "--format=\'%(objectname:short)\' refs/heads/upstream", shell=True ) - commit = commit.decode().strip() + commit = commit if isinstance(commit, str) else commit.decode(ENCODING) + commit = commit.strip() # perform merge-base on most recent default branch commit and pinned mirror commit ancestor_commit = subprocess.check_output( @@ -232,7 +236,8 @@ def get_common_ancestor_commit(dep, deps_list): ), shell=True ) - ancestor_commit = ancestor_commit.decode().strip() + ancestor_commit = ancestor_commit if isinstance(ancestor_commit, str) else ancestor_commit.decode(ENCODING) + ancestor_commit = ancestor_commit.strip() print('Ancestor commit: ' + ancestor_commit) return ancestor_commit except subprocess.CalledProcessError as error: diff --git a/testing/run_tests.py b/testing/run_tests.py index 875b0ef88ccfb..261593a4206fd 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -33,7 +33,7 @@ FONT_SUBSET_DIR = os.path.join(BUILDROOT_DIR, 'flutter', 'tools', 'font-subset') FML_UNITTESTS_FILTER = '--gtest_filter=-*TimeSensitiveTest*' - +ENCODING = 'UTF-8' def print_divider(char='='): print('\n') @@ -598,6 +598,8 @@ def ensure_ios_tests_are_built(ios_out_dir): def assert_expected_xcode_version(): """Checks that the user has a version of Xcode installed""" version_output = subprocess.check_output(['xcodebuild', '-version']) + version_output = version_output if isinstance(version_output, str) else version_output.decode(ENCODING) + version_output = version_output.strip() match = re.match(r'Xcode (\d+)', version_output) message = 'Xcode must be installed to run the iOS embedding unit tests' assert match, message From 4c793b18dcf7081d46deae8268a39ca1a685ebb0 Mon Sep 17 00:00:00 2001 From: Ricardo Amador Date: Wed, 25 Jan 2023 10:12:41 -0800 Subject: [PATCH 02/10] Formatting --- ci/firebase_testlab.py | 6 ++++-- ci/scan_flattened_deps.py | 8 ++++++-- testing/run_tests.py | 5 ++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ci/firebase_testlab.py b/ci/firebase_testlab.py index a9614db43a208..8a9d56c7934d0 100755 --- a/ci/firebase_testlab.py +++ b/ci/firebase_testlab.py @@ -86,7 +86,8 @@ def check_timeline(results_dir): '%s/%s/*/game_loop_results/results_scenario_0.json' % (BUCKET, results_dir) ]) - gsutil_du = gsutil_du if isinstance(gsutil_du, str) else gsutil_du.decode(ENCODING) + gsutil_du = gsutil_du if isinstance(gsutil_du, + str) else gsutil_du.decode(ENCODING) gsutil_du = gsutil_du.strip() if gsutil_du == '0': print('Failed to produce a timeline.') @@ -119,7 +120,8 @@ def main(): git_revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=script_dir) - git_revision = git_revision if isinstance(git_revision, str) else git_revision.decode(ENCODING) + git_revision = git_revision if isinstance(git_revision, str + ) else git_revision.decode(ENCODING) git_revision = git_revision.strip() results = [] apk = None diff --git a/ci/scan_flattened_deps.py b/ci/scan_flattened_deps.py index 7d24a090e009e..09240aad90e34 100644 --- a/ci/scan_flattened_deps.py +++ b/ci/scan_flattened_deps.py @@ -208,7 +208,9 @@ def get_common_ancestor_commit(dep, deps_list): "| sed -n \'/HEAD branch/s/.*: //p\'", shell=True ) - default_branch = default_branch if isinstance(default_branch, str) else default_branch.decode(ENCODING) + default_branch = default_branch if isinstance( + default_branch, str + ) else default_branch.decode(ENCODING) default_branch = default_branch.strip() print( 'default_branch found: {default_branch}'.format( @@ -236,7 +238,9 @@ def get_common_ancestor_commit(dep, deps_list): ), shell=True ) - ancestor_commit = ancestor_commit if isinstance(ancestor_commit, str) else ancestor_commit.decode(ENCODING) + ancestor_commit = ancestor_commit if isinstance( + ancestor_commit, str + ) else ancestor_commit.decode(ENCODING) ancestor_commit = ancestor_commit.strip() print('Ancestor commit: ' + ancestor_commit) return ancestor_commit diff --git a/testing/run_tests.py b/testing/run_tests.py index 261593a4206fd..73c8c0ba9691d 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -35,6 +35,7 @@ FML_UNITTESTS_FILTER = '--gtest_filter=-*TimeSensitiveTest*' ENCODING = 'UTF-8' + def print_divider(char='='): print('\n') for _ in range(4): @@ -598,7 +599,9 @@ def ensure_ios_tests_are_built(ios_out_dir): def assert_expected_xcode_version(): """Checks that the user has a version of Xcode installed""" version_output = subprocess.check_output(['xcodebuild', '-version']) - version_output = version_output if isinstance(version_output, str) else version_output.decode(ENCODING) + version_output = version_output if isinstance( + version_output, str + ) else version_output.decode(ENCODING) version_output = version_output.strip() match = re.match(r'Xcode (\d+)', version_output) message = 'Xcode must be installed to run the iOS embedding unit tests' From d38dc4a64a8d027856b94d5b2103afb759764050 Mon Sep 17 00:00:00 2001 From: Ricardo Amador Date: Wed, 25 Jan 2023 14:35:16 -0800 Subject: [PATCH 03/10] Updated to use a function instead of decoding bytes in multiple places. --- ci/compatibility_helper.py | 23 +++++++++++++++++++++++ ci/firebase_testlab.py | 11 ++++------- ci/scan_flattened_deps.py | 12 ++++-------- testing/run_tests.py | 5 ++--- 4 files changed, 33 insertions(+), 18 deletions(-) create mode 100644 ci/compatibility_helper.py diff --git a/ci/compatibility_helper.py b/ci/compatibility_helper.py new file mode 100644 index 0000000000000..c69f4ba61c94d --- /dev/null +++ b/ci/compatibility_helper.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +# This script contains helper function(s) for supporting both +# python 2 and python 3 infrastructure code. + +ENCODING = 'UTF-8' + +def byte_str_decode(str_or_bytes): + '''Returns a string if given either a string or bytes. + + TODO: This function should be removed when the supported python + version is only python 3. + + Args: + str_or_bytes (string or bytes) we want to convert or return as + the possible value changes depending on the version of python + used. + ''' + return str_or_bytes if isinstance(str_or_bytes, str) else str_or_bytes.decode(ENCODING) diff --git a/ci/firebase_testlab.py b/ci/firebase_testlab.py index 8a9d56c7934d0..510f582e8b82d 100755 --- a/ci/firebase_testlab.py +++ b/ci/firebase_testlab.py @@ -10,6 +10,7 @@ import os import subprocess import sys +from compatibility_helper import byte_str_decode if 'STORAGE_BUCKET' not in os.environ: print('The GCP storage bucket must be provided as an environment variable.') @@ -21,8 +22,6 @@ sys.exit(1) PROJECT = os.environ['GCP_PROJECT'] -ENCODING = 'UTF-8' - script_dir = os.path.dirname(os.path.realpath(__file__)) buildroot_dir = os.path.abspath(os.path.join(script_dir, '..', '..')) out_dir = os.path.join(buildroot_dir, 'out') @@ -69,7 +68,7 @@ def check_logcat(results_dir): 'gsutil', 'cat', '%s/%s/*/logcat' % (BUCKET, results_dir) ]) - logcat = logcat if isinstance(logcat, str) else logcat.decode(ENCODING) + logcat = byte_str_decode(logcat) if not logcat: sys.exit(1) @@ -86,8 +85,7 @@ def check_timeline(results_dir): '%s/%s/*/game_loop_results/results_scenario_0.json' % (BUCKET, results_dir) ]) - gsutil_du = gsutil_du if isinstance(gsutil_du, - str) else gsutil_du.decode(ENCODING) + gsutil_du = byte_str_decode(gsutil_du) gsutil_du = gsutil_du.strip() if gsutil_du == '0': print('Failed to produce a timeline.') @@ -120,8 +118,7 @@ def main(): git_revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=script_dir) - git_revision = git_revision if isinstance(git_revision, str - ) else git_revision.decode(ENCODING) + git_revision = byte_str_decode(git_revision) git_revision = git_revision.strip() results = [] apk = None diff --git a/ci/scan_flattened_deps.py b/ci/scan_flattened_deps.py index 09240aad90e34..67759b3d710a0 100644 --- a/ci/scan_flattened_deps.py +++ b/ci/scan_flattened_deps.py @@ -17,6 +17,7 @@ import subprocess import sys from urllib import request +from compatibility_helper import byte_str_decode SCRIPT_DIR = os.path.dirname(sys.argv[0]) CHECKOUT_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..')) @@ -26,7 +27,6 @@ OSV_VULN_DB_URL = 'https://osv.dev/vulnerability/' SECONDS_PER_YEAR = 31556952 UPSTREAM_PREFIX = 'upstream_' -ENCODING = 'UTF-8' failed_deps = [] # deps which fail to be cloned or git-merge based @@ -208,9 +208,7 @@ def get_common_ancestor_commit(dep, deps_list): "| sed -n \'/HEAD branch/s/.*: //p\'", shell=True ) - default_branch = default_branch if isinstance( - default_branch, str - ) else default_branch.decode(ENCODING) + default_branch = byte_str_decode(default_branch) default_branch = default_branch.strip() print( 'default_branch found: {default_branch}'.format( @@ -228,7 +226,7 @@ def get_common_ancestor_commit(dep, deps_list): "--format=\'%(objectname:short)\' refs/heads/upstream", shell=True ) - commit = commit if isinstance(commit, str) else commit.decode(ENCODING) + commit = byte_str_decode(commit) commit = commit.strip() # perform merge-base on most recent default branch commit and pinned mirror commit @@ -238,9 +236,7 @@ def get_common_ancestor_commit(dep, deps_list): ), shell=True ) - ancestor_commit = ancestor_commit if isinstance( - ancestor_commit, str - ) else ancestor_commit.decode(ENCODING) + ancestor_commit = byte_str_decode(ancestor_commit) ancestor_commit = ancestor_commit.strip() print('Ancestor commit: ' + ancestor_commit) return ancestor_commit diff --git a/testing/run_tests.py b/testing/run_tests.py index 73c8c0ba9691d..01133773a76b6 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -19,6 +19,7 @@ import time import csv import xvfb +from ci.compatibility_helper import byte_str_decode SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) BUILDROOT_DIR = os.path.abspath( @@ -599,9 +600,7 @@ def ensure_ios_tests_are_built(ios_out_dir): def assert_expected_xcode_version(): """Checks that the user has a version of Xcode installed""" version_output = subprocess.check_output(['xcodebuild', '-version']) - version_output = version_output if isinstance( - version_output, str - ) else version_output.decode(ENCODING) + version_output = byte_str_decode(version_output) version_output = version_output.strip() match = re.match(r'Xcode (\d+)', version_output) message = 'Xcode must be installed to run the iOS embedding unit tests' From 08f846af1b558e02772e0d7999052a039d19016a Mon Sep 17 00:00:00 2001 From: Ricardo Amador Date: Wed, 25 Jan 2023 14:37:59 -0800 Subject: [PATCH 04/10] Formatting and whitespace cleanup. --- ci/compatibility_helper.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ci/compatibility_helper.py b/ci/compatibility_helper.py index c69f4ba61c94d..e9eaea68ab14f 100644 --- a/ci/compatibility_helper.py +++ b/ci/compatibility_helper.py @@ -4,20 +4,22 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # -# This script contains helper function(s) for supporting both +# This script contains helper function(s) for supporting both # python 2 and python 3 infrastructure code. ENCODING = 'UTF-8' + def byte_str_decode(str_or_bytes): - '''Returns a string if given either a string or bytes. + '''Returns a string if given either a string or bytes. TODO: This function should be removed when the supported python version is only python 3. Args: str_or_bytes (string or bytes) we want to convert or return as - the possible value changes depending on the version of python + the possible value changes depending on the version of python used. ''' - return str_or_bytes if isinstance(str_or_bytes, str) else str_or_bytes.decode(ENCODING) + return str_or_bytes if isinstance(str_or_bytes, + str) else str_or_bytes.decode(ENCODING) From 4f9434597e288b96e5bc53a40f0e42d8b80f4b40 Mon Sep 17 00:00:00 2001 From: Ricardo Amador Date: Wed, 25 Jan 2023 15:04:25 -0800 Subject: [PATCH 05/10] Fix import statement. --- testing/run_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testing/run_tests.py b/testing/run_tests.py index 01133773a76b6..c396e0f8ef7ec 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -19,7 +19,9 @@ import time import csv import xvfb -from ci.compatibility_helper import byte_str_decode + +sys.path.append('ci') +from compatibility_helper import byte_str_decode SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) BUILDROOT_DIR = os.path.abspath( From 9a82b5ad26ba121784a652f37b0c2bd51f2a7f44 Mon Sep 17 00:00:00 2001 From: Ricardo Amador Date: Wed, 25 Jan 2023 15:18:46 -0800 Subject: [PATCH 06/10] Linter change for docstring. --- ci/compatibility_helper.py | 4 ++-- testing/run_tests.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/compatibility_helper.py b/ci/compatibility_helper.py index e9eaea68ab14f..e1cc61e8d8257 100644 --- a/ci/compatibility_helper.py +++ b/ci/compatibility_helper.py @@ -11,7 +11,7 @@ def byte_str_decode(str_or_bytes): - '''Returns a string if given either a string or bytes. + """Returns a string if given either a string or bytes. TODO: This function should be removed when the supported python version is only python 3. @@ -20,6 +20,6 @@ def byte_str_decode(str_or_bytes): str_or_bytes (string or bytes) we want to convert or return as the possible value changes depending on the version of python used. - ''' + """ return str_or_bytes if isinstance(str_or_bytes, str) else str_or_bytes.decode(ENCODING) diff --git a/testing/run_tests.py b/testing/run_tests.py index c396e0f8ef7ec..0be2ee101f78b 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -19,7 +19,6 @@ import time import csv import xvfb - sys.path.append('ci') from compatibility_helper import byte_str_decode From 8346a1bc8175e3918705d40b515022e7f6678a79 Mon Sep 17 00:00:00 2001 From: Ricardo Amador Date: Wed, 25 Jan 2023 15:21:05 -0800 Subject: [PATCH 07/10] Formatting. --- testing/run_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/run_tests.py b/testing/run_tests.py index 0be2ee101f78b..c396e0f8ef7ec 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -19,6 +19,7 @@ import time import csv import xvfb + sys.path.append('ci') from compatibility_helper import byte_str_decode From 795ca9894bead9efd8abc3823b8f5b412c04b6ce Mon Sep 17 00:00:00 2001 From: Ricardo Amador Date: Wed, 25 Jan 2023 15:34:07 -0800 Subject: [PATCH 08/10] Remove function from run_tests.py since it is a pain to import. --- testing/run_tests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/testing/run_tests.py b/testing/run_tests.py index c396e0f8ef7ec..73c8c0ba9691d 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -20,9 +20,6 @@ import csv import xvfb -sys.path.append('ci') -from compatibility_helper import byte_str_decode - SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) BUILDROOT_DIR = os.path.abspath( os.path.join(os.path.realpath(__file__), '..', '..', '..') @@ -602,7 +599,9 @@ def ensure_ios_tests_are_built(ios_out_dir): def assert_expected_xcode_version(): """Checks that the user has a version of Xcode installed""" version_output = subprocess.check_output(['xcodebuild', '-version']) - version_output = byte_str_decode(version_output) + version_output = version_output if isinstance( + version_output, str + ) else version_output.decode(ENCODING) version_output = version_output.strip() match = re.match(r'Xcode (\d+)', version_output) message = 'Xcode must be installed to run the iOS embedding unit tests' From fca0e99e7bff30e2c876ee01a8b9f37b2de42a28 Mon Sep 17 00:00:00 2001 From: Ricardo Amador Date: Wed, 25 Jan 2023 15:34:55 -0800 Subject: [PATCH 09/10] Add todo message for python 2 deprecation. --- testing/run_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/run_tests.py b/testing/run_tests.py index 73c8c0ba9691d..9e79e4e21e77a 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -599,6 +599,7 @@ def ensure_ios_tests_are_built(ios_out_dir): def assert_expected_xcode_version(): """Checks that the user has a version of Xcode installed""" version_output = subprocess.check_output(['xcodebuild', '-version']) + # TODO ricardoamador: remove this check when python 2 is deprecated. version_output = version_output if isinstance( version_output, str ) else version_output.decode(ENCODING) From f427a6906a2bd6abce51610043f15088cf3839e3 Mon Sep 17 00:00:00 2001 From: Ricardo Amador Date: Wed, 25 Jan 2023 15:37:11 -0800 Subject: [PATCH 10/10] Updated copyright year. --- ci/compatibility_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/compatibility_helper.py b/ci/compatibility_helper.py index e1cc61e8d8257..0d8231ca178e7 100644 --- a/ci/compatibility_helper.py +++ b/ci/compatibility_helper.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2023 The Flutter Authors. All rights reserved. +# Copyright 2013 The Flutter Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. #