From e19d3ccc63665ba843b12ddee3df7b813f483ea5 Mon Sep 17 00:00:00 2001 From: jashook Date: Wed, 4 Mar 2020 13:04:37 -0800 Subject: [PATCH 01/15] Run crossgen in parallel --- .../tests/scripts/crossgen_comparison.py | 96 ++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/src/coreclr/tests/scripts/crossgen_comparison.py b/src/coreclr/tests/scripts/crossgen_comparison.py index 96ced35650d641..4b34db25605a5d 100644 --- a/src/coreclr/tests/scripts/crossgen_comparison.py +++ b/src/coreclr/tests/scripts/crossgen_comparison.py @@ -96,6 +96,7 @@ import glob import json import hashlib +import multiprocessing import os import tarfile import tempfile @@ -152,6 +153,95 @@ def build_argument_parser(): return parser +################################################################################ +# Helper class +################################################################################ + +class AsyncSubprocessHelper: + def __init__(self, items, subproc_count=multiprocessing.cpu_count(), verbose=False): + item_queue = asyncio.Queue() + for item in items: + item_queue.put_nowait(item) + + self.items = items + self.subproc_count = subproc_count + self.verbose = verbose + + if 'win32' in sys.platform: + # Windows specific event-loop policy & cmd + asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + + async def __get_item__(self, item, index, size, async_callback, *extra_args): + """ Wrapper to the async callback which will schedule based on the queue + """ + + # Wait for the queue to become free. Then start + # running the sub process. + subproc_id = await self.subproc_count_queue.get() + + print_prefix = "" + + if self.verbose: + print_prefix = "[{}:{}]: ".format(index, size) + + await async_callback(print_prefix, item, *extra_args) + + # Add back to the queue, incase another process wants to run. + self.subproc_count_queue.put_nowait(subproc_id) + + async def __run_to_completion__(self, async_callback, *extra_args): + """ async wrapper for run_to_completion + """ + + chunk_size = self.subproc_count + + # Create a queue with a chunk size of the cpu count + # + # Each run_pmi invocation will remove an item from the + # queue before running a potentially long running pmi run. + # + # When the queue is drained, we will wait queue.get which + # will wait for when a run_pmi instance has added back to the + subproc_count_queue = asyncio.Queue(chunk_size) + diff_queue = asyncio.Queue() + + for item in self.items: + diff_queue.put_nowait(item) + + for item in range(chunk_size): + subproc_count_queue.put_nowait(item) + + self.subproc_count_queue = subproc_count_queue + tasks = [] + size = diff_queue.qsize() + + count = 1 + item = diff_queue.get_nowait() if not diff_queue.empty() else None + while item is not None: + tasks.append(self.__get_item__(item, count, size, async_callback, *extra_args)) + count += 1 + + item = diff_queue.get_nowait() if not diff_queue.empty() else None + + await asyncio.gather(*tasks) + + def run_to_completion(self, async_callback, *extra_args): + """ Run until the item queue has been depleted + + Notes: + Acts as a wrapper to abstract the async calls to + async_callback. Note that this will allow cpu_count + amount of running subprocesses. Each time the queue + is emptied, another process will start. Note that + the python code is single threaded, it will just + rely on async/await to start subprocesses at + subprocess_count + """ + + reset_env = os.environ.copy() + asyncio.run(self.__run_to_completion__(async_callback, *extra_args)) + os.environ.update(reset_env) + ################################################################################ # Globals ################################################################################ @@ -551,11 +641,15 @@ def crossgen_framework(args): crossgen_results = run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) - for assembly_name in g_Framework_Assemblies: + def run_crossgen(print_prefix, assembly_name): il_filename = os.path.join(args.core_root, assembly_name) ni_filename = os.path.join(ni_files_dirname, add_ni_extension(assembly_name)) crossgen_results = run_crossgen(args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname) save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) + + helper = AsyncSubprocessHelper(g_Framework_Assemblies) + helper.run_to_completion(run_pmi) + shutil.rmtree(ni_files_dirname, ignore_errors=True) def load_crossgen_result_from_json_file(json_filename): From ceaf0344774dfdfa7f170e881c88c8c364a53d6e Mon Sep 17 00:00:00 2001 From: jashook Date: Wed, 4 Mar 2020 14:33:42 -0800 Subject: [PATCH 02/15] Change container to python:3.7 --- eng/pipelines/coreclr/templates/crossgen-comparison-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index c59e3e13afad09..af07e26a243eca 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -77,8 +77,8 @@ jobs: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} # Run all steps in the container. - # Note that the containers are defined in platform-matrix.yml - container: ${{ parameters.container }} + # The test portion of the job only requires python3.7 + container: python:3.7 timeoutInMinutes: 180 # 3 hrs steps: From 76b4c23ff921235b8a14674775d494ea3f321e16 Mon Sep 17 00:00:00 2001 From: jashook Date: Wed, 4 Mar 2020 14:43:57 -0800 Subject: [PATCH 03/15] Undo python3.7 container --- eng/pipelines/coreclr/templates/crossgen-comparison-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index af07e26a243eca..c59e3e13afad09 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -77,8 +77,8 @@ jobs: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} # Run all steps in the container. - # The test portion of the job only requires python3.7 - container: python:3.7 + # Note that the containers are defined in platform-matrix.yml + container: ${{ parameters.container }} timeoutInMinutes: 180 # 3 hrs steps: From 123e5700a4a69ae8e0512d429985ffd7bd19e39b Mon Sep 17 00:00:00 2001 From: jashook Date: Thu, 5 Mar 2020 12:22:59 -0800 Subject: [PATCH 04/15] Update the arm build image and address feedback --- eng/pipelines/common/platform-matrix.yml | 2 +- src/coreclr/tests/scripts/crossgen_comparison.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 38a01a7c5f5f36..353e80222dfeb8 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -38,7 +38,7 @@ jobs: archType: arm platform: Linux_arm container: - image: ubuntu-16.04-cross-14.04-23cacb0-20200121150126 + image: ubuntu-16.04-cross-70a29f1-20200305143320 registry: mcr jobParameters: runtimeFlavor: ${{ parameters.runtimeFlavor }} diff --git a/src/coreclr/tests/scripts/crossgen_comparison.py b/src/coreclr/tests/scripts/crossgen_comparison.py index 4b34db25605a5d..be48f265a56393 100644 --- a/src/coreclr/tests/scripts/crossgen_comparison.py +++ b/src/coreclr/tests/scripts/crossgen_comparison.py @@ -197,11 +197,11 @@ async def __run_to_completion__(self, async_callback, *extra_args): # Create a queue with a chunk size of the cpu count # - # Each run_pmi invocation will remove an item from the + # Each run_crossgen invocation will remove an item from the # queue before running a potentially long running pmi run. # # When the queue is drained, we will wait queue.get which - # will wait for when a run_pmi instance has added back to the + # will wait for when a run_crossgen instance has added back to the subproc_count_queue = asyncio.Queue(chunk_size) diff_queue = asyncio.Queue() @@ -648,7 +648,7 @@ def run_crossgen(print_prefix, assembly_name): save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) helper = AsyncSubprocessHelper(g_Framework_Assemblies) - helper.run_to_completion(run_pmi) + helper.run_to_completion(run_crossgen) shutil.rmtree(ni_files_dirname, ignore_errors=True) From cb3f72c2af525ec7f023461072e6a5d20e2f192b Mon Sep 17 00:00:00 2001 From: jashook Date: Thu, 5 Mar 2020 13:36:38 -0800 Subject: [PATCH 05/15] Default to python 3.x --- .../templates/crossgen-comparison-job.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index c59e3e13afad09..583912b87cd0c4 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -117,6 +117,15 @@ jobs: mkdir $(workItemDirectory)\log\$(crossFlavor) displayName: Create directories + + # Populate Core_Root + - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly + displayName: Populate Core_Root + + - task: UsePythonVersion@0 + inputs: + versionSpec: 3.x + # Create baseline output on the host (x64) machine - task: PythonScript@0 displayName: Create cross-platform crossgen baseline @@ -169,23 +178,23 @@ jobs: WorkItemCommand: chmod +x $HELIX_WORKITEM_PAYLOAD/crossgen; mkdir -p $HELIX_WORKITEM_PAYLOAD/log/$(targetFlavor); - python -u $HELIX_CORRELATION_PAYLOAD/crossgen_comparison.py crossgen_framework + python3 -u $HELIX_CORRELATION_PAYLOAD/crossgen_comparison.py crossgen_framework --crossgen $HELIX_WORKITEM_PAYLOAD/crossgen --il_corelib $HELIX_WORKITEM_PAYLOAD/IL/System.Private.CoreLib.dll --core_root $HELIX_WORKITEM_PAYLOAD --result_dir $HELIX_WORKITEM_PAYLOAD/log/$(targetFlavor); - python -u $HELIX_CORRELATION_PAYLOAD/crossgen_comparison.py compare + python3 -u $HELIX_CORRELATION_PAYLOAD/crossgen_comparison.py compare --base_dir $HELIX_WORKITEM_PAYLOAD/log/$(crossFlavor) --diff_dir $HELIX_WORKITEM_PAYLOAD/log/$(targetFlavor) ${{ if eq(parameters.osName, 'Windows_NT') }}: WorkItemCommand: mkdir %HELIX_WORKITEM_PAYLOAD%\log\$(targetFlavor); - python -u %HELIX_CORRELATION_PAYLOAD%\crossgen_comparison.py crossgen_framework + python3 -u %HELIX_CORRELATION_PAYLOAD%\crossgen_comparison.py crossgen_framework --crossgen %HELIX_WORKITEM_PAYLOAD%\crossgen --il_corelib %HELIX_WORKITEM_PAYLOAD%\IL\System.Private.CoreLib.dll --core_root %HELIX_WORKITEM_PAYLOAD% --result_dir %HELIX_WORKITEM_PAYLOAD%\log\$(targetFlavor); - python -u %HELIX_CORRELATION_PAYLOAD%\crossgen_comparison.py compare + python3 -u %HELIX_CORRELATION_PAYLOAD%\crossgen_comparison.py compare --base_dir %HELIX_WORKITEM_PAYLOAD%\log\$(crossFlavor) --diff_dir %HELIX_WORKITEM_PAYLOAD%\log\$(targetFlavor) From 62cd881c692a1c4ace96aaafe295ef1b1a433f5c Mon Sep 17 00:00:00 2001 From: jashook Date: Thu, 5 Mar 2020 14:56:16 -0800 Subject: [PATCH 06/15] Force python3 --- eng/pipelines/coreclr/templates/crossgen-comparison-job.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index 583912b87cd0c4..ca37add2fbe084 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -122,16 +122,13 @@ jobs: - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly displayName: Populate Core_Root - - task: UsePythonVersion@0 - inputs: - versionSpec: 3.x - # Create baseline output on the host (x64) machine - task: PythonScript@0 displayName: Create cross-platform crossgen baseline inputs: scriptSource: 'filePath' scriptPath: $(coreClrRepoRoot)/tests/scripts/crossgen_comparison.py + pythonInterpreter: /usr/bin/python3 ${{ if ne(parameters.osGroup, 'Windows_NT') }}: arguments: crossgen_framework From 84742dcc80c1f49c3bcaf5264d6813d13529eeea Mon Sep 17 00:00:00 2001 From: jashook Date: Thu, 5 Mar 2020 16:21:31 -0800 Subject: [PATCH 07/15] Small python changes --- .../tests/scripts/crossgen_comparison.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/coreclr/tests/scripts/crossgen_comparison.py b/src/coreclr/tests/scripts/crossgen_comparison.py index be48f265a56393..1fa0cee1d69592 100644 --- a/src/coreclr/tests/scripts/crossgen_comparison.py +++ b/src/coreclr/tests/scripts/crossgen_comparison.py @@ -636,19 +636,22 @@ def crossgen_framework(args): il_corelib_filename = args.il_corelib_filename ni_files_dirname, debugging_files_dirname = create_output_folders() - ni_corelib_filename = os.path.join(ni_files_dirname, os.path.basename(il_corelib_filename)) - platform_assemblies_paths = [args.core_root] - crossgen_results = run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) - save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) + g_Framework_Assemblies = [il_corelib_filename] + g_Framework_Assemblies - def run_crossgen(print_prefix, assembly_name): - il_filename = os.path.join(args.core_root, assembly_name) - ni_filename = os.path.join(ni_files_dirname, add_ni_extension(assembly_name)) - crossgen_results = run_crossgen(args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname) - save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) + def run_crossgen_helper(print_prefix, assembly_name): + if assembly_name == il_corelib_filename: + ni_corelib_filename = os.path.join(ni_files_dirname, os.path.basename(il_corelib_filename)) + platform_assemblies_paths = [args.core_root] + crossgen_results = run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) + save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) + else: + il_filename = os.path.join(args.core_root, assembly_name) + ni_filename = os.path.join(ni_files_dirname, add_ni_extension(assembly_name)) + crossgen_results = run_crossgen(args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname) + save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) helper = AsyncSubprocessHelper(g_Framework_Assemblies) - helper.run_to_completion(run_crossgen) + helper.run_to_completion(run_crossgen_helper) shutil.rmtree(ni_files_dirname, ignore_errors=True) From 71723c3353a344f3e01f0564cbf025b88fd33751 Mon Sep 17 00:00:00 2001 From: jashook Date: Fri, 6 Mar 2020 09:12:07 -0800 Subject: [PATCH 08/15] Define asyncio --- src/coreclr/tests/scripts/crossgen_comparison.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/tests/scripts/crossgen_comparison.py b/src/coreclr/tests/scripts/crossgen_comparison.py index 1fa0cee1d69592..1306594c41fb8f 100644 --- a/src/coreclr/tests/scripts/crossgen_comparison.py +++ b/src/coreclr/tests/scripts/crossgen_comparison.py @@ -93,6 +93,7 @@ import argparse import datetime +import asyncio import glob import json import hashlib From 0752950fe189e15d95ff66c9e532a0762c6d3537 Mon Sep 17 00:00:00 2001 From: jashook Date: Mon, 23 Mar 2020 12:03:18 -0700 Subject: [PATCH 09/15] Update to run arm 18.04 cross --- eng/pipelines/common/platform-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 353e80222dfeb8..11770fd662119e 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -38,7 +38,7 @@ jobs: archType: arm platform: Linux_arm container: - image: ubuntu-16.04-cross-70a29f1-20200305143320 + image: ubuntu-18.04-cross-arm-16.04-09ec757-20200320131345 registry: mcr jobParameters: runtimeFlavor: ${{ parameters.runtimeFlavor }} From 151c20628dcffe027531fb74cc6cc2eff280d433 Mon Sep 17 00:00:00 2001 From: jashook Date: Mon, 23 Mar 2020 12:08:05 -0700 Subject: [PATCH 10/15] Remove dup core_root generation --- eng/pipelines/coreclr/templates/crossgen-comparison-job.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index ca37add2fbe084..9a1e5d482ac557 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -116,11 +116,7 @@ jobs: - script: | mkdir $(workItemDirectory)\log\$(crossFlavor) displayName: Create directories - - - # Populate Core_Root - - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly - displayName: Populate Core_Root + # Create baseline output on the host (x64) machine - task: PythonScript@0 From 695138ba2d4e9a86ae36f3e71004d0ad4e9d488d Mon Sep 17 00:00:00 2001 From: jashook Date: Tue, 24 Mar 2020 14:06:03 -0700 Subject: [PATCH 11/15] Update arm container --- eng/pipelines/common/platform-matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 11770fd662119e..ed0bcf606db795 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -38,7 +38,7 @@ jobs: archType: arm platform: Linux_arm container: - image: ubuntu-18.04-cross-arm-16.04-09ec757-20200320131345 + image: ubuntu-18.04-cross-arm-16.04-09ec757-20200324125113 registry: mcr jobParameters: runtimeFlavor: ${{ parameters.runtimeFlavor }} From 1d88239a6e6199470056fa767d560bc37f2c22bb Mon Sep 17 00:00:00 2001 From: jashook Date: Wed, 25 Mar 2020 12:06:08 -0700 Subject: [PATCH 12/15] Use 3.6 for crossgen_comparison --- .../tests/scripts/crossgen_comparison.py | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/src/coreclr/tests/scripts/crossgen_comparison.py b/src/coreclr/tests/scripts/crossgen_comparison.py index 1306594c41fb8f..395478eb321771 100644 --- a/src/coreclr/tests/scripts/crossgen_comparison.py +++ b/src/coreclr/tests/scripts/crossgen_comparison.py @@ -125,6 +125,7 @@ def build_argument_parser(): corelib_parser.add_argument('--crossgen', dest='crossgen_executable_filename', required=True) corelib_parser.add_argument('--il_corelib', dest='il_corelib_filename', required=True) corelib_parser.add_argument('--result_dir', dest='result_dirname', required=True) + corelib_parser.add_argument('--run_serial', dest='run_serial', action="store_true", default=False) corelib_parser.set_defaults(func=crossgen_corelib) framework_parser_description = """Runs crossgen on each assembly in Core_Root and @@ -135,6 +136,7 @@ def build_argument_parser(): framework_parser.add_argument('--il_corelib', dest='il_corelib_filename', required=True) framework_parser.add_argument('--core_root', dest='core_root', required=True) framework_parser.add_argument('--result_dir', dest='result_dirname', required=True) + framework_parser.add_argument('--run_serial', dest='run_serial', action="store_true", default=False) framework_parser.set_defaults(func=crossgen_framework) dotnet_sdk_parser_description = "Unpack .NET Core SDK archive file and runs crossgen on each assembly." @@ -143,6 +145,7 @@ def build_argument_parser(): dotnet_sdk_parser.add_argument('--il_corelib', dest='il_corelib_filename', required=True) dotnet_sdk_parser.add_argument('--dotnet_sdk', dest='dotnet_sdk_filename', required=True) dotnet_sdk_parser.add_argument('--result_dir', dest='result_dirname', required=True) + dotnet_sdk_parser.add_argument('--run_serial', dest='run_serial', action="store_true", default=False) dotnet_sdk_parser.set_defaults(func=crossgen_dotnet_sdk) compare_parser_description = "Compares crossgen results in directories {base_dir} and {diff_dir}" @@ -150,6 +153,7 @@ def build_argument_parser(): compare_parser = subparsers.add_parser('compare', description=compare_parser_description) compare_parser.add_argument('--base_dir', dest='base_dirname', required=True) compare_parser.add_argument('--diff_dir', dest='diff_dirname', required=True) + compare_parser.add_argument('--run_serial', dest='run_serial', action="store_true", default=False) compare_parser.set_defaults(func=compare_results) return parser @@ -240,7 +244,8 @@ def run_to_completion(self, async_callback, *extra_args): """ reset_env = os.environ.copy() - asyncio.run(self.__run_to_completion__(async_callback, *extra_args)) + loop = asyncio.get_event_loop() + loop.run_until_complete(self.__run_to_completion__(async_callback, *extra_args)) os.environ.update(reset_env) ################################################################################ @@ -639,10 +644,26 @@ def crossgen_framework(args): ni_files_dirname, debugging_files_dirname = create_output_folders() g_Framework_Assemblies = [il_corelib_filename] + g_Framework_Assemblies - def run_crossgen_helper(print_prefix, assembly_name): + async def run_crossgen_helper(print_prefix, assembly_name): + platform_assemblies_paths = [args.core_root] + print("{}{} {}".format(print_prefix, args.crossgen_executable_filename, assembly_name)) + + if assembly_name == il_corelib_filename: + ni_corelib_filename = os.path.join(ni_files_dirname, os.path.basename(il_corelib_filename)) + crossgen_results = run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) + save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) + else: + il_filename = os.path.join(args.core_root, assembly_name) + ni_filename = os.path.join(ni_files_dirname, add_ni_extension(assembly_name)) + crossgen_results = run_crossgen(args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname) + save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) + + def run_crossgen_helper_serial(print_prefix, assembly_name): + platform_assemblies_paths = [args.core_root] + print("{}{} {}".format(print_prefix, args.crossgen_executable_filename, assembly_name)) + if assembly_name == il_corelib_filename: ni_corelib_filename = os.path.join(ni_files_dirname, os.path.basename(il_corelib_filename)) - platform_assemblies_paths = [args.core_root] crossgen_results = run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) else: @@ -651,8 +672,13 @@ def run_crossgen_helper(print_prefix, assembly_name): crossgen_results = run_crossgen(args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname) save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) - helper = AsyncSubprocessHelper(g_Framework_Assemblies) - helper.run_to_completion(run_crossgen_helper) + if args.run_serial: + for index, item in enumerate(g_Framework_Assemblies): + run_crossgen_helper("[{}:{}]".format(index, len(g_Framework_Assemblies)), item) + + else: + helper = AsyncSubprocessHelper(g_Framework_Assemblies, verbose=True) + helper.run_to_completion(run_crossgen_helper) shutil.rmtree(ni_files_dirname, ignore_errors=True) From 7341c2f7ffb85a813c2a57c0577d1e59a7312926 Mon Sep 17 00:00:00 2001 From: jashook Date: Wed, 25 Mar 2020 14:23:04 -0700 Subject: [PATCH 13/15] Finish adding async defintions --- .../tests/scripts/crossgen_comparison.py | 73 ++++++++----------- 1 file changed, 30 insertions(+), 43 deletions(-) diff --git a/src/coreclr/tests/scripts/crossgen_comparison.py b/src/coreclr/tests/scripts/crossgen_comparison.py index 395478eb321771..a8d2903cc75355 100644 --- a/src/coreclr/tests/scripts/crossgen_comparison.py +++ b/src/coreclr/tests/scripts/crossgen_comparison.py @@ -125,7 +125,6 @@ def build_argument_parser(): corelib_parser.add_argument('--crossgen', dest='crossgen_executable_filename', required=True) corelib_parser.add_argument('--il_corelib', dest='il_corelib_filename', required=True) corelib_parser.add_argument('--result_dir', dest='result_dirname', required=True) - corelib_parser.add_argument('--run_serial', dest='run_serial', action="store_true", default=False) corelib_parser.set_defaults(func=crossgen_corelib) framework_parser_description = """Runs crossgen on each assembly in Core_Root and @@ -136,7 +135,6 @@ def build_argument_parser(): framework_parser.add_argument('--il_corelib', dest='il_corelib_filename', required=True) framework_parser.add_argument('--core_root', dest='core_root', required=True) framework_parser.add_argument('--result_dir', dest='result_dirname', required=True) - framework_parser.add_argument('--run_serial', dest='run_serial', action="store_true", default=False) framework_parser.set_defaults(func=crossgen_framework) dotnet_sdk_parser_description = "Unpack .NET Core SDK archive file and runs crossgen on each assembly." @@ -145,7 +143,6 @@ def build_argument_parser(): dotnet_sdk_parser.add_argument('--il_corelib', dest='il_corelib_filename', required=True) dotnet_sdk_parser.add_argument('--dotnet_sdk', dest='dotnet_sdk_filename', required=True) dotnet_sdk_parser.add_argument('--result_dir', dest='result_dirname', required=True) - dotnet_sdk_parser.add_argument('--run_serial', dest='run_serial', action="store_true", default=False) dotnet_sdk_parser.set_defaults(func=crossgen_dotnet_sdk) compare_parser_description = "Compares crossgen results in directories {base_dir} and {diff_dir}" @@ -153,7 +150,6 @@ def build_argument_parser(): compare_parser = subparsers.add_parser('compare', description=compare_parser_description) compare_parser.add_argument('--base_dir', dest='base_dirname', required=True) compare_parser.add_argument('--diff_dir', dest='diff_dirname', required=True) - compare_parser.add_argument('--run_serial', dest='run_serial', action="store_true", default=False) compare_parser.set_defaults(func=compare_results) return parser @@ -244,8 +240,11 @@ def run_to_completion(self, async_callback, *extra_args): """ reset_env = os.environ.copy() + loop = asyncio.get_event_loop() loop.run_until_complete(self.__run_to_completion__(async_callback, *extra_args)) + loop.close() + os.environ.update(reset_env) ################################################################################ @@ -444,22 +443,22 @@ def __init__(self, crossgen_executable_filename): self.crossgen_executable_filename = crossgen_executable_filename self.platform_assemblies_paths_sep = ';' if sys.platform == 'win32' else ':' - def crossgen_il_file(self, il_filename, ni_filename, platform_assemblies_paths): + async def crossgen_il_file(self, il_filename, ni_filename, platform_assemblies_paths): """ Runs a subprocess "{crossgen_executable_filename} /nologo /Platform_Assemblies_Paths /out {ni_filename} /in {il_filename}" and returns returncode, stdour, stderr. """ args = self._build_args_crossgen_il_file(il_filename, ni_filename, platform_assemblies_paths) - return self._run_with_args(args) + return await self._run_with_args(args) - def create_debugging_file(self, ni_filename, debugging_files_dirname, platform_assemblies_paths): + async def create_debugging_file(self, ni_filename, debugging_files_dirname, platform_assemblies_paths): """ Runs a subprocess "{crossgen_executable_filename} /nologo /Platform_Assemblies_Paths /CreatePerfMap {debugging_files_dirname} /in {il_filename}" on Unix or "{crossgen_executable_filename} /nologo /Platform_Assemblies_Paths /CreatePdb {debugging_files_dirname} /in {il_filename}" on Windows and returns returncode, stdout, stderr. """ args = self._build_args_create_debugging_file(ni_filename, debugging_files_dirname, platform_assemblies_paths) - return self._run_with_args(args) + return await self._run_with_args(args) def _build_args_crossgen_il_file(self, il_filename, ni_filename, platform_assemblies_paths): args = [] @@ -485,15 +484,22 @@ def _build_args_create_debugging_file(self, ni_filename, debugging_files_dirname args.append(ni_filename) return args - def _run_with_args(self, args): + async def _run_with_args(self, args): """ Creates a subprocess running crossgen with specified set of arguments, communicates with the owner process - waits for its termination and pulls returncode, stdour, stderr. """ - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() - return (p.returncode, stdout.decode(), stderr.decode()) + stdout = None + stderr = None + + proc = await asyncio.create_subprocess_shell(" ".join(args), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE) + stdout, stderr = await proc.communicate() + + return (proc.returncode, stdout.decode(), stderr.decode()) def compute_file_hashsum(filename): @@ -574,9 +580,9 @@ class FileTypes: NativeOrReadyToRunImage = 'NativeOrReadyToRunImage' DebuggingFile = 'DebuggingFile' -def run_crossgen(crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname): +async def run_crossgen(crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname): runner = CrossGenRunner(crossgen_executable_filename) - returncode, stdout, stderr = runner.crossgen_il_file(il_filename, ni_filename, platform_assemblies_paths) + returncode, stdout, stderr = await runner.crossgen_il_file(il_filename, ni_filename, platform_assemblies_paths) ni_file_hashsum = compute_file_hashsum(ni_filename) if returncode == 0 else None ni_file_size_in_bytes = os.path.getsize(ni_filename) if returncode == 0 else None assembly_name = get_assembly_name(il_filename) @@ -586,7 +592,7 @@ def run_crossgen(crossgen_executable_filename, il_filename, ni_filename, platfor return [crossgen_assembly_result] platform_assemblies_paths = platform_assemblies_paths + [os.path.dirname(ni_filename)] - returncode, stdout, stderr = runner.create_debugging_file(ni_filename, debugging_files_dirname, platform_assemblies_paths) + returncode, stdout, stderr = await runner.create_debugging_file(ni_filename, debugging_files_dirname, platform_assemblies_paths) if returncode == 0: filenames = list(filter(lambda filename: not re.match("^{0}\.ni\.".format(assembly_name), filename, re.IGNORECASE) is None, os.listdir(debugging_files_dirname))) @@ -617,7 +623,7 @@ def create_output_folders(): os.mkdir(debugging_files_dirname) return ni_files_dirname, debugging_files_dirname -def crossgen_corelib(args): +async def crossgen_corelib(args): il_corelib_filename = args.il_corelib_filename assembly_name = os.path.basename(il_corelib_filename) ni_corelib_dirname, debugging_files_dirname = create_output_folders() @@ -629,7 +635,7 @@ def crossgen_corelib(args): print("IL Corelib path does not exist.") sys.exit(1) - crossgen_results = run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) + crossgen_results = await run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) shutil.rmtree(ni_corelib_dirname, ignore_errors=True) save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) @@ -650,35 +656,16 @@ async def run_crossgen_helper(print_prefix, assembly_name): if assembly_name == il_corelib_filename: ni_corelib_filename = os.path.join(ni_files_dirname, os.path.basename(il_corelib_filename)) - crossgen_results = run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) - save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) - else: - il_filename = os.path.join(args.core_root, assembly_name) - ni_filename = os.path.join(ni_files_dirname, add_ni_extension(assembly_name)) - crossgen_results = run_crossgen(args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname) - save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) - - def run_crossgen_helper_serial(print_prefix, assembly_name): - platform_assemblies_paths = [args.core_root] - print("{}{} {}".format(print_prefix, args.crossgen_executable_filename, assembly_name)) - - if assembly_name == il_corelib_filename: - ni_corelib_filename = os.path.join(ni_files_dirname, os.path.basename(il_corelib_filename)) - crossgen_results = run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) + crossgen_results = await run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) else: il_filename = os.path.join(args.core_root, assembly_name) ni_filename = os.path.join(ni_files_dirname, add_ni_extension(assembly_name)) - crossgen_results = run_crossgen(args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname) + crossgen_results = await run_crossgen(args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname) save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) - if args.run_serial: - for index, item in enumerate(g_Framework_Assemblies): - run_crossgen_helper("[{}:{}]".format(index, len(g_Framework_Assemblies)), item) - - else: - helper = AsyncSubprocessHelper(g_Framework_Assemblies, verbose=True) - helper.run_to_completion(run_crossgen_helper) + helper = AsyncSubprocessHelper(g_Framework_Assemblies, verbose=True) + helper.run_to_completion(run_crossgen_helper) shutil.rmtree(ni_files_dirname, ignore_errors=True) @@ -702,7 +689,7 @@ def dotnet_sdk_enumerate_assemblies(dotnet_sdk_dirname): filenames = filter(lambda filename: filename != 'System.Private.CoreLib.dll', filenames) yield (dirpath, filenames) -def crossgen_dotnet_sdk(args): +async def crossgen_dotnet_sdk(args): dotnet_sdk_dirname = tempfile.mkdtemp() with tarfile.open(args.dotnet_sdk_filename) as dotnet_sdk_tarfile: dotnet_sdk_tarfile.extractall(dotnet_sdk_dirname) @@ -711,7 +698,7 @@ def crossgen_dotnet_sdk(args): ni_files_dirname, debugging_files_dirname = create_output_folders() ni_corelib_filename = os.path.join(ni_files_dirname, os.path.basename(il_corelib_filename)) platform_assemblies_paths = [os.path.dirname(il_corelib_filename)] - crossgen_results = run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) + crossgen_results = await run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) platform_assemblies_paths = [ni_files_dirname] @@ -723,7 +710,7 @@ def crossgen_dotnet_sdk(args): for assembly_name in assembly_names: il_filename = os.path.join(il_files_dirname, assembly_name) ni_filename = os.path.join(ni_files_dirname, add_ni_extension(assembly_name)) - crossgen_results = run_crossgen(args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname) + crossgen_results = await run_crossgen(args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname) save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) shutil.rmtree(ni_files_dirname, ignore_errors=True) From c19887dcc1646e67de4c505bb07a70abedab234b Mon Sep 17 00:00:00 2001 From: jashook Date: Wed, 25 Mar 2020 14:44:34 -0700 Subject: [PATCH 14/15] Fix spacing in file --- eng/pipelines/coreclr/templates/crossgen-comparison-job.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index 9a1e5d482ac557..cfecf9bbd5c06f 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -101,7 +101,6 @@ jobs: artifactName: '$(librariesBuildArtifactName)' displayName: 'live-built libraries' - # Populate Core_Root - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly displayName: Populate Core_Root @@ -116,7 +115,6 @@ jobs: - script: | mkdir $(workItemDirectory)\log\$(crossFlavor) displayName: Create directories - # Create baseline output on the host (x64) machine - task: PythonScript@0 @@ -140,7 +138,6 @@ jobs: --core_root $(workItemDirectory) --result_dir $(workItemDirectory)\log\$(crossFlavor) - # Dump contents and payload information - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - script: | @@ -152,7 +149,6 @@ jobs: dir $(workItemDirectory) displayName: Dump contents and payload information - # Send payload to Helix where the native output is generated and compared to the baseline - template: /eng/common/templates/steps/send-to-helix.yml parameters: From 750e14b2ef5042fd5a2c8dca25809e6d1713f87c Mon Sep 17 00:00:00 2001 From: jashook Date: Wed, 25 Mar 2020 14:46:29 -0700 Subject: [PATCH 15/15] Update docker image used in linux-requirements --- docs/workflow/requirements/linux-requirements.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/workflow/requirements/linux-requirements.md b/docs/workflow/requirements/linux-requirements.md index ec65029cdb6f42..f7180dcf77e186 100644 --- a/docs/workflow/requirements/linux-requirements.md +++ b/docs/workflow/requirements/linux-requirements.md @@ -48,7 +48,7 @@ These instructions might fall stale often enough as we change our images as our | Alpine | x64 | `mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-WithNode-0fc54a3-20190918214015` | - | | CentOS 6 (build for RHEL 6) | x64 | `mcr.microsoft.com/dotnet-buildtools/prereqs:centos-6-6aaa05d-20191106231336` | - | | CentOS 7 (build for Linux x64) | x64 | `mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-6aaa05d-20191106231356` | - | -| Ubuntu 16.04 | arm32(armhf) | `mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-14.04-23cacb0-20191023143847` | `/crossrootfs/arm` | +| Ubuntu 16.04 | arm32(armhf) | `mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm-16.04-09ec757-20200324125113` | `/crossrootfs/arm` | | Ubuntu 16.04 | arm64 (arm64v8) | `mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-cfdd435-20191023143847` | `/crossrootfs/arm64` | | Alpine | arm64 (arm64v8) | `mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-alpine-406629a-20191023143847` | `/crossrootfs/arm64` |