From 161dde2b8389cf7eadbcb3d5a066e740af74dba3 Mon Sep 17 00:00:00 2001 From: Adeel <3840695+am11@users.noreply.github.com> Date: Tue, 13 Sep 2022 08:17:41 +0300 Subject: [PATCH 1/7] Remove usages of Ubuntu 16.04 from pipelines --- eng/build.sh | 2 +- eng/pipelines/common/global-build-job.yml | 11 +++++----- eng/pipelines/common/platform-matrix.yml | 4 ++-- .../templates/runtimes/build-test-job.yml | 8 ++++---- .../coreclr/templates/build-jit-job.yml | 4 ++-- eng/pipelines/coreclr/templates/build-job.yml | 4 ++-- .../coreclr/templates/helix-queues-setup.yml | 4 ++-- eng/pipelines/installer/jobs/build-job.yml | 20 +++++-------------- eng/pipelines/libraries/base-job.yml | 14 +------------ .../libraries/helix-queues-setup.yml | 6 +++--- eng/testing/tests.singlefile.targets | 4 +++- .../Microsoft.NETCore.Native.Unix.targets | 4 ++++ .../Microsoft.NETCore.Native.targets | 5 ++++- .../tools/aot/crossgen2/crossgen2.csproj | 4 +++- .../NativeExports/NativeExports.csproj | 9 ++++++++- 15 files changed, 50 insertions(+), 53 deletions(-) diff --git a/eng/build.sh b/eng/build.sh index f9ef572f3a06ca..fdba03186cd66a 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -397,7 +397,7 @@ while [[ $# > 0 ]]; do shift 2 ;; - -cross) + -cross|*crossbuild=true*) crossBuild=1 arguments="$arguments /p:CrossBuild=True" shift 1 diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index e1a8e45ba97bc7..38fbf87810b345 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -75,18 +75,19 @@ jobs: - name: _osParameter value: -os ${{ parameters.osGroup }} - - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, '_musl')) }}: - - name: _osParameter - value: /p:RuntimeOS=linux-musl /p:OutputRid=linux-musl-${{ parameters.archType }} - - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, '_bionic')) }}: - name: _osParameter - value: /p:RuntimeOS=linux-bionic /p:OutputRid=linux-bionic-${{ parameters.archType }} + value: /p:RuntimeOS=linux-bionic # Do not rename as it clashes with MSBuild property in libraries/build-native.proj - name: _crossBuildPropertyArg value: /p:CrossBuild=${{ parameters.crossBuild }} + - ${{ if ne(parameters.jobParameters.crossrootfsDir, '') }}: + # This is only required for cross builds. + - name: ROOTFS_DIR + value: ${{ parameters.jobParameters.crossrootfsDir }} + - name: _cxx11Parameter ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.archType, 'arm64')) }}: value: /p:MonoLLVMUseCxx11Abi=true diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 3dc09537df68e2..50875f25bb07d6 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -139,7 +139,7 @@ jobs: platform: Linux_musl_arm shouldContinueOnError: ${{ parameters.shouldContinueOnError }} container: - image: ubuntu-16.04-cross-arm-alpine-20210923140502-78f7860 + image: ubuntu-18.04-cross-arm-alpine-20220912173203-78f7860 registry: mcr jobParameters: runtimeFlavor: ${{ parameters.runtimeFlavor }} @@ -165,7 +165,7 @@ jobs: platform: Linux_musl_arm64 shouldContinueOnError: ${{ parameters.shouldContinueOnError }} container: - image: ubuntu-16.04-cross-arm64-alpine-20210923140502-78f7860 + image: ubuntu-22.04-cross-arm64-alpine-20220912173307-b016fdd registry: mcr jobParameters: runtimeFlavor: ${{ parameters.runtimeFlavor }} diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index 764446c77e86b9..1e8e33277b8cdf 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -39,9 +39,9 @@ jobs: pool: ${{ parameters.pool }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc'), not(and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'), eq(parameters.archType, 'x64'))), not(eq(parameters.osGroup, 'OSX'))) }}: + ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc'), not(and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'))), not(eq(parameters.osGroup, 'OSX'))) }}: compilerArg: '-clang9' - ${{ if not(and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc'), not(and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'), eq(parameters.archType, 'x64'))), not(eq(parameters.osGroup, 'OSX')))) }}: + ${{ if not(and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc'), not(and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'))), not(eq(parameters.osGroup, 'OSX')))) }}: compilerArg: '' # Test jobs should continue on error for internal builds @@ -79,8 +79,8 @@ jobs: - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc')) }}: - name: compilerArg value: '-clang9' - # Building for x64 MUSL happens on Alpine Linux and we need to use the stable version available there - - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'), eq(parameters.archType, 'x64')) }}: + # We need to use the stable version available on Alpine Linux + - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl')) }}: - name: compilerArg value: '' # AppleClang has different version scheme, so we let compiler introspection pick up the available clang from PATH diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index bdd1a29ac8b997..9f3beb3f0f40a5 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -58,8 +58,8 @@ jobs: - ${{ if ne(parameters.osGroup, 'windows') }}: - name: compilerArg value: '-clang9' - # Building for x64 MUSL happens on Alpine Linux and we need to use the stable version available there - - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'), eq(parameters.archType, 'x64')) }}: + # We need to use the stable version available on Alpine Linux + - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl')) }}: - name: compilerArg value: '' # AppleClang has different version scheme, so we let compiler introspection pick up the available clang from PATH diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index 930580eae47391..98ae1b60bba531 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -93,8 +93,8 @@ jobs: - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc')) }}: - name: compilerArg value: '-clang9' - # Building for x64 MUSL happens on Alpine Linux and we need to use the stable version available there - - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'), eq(parameters.archType, 'x64')) }}: + # We need to use the stable version available on Alpine Linux + - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl')) }}: - name: compilerArg value: '' # AppleClang has different version scheme, so we let compiler introspection pick up the available clang from PATH diff --git a/eng/pipelines/coreclr/templates/helix-queues-setup.yml b/eng/pipelines/coreclr/templates/helix-queues-setup.yml index 796a36abbda258..6948e5bb4e901c 100644 --- a/eng/pipelines/coreclr/templates/helix-queues-setup.yml +++ b/eng/pipelines/coreclr/templates/helix-queues-setup.yml @@ -78,7 +78,7 @@ jobs: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - (Alpine.314.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64-20210910135833-1848e19 - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - (Alpine.314.Amd64)ubuntu.1604.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64-20210910135833-1848e19 + - (Alpine.314.Amd64)ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64-20210910135833-1848e19 # Linux musl arm32 - ${{ if eq(parameters.platform, 'Linux_musl_arm') }}: @@ -110,7 +110,7 @@ jobs: - (Debian.10.Amd64)Ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64-20220810215022-f344011 - (Debian.11.Amd64)Ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64-20220810215032-f344011 - Ubuntu.1804.Amd64 - - (Centos.8.Amd64)Ubuntu.1604.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-helix-20201229003624-c1bf759 + - (Centos.8.Amd64)Ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-helix-20201229003624-c1bf759 - (Fedora.34.Amd64)Ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix-20220809205730-e7e8d1c - RedHat.7.Amd64 diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index 44abaeddb39c1d..96af363b9f0895 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -194,34 +194,24 @@ jobs: value: ${{ eq(parameters.osSubgroup, '') }} - ${{ if and(eq(parameters.osSubgroup, '_musl'), eq(parameters.osGroup, 'Linux')) }}: - # Set output RID manually: musl isn't properly detected. Make sure to also convert linux to - # lowercase for RID format. (Detection normally converts, but we're preventing it.) - - name: OutputRidArg - value: /p:OutputRid=linux-musl-${{ parameters.archType }} - - name: RuntimeOSArg - value: /p:RuntimeOS=linux-musl - name: _PortableBuild value: true + - ${{ if eq(parameters.crossBuild, true) }}: + - name: ArchArg + value: -arch ${{ parameters.archType }} + - name: BaseJobBuildCommand value: >- $(Build.SourcesDirectory)/build.sh -subset host+packs -ci $(BuildAction) + $(ArchArg) /p:CrossBuild=${{ parameters.crossBuild }} /p:PortableBuild=$(_PortableBuild) /p:SkipTests=$(SkipTests) $(pgoInstrumentArg) $(LiveOverridePathArgs) $(CommonMSBuildArgs) - $(OutputRidArg) - $(RuntimeOSArg) - - - name: PublishArguments - value: >- - /p:PortableBuild=$(_PortableBuild) - $(CommonMSBuildArgs) - $(OutputRidArg) - /bl:msbuild.publish.binlog - name: installersSubsetArg value: --subset packs.installers diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 3e5119b3e8a83f..95d96c02745ca5 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -53,7 +53,6 @@ jobs: - _msbuildCommonParameters: '' # rename this variable, due to collision with build-native.proj - - _runtimeOSArg: '' - _osArg: '' - _finalFrameworkArg: '' - _testModeArg: '' @@ -71,12 +70,6 @@ jobs: - ${{ if eq(parameters.crossBuild, true) }}: - _crossBuildPropertyArg: /p:CrossBuild=true - - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, '_musl')) }}: - - _runtimeOSArg: /p:RuntimeOS=linux-musl - - - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubGroup, ''), eq(parameters.archType, 'arm')) }}: - - _runtimeOSArg: /p:RuntimeOS=ubuntu.16.04 - # force a value for OS when cross-building - ${{ if in(parameters.osGroup, 'Browser', 'iOS', 'tvOS', 'Android', 'FreeBSD') }}: - _osArg: -os ${{ parameters.osGroup }} @@ -111,15 +104,10 @@ jobs: - ${{ if ne(parameters.testDisplayName, '') }}: - _testRunNamePrefixSuffix: ${{ parameters.testDisplayName }} - # Windows variables - - ${{ if eq(parameters.osGroup, 'windows') }}: - - _runtimeOSArg: /p:RuntimeOS=win10 - - # Non-Windows variables - ${{ if ne(parameters.osGroup, 'windows') }}: - _buildScript: ./$(_buildScriptFileName)$(scriptExt) - - _buildArguments: $(_runtimeConfigurationArg) -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_testModeArg) $(_testScopeArg) $(_osArg) $(_runtimeOSArg) $(_msbuildCommonParameters) $(_runtimeArtifactsPathArg) $(_crossBuildPropertyArg) + - _buildArguments: $(_runtimeConfigurationArg) -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_testModeArg) $(_testScopeArg) $(_osArg) $(_msbuildCommonParameters) $(_runtimeArtifactsPathArg) $(_crossBuildPropertyArg) - ${{ parameters.variables }} # we need to override this value to support build-coreclr-and-libraries-job.yml diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index 86c3b7cf491087..e76f18eff36963 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -74,10 +74,10 @@ jobs: - (Fedora.34.Amd64.Open)ubuntu.1804.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix-20220908220224-98605e8 - Ubuntu.2204.Amd64.Open - (Debian.11.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64-20220908132354-59ee6b3 - - (Mariner.1.0.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-1.0-helix-20210528192219-92bf620 - - (openSUSE.15.2.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:opensuse-15.2-helix-amd64-20220912173041-1d2766c + - (Mariner.1.0.Amd64.Open)ubuntu.1804.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-1.0-helix-20210528192219-92bf620 + - (openSUSE.15.2.Amd64.Open)ubuntu.1804.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:opensuse-15.2-helix-amd64-20220912173041-1d2766c - ${{ if or(ne(parameters.jobParameters.isExtraPlatforms, true), eq(parameters.jobParameters.includeAllPlatforms, true)) }}: - - (Centos.7.Amd64.Open)Ubuntu.1604.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-helix-20220601183719-dde38af + - (Centos.7.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-helix-20220601183719-dde38af - RedHat.7.Amd64.Open - (Debian.10.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64-20220908132009-59ee6b3 - Ubuntu.1804.Amd64.Open diff --git a/eng/testing/tests.singlefile.targets b/eng/testing/tests.singlefile.targets index d307e837e6235a..8c249d575fb703 100644 --- a/eng/testing/tests.singlefile.targets +++ b/eng/testing/tests.singlefile.targets @@ -21,7 +21,9 @@ $(CoreCLRILCompilerDir) $(CoreCLRCrossILCompilerDir) - clang-9 + clang-15 + clang-9 + true $(ROOTFS_DIR) $(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll $(CoreCLRAotSdkDir) diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets index 58937c04c18da6..5ea5c13908a171 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets @@ -20,6 +20,7 @@ The .NET Foundation licenses this file to you under the MIT license. $(CppCompilerAndLinker) ar --flat + false @@ -110,6 +111,9 @@ The .NET Foundation licenses this file to you under the MIT license. + + + diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets index 417d4567a95189..90688aec24a047 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets @@ -340,9 +340,12 @@ The .NET Foundation licenses this file to you under the MIT license. <_IgnoreLinkerWarnings>false <_IgnoreLinkerWarnings Condition="'$(TargetOS)' == 'OSX'">true + + + <_UnsupportedLinkerOptionRegex>.*(nostart-stop-gc).* - + diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2.csproj b/src/coreclr/tools/aot/crossgen2/crossgen2.csproj index 8696352abc03e8..594636eb2f474f 100644 --- a/src/coreclr/tools/aot/crossgen2/crossgen2.csproj +++ b/src/coreclr/tools/aot/crossgen2/crossgen2.csproj @@ -32,7 +32,9 @@ $(CoreCLRILCompilerDir) $(CoreCLRCrossILCompilerDir) - clang-9 + clang-15 + clang-9 + true $(ROOTFS_DIR) $(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll $(CoreCLRAotSdkDir) diff --git a/src/libraries/System.Runtime.InteropServices/tests/TestAssets/NativeExports/NativeExports.csproj b/src/libraries/System.Runtime.InteropServices/tests/TestAssets/NativeExports/NativeExports.csproj index d420161d1a6ebe..ef29b282725ca3 100644 --- a/src/libraries/System.Runtime.InteropServices/tests/TestAssets/NativeExports/NativeExports.csproj +++ b/src/libraries/System.Runtime.InteropServices/tests/TestAssets/NativeExports/NativeExports.csproj @@ -35,6 +35,13 @@ StandardOutputImportance="Low"> + + + + --target=$(TargetTriple) --gcc-toolchain=$(ROOTFS_DIR)/usr --sysroot=$(ROOTFS_DIR) - $(CommonToolchainArgs) $(DnneLinkerUserFlags.Replace(';',' ')) + $(CommonToolchainArgs) $(DnneLinkerCommand) $(DnneLinkerUserFlags.Replace(';',' ')) $(CommonToolchainArgs) $(DnneCompilerUserFlags.Replace(';',' ')) From a4f9bb00d767bb41849dd2a54fe54e885d275990 Mon Sep 17 00:00:00 2001 From: Adeel <3840695+am11@users.noreply.github.com> Date: Tue, 13 Sep 2022 08:12:21 +0300 Subject: [PATCH 2/7] Delete old jenkins scripts --- src/tests/Common/scripts/arm32_ci_script.sh | 611 ----------------- src/tests/Common/scripts/arm32_ci_test.sh | 115 ---- src/tests/Common/scripts/run-pmi-diffs.py | 719 -------------------- src/tests/Common/scripts/x86_ci_script.sh | 58 -- 4 files changed, 1503 deletions(-) delete mode 100755 src/tests/Common/scripts/arm32_ci_script.sh delete mode 100755 src/tests/Common/scripts/arm32_ci_test.sh delete mode 100755 src/tests/Common/scripts/run-pmi-diffs.py delete mode 100755 src/tests/Common/scripts/x86_ci_script.sh diff --git a/src/tests/Common/scripts/arm32_ci_script.sh b/src/tests/Common/scripts/arm32_ci_script.sh deleted file mode 100755 index 08892b0ffb01e9..00000000000000 --- a/src/tests/Common/scripts/arm32_ci_script.sh +++ /dev/null @@ -1,611 +0,0 @@ -#!/usr/bin/env bash - -#Usage message -function usage { - echo 'ARM Emulator Cross Build and Test Script' - echo 'This script cross builds coreclr source and tests the binaries generated' - echo '' - echo 'Typical usage:' - echo ' coreclr source is at ~/clr' - echo ' corefx source is at ~/cfx' - echo ' --testRootDir and --mscorlibDir have been built on Windows/downloaded from dotnet-ci.cloudapp.net' - echo ' --coreFxNativeBinDir has been built using cross build' - echo ' --coreFxBinDir has been built on Linux' - echo '$ cd ~/clr' - echo '$ ./tests/scripts/arm32_ci_script.sh' - echo ' --emulatorPath=/opt/linux-arm-emulator' - echo ' --mountPath=/opt/linux-arm-emulator-root' - echo ' --buildConfig=Release' - echo ' --testRootDir=~/Downloads/windows.x64.Release' - echo ' --mscorlibDir=~/clr/bin/bin/coreclr/Linux.armel.Release' - echo ' --coreFxNativeBinDir=~/cfx/bin/Linux.armel.Release' - echo ' --coreFxBinDir="~/cfx/bin/Linux.AnyCPU.Release;~/cfx/bin/Unix.AnyCPU.Release;~/cfx/bin/AnyOS.AnyCPU.Release"' - echo ' --testDirFile=~/clr/tests/testsRunningInsideARM.txt' - echo '' - echo 'Required Arguments:' - echo ' --emulatorPath= : Path of the emulator folder (without ending /)' - echo ' /platform/rootfs-t30.ext4 should exist' - echo ' --mountPath= : The desired path for mounting the emulator rootfs (without ending /)' - echo ' This path is created if not already present' - echo ' --buildConfig= : The value of config should be either Debug, Checked or Release' - echo ' Any other value is not accepted' - echo 'Optional Arguments:' - echo ' --mode= : docker or emulator (default)' - echo ' --arm : Build using hard ABI' - echo ' --armel : Build using softfp ABI (default)' - echo ' --linuxCodeName= : Code name for Linux: For arm, trusty (default) and xenial. For armel, tizen' - echo ' --skipRootFS : Skip building rootfs' - echo ' --skipTests : Presenting this option skips testing the generated binaries' - echo ' If this option is not presented, then tests are run by default' - echo ' using the other test related options' - echo ' --skipmscorlib : Skips generating mscorlib.dll on Linux' - echo ' If tests are run and this option is not used,' - echo ' then --mscorlibDir option to this script is mandatory' - echo ' -v --verbose : Build made verbose' - echo ' -h --help : Prints this usage message and exits' - echo '' - echo 'Test related Arguments (mandatory if --skipTests is not used):' - echo ' --testRootDir= : The root directory of the test build' - echo ' --mscorlibDir= : The directory containing the mscorlib.dll binary' - echo ' If provided, then the mscorlib.dll in this directory is' - echo ' used for tests instead of the built mscorlib.dll' - echo ' --coreFxNativeBinDir= : The directory of the CoreFX native build' - echo ' --coreFxBinDir="[;]" : List one or more directories with CoreFX managed build binaries' - echo ' --testDirFile= : Runs tests only in the directories specified by the file at ' - echo ' The directories are listed in lines in the file at ' - echo '' - echo 'Any other argument triggers an error and this usage message is displayed' - exit 1 -} - -#Display error message and exit -function exit_with_error { - set +x - - local errorMessage="$1" - local printUsage=$2 - - echo "ERROR: $errorMessage" - if [[ "$printUsage" == "true" ]]; then - echo '' - usage - fi - exit 1 -} - -#Exit if input string is empty -function exit_if_empty { - local inputString="$1" - local errorMessage="$2" - local printUsage=$3 - - if [ -z "$inputString" ]; then - exit_with_error "$errorMessage" $printUsage - fi -} - -#Exit if the input path does not exist -function exit_if_path_absent { - local path="$1" - local errorMessage="$2" - local printUsage=$3 - - if [ ! -f "$path" -a ! -d "$path" ]; then - exit_with_error "$errorMessage" $printUsage - fi -} - -#Check if the git changes were reverted completely -function check_git_head { - local currentGitHead=`git rev-parse --verify HEAD` - - if [[ "$__initialGitHead" != "$currentGitHead" ]]; then - exit_with_error "Some changes made to the code history were not completely reverted. Intial Git HEAD: $__initialGitHead, current Git HEAD: $currentGitHead" false - fi -} - -function unmount_rootfs { - local rootfsFolder="$1" - - #Check if there are any open files in this directory. - if [ -d $rootfsFolder ]; then - #If we find information about the file - if sudo lsof +D $rootfsFolder; then - (set +x; echo 'See above for lsof information. Continuing with the build.') - fi - fi - - if mountpoint -q -- "$rootfsFolder"; then - sudo umount "$rootfsFolder" - fi -} - -#Clean the previous build files inside the emulator -function clean_emulator { - #Remove any previous copies of the coreclr and the corefx directories in the emulator - sudo rm -rf "$__ARMRootfsCoreclrPath" "$__ARMRootfsCorefxPath" -} - -#Clean the changes made to the environment by the script -function clean_env { - #Clean the emulator - clean_emulator - - #Check for revert of git changes - check_git_head - - sudo rm -rf "/mnt/arm32_ci_temp" -} - -#Trap Ctrl-C and handle it -function handle_ctrl_c { - set +x - - echo 'ERROR: Ctrl-C handled. Script aborted before complete execution.' - - exit 1 -} -trap handle_ctrl_c INT - -#Trap Exit and handle it -function handle_exit { - set +x - - echo 'The script is exited. Cleaning environment..' - - if [[ "$__ciMode" == "emulator" ]]; then - clean_env - fi -} -trap handle_exit EXIT - - -#Mount with checking to be already existed -function mount_with_checking { - set +x - local options="$1" - local from="$2" - local rootfsFolder="$3" - - if mountpoint -q -- "$rootfsFolder"; then - (set +x; echo "$rootfsFolder is already mounted.") - else { - (set -x; sudo mount $options "$from" "$rootfsFolder") - } - fi -} - -#Mount emulator to the target mount path -function mount_emulator { - #Check if the mount path exists and create if necessary - if [ ! -d "$__ARMRootfsMountPath" ]; then - sudo mkdir "$__ARMRootfsMountPath" - fi - - if [ ! -d "$__ARMEmulRootfs" ]; then - sudo mkdir "$__ARMEmulRootfs" - fi - - if [ ! -f "$__ARMEmulRootfs/arm-emulator-rootfs.tar" ]; then - if mountpoint -q -- "$__ARMRootfsMountPath"; then - sudo umount -l $__ARMRootfsMountPath - fi - mount_with_checking "" "$__ARMEmulPath/platform/rootfs-t30.ext4" "$__ARMRootfsMountPath" - - cd $__ARMRootfsMountPath - sudo tar -cf "$__ARMEmulRootfs/arm-emulator-rootfs.tar" * - cd - - fi - - sudo tar -xf "$__ARMEmulRootfs/arm-emulator-rootfs.tar" -C "$__ARMEmulRootfs" - - mount_with_checking "-t proc" "/proc" "$__ARMEmulRootfs/proc" - mount_with_checking "-o bind" "/dev/" "$__ARMEmulRootfs/dev" - mount_with_checking "-o bind" "/dev/pts" "$__ARMEmulRootfs/dev/pts" - mount_with_checking "-t tmpfs" "shm" "$__ARMEmulRootfs/run/shm" - mount_with_checking "-o bind" "/sys" "$__ARMEmulRootfs/sys" - if [ ! -d "$__ARMEmulRootfs/bindings/tmp" ]; then - sudo mkdir -p "$__ARMEmulRootfs/bindings/tmp" - fi - mount_with_checking "-o bind" "/mnt" "$__ARMEmulRootfs/bindings/tmp" - - if [ ! -d "$__ARMEmulRootfs/$__TempFolder" ]; then - sudo mkdir "$__ARMEmulRootfs/$__TempFolder" - fi -} - -#Cross builds coreclr -function cross_build_coreclr { -#Export the needed environment variables - (set +x; echo 'Exporting LINUX_ARM_* environment variable') - source "$__ARMEmulRootfs"/dotnet/setenv/setenv_incpath.sh "$__ARMEmulRootfs" - - #Apply release optimization patch if needed - if [[ "$__buildConfig" == "Release" ]]; then - (set +x; echo 'Applying release optimization patch to build in Release mode') - git am < "$__ARMEmulRootfs"/dotnet/setenv/coreclr_release.patch - fi - - #Cross building for emulator rootfs - ROOTFS_DIR="$__ARMEmulRootfs" CPLUS_INCLUDE_PATH=$LINUX_ARM_INCPATH CXXFLAGS=$LINUX_ARM_CXXFLAGS ./build.sh $__buildArch cross $__verboseFlag $__skipMscorlib clang3.5 $__buildConfig - - #Reset the code to the upstream version - (set +x; echo 'Rewinding HEAD to main code') - if [[ "$__buildConfig" == "Release" ]]; then - git reset --hard HEAD^ - fi -} - -#Cross builds coreclr using Docker -function cross_build_coreclr_with_docker { - __currentWorkingDirectory=`pwd` - - # Check build configuration and choose Docker image - __dockerEnvironmentVariables="" - if [[ "$__buildArch" == "arm" ]]; then - # TODO: For arm, we are going to embed RootFS inside Docker image. - case $__linuxCodeName in - trusty) - __dockerImage=" mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-14.04-cross-0cd4667-20172211042239" - __skipRootFS=1 - __dockerEnvironmentVariables+=" -e ROOTFS_DIR=/crossrootfs/arm" - __runtimeOS="ubuntu.14.04" - ;; - xenial) - __dockerImage=" mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548" - __skipRootFS=1 - __dockerEnvironmentVariables+=" -e ROOTFS_DIR=/crossrootfs/arm" - __runtimeOS="ubuntu.16.04" - ;; - *) - exit_with_error "ERROR: $__linuxCodeName is not a supported linux name for $__buildArch" false - ;; - esac - elif [[ "$__buildArch" == "armel" ]]; then - # For armel Tizen, we are going to construct RootFS on the fly. - case $__linuxCodeName in - tizen) - __dockerImage=" tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1" - __skipRootFS=1 - __dockerEnvironmentVariables+=" -e ROOTFS_DIR=/crossrootfs/armel.tizen.build" - __runtimeOS="tizen.5.0.0" - ;; - *) - echo "ERROR: $__linuxCodeName is not a supported linux name for $__buildArch" - exit_with_error "ERROR: $__linuxCodeName is not a supported linux name for $__buildArch" false - ;; - esac - else - exit_with_error "ERROR: unknown buildArch $__buildArch" false - fi - __dockerCmd="sudo docker run ${__dockerEnvironmentVariables} --privileged -i --rm -v $__currentWorkingDirectory:/opt/code -w /opt/code $__dockerImage" - - if [[ "$__skipRootFS" == 0 ]]; then - # Build rootfs - __buildRootfsCmd="$__RepoRootDir/eng/common/cross/build-rootfs.sh $__buildArch $__linuxCodeName --skipunmount" - - (set +x; echo "Build RootFS for $__buildArch $__linuxCodeName") - $__dockerCmd $__buildRootfsCmd - sudo chown -R $(id -u -n) $__RepoRootDir/eng/common/cross/rootfs - fi - - __extraArgs="" - if [[ "$__buildArch" == "armel" && "$__linuxCodeName" == "tizen" ]]; then - __extraArgs="cmakeargs -DFEATURE_NGEN_RELOCS_OPTIMIZATIONS=true cmakeargs -DFEATURE_GDBJIT=true cmakeargs -DFEATURE_PREJIT=true -PortableBuild=false" - fi - - # Cross building coreclr with rootfs in Docker - (set +x; echo "Start cross build coreclr for $__buildArch $__linuxCodeName") - __buildCmd="./build.sh $__buildArch cross $__verboseFlag $__skipMscorlib $__buildConfig $__extraArgs" - $__dockerCmd $__buildCmd - sudo chown -R $(id -u -n) ./bin -} - -#Copy the needed files to the emulator to run tests -function copy_to_emulator { - - #Create the coreclr and corefx directories in the emulator - sudo mkdir -p "$__ARMRootfsCoreclrPath/bin/obj/$__buildDirName" - sudo mkdir -p "$__ARMRootfsCoreclrPath/bin/Product" - sudo mkdir "$__ARMRootfsCorefxPath" - - #Copy all coreclr files to the coreclr root in the emulator and set the paths accordingly - local testRootDirBase=`basename "$__testRootDir"` - sudo cp -R "$__testRootDir" "$__ARMRootfsCoreclrPath/$testRootDirBase" - __testRootDirBase="$__ARMEmulCoreclr/$testRootDirBase" - - sudo cp -R "./$__testNativeBinDirBase" "$__ARMRootfsCoreclrPath/$__testNativeBinDirBase" - __testNativeBinDirBase="$__ARMEmulCoreclr/$__testNativeBinDirBase" - - sudo cp -R "./$__coreClrBinDirBase" "$__ARMRootfsCoreclrPath/$__coreClrBinDirBase" - __coreClrBinDirBase="$__ARMEmulCoreclr/$__coreClrBinDirBase" - __mscorlibDirBase="$__coreClrBinDirBase" - - local testDirFileBase=`basename "$__testDirFile"` - sudo cp "$__testDirFile" "$__ARMRootfsCoreclrPath/$testDirFileBase" - __testDirFileBase="$__ARMEmulCoreclr/$testDirFileBase" - - sudo cp -R ./tests "$__ARMRootfsCoreclrPath/" - sudo cp -R ./.packages "$__ARMRootfsCoreclrPath/" - sudo cp -R ./Tools "$__ARMRootfsCoreclrPath/" - - #Copy corefx binary directories to the corefx root in the emulator (first native and then managed) - local coreFxNativeBinDirBase=`basename "$__coreFxNativeBinDir"` - sudo cp -R "$__coreFxNativeBinDir" "$__ARMRootfsCorefxPath/$coreFxNativeBinDirBase" - __coreFxNativeBinDirBase="$__ARMEmulCorefx/$coreFxNativeBinDirBase" - - __coreFxBinDirBase= - while IFS=';' read -ra coreFxBinDirectories; do - for currDir in "${coreFxBinDirectories[@]}"; do - local currDirBase=`basename "$currDir"` - sudo cp -R "$currDir" "$__ARMRootfsCorefxPath/$currDirBase" - - if [ -z "$__coreFxBinDirBase" ]; then - __coreFxBinDirBase="$__ARMEmulCorefx/$currDirBase" - else - __coreFxBinDirBase="$__coreFxBinDirBase;$__ARMEmulCorefx/$currDirBase" - fi - done - done <<< "$__coreFxBinDir" -} - -#Runs tests in an emulated mode -function run_tests { - sudo chroot $__ARMEmulRootfs /bin/bash -x < : arm (default) or armel' - echo ' --buildConfig= : Release (default) Checked, or Debug' -} - -# Display error message and exit -function exit_with_error { - set +x - - local errorMessage="$1" - local printUsage=$2 - - echo "ERROR: $errorMessage" - if [[ "$printUsage" == "true" ]]; then - echo '' - usage - fi - exit 1 -} - -# Exit if the input path does not exist -function exit_if_path_absent { - local path="$1" - local errorMessage="$2" - local printUsage=$3 - - if [ ! -f "$path" -a ! -d "$path" ]; then - exit_with_error "$errorMessage" $printUsage - fi -} - -__abi="arm" -__buildConfig="Release" - -# Parse command line arguments -for arg in "$@" -do - case $arg in - --abi=*) - __abi=${arg#*=} - if [[ "$__abi" != "arm" && "$__abi" != "armel" ]]; then - exit_with_error "--abi can be either arm or armel" true - fi - ;; - --buildConfig=*) - __buildConfig=${arg#*=} - if [[ "$__buildConfig" != "Debug" && "$__buildConfig" != "Release" && "$__buildConfig" != "Checked" ]]; then - exit_with_error "--buildConfig can be Debug, Checked or Release" true - fi - ;; - -v|--verbose) - __verboseFlag="verbose" - ;; - -h|--help) - usage - exit 0 - ;; - *) - exit_with_error "$arg not a recognized argument" true - ;; - esac -done -__buildDirName="Linux.${__abi}.${__buildConfig}" - -CORECLR_DIR=/opt/code -ARM_CHROOT_HOME_DIR=/home/coreclr - -if [ -z "${ROOTFS_DIR}" ]; then - __ROOTFS_DIR=${CORECLR_DIR}/eng/common/cross/rootfs/${__abi} -else - __ROOTFS_DIR=${ROOTFS_DIR} -fi - -if [[ "$__abi" == "armel" ]]; then - # Prepare armel emulation environment - pushd ${CORECLR_DIR}/eng/common/cross/armel/tizen - apt-get update - apt-get -y -qq --force-yes --reinstall install qemu binfmt-support qemu-user-static - __qemuARM=$(which qemu-arm-static) - cp $__qemuARM ${__ROOTFS_DIR}/usr/bin/ - popd -fi - -# Mount -mkdir -p ${__ROOTFS_DIR}${ARM_CHROOT_HOME_DIR} -mount -t proc /proc ${__ROOTFS_DIR}/proc -mount -o bind /dev ${__ROOTFS_DIR}/dev -mount -o bind /dev/pts ${__ROOTFS_DIR}/dev/pts -mount -o bind /sys ${__ROOTFS_DIR}/sys -mount -o bind ${CORECLR_DIR} ${__ROOTFS_DIR}${ARM_CHROOT_HOME_DIR} - -# Test environment emulation using docker and qemu has some problem to use lttng library. -# We should remove libcoreclrtraceptprovider.so to avoid test hang. -rm -f -v ${__ROOTFS_DIR}${ARM_CHROOT_HOME_DIR}/bin/bin/coreclr/${__buildDirName}/libcoreclrtraceptprovider.so -rm -f -v ${__ROOTFS_DIR}${ARM_CHROOT_HOME_DIR}/bin/CoreFxBinDir/libcoreclrtraceptprovider.so - -chroot ${__ROOTFS_DIR} /bin/bash -x <= 0 - - for i in range(0, n): - path = os.path.dirname(path) - - return path - -def log(message): - """ Print logging information - Args: - message (str): message to be printed - """ - - print('[%s]: %s' % (sys.argv[0], message)) - -def copy_files(source_dir, target_dir): - """ Copy any files in the source_dir to the target_dir. - The copy is not recursive. - The directories must already exist. - Args: - source_dir (str): source directory path - target_dir (str): target directory path - Returns: - Nothing - """ - - global testing - assert os.path.isdir(source_dir) - assert os.path.isdir(target_dir) - - for source_filename in os.listdir(source_dir): - source_pathname = os.path.join(source_dir, source_filename) - if os.path.isfile(source_pathname): - target_pathname = os.path.join(target_dir, source_filename) - log('Copy: %s => %s' % (source_pathname, target_pathname)) - if not testing: - shutil.copy2(source_pathname, target_pathname) - -def run_command(command, command_env): - """ Run a command (process) in a given environment. stdout/stderr are output piped through. - Args: - command (array): the command to run, with components of the command as separate elements. - command_env (map): environment in which the command should be run - Returns: - The return code of the command. - """ - - returncode = 0 - - log('Invoking: %s' % (' '.join(command))) - if not testing: - proc = subprocess.Popen(command, env=command_env) - output,error = proc.communicate() - returncode = proc.returncode - if returncode != 0: - log('Return code = %s' % returncode) - - return returncode - -########################################################################## -# Do baseline build: -# 1. determine appropriate commit, -# 2. clone coreclr, -# 3. do build -########################################################################## - -def baseline_build(): - - if not testing: - if os.path.isdir(baseCoreClrPath): - log('Removing existing tree: %s' % baseCoreClrPath) - shutil.rmtree(baseCoreClrPath, onerror=del_rw) - - # Find the baseline commit - - # Clone at that commit - - command = 'git clone -b %s --single-branch %s %s' % ( - target_branch, Coreclr_url, baseCoreClrPath) - log(command) - returncode = 0 if testing else os.system(command) - if returncode != 0: - log('ERROR: git clone failed') - return 1 - - # Change directory to the baseline root - - with ChangeDir(baseCoreClrPath): - - # Set up for possible docker usage - - scriptPath = '.' - buildOpts = '' - dockerCmd = '' - if not Is_windows and (arch == 'arm' or arch == 'arm64'): - # Linux arm and arm64 builds are cross-compilation builds using Docker. - if arch == 'arm': - dockerFile = Docker_name_arm32 - dockerOpts = Docker_opts_arm32 - else: - # arch == 'arm64' - dockerFile = Docker_name_arm64 - dockerOpts = Docker_opts_arm64 - - dockerCmd = 'docker run -i --rm -v %s:%s -w %s %s %s ' % (baseCoreClrPath, baseCoreClrPath, baseCoreClrPath, dockerOpts, dockerFile) - buildOpts = 'cross' - scriptPath = baseCoreClrPath - - # Build a checked baseline jit - - if Is_windows: - command = 'set __TestIntermediateDir=int&&build.cmd %s checked skiptests skipbuildpackages' % arch - else: - command = '%s%s/build.sh %s checked skipbuildpackages %s' % (dockerCmd, scriptPath, arch, buildOpts) - log(command) - returncode = 0 if testing else os.system(command) - if returncode != 0: - log('ERROR: build failed') - return 1 - - # Build the layout (Core_Root) directory - # For Windows, invoke build-test.cmd to restore packages before generating the layout. - - if Is_windows: - command = 'build-test.cmd %s %s skipmanaged skipnative' % (build_type, arch) - log(command) - returncode = 0 if testing else os.system(command) - if returncode != 0: - log('ERROR: restoring packages failed') - return 1 - - if Is_windows: - command = 'tests\\runtest.cmd %s checked GenerateLayoutOnly' % arch - else: - command = '%s%s/build-test.sh %s checked generatelayoutonly' % (dockerCmd, scriptPath, arch) - log(command) - returncode = 0 if testing else os.system(command) - if returncode != 0: - log('ERROR: generating layout failed') - return 1 - - return 0 - -########################################################################## -# Do PMI diff run: -# 1. download dotnet CLI (needed by jitutils) -# 2. clone jitutils repo -# 3. build jitutils -# 4. run PMI asm generation on baseline and diffs -# 5. run jit-analyze to compare baseline and diff -########################################################################## - -def do_pmi_diffs(): - global baseCoreClrPath - - # Setup scratch directories. Names are short to avoid path length problems on Windows. - dotnetcliPath = os.path.abspath(os.path.join(scratch_root, 'cli')) - jitutilsPath = os.path.abspath(os.path.join(scratch_root, 'jitutils')) - asmRootPath = os.path.abspath(os.path.join(scratch_root, 'asm')) - - dotnet_tool = 'dotnet.exe' if Is_windows else 'dotnet' - - # Make sure the temporary directories do not exist. If they do already, delete them. - - if not testing: - # If we can't delete the dotnet tree, it might be because a previous run failed or was - # cancelled, and the build servers are still running. Try to stop it if that happens. - if os.path.isdir(dotnetcliPath): - try: - log('Removing existing tree: %s' % dotnetcliPath) - shutil.rmtree(dotnetcliPath, onerror=del_rw) - except OSError: - if os.path.isfile(os.path.join(dotnetcliPath, dotnet_tool)): - log('Failed to remove existing tree; trying to shutdown the dotnet build servers before trying again.') - - # Looks like the dotnet too is still there; try to run it to shut down the build servers. - temp_env = my_env - temp_env["PATH"] = dotnetcliPath + os.pathsep + my_env["PATH"] - log('Shutting down build servers') - command = ["dotnet", "build-server", "shutdown"] - returncode = run_command(command, temp_env) - - # Try again - log('Trying again to remove existing tree: %s' % dotnetcliPath) - shutil.rmtree(dotnetcliPath, onerror=del_rw) - else: - log('Failed to remove existing tree') - return 1 - - if os.path.isdir(jitutilsPath): - log('Removing existing tree: %s' % jitutilsPath) - shutil.rmtree(jitutilsPath, onerror=del_rw) - if os.path.isdir(asmRootPath): - log('Removing existing tree: %s' % asmRootPath) - shutil.rmtree(asmRootPath, onerror=del_rw) - - try: - os.makedirs(dotnetcliPath) - os.makedirs(jitutilsPath) - os.makedirs(asmRootPath) - except OSError: - if not os.path.isdir(dotnetcliPath): - log('ERROR: cannot create CLI install directory %s' % dotnetcliPath) - return 1 - if not os.path.isdir(jitutilsPath): - log('ERROR: cannot create jitutils install directory %s' % jitutilsPath) - return 1 - if not os.path.isdir(asmRootPath): - log('ERROR: cannot create asm directory %s' % asmRootPath) - return 1 - - log('dotnet CLI install directory: %s' % dotnetcliPath) - log('jitutils install directory: %s' % jitutilsPath) - log('asm directory: %s' % asmRootPath) - - # Download .NET CLI - - log('Downloading .NET CLI') - - dotnetcliUrl = "" - dotnetcliFilename = "" - - if Clr_os == 'Linux' and arch == 'x64': - dotnetcliUrl = "https://dotnetcli.azureedge.net/dotnet/Sdk/2.1.402/dotnet-sdk-2.1.402-linux-x64.tar.gz" - elif Clr_os == 'Linux' and arch == 'arm': - dotnetcliUrl = "https://dotnetcli.blob.core.windows.net/dotnet/Sdk/release/2.1.4xx/dotnet-sdk-latest-linux-arm.tar.gz" - elif Clr_os == 'Linux' and arch == 'arm64': - # Use the latest (3.0) dotnet SDK. Earlier versions don't work. - dotnetcliUrl = "https://dotnetcli.blob.core.windows.net/dotnet/Sdk/master/dotnet-sdk-latest-linux-arm64.tar.gz" - elif Clr_os == 'OSX': - dotnetcliUrl = "https://dotnetcli.azureedge.net/dotnet/Sdk/2.1.402/dotnet-sdk-2.1.402-osx-x64.tar.gz" - elif Clr_os == 'windows': - dotnetcliUrl = "https://dotnetcli.azureedge.net/dotnet/Sdk/2.1.402/dotnet-sdk-2.1.402-win-x64.zip" - else: - log('ERROR: unknown or unsupported OS (%s) architecture (%s) combination' % (Clr_os, arch)) - return 1 - - if Is_windows: - dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.zip') - else: - dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.tar.gz') - - log('Downloading: %s => %s' % (dotnetcliUrl, dotnetcliFilename)) - - if not testing: - urlretrieve = urllib.urlretrieve if sys.version_info.major < 3 else urllib.request.urlretrieve - urlretrieve(dotnetcliUrl, dotnetcliFilename) - - if not os.path.isfile(dotnetcliFilename): - log('ERROR: Did not download .NET CLI') - return 1 - - # Install .NET CLI - - log('Unpacking .NET CLI') - - if not testing: - if Is_windows: - with zipfile.ZipFile(dotnetcliFilename, "r") as z: - z.extractall(dotnetcliPath) - else: - tar = tarfile.open(dotnetcliFilename) - tar.extractall(dotnetcliPath) - tar.close() - - if not os.path.isfile(os.path.join(dotnetcliPath, dotnet_tool)): - log('ERROR: did not extract .NET CLI from download') - return 1 - - # Add dotnet CLI to PATH we'll use to spawn processes. - - log('Add %s to my PATH' % dotnetcliPath) - my_env["PATH"] = dotnetcliPath + os.pathsep + my_env["PATH"] - - # To aid diagnosing problems, do "dotnet --info" to output to any capturing logfile. - - command = ["dotnet", "--info"] - returncode = run_command(command, my_env) - - # Clone jitutils - - command = 'git clone -b main --single-branch %s %s' % (Jitutils_url, jitutilsPath) - log(command) - returncode = 0 if testing else os.system(command) - if returncode != 0: - log('ERROR: cannot clone jitutils'); - return 1 - - # We're going to start running dotnet CLI commands. Unfortunately, once you've done that, - # the dotnet CLI sticks around with a set of build server processes running. Put all this - # in a try/finally, and stop the build servers under any circumstance. - - try: - - # - # Build jitutils, including "dotnet restore" - # - - # Change directory to the jitutils root - - with ChangeDir(jitutilsPath): - - # Do "dotnet restore" - - command = ["dotnet", "restore"] - returncode = run_command(command, my_env) - - # Do build - - command = ['build.cmd', '-p'] if Is_windows else ['bash', './build.sh', '-p'] - returncode = run_command(command, my_env) - if returncode != 0: - log('ERROR: jitutils build failed') - return 1 - - jitutilsBin = os.path.join(jitutilsPath, "bin") - - if not testing and not os.path.isdir(jitutilsBin): - log("ERROR: jitutils not correctly built") - return 1 - - jitDiffPath = os.path.join(jitutilsBin, "jit-diff.dll") - if not testing and not os.path.isfile(jitDiffPath): - log("ERROR: jit-diff.dll not built") - return 1 - - jitAnalyzePath = os.path.join(jitutilsBin, "jit-analyze.dll") - if not testing and not os.path.isfile(jitAnalyzePath): - log("ERROR: jit-analyze.dll not built") - return 1 - - # Add jitutils bin to path for spawned processes - - log('Add %s to my PATH' % jitutilsBin) - my_env["PATH"] = jitutilsBin + os.pathsep + my_env["PATH"] - - # - # Run PMI asm diffs - # - - # We want this script as a whole to return 0 if it succeeds (even if there are diffs) and only - # return non-zero if there are any fatal errors. - # - # TO DO: figure out how to differentiate fatal errors and a return code indicating there are diffs, - # and have the invoking netci.groovy code act differently for each case. - - # Generate the diffs - # - # Invoke command like: - # dotnet c:\gh\jitutils\artifacts\jit-diff.dll diff --pmi --base --base_root f:\gh\coreclr12 --diff --diff_root f:\gh\coreclr10 --arch x64 --build Checked --tag 1 --noanalyze --output f:\output --corelib - # - # We pass --noanalyze and call jit-analyze manually. This isn't really necessary, but it does give us better output - # due to https://github.com/dotnet/jitutils/issues/175. - - altjit_args = [] - if ci_arch is not None and (ci_arch == 'x86_arm_altjit' or ci_arch == 'x64_arm64_altjit'): - altjit_args = ["--altjit", "protononjit.dll"] - - # Over which set of assemblies should we generate asm? - # TODO: parameterize this - asm_source_args = ["--frameworks", "--benchmarks"] - - command = ["dotnet", jitDiffPath, "diff", "--pmi", "--base", "--base_root", baseCoreClrPath, "--diff", "--diff_root", diff_root, "--arch", arch, "--build", build_type, "--tag", "1", "--noanalyze", "--output", asmRootPath] + asm_source_args + altjit_args - returncode = run_command(command, my_env) - - # We ignore the return code: it is non-zero if there are any diffs. If there are fatal errors here, we will miss them. - # Question: does jit-diff distinguish between non-zero fatal error code and the existence of diffs? - - # Did we get any diffs? - - baseOutputDir = os.path.join(asmRootPath, "1", "base") - if not testing and not os.path.isdir(baseOutputDir): - log("ERROR: base asm not generated") - return 1 - - diffOutputDir = os.path.join(asmRootPath, "1", "diff") - if not testing and not os.path.isdir(diffOutputDir): - log("ERROR: diff asm not generated") - return 1 - - # Do the jit-analyze comparison: - # dotnet c:\gh\jitutils\artifacts\jit-analyze.dll --base f:\output\diffs\1\base --recursive --diff f:\output\diffs\1\diff - - command = ["dotnet", jitAnalyzePath, "--recursive", "--base", baseOutputDir, "--diff", diffOutputDir] - returncode = run_command(command, my_env) - if returncode != 0: - # This is not a fatal error. - log('Compare: %s %s' % (baseOutputDir, diffOutputDir)) - - finally: - - # Shutdown the dotnet build servers before cleaning things up - # TODO: make this shutdown happen anytime after we've run any 'dotnet' commands. I.e., try/finally style. - - log('Shutting down build servers') - command = ["dotnet", "build-server", "shutdown"] - returncode = run_command(command, my_env) - - return 0 - -########################################################################## -# Main -########################################################################## - -def main(args): - - global arch, ci_arch, build_type, base_root, diff_root, scratch_root, skip_baseline_build, skip_diffs, target_branch, commit_hash - global my_env - global base_layout_root - global diff_layout_root - global baseCoreClrPath - global testing - - arch, ci_arch, build_type, base_root, diff_root, scratch_root, skip_baseline_build, skip_diffs, target_branch, commit_hash = validate_args(args) - - my_env = os.environ - - if not testing and not os.path.isdir(diff_root): - log('ERROR: root directory for coreclr diff tree not found: %s' % diff_root) - return 1 - - # Check the diff layout directory before going too far. - - diff_layout_root = os.path.join(diff_root, - 'bin', - 'tests', - '%s.%s.%s' % (Clr_os, arch, build_type), - 'Tests', - 'Core_Root') - - if not testing and not os.path.isdir(diff_layout_root): - log('ERROR: diff test overlay not found or is not a directory: %s' % diff_layout_root) - return 1 - - # Create the scratch root directory - - if not testing: - try: - os.makedirs(scratch_root) - except OSError: - if not os.path.isdir(scratch_root): - log('ERROR: cannot create scratch directory %s' % scratch_root) - return 1 - - # Set up baseline root directory. If one is passed to us, we use it. Otherwise, we create - # a temporary directory. - - if base_root is None: - # Setup scratch directories. Names are short to avoid path length problems on Windows. - # No need to create this directory now, as the "git clone" will do it later. - baseCoreClrPath = os.path.abspath(os.path.join(scratch_root, 'base')) - else: - baseCoreClrPath = os.path.abspath(base_root) - if not testing and not os.path.isdir(baseCoreClrPath): - log('ERROR: base root directory not found or is not a directory: %s' % baseCoreClrPath) - return 1 - - # Do the baseline build, if needed - - if not skip_baseline_build and base_root is None: - returncode = baseline_build() - if returncode != 0: - return 1 - - # Check that the baseline root directory was created. - - base_layout_root = os.path.join(baseCoreClrPath, - 'bin', - 'tests', - '%s.%s.%s' % (Clr_os, arch, build_type), - 'Tests', - 'Core_Root') - - if not testing and not os.path.isdir(base_layout_root): - log('ERROR: baseline test overlay not found or is not a directory: %s' % base_layout_root) - return 1 - - # Do the diff run, if needed - - if not skip_diffs: - returncode = do_pmi_diffs() - if returncode != 0: - return 1 - - return 0 - - -########################################################################## -# setup for Main -########################################################################## - -if __name__ == '__main__': - Args = parser.parse_args(sys.argv[1:]) - return_code = main(Args) - log('Exit code: %s' % return_code) - sys.exit(return_code) diff --git a/src/tests/Common/scripts/x86_ci_script.sh b/src/tests/Common/scripts/x86_ci_script.sh deleted file mode 100755 index f4e0fee25bcebc..00000000000000 --- a/src/tests/Common/scripts/x86_ci_script.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -#Parse command line arguments -__buildConfig= -for arg in "$@" -do - case $arg in - --buildConfig=*) - __buildConfig="$(echo ${arg#*=} | tr "[:upper:]" "[:lower:]")" - if [[ "$__buildConfig" != "debug" && "$__buildConfig" != "release" && "$__buildConfig" != "checked" ]]; then - exit_with_error "--buildConfig can be only Debug or Release" true - fi - ;; - *) - ;; - esac -done - -#Check if there are any uncommitted changes in the source directory as git adds and removes patches -if [[ -n $(git status -s) ]]; then - echo 'ERROR: There are some uncommitted changes. To avoid losing these changes commit them and try again.' - echo '' - git status - exit 1 -fi - -#Change build configuration to the capitalized form to create build product paths correctly -if [[ "$__buildConfig" == "release" ]]; then - __buildConfig="Release" -elif [[ "$__buildConfig" == "checked" ]]; then - __buildConfig="Checked" -else - __buildConfig="Debug" -fi -__buildDirName="$__targetOS.$__buildArch.$__buildConfig" - -set -x -set -e - -__dockerImage="hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548" - -# Begin cross build -# We cannot build nuget package yet -__dockerEnvironmentSet="-e ROOTFS_DIR=/crossrootfs/x86" -__currentWorkingDir=`pwd` -__dockerCmd="docker run -i --rm ${__dockerEnvironmentSet} -v $__currentWorkingDir:/opt/code -w /opt/code $__dockerImage" -__buildCmd="./build.sh x86 cross skipnuget $__buildConfig" -$__dockerCmd $__buildCmd - -# Begin PAL test -__dockerImage="hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test" -__dockerCmd="docker run -i --rm -v $__currentWorkingDir:/opt/code -w /opt/code $__dockerImage" -__palTestCmd="./src/pal/tests/palsuite/runpaltests.sh /opt/code/artifacts/obj/Linux.x86.${__buildConfig} /opt/code/artifacts/paltestout" -$__dockerCmd $__palTestCmd - -sudo chown -R $(id -u -n) artifacts/ - -(set +x; echo 'Build complete') From 89a64564bca412bf2c538eec30c209658bfad2bc Mon Sep 17 00:00:00 2001 From: Adeel <3840695+am11@users.noreply.github.com> Date: Wed, 14 Sep 2022 15:04:01 +0300 Subject: [PATCH 3/7] Try with grep --- .../BuildIntegration/Microsoft.NETCore.Native.targets | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets index 90688aec24a047..3b79252945e7c3 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets @@ -342,10 +342,12 @@ The .NET Foundation licenses this file to you under the MIT license. <_IgnoreLinkerWarnings Condition="'$(TargetOS)' == 'OSX'">true - <_UnsupportedLinkerOptionRegex>.*(nostart-stop-gc).* + <_UnsupportedLinkerOptionPattern>(nostart-stop-gc) - + + From 7d509758cde77fa20e5ebda77ed55ff5d2036364 Mon Sep 17 00:00:00 2001 From: Adeel <3840695+am11@users.noreply.github.com> Date: Wed, 14 Sep 2022 22:37:08 +0300 Subject: [PATCH 4/7] Use attribute(retain) --- src/coreclr/nativeaot/Bootstrap/main.cpp | 12 ++++++++++++ .../Microsoft.NETCore.Native.Unix.targets | 2 -- .../Microsoft.NETCore.Native.targets | 7 +------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/coreclr/nativeaot/Bootstrap/main.cpp b/src/coreclr/nativeaot/Bootstrap/main.cpp index fed5a155a0c5b9..c61cd7c9ffa928 100644 --- a/src/coreclr/nativeaot/Bootstrap/main.cpp +++ b/src/coreclr/nativeaot/Bootstrap/main.cpp @@ -74,18 +74,30 @@ extern char __unbox_z __asm("section$end$__TEXT$__unbox"); #else // __APPLE__ +#if defined(__has_attribute) && __has_attribute(retain) +#define RETAIN_SECTION(name) __attribute__((retain,used,section(name))) +#else +#define RETAIN_SECTION(name) +#endif + extern "C" void * __start___modules[]; extern "C" void * __stop___modules[]; + +RETAIN_SECTION("__modules") static void * (&__modules_a)[] = __start___modules; static void * (&__modules_z)[] = __stop___modules; extern "C" char __start___managedcode; extern "C" char __stop___managedcode; + +RETAIN_SECTION("__managedcode") static char& __managedcode_a = __start___managedcode; static char& __managedcode_z = __stop___managedcode; extern "C" char __start___unbox; extern "C" char __stop___unbox; + +RETAIN_SECTION("__unbox") static char& __unbox_a = __start___unbox; static char& __unbox_z = __stop___unbox; diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets index 5ea5c13908a171..b112715b846770 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets @@ -112,8 +112,6 @@ The .NET Foundation licenses this file to you under the MIT license. - - diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets index 3b79252945e7c3..417d4567a95189 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets @@ -340,14 +340,9 @@ The .NET Foundation licenses this file to you under the MIT license. <_IgnoreLinkerWarnings>false <_IgnoreLinkerWarnings Condition="'$(TargetOS)' == 'OSX'">true - - - <_UnsupportedLinkerOptionPattern>(nostart-stop-gc) - - + From d1a757055a7b8ced66d822f07a66d8a0e061823e Mon Sep 17 00:00:00 2001 From: Adeel <3840695+am11@users.noreply.github.com> Date: Thu, 15 Sep 2022 00:28:50 +0300 Subject: [PATCH 5/7] Improve handling of /p:CrossBuild=true --- eng/build.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/eng/build.sh b/eng/build.sh index fdba03186cd66a..b7737f4d5f73ff 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -397,12 +397,18 @@ while [[ $# > 0 ]]; do shift 2 ;; - -cross|*crossbuild=true*) + -cross) crossBuild=1 arguments="$arguments /p:CrossBuild=True" shift 1 ;; + *crossbuild=true*) + crossBuild=1 + extraargs="$extraargs $1" + shift 1 + ;; + -clang*) compiler="${opt/#-/}" # -clang-9 => clang-9 or clang-9 => (unchanged) arguments="$arguments /p:Compiler=$compiler /p:CppCompilerAndLinker=$compiler" From 1a63b333643c18aa78cd0ca8ca66c9216322d9fe Mon Sep 17 00:00:00 2001 From: Adeel <3840695+am11@users.noreply.github.com> Date: Thu, 15 Sep 2022 20:04:09 +0300 Subject: [PATCH 6/7] Update images and revert to binutils linker --- eng/pipelines/common/platform-matrix.yml | 4 ++-- eng/testing/tests.singlefile.targets | 1 - .../BuildIntegration/Microsoft.NETCore.Native.Unix.targets | 2 -- src/coreclr/tools/aot/crossgen2/crossgen2.csproj | 1 - 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 50875f25bb07d6..1dca99ef9d58be 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -139,7 +139,7 @@ jobs: platform: Linux_musl_arm shouldContinueOnError: ${{ parameters.shouldContinueOnError }} container: - image: ubuntu-18.04-cross-arm-alpine-20220912173203-78f7860 + image: ubuntu-18.04-cross-arm-alpine-20220915134743-78f7860 registry: mcr jobParameters: runtimeFlavor: ${{ parameters.runtimeFlavor }} @@ -165,7 +165,7 @@ jobs: platform: Linux_musl_arm64 shouldContinueOnError: ${{ parameters.shouldContinueOnError }} container: - image: ubuntu-22.04-cross-arm64-alpine-20220912173307-b016fdd + image: ubuntu-22.04-cross-arm64-alpine-20220915142421-44c622d registry: mcr jobParameters: runtimeFlavor: ${{ parameters.runtimeFlavor }} diff --git a/eng/testing/tests.singlefile.targets b/eng/testing/tests.singlefile.targets index 8c249d575fb703..5081048c8032ca 100644 --- a/eng/testing/tests.singlefile.targets +++ b/eng/testing/tests.singlefile.targets @@ -23,7 +23,6 @@ $(CoreCLRCrossILCompilerDir) clang-15 clang-9 - true $(ROOTFS_DIR) $(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll $(CoreCLRAotSdkDir) diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets index b112715b846770..58937c04c18da6 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets @@ -20,7 +20,6 @@ The .NET Foundation licenses this file to you under the MIT license. $(CppCompilerAndLinker) ar --flat - false @@ -111,7 +110,6 @@ The .NET Foundation licenses this file to you under the MIT license. - diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2.csproj b/src/coreclr/tools/aot/crossgen2/crossgen2.csproj index 594636eb2f474f..12864a2341ee8e 100644 --- a/src/coreclr/tools/aot/crossgen2/crossgen2.csproj +++ b/src/coreclr/tools/aot/crossgen2/crossgen2.csproj @@ -34,7 +34,6 @@ $(CoreCLRCrossILCompilerDir) clang-15 clang-9 - true $(ROOTFS_DIR) $(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll $(CoreCLRAotSdkDir) From fb3838c50e898528195ae9a38fab26601a6f6748 Mon Sep 17 00:00:00 2001 From: Adeel <3840695+am11@users.noreply.github.com> Date: Fri, 16 Sep 2022 00:02:45 +0300 Subject: [PATCH 7/7] Revert __attribute__((retain)) for now --- src/coreclr/nativeaot/Bootstrap/main.cpp | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/coreclr/nativeaot/Bootstrap/main.cpp b/src/coreclr/nativeaot/Bootstrap/main.cpp index c61cd7c9ffa928..fed5a155a0c5b9 100644 --- a/src/coreclr/nativeaot/Bootstrap/main.cpp +++ b/src/coreclr/nativeaot/Bootstrap/main.cpp @@ -74,30 +74,18 @@ extern char __unbox_z __asm("section$end$__TEXT$__unbox"); #else // __APPLE__ -#if defined(__has_attribute) && __has_attribute(retain) -#define RETAIN_SECTION(name) __attribute__((retain,used,section(name))) -#else -#define RETAIN_SECTION(name) -#endif - extern "C" void * __start___modules[]; extern "C" void * __stop___modules[]; - -RETAIN_SECTION("__modules") static void * (&__modules_a)[] = __start___modules; static void * (&__modules_z)[] = __stop___modules; extern "C" char __start___managedcode; extern "C" char __stop___managedcode; - -RETAIN_SECTION("__managedcode") static char& __managedcode_a = __start___managedcode; static char& __managedcode_z = __stop___managedcode; extern "C" char __start___unbox; extern "C" char __stop___unbox; - -RETAIN_SECTION("__unbox") static char& __unbox_a = __start___unbox; static char& __unbox_z = __stop___unbox;