diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6d551c47ed7..996ba48a44c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -120,6 +120,8 @@ stages: - Codeql.TSAEnabled: ${{ eq(variables['Build.Reason'], 'Schedule') }} # Default expects tsaoptions.json under SourceRoot. - Codeql.TSAOptionsPath: '$(Build.SourcesDirectory)/.config/tsaoptions.json' + - ${{ else }}: + - _AdditionalBuildArgs: '' steps: - task: NuGetCommand@2 displayName: 'Clear NuGet caches' diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 04d7c34d164..ea8cc9ef527 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -39,17 +39,17 @@ - + https://github.com/dotnet/arcade - e82404fca08383513e0b0b3c5308d4a9b18b7c7a + 96d8be74c39a4765ec919ff9bebf9e0c875fc195 - + https://github.com/dotnet/arcade - e82404fca08383513e0b0b3c5308d4a9b18b7c7a + 96d8be74c39a4765ec919ff9bebf9e0c875fc195 - + https://github.com/dotnet/arcade - e82404fca08383513e0b0b3c5308d4a9b18b7c7a + 96d8be74c39a4765ec919ff9bebf9e0c875fc195 diff --git a/eng/Versions.props b/eng/Versions.props index 15c542dbe37..c33f4ca79c2 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -26,7 +26,7 @@ 8.0.0-preview.2.23105.4 - 8.0.0-beta.22630.1 + 8.0.0-beta.23110.3 4.2.0 diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index b04d3b38065..1ebf454f3cf 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -48,12 +48,14 @@ __UbuntuPackages+=" symlinks" __UbuntuPackages+=" libicu-dev" __UbuntuPackages+=" liblttng-ust-dev" __UbuntuPackages+=" libunwind8-dev" +__UbuntuPackages+=" libnuma-dev" __AlpinePackages+=" gettext-dev" __AlpinePackages+=" icu-dev" __AlpinePackages+=" libunwind-dev" __AlpinePackages+=" lttng-ust-dev" __AlpinePackages+=" compiler-rt-static" +__AlpinePackages+=" numactl-dev" # runtime libraries' dependencies __UbuntuPackages+=" libcurl4-openssl-dev" @@ -147,9 +149,9 @@ while :; do __BuildArch=ppc64le __UbuntuArch=ppc64el __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//') - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') + __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" + __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" + __UbuntuPackages="${__UbuntuPackages// libomp5/}" unset __LLDB_Package ;; riscv64) @@ -157,7 +159,7 @@ while :; do __UbuntuArch=riscv64 __UbuntuRepo="http://deb.debian.org/debian-ports" __CodeName=sid - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') + __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" unset __LLDB_Package if [[ -e "/usr/share/keyrings/debian-ports-archive-keyring.gpg" ]]; then @@ -168,9 +170,9 @@ while :; do __BuildArch=s390x __UbuntuArch=s390x __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//') - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') + __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" + __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" + __UbuntuPackages="${__UbuntuPackages// libomp5/}" unset __LLDB_Package ;; x64) @@ -310,6 +312,8 @@ done if [[ "$__BuildArch" == "armel" ]]; then __LLDB_Package="lldb-3.5-dev" +elif [[ "$__BuildArch" == "arm" && "$__AlpineVersion" == "3.13" ]]; then + __AlpinePackages="${__AlpinePackages//numactl-dev/}" fi __UbuntuPackages+=" ${__LLDB_Package:-}" diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index dbf2ab4ee7d..bcb579e37a9 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -34,6 +34,25 @@ $jsonTemplateFiles | ForEach-Object { $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern $wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them +if (-not $wxlFiles) { + $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files + if ($wxlEnFiles) { + $wxlFiles = @() + $wxlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru + } + } +} + +$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html" } # add installer HTML files +$macosHtmlFiles = @() +if ($macosHtmlEnFiles) { + $macosHtmlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru + } +} $xlfFiles = @() @@ -99,8 +118,7 @@ $locJson = @{ $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" $continue = $true foreach ($exclusion in $exclusions.Exclusions) { - if ($_.FullName.Contains($exclusion)) - { + if ($_.FullName.Contains($exclusion)) { $continue = $false } } @@ -115,6 +133,30 @@ $locJson = @{ } } ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "VS_macOS_CloneLanguages" + LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" ) + LocItems = @( + $macosHtmlFiles | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if ($continue) { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + } + } + ) } ) } diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index fbc67effc36..27ccdb9ecc9 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -83,7 +83,8 @@ try { Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue if ($NativeTools) { if ($PathPromotion -eq $True) { - if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build + $ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools" + if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine $NativeTools.PSObject.Properties | ForEach-Object { $ToolName = $_.Name $ToolVersion = $_.Value @@ -93,11 +94,6 @@ try { if ($ToolVersion -eq "latest") { $ToolVersion = "" } - $ArcadeToolsDirectory = "C:\arcade-tools" - if (-not (Test-Path $ArcadeToolsDirectory)) { - Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed." - exit 1 - } $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending) if ($ToolDirectories -eq $null) { Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." @@ -125,6 +121,7 @@ try { if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image." } } exit 0 diff --git a/eng/common/loc/P22DotNetHtmlLocalization.lss b/eng/common/loc/P22DotNetHtmlLocalization.lss new file mode 100644 index 00000000000..6661fed566e Binary files /dev/null and b/eng/common/loc/P22DotNetHtmlLocalization.lss differ diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index c670cb79689..7aee4213e1b 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -63,7 +63,7 @@ if [ -z "$CLR_CC" ]; then # Set default versions if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions="15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" + if [ "$compiler" = "clang" ]; then versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 0549e4c71ce..61914a1fbcd 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -88,6 +88,16 @@ jobs: - ${{ if ne(variable.group, '') }}: - group: ${{ variable.group }} + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + # handle key-value variable syntax. # example: # - [key]: [value] @@ -124,7 +134,7 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - task: NuGetAuthenticate@0 - - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}: + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - task: DownloadPipelineArtifact@2 inputs: buildType: current @@ -161,7 +171,7 @@ jobs: TeamName: $(_TeamName) - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - task: CopyFiles@2 displayName: Gather binaries for publish to artifacts inputs: @@ -182,7 +192,7 @@ jobs: ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} continueOnError: true condition: always() - - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}: + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - publish: artifacts/log artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} displayName: Publish logs diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index e40bf35203b..8a3deef2b72 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -46,20 +46,12 @@ jobs: # source-build builds run in Docker, including the default managed platform. # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic pool: - # Main environments - ${{ if and(eq(variables['System.TeamProject'], 'public'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}: - name: NetCore-Public + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - # Servicing build environments - ${{ if and(eq(variables['System.TeamProject'], 'public'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}: - name: NetCore-Svc-Public - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}: - name: NetCore1ESPool-Svc-Internal + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 ${{ if ne(parameters.platform.pool, '') }}: diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 09c506d1185..71b66aeb8a5 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,6 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20220804.1 + sourceIndexPackageVersion: 1.0.1-20221220.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -40,10 +40,10 @@ jobs: - ${{ preStep }} - task: UseDotNet@2 - displayName: Use .NET Core sdk 3.1 + displayName: Use .NET Core SDK 6 inputs: packageType: sdk - version: 3.1.x + version: 6.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index a7b943c2fa4..9cc5c550d3b 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -16,7 +16,7 @@ # First, import the template in an arcade-ified repo to pick up the variables, e.g.: # # variables: -# - template: eng/common/templates/variables/pool-providers.yml +# - template: /eng/common/templates/variables/pool-providers.yml # # ... then anywhere specifying the pool provider use the runtime variables, # $(DncEngInternalBuildPool) and $ (DncEngPublicBuildPool), e.g.: @@ -26,23 +26,32 @@ # demands: ImageOverride -equals windows.vs2019.amd64 variables: -# Coalesce the target and source branches so we know when a PR targets a release branch -# If these variables are somehow missing, fall back to main (tends to have more capacity) -- name: BranchNameForPoolSelection - value: ${{ coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main') }} + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) -# Any new -Svc alternative pools should have variables added here to allow for splitting work - -# Main branch pools -- ${{ if ne(contains(variables['BranchNameForPoolSelection'], 'release'), true) }}: + # Any new -Svc alternative pools should have variables added here to allow for splitting work - name: DncEngPublicBuildPool - value: NetCore-Public - - name: DncEngInternalBuildPool - value: NetCore1ESPool-Internal + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore-Svc-Public' + ), + False, + 'NetCore-Public' + ) + ] -# Release branch pools -- ${{ if contains(variables['BranchNameForPoolSelection'], 'release') }}: - - name: DncEngPublicBuildPool - value: NetCore-Svc-Public - name: DncEngInternalBuildPool - value: NetCore1ESPool-Svc-Internal \ No newline at end of file + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 1a8c16c56e5..7caacc6de2e 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -581,7 +581,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') - $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net7.0' } + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net8.0' } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore @@ -743,6 +743,8 @@ function MSBuild() { (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll')) (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.ArcadeLogging.dll')), (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.Arcade.Sdk.dll')) + (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')), + (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll')) ) $selectedPath = $null foreach ($path in $possiblePaths) { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 2f27d7453cd..cf9fb1ea2d3 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -312,7 +312,7 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" - _InitializeBuildToolFramework="net7.0" + _InitializeBuildToolFramework="net8.0" } # Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 @@ -428,6 +428,8 @@ function MSBuild { possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" ) possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.ArcadeLogging.dll" ) possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.Arcade.Sdk.dll" ) + possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.ArcadeLogging.dll" ) + possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.Arcade.Sdk.dll" ) for path in "${possiblePaths[@]}"; do if [[ -f $path ]]; then selectedPath=$path diff --git a/global.json b/global.json index dc49b867c84..76f2baa9946 100644 --- a/global.json +++ b/global.json @@ -1,14 +1,22 @@ { - "tools": { - "dotnet": "7.0.103" - }, "sdk": { - "version": "7.0.103", + "version": "8.0.100-alpha.1.23061.8", "allowPrerelease": true, "rollForward": "latestMajor" }, + "tools": { + "dotnet": "8.0.100-alpha.1.23061.8", + "runtimes": { + "aspnetcore": [ + "7.0.3" + ], + "dotnet": [ + "7.0.3" + ] + } + }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.22630.1", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.22630.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23110.3", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23110.3" } }