From fb0902bab405ffef6f2e00cf36bfe619a7f02a7b Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Tue, 6 Jun 2023 16:46:53 -0400 Subject: [PATCH 1/3] [ci] Fix nightly test environment provisioning Commit 475b912a improved Android SDK dependency provisioning steps by combining the install of emulator and non emulator related dependencies. These changes also moved emulator component provisioning to a step that includes preview .NET SDK provisioning. In the nightly build this step was running _after_ we installed our workload packages, and our workload install would be removed as part of SDK re-provisioning. This caused ~all of our nightly tests to fail with: /Users/runner/work/1/s/xamarin-android/bin/Release/dotnet/sdk/8.0.100-preview.6.23276.3/Sdks/Microsoft.NET.Sdk/targets/Microsoft.NET.Sdk.ImportWorkloads.targets(38,5): error NETSDK1147: To build this project, the following workloads must be installed: android or System.IO.DirectoryNotFoundException : Unable to locate a Microsoft.Android.Sdk in either '/Users/runner/work/1/s/xamarin-android/bin/Release/dotnet/packs' or '/Users/runner/work/1/s/xamarin-android/bin/Release/lib/packs'. Fix this by updating the nightly build to bring in the handful of recent improvements that were made to our to emulator test jobs. Emulator component provisioning will now happen _before_ .NET SDK and workload installation. The localization test job template has also been removed as it is no longer in use. --- .../automation/azure-pipelines-nightly.yaml | 30 ++++------ .../yaml-templates/run-emulator-tests.yaml | 27 +++------ .../run-localization-tests.yaml | 56 ------------------- .../yaml-templates/start-stop-emulator.yaml | 30 ++++++++-- 4 files changed, 43 insertions(+), 100 deletions(-) delete mode 100644 build-tools/automation/yaml-templates/run-localization-tests.yaml diff --git a/build-tools/automation/azure-pipelines-nightly.yaml b/build-tools/automation/azure-pipelines-nightly.yaml index 0f7baf8987b..029a3992395 100644 --- a/build-tools/automation/azure-pipelines-nightly.yaml +++ b/build-tools/automation/azure-pipelines-nightly.yaml @@ -110,12 +110,10 @@ stages: steps: - template: yaml-templates/setup-test-environment.yaml parameters: - configuration: $(XA.Build.Configuration) - - - template: yaml-templates/run-xaprepare.yaml - parameters: - displayName: install emulator - arguments: --s=EmulatorTestDependencies + installLegacyDotNet: false + restoreNUnitConsole: false + updateMono: false + xaprepareScenario: EmulatorTestDependencies - template: yaml-templates/run-dotnet-preview.yaml parameters: @@ -127,12 +125,12 @@ stages: - script: echo "##vso[task.setvariable variable=Java8SdkDirectory]$JAVA_HOME_8_X64" displayName: set Java8SdkDirectory - - task: MSBuild@1 - displayName: install and launch emulator - inputs: - solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj - configuration: $(XA.Build.Configuration) - msbuildArguments: /t:InstallAvdImage;AcquireAndroidTarget /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /p:TestAvdType=$(avdType) /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/install-emulator-$(avdApiLevel).binlog + - template: yaml-templates/start-stop-emulator.yaml + parameters: + specificImage: true + avdApiLevel: $(avdApiLevel) + avdAbi: $(avdAbi) + avdType: $(avdType) - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -144,14 +142,6 @@ stages: artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.aab artifactFolder: Mono.Android-$(XA.Build.Configuration)-$(avdApiLevel) - - task: MSBuild@1 - displayName: shut down emulator - inputs: - solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj - configuration: $(XA.Build.Configuration) - msbuildArguments: /t:AcquireAndroidTarget,ReleaseAndroidTarget /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /p:TestAvdType=$(avdType) /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog - condition: always() - - template: yaml-templates/upload-results.yaml parameters: configuration: $(XA.Build.Configuration) diff --git a/build-tools/automation/yaml-templates/run-emulator-tests.yaml b/build-tools/automation/yaml-templates/run-emulator-tests.yaml index 8d1b4f7fd11..81b115b8c06 100644 --- a/build-tools/automation/yaml-templates/run-emulator-tests.yaml +++ b/build-tools/automation/yaml-templates/run-emulator-tests.yaml @@ -16,36 +16,23 @@ jobs: clean: all steps: - template: setup-test-environment.yaml - - - template: run-xaprepare.yaml parameters: - arguments: --s=EmulatorTestDependencies + installLegacyDotNet: false + restoreNUnitConsole: false + updateMono: false + xaprepareScenario: EmulatorTestDependencies - task: DownloadPipelineArtifact@2 inputs: artifactName: $(TestAssembliesArtifactName) downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration) - - task: MSBuild@1 - displayName: start emulator - inputs: - solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj - configuration: $(XA.Build.Configuration) - msbuildArguments: >- - /t:AcquireAndroidTarget ${{ parameters.emulatorMSBuildArgs }} /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog + - template: start-stop-emulator.yaml + parameters: + emulatorMSBuildArgs: ${{ parameters.emulatorMSBuildArgs }} - ${{ parameters.testSteps }} - - task: MSBuild@1 - displayName: shut down emulator - inputs: - solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj - configuration: $(XA.Build.Configuration) - msbuildArguments: >- - /t:AcquireAndroidTarget,ReleaseAndroidTarget - /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog - condition: always() - - template: upload-results.yaml parameters: artifactName: Test Results - ${{ parameters.jobName }} With Emulator - macOS diff --git a/build-tools/automation/yaml-templates/run-localization-tests.yaml b/build-tools/automation/yaml-templates/run-localization-tests.yaml deleted file mode 100644 index 2376c0e3dfb..00000000000 --- a/build-tools/automation/yaml-templates/run-localization-tests.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Runs Localization tests against an emulator running on macOS - -parameters: - node_id: 0 - -jobs: - - job: mac_localization_tests_${{ parameters.node_id }} - displayName: Localization Emulator Tests ${{ parameters.node_id }} - pool: - vmImage: $(HostedMacImage) - timeoutInMinutes: 180 - cancelTimeoutInMinutes: 5 - workspace: - clean: all - steps: - - template: setup-test-environment.yaml - - - template: run-xaprepare.yaml - parameters: - arguments: --s=EmulatorTestDependencies - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: $(TestAssembliesArtifactName) - downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration) - - - task: MSBuild@1 - displayName: start emulator - inputs: - solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj - configuration: $(XA.Build.Configuration) - msbuildArguments: >- - /t:AcquireAndroidTarget /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog - - - template: run-nunit-tests.yaml - parameters: - testRunTitle: LocalizationTests On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name~CheckLocalizationIsCorrectNode${{ parameters.node_id }}" - testResultsFile: TestResult-LocalizationTests-Node${{ parameters.node_id }}-$(XA.Build.Configuration).xml - - - task: MSBuild@1 - displayName: shut down emulator - inputs: - solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj - configuration: $(XA.Build.Configuration) - msbuildArguments: >- - /t:AcquireAndroidTarget,ReleaseAndroidTarget - /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog - condition: always() - - - template: upload-results.yaml - parameters: - artifactName: Test Results - TimeZoneInfo With Emulator - macOS - ${{ parameters.node_id }} - - - template: fail-on-issue.yaml diff --git a/build-tools/automation/yaml-templates/start-stop-emulator.yaml b/build-tools/automation/yaml-templates/start-stop-emulator.yaml index 31c9e876c2d..1817d68ed64 100644 --- a/build-tools/automation/yaml-templates/start-stop-emulator.yaml +++ b/build-tools/automation/yaml-templates/start-stop-emulator.yaml @@ -5,6 +5,7 @@ parameters: avdApiLevel: # Device API level, like '30', required if 'specificImage' is 'true' avdAbi: # Device ABI, like 'x86', required if 'specificImage' is 'true' avdType: # Device AVD, like 'android-wear', required if 'specificImage' is 'true' + emulatorMSBuildArgs: # Extra args to pass to the emulator configuration like '-p:TestAvdExtraBootArgs=-writable-system', optional launchTimeoutMin: 20 # Minutes to wait for the emulator to start xaSourcePath: $(System.DefaultWorkingDirectory) # working directory @@ -16,9 +17,20 @@ steps: inputs: projects: ${{ parameters.xaSourcePath }}/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Emulator.csproj ${{ if eq(parameters.specificImage, true) }}: - arguments: -c $(XA.Build.Configuration) -t:"InstallAvdImage;AcquireAndroidTarget" -p:TestDeviceName=${{ parameters.deviceName }} -p:TestAvdApiLevel=${{ parameters.avdApiLevel }} -p:TestAvdAbi=${{ parameters.avdAbi }} -p:TestAvdType=${{ parameters.avdType }} -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/install-emulator-${{ parameters.avdApiLevel }}.binlog + arguments: >- + -c $(XA.Build.Configuration) -t:"InstallAvdImage;AcquireAndroidTarget" + -p:TestDeviceName=${{ parameters.deviceName }} + -p:TestAvdApiLevel=${{ parameters.avdApiLevel }} + -p:TestAvdAbi=${{ parameters.avdAbi }} + -p:TestAvdType=${{ parameters.avdType }} + ${{ parameters.emulatorMSBuildArgs }} + -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} + -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/install-emulator-${{ parameters.avdApiLevel }}.binlog ${{ else }}: - arguments: -c $(XA.Build.Configuration) -t:AcquireAndroidTarget -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog + arguments: >- + -c $(XA.Build.Configuration) -t:AcquireAndroidTarget + -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} + -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog - ${{ if eq(parameters.command, 'stop') }}: - task: DotNetCoreCLI@2 @@ -28,6 +40,16 @@ steps: inputs: projects: ${{ parameters.xaSourcePath }}/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Emulator.csproj ${{ if eq(parameters.specificImage, true) }}: - arguments: -c $(XA.Build.Configuration) -t:"AcquireAndroidTarget,ReleaseAndroidTarget" -p:TestDeviceName=${{ parameters.deviceName }} -p:TestAvdApiLevel=${{ parameters.avdApiLevel }} -p:TestAvdAbi=${{ parameters.avdAbi }} -p:TestAvdType=${{ parameters.avdType }} -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog + arguments: >- + -c $(XA.Build.Configuration) -t:"AcquireAndroidTarget,ReleaseAndroidTarget" + -p:TestDeviceName=${{ parameters.deviceName }} + -p:TestAvdApiLevel=${{ parameters.avdApiLevel }} + -p:TestAvdAbi=${{ parameters.avdAbi }} + -p:TestAvdType=${{ parameters.avdType }} + -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} + -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog ${{ else }}: - arguments: -c $(XA.Build.Configuration) -t:"AcquireAndroidTarget,ReleaseAndroidTarget" -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog + arguments: >- + -c $(XA.Build.Configuration) -t:"AcquireAndroidTarget,ReleaseAndroidTarget" + -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} + -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog From 53a00641ccd10c7afdd8759e1cb684e77165c2f2 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Tue, 6 Jun 2023 16:52:11 -0400 Subject: [PATCH 2/3] Fix spacing --- .../yaml-templates/start-stop-emulator.yaml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/build-tools/automation/yaml-templates/start-stop-emulator.yaml b/build-tools/automation/yaml-templates/start-stop-emulator.yaml index 1817d68ed64..3bede38ec6d 100644 --- a/build-tools/automation/yaml-templates/start-stop-emulator.yaml +++ b/build-tools/automation/yaml-templates/start-stop-emulator.yaml @@ -18,19 +18,19 @@ steps: projects: ${{ parameters.xaSourcePath }}/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Emulator.csproj ${{ if eq(parameters.specificImage, true) }}: arguments: >- - -c $(XA.Build.Configuration) -t:"InstallAvdImage;AcquireAndroidTarget" - -p:TestDeviceName=${{ parameters.deviceName }} - -p:TestAvdApiLevel=${{ parameters.avdApiLevel }} - -p:TestAvdAbi=${{ parameters.avdAbi }} - -p:TestAvdType=${{ parameters.avdType }} - ${{ parameters.emulatorMSBuildArgs }} - -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} - -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/install-emulator-${{ parameters.avdApiLevel }}.binlog + -c $(XA.Build.Configuration) -t:"InstallAvdImage;AcquireAndroidTarget" + -p:TestDeviceName=${{ parameters.deviceName }} + -p:TestAvdApiLevel=${{ parameters.avdApiLevel }} + -p:TestAvdAbi=${{ parameters.avdAbi }} + -p:TestAvdType=${{ parameters.avdType }} + ${{ parameters.emulatorMSBuildArgs }} + -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} + -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/install-emulator-${{ parameters.avdApiLevel }}.binlog ${{ else }}: arguments: >- - -c $(XA.Build.Configuration) -t:AcquireAndroidTarget - -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} - -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog + -c $(XA.Build.Configuration) -t:AcquireAndroidTarget + -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} + -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog - ${{ if eq(parameters.command, 'stop') }}: - task: DotNetCoreCLI@2 From b4a163b201a515a146f14ec45bf4a2daa36d57a8 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Wed, 7 Jun 2023 11:41:42 -0400 Subject: [PATCH 3/3] Pass extra launch args to both start steps --- build-tools/automation/yaml-templates/start-stop-emulator.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/build-tools/automation/yaml-templates/start-stop-emulator.yaml b/build-tools/automation/yaml-templates/start-stop-emulator.yaml index 3bede38ec6d..afaa89cc660 100644 --- a/build-tools/automation/yaml-templates/start-stop-emulator.yaml +++ b/build-tools/automation/yaml-templates/start-stop-emulator.yaml @@ -29,6 +29,7 @@ steps: ${{ else }}: arguments: >- -c $(XA.Build.Configuration) -t:AcquireAndroidTarget + ${{ parameters.emulatorMSBuildArgs }} -p:AvdLaunchTimeoutMinutes=${{ parameters.launchTimeoutMin }} -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog