diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index d6e75be1a891..4c9915859442 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -49,10 +49,99 @@ variables: - template: /eng/common/templates/variables/pool-providers.yml jobs: +# CoreCLR runtime E2E tests (UseMonoRuntime=false) — runs first for faster feedback +# Threading tests are excluded via the csproj when UseMonoRuntime=false +- template: jobs/default-build.yml + parameters: + jobName: Components_E2E_Test_CoreCLR + jobDisplayName: "Test: Blazor E2E tests on Linux (CoreCLR)" + agentOs: Linux + use1ESUbuntu: true + isAzDOTestingJob: true + enablePublishTestResults: false + timeoutInMinutes: 120 + steps: + - script: git submodule update --init + displayName: Update submodules + - script: ./restore.sh + displayName: Run restore.sh + - script: npm ci + displayName: NPM install + - script: npm run build + displayName: Build JS + - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) /p:UseMonoRuntime=false + displayName: Build (CoreCLR) + - script: | + set -eo pipefail + + .dotnet/dotnet test ./src/Components/test/E2ETest \ + -c $(BuildConfiguration) \ + --no-build \ + /p:UseMonoRuntime=false \ + --filter 'Quarantined!=true|Quarantined=false' \ + -p:VsTestUseMSBuildOutput=false \ + --logger:"trx%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ + --logger:"html%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.html" \ + --results-directory $(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined \ + | tee e2e-test-output.log + + if grep -q "No test matches the given testcase filter" e2e-test-output.log + then + echo "##vso[task.logissue type=error] No tests matched the filter." + + exit 1 + fi + + # Check total tests run to detect abnormalities. + # Lower than Mono threshold since server-only tests (ServerExecutionTests, ServerRenderingTests) are excluded. + total=$(sed -nE 's/.*Failed:[[:space:]]*[0-9]+,[[:space:]]*Passed:[[:space:]]*[0-9]+,[[:space:]]*Skipped:[[:space:]]*[0-9]+,[[:space:]]*Total:[[:space:]]*([0-9]+).*/\1/p' e2e-test-output.log) + min_total=500 + if [ -z "$total" ] || [ "$total" -lt "$min_total" ] + then + echo "##vso[task.logissue type=error] Insufficient total test count: $total. We expect at least $min_total tests to run." + exit 1 + fi + displayName: Run E2E tests (CoreCLR) + - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build /p:UseMonoRuntime=false --filter 'Quarantined=true' -p:RunQuarantinedTests=true + -p:VsTestUseMSBuildOutput=false + --logger:"trx%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" + --logger:"html%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.html" + --results-directory $(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Quarantined + displayName: Run Quarantined E2E tests (CoreCLR) + continueOnError: true + - task: PublishTestResults@2 + displayName: Publish E2E Test Results (CoreCLR) + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined' + testRunTitle: ComponentsE2E-CoreCLR-$(AgentOsName)-$(BuildConfiguration)-xunit + condition: always() + - task: PublishTestResults@2 + displayName: Publish Quarantined E2E Test Results (CoreCLR) + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Quarantined' + testRunTitle: Quarantine-CoreCLR-$(AgentOsName)-$(BuildConfiguration)-xunit + mergeTestResults: true + condition: always() + + artifacts: + - name: Components_E2E_CoreCLR_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Components_E2E_CoreCLR_Test_Logs + path: '$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)' + includeForks: true + publishOnError: true + +# Mono runtime (default) E2E tests - template: jobs/default-build.yml parameters: jobName: Components_E2E_Test - jobDisplayName: "Test: Blazor E2E tests on Linux" + jobDisplayName: "Test: Blazor E2E tests on Linux (Mono)" agentOs: Linux use1ESUbuntu: true isAzDOTestingJob: true @@ -121,7 +210,7 @@ jobs: testResultsFormat: 'VSTest' testResultsFiles: '*.trx' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined' - testRunTitle: ComponentsE2E-$(AgentOsName)-$(BuildConfiguration)-xunit + testRunTitle: ComponentsE2E-Mono-$(AgentOsName)-$(BuildConfiguration)-xunit condition: always() - task: PublishTestResults@2 displayName: Publish Quarantined E2E Test Results @@ -129,7 +218,7 @@ jobs: testResultsFormat: 'VSTest' testResultsFiles: '*.trx' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Quarantined' - testRunTitle: Quarantine-$(AgentOsName)-$(BuildConfiguration)-xunit + testRunTitle: Quarantine-Mono-$(AgentOsName)-$(BuildConfiguration)-xunit mergeTestResults: true condition: always() diff --git a/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj b/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj index 48c04184707e..a1edc88a05cb 100644 --- a/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj +++ b/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj @@ -42,6 +42,16 @@ + + + + + + + + + @@ -74,13 +84,14 @@ - + + - + @@ -97,7 +108,7 @@ - +