diff --git a/eng/pipelines/common/templates/wasm-build-only.yml b/eng/pipelines/common/templates/wasm-build-only.yml index 761de04536d07c..32e17588a52928 100644 --- a/eng/pipelines/common/templates/wasm-build-only.yml +++ b/eng/pipelines/common/templates/wasm-build-only.yml @@ -27,16 +27,6 @@ jobs: value: ${{ parameters.alwaysRun }} - name: allWasmContainsChange value: $[ dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'] ] - - name: isDefaultPipeline - # default pipeline, so not runtime-wasm, and not - # runtime-extra-platforms+rolling build - value: ${{ - and( - ne(parameters.isWasmOnlyBuild, true), - or( - ne(parameters.isExtraPlatformsBuild, true), - eq(variables['isRollingBuild'], true))) - }} - name: shouldRunOnDefaultPipelines value: $[ or( diff --git a/eng/pipelines/common/templates/wasm-build-tests.yml b/eng/pipelines/common/templates/wasm-build-tests.yml index 7ce2176f19a84d..07c253ed80bf1c 100644 --- a/eng/pipelines/common/templates/wasm-build-tests.yml +++ b/eng/pipelines/common/templates/wasm-build-tests.yml @@ -26,16 +26,6 @@ jobs: value: $[ dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'] ] - name: alwaysRunVar value: ${{ parameters.alwaysRun }} - - name: isDefaultPipeline - # default pipeline, so not runtime-wasm, and not - # runtime-extra-platforms+rolling build - value: ${{ - and( - ne(parameters.isWasmOnlyBuild, true), - or( - ne(parameters.isExtraPlatformsBuild, true), - eq(variables['isRollingBuild'], true))) - }} - name: shouldRunOnDefaultPipelines value: $[ or( diff --git a/eng/pipelines/common/templates/wasm-debugger-tests.yml b/eng/pipelines/common/templates/wasm-debugger-tests.yml index 2bd104a6592036..29e29e2a6aa27f 100644 --- a/eng/pipelines/common/templates/wasm-debugger-tests.yml +++ b/eng/pipelines/common/templates/wasm-debugger-tests.yml @@ -25,16 +25,6 @@ jobs: value: $[ dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'] ] - name: alwaysRunVar value: ${{ parameters.alwaysRun }} - - name: isDefaultPipeline - # default pipeline, so not runtime-wasm, and not - # runtime-extra-platforms+rolling build - value: ${{ - and( - ne(parameters.isWasmOnlyBuild, true), - or( - ne(parameters.isExtraPlatformsBuild, true), - eq(variables['isRollingBuild'], true))) - }} - name: shouldRunOnDefaultPipelines value: $[ or( diff --git a/eng/pipelines/common/templates/wasm-library-tests.yml b/eng/pipelines/common/templates/wasm-library-tests.yml index f96ca8aa1aabeb..9981d8727a85c5 100644 --- a/eng/pipelines/common/templates/wasm-library-tests.yml +++ b/eng/pipelines/common/templates/wasm-library-tests.yml @@ -34,16 +34,6 @@ jobs: value: ${{ parameters.alwaysRun }} - name: allWasmContainsChange value: $[ dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'] ] - - name: isDefaultPipeline - # default pipeline, so not runtime-wasm, and not - # runtime-extra-platforms+rolling build - value: ${{ - and( - ne(parameters.isWasmOnlyBuild, true), - or( - ne(parameters.isExtraPlatformsBuild, true), - eq(variables['isRollingBuild'], true))) - }} - name: shouldRunOnDefaultPipelines value: $[ or( diff --git a/eng/pipelines/common/templates/wasm-runtime-tests.yml b/eng/pipelines/common/templates/wasm-runtime-tests.yml index b244949340241c..903235e75c1021 100644 --- a/eng/pipelines/common/templates/wasm-runtime-tests.yml +++ b/eng/pipelines/common/templates/wasm-runtime-tests.yml @@ -25,16 +25,6 @@ jobs: value: 10 - name: timeoutPerTestCollectionInMinutes value: 200 - - name: isDefaultPipeline - # default pipeline, so not runtime-wasm, and not - # runtime-extra-platforms+!rolling build - value: ${{ - and( - ne(parameters.isWasmOnlyBuild, true), - or( - ne(parameters.isExtraPlatformsBuild, true), - eq(variables['isRollingBuild'], true))) - }} - name: shouldRunOnDefaultPipelines value: $[ or( @@ -49,6 +39,7 @@ jobs: nameSuffix: AllSubsets_Mono_RuntimeTests runtimeVariant: monointerpreter buildArgs: -s mono+libs -c $(_BuildConfig) + timeoutInMinutes: 180 # if !alwaysRun, then: # if this is runtime-wasm (isWasmOnlyBuild): # - then run only if it would not have run on default pipelines (based diff --git a/eng/pipelines/common/variables.yml b/eng/pipelines/common/variables.yml index ea0836fc802440..0798b9fcd2e589 100644 --- a/eng/pipelines/common/variables.yml +++ b/eng/pipelines/common/variables.yml @@ -18,20 +18,27 @@ variables: - name: isNotExtraPlatformsBuild value: ${{ ne(variables['Build.DefinitionName'], 'runtime-extra-platforms') }} - name: isWasmOnlyBuild - value: ${{ eq(variables['Build.DefinitionName'], 'runtime-wasm') }} + value: ${{ in(variables['Build.DefinitionName'], 'runtime-wasm', 'runtime-wasm-libtests', 'runtime-wasm-non-libtests') }} - name: isRunSmokeTestsOnly - value: ${{ and(ne(variables['Build.DefinitionName'], 'runtime-extra-platforms'), ne(variables['Build.DefinitionName'], 'runtime-wasm')) }} + value: ${{ notin(variables['Build.DefinitionName'], 'runtime-extra-platforms', 'runtime-wasm', 'runtime-wasm-libtests', 'runtime-wasm-non-libtests') }} - name: isNotSpecificPlatformOnlyBuild - value: ${{ ne(variables['Build.DefinitionName'], 'runtime-wasm') }} + value: ${{ notin(variables['Build.DefinitionName'], 'runtime-wasm', 'runtime-wasm-libtests', 'runtime-wasm-non-libtests') }} # We only run evaluate paths on runtime, runtime-staging and runtime-community pipelines on PRs # keep in sync with /eng/pipelines/common/xplat-setup.yml - name: dependOnEvaluatePaths - value: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging', 'runtime-community', 'runtime-extra-platforms', 'runtime-wasm')) }} + value: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging', 'runtime-community', 'runtime-extra-platforms', 'runtime-wasm', 'runtime-wasm-libtests', 'runtime-wasm-non-libtests', 'dotnet-linker-tests', 'runtime-dev-innerloop')) }} - name: debugOnPrReleaseOnRolling ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: value: Release ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: value: Debug +- name: isDefaultPipeline + value: $[ and( + ne(variables['isWasmOnlyBuild'], true), + or( + ne(variables['isExtraPlatformsBuild'], true), + eq(variables['isRollingBuild'], true))) ] + - template: /eng/pipelines/common/perf-variables.yml diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index 9d91ccd891da39..53c43c02ce88da 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -18,7 +18,7 @@ jobs: shouldContinueOnError: ${{ or(eq(parameters.shouldContinueOnError, true), and(ne(parameters.shouldContinueOnError, 'forceFalse'), endsWith(variables['Build.DefinitionName'], 'staging'), eq(variables['Build.Reason'], 'PullRequest'))) }} # keep in sync with /eng/pipelines/common/variables.yml - dependOnEvaluatePaths: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging', 'runtime-community', 'runtime-extra-platforms', 'runtime-wasm')) }} + dependOnEvaluatePaths: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-staging', 'runtime-community', 'runtime-extra-platforms', 'runtime-wasm', 'runtime-wasm-libtests', 'runtime-wasm-non-libtests', 'dotnet-linker-tests', 'runtime-dev-innerloop')) }} variables: # Disable component governance in our CI builds. These builds are not shipping nor diff --git a/eng/pipelines/runtime-extra-platforms-wasm.yml b/eng/pipelines/runtime-extra-platforms-wasm.yml index 26152f51cf3a47..60d0105dedcc59 100644 --- a/eng/pipelines/runtime-extra-platforms-wasm.yml +++ b/eng/pipelines/runtime-extra-platforms-wasm.yml @@ -7,6 +7,8 @@ parameters: isExtraPlatformsBuild: false isWasmOnlyBuild: false isRollingBuild: false + excludeLibTests: false + excludeNonLibTests: false jobs: @@ -60,7 +62,7 @@ jobs: # - run everything, if relevant paths changed # - For runtime-wasm, force run all the jobs # -- ${{ if ne(parameters.isRollingBuild, true) }}: +- ${{ if and(ne(parameters.isRollingBuild, true), ne(parameters.excludeLibTests, true)) }}: # Library tests # these run on runtime also - template: /eng/pipelines/common/templates/wasm-library-tests.yml @@ -174,6 +176,7 @@ jobs: isWasmOnlyBuild: ${{ parameters.isWasmOnlyBuild }} alwaysRun: ${{ parameters.isWasmOnlyBuild }} +- ${{ if and(ne(parameters.isRollingBuild, true), ne(parameters.excludeNonLibTests, true)) }}: # Wasm.Build.Tests - template: /eng/pipelines/common/templates/wasm-build-tests.yml parameters: diff --git a/eng/pipelines/runtime-wasm-libtests.yml b/eng/pipelines/runtime-wasm-libtests.yml new file mode 100644 index 00000000000000..573dc4f3338587 --- /dev/null +++ b/eng/pipelines/runtime-wasm-libtests.yml @@ -0,0 +1,19 @@ +trigger: none + +variables: + - template: /eng/pipelines/common/variables.yml + +jobs: + +# +# Evaluate paths +# +- template: /eng/pipelines/common/evaluate-default-paths.yml + +- template: /eng/pipelines/runtime-extra-platforms-wasm.yml + parameters: + isExtraPlatformsBuild: ${{ variables.isExtraPlatformsBuild }} + isWasmOnlyBuild: ${{ variables.isWasmOnlyBuild }} + isRollingBuild: ${{ variables.isRollingBuild }} + excludeNonLibTests: true + excludeLibTests: false diff --git a/eng/pipelines/runtime-wasm-non-libtests.yml b/eng/pipelines/runtime-wasm-non-libtests.yml new file mode 100644 index 00000000000000..c4f5e43c3b62a9 --- /dev/null +++ b/eng/pipelines/runtime-wasm-non-libtests.yml @@ -0,0 +1,19 @@ +trigger: none + +variables: + - template: /eng/pipelines/common/variables.yml + +jobs: + +# +# Evaluate paths +# +- template: /eng/pipelines/common/evaluate-default-paths.yml + +- template: /eng/pipelines/runtime-extra-platforms-wasm.yml + parameters: + isExtraPlatformsBuild: ${{ variables.isExtraPlatformsBuild }} + isWasmOnlyBuild: ${{ variables.isWasmOnlyBuild }} + isRollingBuild: ${{ variables.isRollingBuild }} + excludeNonLibTests: false + excludeLibTests: true diff --git a/src/mono/wasm/README.md b/src/mono/wasm/README.md index f8e2344fbf3295..221fce0e52b611 100644 --- a/src/mono/wasm/README.md +++ b/src/mono/wasm/README.md @@ -230,38 +230,60 @@ npm update ### How do I know which jobs run on CI, and when? ## PR: -* `runtime-extra-platforms`, and `runtime-wasm` run only when manually triggered with a comment - `/azp run ` -* `runtime`, and `runtime-staging`, run jobs only when relevant paths change. And for `EAT`, and `AOT`, only smoke tests are run. -* And when `runtime-wasm` is triggered manually, it runs *all* the wasm jobs completely - -| . | runtime | runtime-staging | runtime-extra-platforms(manual only) | runtime-wasm (manual only) | -| ----------------- | -------------------- | --------------- | ------------------------------------ | ------- | -| libtests | linux: all, only-pc | windows: all, only-pc | linux+windows: all, only-pc | linux+windows: all, always | -| libtests eat | linux: smoke, only-pc | - | linux: all, only-pc | linux: all, always | -| libtests aot | linux: smoke, only-pc | windows: smoke, only-pc | linux+windows: all, only-pc | linux+windows: all, always | -| high resource aot | none | none | linux+windows: all, only-pc | linux+windows: all, always | -| Wasm.Build.Tests | linux: only-pc | windows: only-pc | linux+windows: only-pc | linux+windows | -| Debugger tests | - | linux+windows: only-pc | linux+windows: only-pc | linux+windows | -| Runtime tests | linux: only-pc | - | linux: only-pc | linux | +* `only-pc` means `only on relevant path changes` + +### Run by default + +* `runtime` runs jobs only when relevant paths change. And for `AOT`, only smoke tests are run. + +| . | runtime | +| ----------------- | -------------------- | +| libtests | linux+windows: all, only-pc | +| libtests eat | linux+windows: smoke, only-pc | +| libtests aot | linux+windows: smoke, only-pc | +| high resource aot | none | +| Wasm.Build.Tests | linux+windows: only-pc | +| Debugger tests | linux+windows: only-pc | +| Runtime tests | linux+windows: only-pc | + + +### Run manually with `/azp run ..` + +* `runtime-wasm*` pipelines are triggered manually, and they only run the jobs that would not run on any default pipelines based on path changes. +* The `AOT` jobs run only smoke tests on `runtime`, and on `runtime-wasm*` pipelines all the `AOT` tests are run. + +| . | runtime-wasm | runtime-wasm-libtests | runtime-wasm-non-libtests | +| ----------------- | -------------------------- | -------------------- | -------------------- | +| libtests | linux+windows: all | linux+windows: all | none | +| libtests eat | linux: all | linux: all | none | +| libtests aot | linux+windows: all | linux+windows: all | none | +| high resource aot | linux+windows: all | linux+windows: all | none | +| Wasm.Build.Tests | linux+windows | none | linux+windows | +| Debugger tests | linux+windows | none | linux+windows | +| Runtime tests | linux | none | linux | +| Perftrace | linux: all tests | linux: all tests | none | +| Multi-thread | linux: all tests | linux: all tests | none | + +* `runtime-extra-platforms` does not run any wasm jobs on PRs * `high resource aot` runs a few specific library tests with AOT, that require more memory to AOT. ## Rolling build (twice a day): -* `runtime`, and `runtime-staging`, run all the wasm jobs unconditionally, but `EAT`, and `AOT` still run only smoke tests. -* `runtime-extra-platforms` also runs by default. And it runs only the cases not covered by the above two pipelines. - -* jobs w/o `only-pc` are always run - -| . | runtime | runtime-staging | runtime-extra-platforms (always run) | runtime-wasm (manual only) | -| ----------------- | ------------- | --------------- | ------------------------------------ | ------ | -| libtests | linux: all(v8/chr) | windows: all | none | N/A | -| libtests eat | linux: smoke | - | linux: all | | -| libtests aot | linux: smoke | windows: smoke | linux+windows: all | | -| high resource aot | none | none | linux+windows: all | | -| | | | | | -| Wasm.Build.Tests | linux: always | windows: always | none | | -| Debugger tests | - | linux+windows: always | none | | -| Runtime tests | linux: always | - | none | | +* `runtime` runs all the wasm jobs, but `AOT` still only runs smoke tests. +* `runtime-extra-platforms` also runs by default. And it runs only the cases not covered by `runtime`. + +| . | runtime | runtime-extra-platforms (always run) | +| ----------------- | ------------- | ------------------------------------ | +| libtests | linux+windows: all | none | +| libtests eat | linux: all | none | +| libtests aot | linux+windows: smoke | linux+windows: all | +| high resource aot | none | linux+windows: all | +| | | | +| Wasm.Build.Tests | linux+windows | none | +| Debugger tests | linux+windows | none | +| Runtime tests | linux | none | +| Perftrace | linux: build only | none | +| Multi-thread | linux: build only | none | * `high resource aot` runs a few specific library tests with AOT, that require more memory to AOT.