diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml
index d52240be644def..42cb3758d4db30 100644
--- a/eng/pipelines/common/templates/runtimes/run-test-job.yml
+++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml
@@ -58,10 +58,11 @@ jobs:
 
     dependsOn:
     - ${{ if ne(parameters.corefxTests, true) }}:
-      - ${{ if eq(parameters.testGroup, 'innerloop') }}:
-        - '${{ parameters.runtimeFlavor }}_common_test_build_p0_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
-      - ${{ if ne(parameters.testGroup, 'innerloop') }}:
-        - '${{ parameters.runtimeFlavor }}_common_test_build_p1_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
+      - ${{ if ne(parameters.testBuildPhased, true) }}:
+        - ${{ if eq(parameters.testGroup, 'innerloop') }}:
+          - '${{ parameters.runtimeFlavor }}_common_test_build_p0_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
+        - ${{ if ne(parameters.testGroup, 'innerloop') }}:
+          - '${{ parameters.runtimeFlavor }}_common_test_build_p1_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
       - ${{ if eq(parameters.testBuildPhased, true) }}:
         # Also depend on AnyOS AnyCPU tests
         - ${{ if eq(parameters.testGroup, 'innerloop') }}:
@@ -209,13 +210,14 @@ jobs:
             artifactName: '$(managedGenericTestArtifactName)'
             displayName: 'generic managed test artifacts'
 
-      # Download and unzip target specific tests
-      - template: /eng/pipelines/common/download-artifact-step.yml
-        parameters:
-          unpackFolder: '$(managedTestArtifactRootFolderPath)'
-          artifactFileName: '$(managedTestArtifactName)$(archiveExtension)'
-          artifactName: '$(managedTestArtifactName)'
-          displayName: 'managed test artifacts (built on ${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }})'
+      # Download and unzip target specific tests (unless we build them below)
+      - ${{ if ne(parameters.testBuildPhased, true) }}:
+        - template: /eng/pipelines/common/download-artifact-step.yml
+          parameters:
+            unpackFolder: '$(managedTestArtifactRootFolderPath)'
+            artifactFileName: '$(managedTestArtifactName)$(archiveExtension)'
+            artifactName: '$(managedTestArtifactName)'
+            displayName: 'target specific managed test artifacts (built on ${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }})'
 
 
     # Download product binaries directory
@@ -236,9 +238,27 @@ jobs:
           artifactName: '$(coreClrProductArtifactName)'
           displayName: 'CoreCLR product download for Mono'
 
+
+    # Build target specific tests
+    - ${{ if and(ne(parameters.corefxTests, true), eq(parameters.testBuildPhased, true)) }}:
+      # Install test build dependencies
+      - ${{ if eq(parameters.osGroup, 'OSX') }}:
+        - script: sh $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup)
+          displayName: Install native dependencies
+      - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
+        # Necessary to install correct cmake version
+        - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
+          displayName: Install native dependencies
+
+      # Build targetSpecific managed test components
+      - script: $(coreClrRepoRootDir)build-test$(scriptExt) targetSpecific skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg)
+        displayName: Build target specific managed test components
+
+
+    # If we didn't build target specific tests here
     # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing
     # ilproj test projects during copynativeonly.
-    - ${{ if ne(parameters.corefxTests, true) }}:
+    - ${{ if and(ne(parameters.corefxTests, true), ne(parameters.testBuildPhased, true)) }}:
       - template: /eng/pipelines/common/download-artifact-step.yml
         parameters:
           unpackFolder: '$(microsoftNetSdkIlFolderPath)'
diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml
index 51a85c2f4558df..4028d51a5e222a 100644
--- a/eng/pipelines/coreclr/ci.yml
+++ b/eng/pipelines/coreclr/ci.yml
@@ -103,23 +103,6 @@ jobs:
       liveLibrariesBuildConfig: Release
       testBuildPhase: targetGeneric
 
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
-    buildConfig: checked
-    platforms:
-    - Linux_arm
-    - Linux_arm64
-    - OSX_x64
-    - Windows_NT_arm
-    - Windows_NT_arm64
-    - Windows_NT_x64
-    - Windows_NT_x86
-    testGroup: outerloop
-    jobParameters:
-      liveLibrariesBuildConfig: Release
-      testBuildPhase: targetSpecific
-
 #
 # Checked JIT test runs
 #
diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
index da37549ddfd373..af93039f2173f7 100644
--- a/eng/pipelines/runtime.yml
+++ b/eng/pipelines/runtime.yml
@@ -592,46 +592,6 @@ jobs:
           eq(dependencies.checkout.outputs['SetPathVars_coreclr.containsChange'], true),
           eq(variables['isFullMatrix'], true))
 
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
-    buildConfig: checked
-    platforms:
-    - Linux_arm 
-    - Windows_NT_x86
-    - Windows_NT_arm
-    - Windows_NT_arm64
-    jobParameters:
-      testGroup: innerloop
-      liveLibrariesBuildConfig: Release
-      testBuildPhase: targetSpecific
-      condition: >-
-        or(
-          eq(dependencies.checkout.outputs['SetPathVars_coreclr.containsChange'], true),
-          eq(variables['isFullMatrix'], true))
-
-#
-# CoreCLR Test builds using live libraries debug build
-# Only when CoreCLR is changed
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
-    buildConfig: checked
-    platforms:
-    - OSX_x64
-    - Linux_x64
-    - Linux_arm64
-    - Windows_NT_x64
-    jobParameters:
-      testGroup: innerloop
-      liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
-      testBuildPhase: targetSpecific
-      condition: >-
-        or(
-          eq(dependencies.checkout.outputs['SetPathVars_coreclr.containsChange'], true),
-          eq(variables['isFullMatrix'], true))
-
 #
 # CoreCLR Test executions using live libraries
 # Only when CoreCLR is changed
diff --git a/src/coreclr/tests/src/readytorun/coreroot_determinism/coreroot_determinism.csproj b/src/coreclr/tests/src/readytorun/coreroot_determinism/coreroot_determinism.csproj
index 3fbc7d2691fd51..541698b29206c5 100644
--- a/src/coreclr/tests/src/readytorun/coreroot_determinism/coreroot_determinism.csproj
+++ b/src/coreclr/tests/src/readytorun/coreroot_determinism/coreroot_determinism.csproj
@@ -5,6 +5,7 @@
     <CLRTestPriority>0</CLRTestPriority>
     <!-- Crossgen2 currently targets only x64 -->
     <DisableProjectBuild Condition="'$(TargetArchitecture)' != 'x64'">true</DisableProjectBuild>
+    <CLRTestNeedTarget>1</CLRTestNeedTarget>
     <!-- Known not to work with GCStress for now: https://github.com/dotnet/coreclr/issues/26633 -->
     <GCStressIncompatible>true</GCStressIncompatible>
     <!-- This is an explicit crossgen test -->
diff --git a/src/coreclr/tests/src/readytorun/crossgen2/crossgen2smoke.csproj b/src/coreclr/tests/src/readytorun/crossgen2/crossgen2smoke.csproj
index e97fc45f15cdfd..ae16127cb33b50 100644
--- a/src/coreclr/tests/src/readytorun/crossgen2/crossgen2smoke.csproj
+++ b/src/coreclr/tests/src/readytorun/crossgen2/crossgen2smoke.csproj
@@ -7,6 +7,7 @@
 
     <!-- Crossgen2 currently targets x64 and ARM64 only -->
     <DisableProjectBuild Condition="'$(TargetArchitecture)' != 'x64' and '$(TargetArchitecture)' != 'arm64'">true</DisableProjectBuild>
+    <CLRTestNeedTarget>1</CLRTestNeedTarget>
 
     <!-- Generate ILDLL so that the DLL can be the crossgenned assembly -->
     <TargetExt>.ildll</TargetExt>
diff --git a/src/coreclr/tests/src/readytorun/determinism/crossgen2determinism.csproj b/src/coreclr/tests/src/readytorun/determinism/crossgen2determinism.csproj
index 52d6c1ba8357d8..558d3b8e7058d4 100644
--- a/src/coreclr/tests/src/readytorun/determinism/crossgen2determinism.csproj
+++ b/src/coreclr/tests/src/readytorun/determinism/crossgen2determinism.csproj
@@ -5,6 +5,7 @@
     <CLRTestPriority>0</CLRTestPriority>
     <!-- Crossgen2 currently targets only x64 -->
     <DisableProjectBuild Condition="'$(TargetArchitecture)' != 'x64'">true</DisableProjectBuild>
+    <CLRTestNeedTarget>1</CLRTestNeedTarget>
     <!-- Known not to work with GCStress for now: https://github.com/dotnet/coreclr/issues/26633 -->
     <GCStressIncompatible>true</GCStressIncompatible>
     <!-- This is an explicit crossgen test -->
diff --git a/src/coreclr/tests/src/readytorun/tests/fieldgetter.ilproj b/src/coreclr/tests/src/readytorun/tests/fieldgetter.ilproj
index 45d63e135ee202..3a93b6c81d08af 100644
--- a/src/coreclr/tests/src/readytorun/tests/fieldgetter.ilproj
+++ b/src/coreclr/tests/src/readytorun/tests/fieldgetter.ilproj
@@ -2,6 +2,8 @@
   <PropertyGroup>
     <OutputType>Library</OutputType>
     <CLRTestKind>SharedLibrary</CLRTestKind>
+    <!-- This requires a specific targetis OS because it fails when built on OSX and run on Windows -->
+    <CLRTestNeedTarget>1</CLRTestNeedTarget>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="fieldgetter.il" />
diff --git a/src/coreclr/tests/src/readytorun/tests/mainv1.csproj b/src/coreclr/tests/src/readytorun/tests/mainv1.csproj
index 8587289faf1054..91d1da34ed91f6 100644
--- a/src/coreclr/tests/src/readytorun/tests/mainv1.csproj
+++ b/src/coreclr/tests/src/readytorun/tests/mainv1.csproj
@@ -3,6 +3,8 @@
     <OutputType>exe</OutputType>
     <CLRTestKind>BuildAndRun</CLRTestKind>
     <CrossGenTest>false</CrossGenTest>
+    <!-- This requires a specific targetis OS because it fails when built on OSX and run on Windows -->
+    <CLRTestNeedTarget>1</CLRTestNeedTarget>
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="fieldgetter.ilproj" />
diff --git a/src/coreclr/tests/src/readytorun/tests/mainv2.csproj b/src/coreclr/tests/src/readytorun/tests/mainv2.csproj
index 9207507d02e13b..25a6b98afcc38f 100644
--- a/src/coreclr/tests/src/readytorun/tests/mainv2.csproj
+++ b/src/coreclr/tests/src/readytorun/tests/mainv2.csproj
@@ -3,6 +3,8 @@
     <OutputType>exe</OutputType>
     <CLRTestKind>BuildAndRun</CLRTestKind>
     <CrossGenTest>false</CrossGenTest>
+    <!-- This requires a specific targetis OS because it fails when built on OSX and run on Windows -->
+    <CLRTestNeedTarget>1</CLRTestNeedTarget>
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="fieldgetter.ilproj" />
diff --git a/src/coreclr/tests/src/readytorun/tests/testv1/test.csproj b/src/coreclr/tests/src/readytorun/tests/testv1/test.csproj
index 970a6904e4f11b..783ed6a1e94c0f 100644
--- a/src/coreclr/tests/src/readytorun/tests/testv1/test.csproj
+++ b/src/coreclr/tests/src/readytorun/tests/testv1/test.csproj
@@ -2,6 +2,8 @@
   <PropertyGroup>
     <OutputType>library</OutputType>
     <CLRTestKind>SharedLibrary</CLRTestKind>
+    <!-- This requires a specific targetis OS because it fails when built on OSX and run on Windows -->
+    <CLRTestNeedTarget>1</CLRTestNeedTarget>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="..\test.cs" />