diff --git a/build-tools/scripts/UnitTestApks.targets b/build-tools/scripts/UnitTestApks.targets
index 9bb4d6590f3..fdeaef1cf73 100644
--- a/build-tools/scripts/UnitTestApks.targets
+++ b/build-tools/scripts/UnitTestApks.targets
@@ -63,6 +63,10 @@
         ToolExe="$(AdbToolExe)"
         ToolPath="$(AdbToolPath)"
     />
+    <Error
+        Condition="'@(_FailedComponent)' != ''"
+        Text="Execution of the following components did not complete successfully: @(_FailedComponent->'%(Identity)', ', ')"
+    />
   </Target>
 
   <!--
@@ -96,7 +100,6 @@
   <Target Name="RunUnitTestApks"
       Condition=" '@(UnitTestApk)' != '' ">
     <RunInstrumentationTests
-        ContinueOnError="True"
         AdbTarget="$(_AdbTarget)"
         AdbOptions="$(AdbOptions)"
         Component="%(UnitTestApk.Package)/%(UnitTestApk.InstrumentationType)"
@@ -104,7 +107,7 @@
         TestFixture="$(TestFixture)"
         ToolExe="$(AdbToolExe)"
         ToolPath="$(AdbToolPath)">
-      <Output TaskParameter="NUnit2TestResultsFile" PropertyName="_ResultsFile "/>
+      <Output TaskParameter="FailedToRun" ItemName="_FailedComponent"/>
     </RunInstrumentationTests>
   </Target>
 </Project>
diff --git a/src/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunInstrumentationTests.cs b/src/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunInstrumentationTests.cs
index 23d195e3a3b..151ea7b4403 100644
--- a/src/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunInstrumentationTests.cs
+++ b/src/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunInstrumentationTests.cs
@@ -21,9 +21,11 @@ public class RunInstrumentationTests : Adb
 
 		public                  string[]            InstrumentationArguments    { get; set; }
 
-		[Output]
 		public                  string              NUnit2TestResultsFile       { get; set; }
 
+		[Output]
+		public                  string              FailedToRun                 { get; set; }
+
 		protected   override    bool                LogTaskMessages {
 			get { return false; }
 		}
@@ -63,15 +65,19 @@ public override bool Execute ()
 			base.Execute ();
 
 			if (string.IsNullOrEmpty (targetTestResultsPath)) {
-				Log.LogError ("No `nunit2-results-path` bundle value found in command output! Cannot find NUnit2 XML output!");
-				return false;
+				FailedToRun = Component;
+				Log.LogError (
+						"Could not find NUnit2 results file after running component `{0}`: " +
+						"no `nunit2-results-path` bundle value found in command output!",
+						Component);
+				// Can return false once we use MSBuild and not xbuild
+				// return false;
+				return true;
 			}
 
 			executionState  = ExecuteState.PullFiles;
 			base.Execute ();
 
-			Log.LogMessage (MessageImportance.Low, $"  [Output] {nameof (NUnit2TestResultsFile)}: {NUnit2TestResultsFile}");
-
 			return !Log.HasLoggedErrors;
 		}