diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ResolveSdksTask.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ResolveSdksTask.cs
index bd58f36b941..d41eb4d4daf 100644
--- a/src/Xamarin.Android.Build.Tasks/Tasks/ResolveSdksTask.cs
+++ b/src/Xamarin.Android.Build.Tasks/Tasks/ResolveSdksTask.cs
@@ -155,7 +155,8 @@ public bool RunTask ()
 			this.AndroidSdkPath = AndroidSdk.AndroidSdkPath;
 			this.JavaSdkPath = AndroidSdk.JavaSdkPath;
 
-			ValidateJavaVersion (TargetFrameworkVersion, AndroidSdkBuildToolsVersion);
+			if (!ValidateJavaVersion (TargetFrameworkVersion, AndroidSdkBuildToolsVersion))
+				return false;
 
 			if (string.IsNullOrEmpty (AndroidSdkPath)) {
 				Log.LogCodedError ("XA5205", "The Android SDK Directory could not be found. Please set via /p:AndroidSdkDirectory.");
@@ -336,7 +337,7 @@ Version GetJavaVersionForBuildTools (string buildToolsVersion)
 			return Version.Parse (MinimumSupportedJavaVersion);
 		}
 
-		void ValidateJavaVersion (string targetFrameworkVersion, string buildToolsVersion)
+		bool ValidateJavaVersion (string targetFrameworkVersion, string buildToolsVersion)
 		{
 			Version requiredJavaForFrameworkVersion = GetJavaVersionForFramework (targetFrameworkVersion);
 			Version requiredJavaForBuildTools = GetJavaVersionForBuildTools (buildToolsVersion);
@@ -358,7 +359,7 @@ void ValidateJavaVersion (string targetFrameworkVersion, string buildToolsVersio
 			} catch (Exception ex) {
 				Log.LogWarningFromException (ex);
 				Log.LogWarning ($"Failed to get the Java SDK version. Please ensure you have Java {required} or above installed.");
-				return;
+				return false;
 			}
 			var versionInfo = sb.ToString ();
 			var versionNumberMatch = javaVersionRegex.Match (versionInfo);
@@ -373,6 +374,7 @@ void ValidateJavaVersion (string targetFrameworkVersion, string buildToolsVersio
 				}
 			} else
 				Log.LogWarning ($"Failed to get the Java SDK version. Found {versionInfo} but this does not seem to contain a valid version number.");
+			return !Log.HasLoggedErrors;
 		}
 
 		bool ValidateApiLevels ()
diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs
index fb579980427..c5630964510 100644
--- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs
+++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs
@@ -346,16 +346,15 @@ public void CreateMultiDexWithSpacesInConfig ()
 		public void BuildMultiDexApplication (bool useJackAndJill, string fxVersion)
 		{
 			var proj = CreateMultiDexRequiredApplication ();
-			proj.UseJackAndJill = useJackAndJill;
-			proj.TargetFrameworkVersion = fxVersion;
 			proj.UseLatestPlatformSdk = false;
 			proj.SetProperty ("AndroidEnableMultiDex", "True");
 
 			using (var b = CreateApkBuilder ("temp/BuildMultiDexApplication")) {
+				proj.TargetFrameworkVersion = b.LatestTargetFrameworkVersion ();
 				Assert.IsTrue (b.Build (proj), "Build should have succeeded.");
 				Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android/bin/classes.dex")),
 					"multidex-ed classes.zip exists");
-				Assert.IsTrue (b.LastBuildOutput.Contains (Path.Combine (fxVersion, "mono.android.jar")), fxVersion + "/mono.android.jar should be used.");
+				Assert.IsTrue (b.LastBuildOutput.Contains (Path.Combine (proj.TargetFrameworkVersion, "mono.android.jar")), proj.TargetFrameworkVersion + "/mono.android.jar should be used.");
 			}
 		}
 
@@ -1881,8 +1880,11 @@ public void ValidateJavaVersion (string targetFrameworkVersion, string buildTool
 			var proj = new XamarinAndroidApplicationProject () {
 				IsRelease = true,
 				TargetFrameworkVersion = targetFrameworkVersion,
+				UseLatestPlatformSdk = false,
 			};
 			using (var builder = CreateApkBuilder (Path.Combine (path, proj.ProjectName), false, false)) {
+				if (!Directory.Exists(Path.Combine (builder.FrameworkLibDirectory, "xbuild-frameworks", "MonoAndroid", targetFrameworkVersion)))
+					Assert.Ignore ("This is a Pull Request Build. Ignoring test.");
 				builder.ThrowOnBuildFailure = false;
 				builder.Target = "_SetLatestTargetFrameworkVersion";
 				Assert.AreEqual (expectedResult, builder.Build (proj, parameters: new string[] {
@@ -1890,7 +1892,7 @@ public void ValidateJavaVersion (string targetFrameworkVersion, string buildTool
 					$"JavaToolExe={javaExe}",
 					$"AndroidSdkBuildToolsVersion={buildToolsVersion}",
 					$"AndroidSdkDirectory={AndroidSdkDirectory}",
-				} ), string.Format ("Build should have {0}", expectedResult ? "succeeded" : "failed"));
+				}), string.Format ("Build should have {0}", expectedResult ? "succeeded" : "failed"));
 			}
 		}
 
diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs
index 412d251ca98..649ce7be04c 100644
--- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs
+++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs
@@ -107,6 +107,18 @@ protected string CreateFauxAndroidSdkDirectory (string path, string buildToolsVe
 			return androidSdkDirectory;
 		}
 
+		protected string CreateFauxReferencesDirectory (string path, string[] versions)
+		{
+			string referencesDirectory = Path.Combine (Root, path);
+			Directory.CreateDirectory (referencesDirectory);
+			Directory.CreateDirectory (Path.Combine (referencesDirectory, "v1.0"));
+			File.WriteAllText (Path.Combine (referencesDirectory, "v1.0", "mscorlib.dll"), "");
+			foreach (var v in versions){
+				Directory.CreateDirectory (Path.Combine (referencesDirectory, v));
+			}
+			return referencesDirectory;
+		}
+
 		protected string CreateFauxJavaSdkDirectory (string path, string javaVersion, out string javaExe)
 		{
 			javaExe = IsWindows ? "Java.cmd" : "java.bash";
diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs
index 353387d456a..851ae7d271d 100644
--- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs
+++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using System.IO;
 using System.Diagnostics;
 using Microsoft.Build.Framework;
@@ -62,6 +63,22 @@ public string FrameworkLibDirectory {
 			}
 		}
 
+		public string LatestTargetFrameworkVersion () {
+			Version latest = new Version (1, 0);
+			var outdir = FrameworkLibDirectory;
+			var path = Path.Combine (outdir, IsUnix ? Path.Combine ("xbuild-frameworks", "MonoAndroid") : "");
+			foreach (var dir in Directory.EnumerateDirectories (path, "v*", SearchOption.TopDirectoryOnly)) {
+				Version version;
+				string v = Path.GetFileName (dir).Replace ("v", "");
+				if (!Version.TryParse (v, out version))
+					continue;
+				if (latest.Major < version.Major && latest.Minor <= version.Minor)
+					latest = version;
+			}
+			return "v" + latest.ToString (2);
+		}
+
+
 		public string Root {
 			get {
 				return Path.GetDirectoryName (new Uri (typeof (XamarinProject).Assembly.CodeBase).LocalPath);
@@ -143,8 +160,8 @@ protected bool BuildInternal (string projectOrSolution, string target, string []
 				if (Directory.Exists (outdir)) {
 					var frameworksPath = Path.Combine (outdir, "lib", "xamarin.android", "xbuild-frameworks");
 					psi.EnvironmentVariables ["MONO_ANDROID_PATH"] = outdir;
-					psi.EnvironmentVariables ["XBUILD_FRAMEWORK_FOLDERS_PATH"] = frameworksPath;
 					args.AppendFormat ("/p:MonoDroidInstallDirectory=\"{0}\" ", outdir);
+					psi.EnvironmentVariables ["XBUILD_FRAMEWORK_FOLDERS_PATH"] = frameworksPath;
 					if (RunningMSBuild)
 						args.AppendFormat ($"/p:TargetFrameworkRootPath={frameworksPath} ");
 				}