Remove Mono submodule

.external

@@ -1 +1,2 @@
 xamarin/monodroid:master@237e0bd9f105b9842778ef82161a20c6d4497a40
+mono/mono:2019-06@afcf28a3660b069721e5c441dfe5bb7548e3204f

.gitignore

@@ -3,7 +3,6 @@
 bin
 Configuration.Override.props
 Configuration.OperatingSystem.props
-msfinal.pub
 obj
 packages
 .DS_Store

.gitmodules

@@ -22,10 +22,6 @@
     path = external/mman-win32
     url = https://github.com/witwall/mman-win32.git
     branch = master
-[submodule "external/mono"]
-    path = external/mono
-    url = https://github.com/mono/mono.git
-    branch = 2019-06
 [submodule "external/mxe"]
     path = external/mxe
     url = https://github.com/xamarin/mxe.git

Configuration.props

@@ -88,7 +88,7 @@
     <MonoRequiredMaximumVersion Condition=" '$(MonoRequiredMaximumVersion)' == '' ">6.5.0</MonoRequiredMaximumVersion>
     <IgnoreMaxMonoVersion Condition=" '$(IgnoreMaxMonoVersion)' == '' ">True</IgnoreMaxMonoVersion>
     <MonoRequiredDarwinMinimumVersion>$(MonoRequiredMinimumVersion).98</MonoRequiredDarwinMinimumVersion>
-    <LinkerSourceDirectory>$(MSBuildThisFileDirectory)external\mono\external\linker\src</LinkerSourceDirectory>
+    <LinkerSourceDirectory>$(MSBuildThisFileDirectory)external\mono\sdks\out\android-sources\external\linker\src</LinkerSourceDirectory>
     <OpenTKSourceDirectory>$(MSBuildThisFileDirectory)external\opentk</OpenTKSourceDirectory>
     <MingwZlibRootDirectory Condition=" '$(ZlibRootDirectory)' == '' And '$(HostOS)' == 'Linux' ">\usr</MingwZlibRootDirectory>
     <MingwZlibRootDirectory Condition=" '$(ZlibRootDirectory)' == '' And '$(HostOS)' == 'Darwin' ">$(HostHomebrewPrefix)\opt\mingw-zlib\usr</MingwZlibRootDirectory>

build-tools/conjure-xamarin-android-cecil/conjure-xamarin-android-cecil.cs

@@ -0,0 +1,63 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+using Mono.Cecil;
+
+public class ConjureXamarinAndroidCecil
+{
+	const string BaseNameReplacement = "Xamarin.Android.Cecil";
+	const string CecilAssemblyName = BaseNameReplacement;
+	const string CecilMdbAssemblyName = BaseNameReplacement + ".Mdb";
+
+	static readonly List<string> internalsVisibleTo = new List<string> {
+		"Xamarin.Android.Cecil.Pdb, PublicKey=0024000004800000940000000602000000240000525341310004000011000000438ac2a5acfbf16cbd2b2b47a62762f273df9cb2795ceccdf77d10bf508e69e7a362ea7a45455bbf3ac955e1f2e2814f144e5d817efc4c6502cc012df310783348304e3ae38573c6d658c234025821fda87a0be8a0d504df564e2c93b2b878925f42503e9d54dfef9f9586d9e6f38a305769587b1de01f6c0410328b2c9733db",
+		"Xamarin.Android.Cecil.Mdb, PublicKey=0024000004800000940000000602000000240000525341310004000011000000438ac2a5acfbf16cbd2b2b47a62762f273df9cb2795ceccdf77d10bf508e69e7a362ea7a45455bbf3ac955e1f2e2814f144e5d817efc4c6502cc012df310783348304e3ae38573c6d658c234025821fda87a0be8a0d504df564e2c93b2b878925f42503e9d54dfef9f9586d9e6f38a305769587b1de01f6c0410328b2c9733db"
+	};
+
+	public static int Main (string[] args)
+	{
+		if (args.Length < 2) {
+			Console.WriteLine ("Usage: <input directory> <output directory>");
+			Console.WriteLine ("  <input directory> must have Mono.Cecil.dll and Mono.Cecil.Mdb.dll assemblies");
+			return 1;
+		}
+
+		string inputDir = args [0];
+		string inputFilePath = Path.Combine (inputDir, "Mono.Cecil.dll");
+		string outputDirPath = args [1];
+
+		var resolver = new DefaultAssemblyResolver ();
+		resolver.AddSearchDirectory (Path.GetDirectoryName (inputFilePath));
+		var rp = new ReaderParameters () {
+			AssemblyResolver = resolver,
+			ReadSymbols = true
+		};
+		var monoCecil = AssemblyDefinition.ReadAssembly (inputFilePath, rp);
+		monoCecil.Name.Name = CecilAssemblyName;
+
+		var ivtCtor = monoCecil.MainModule.ImportReference (typeof (System.Runtime.CompilerServices.InternalsVisibleToAttribute).GetConstructor (new []{typeof(string)}));
+		foreach (string ivtParam in internalsVisibleTo) {
+			var ca = new CustomAttribute (ivtCtor);
+			ca.ConstructorArguments.Add (new CustomAttributeArgument (monoCecil.MainModule.TypeSystem.String, ivtParam));
+			monoCecil.CustomAttributes.Add (ca);
+		}
+
+		var wp = new WriterParameters {
+			WriteSymbols = true
+		};
+
+		monoCecil.Write (Path.Combine (outputDirPath, $"{CecilAssemblyName}.dll"), wp);
+
+		inputFilePath = Path.Combine (inputDir, "Mono.Cecil.Mdb.dll");
+		var monoCecilMdb = AssemblyDefinition.ReadAssembly (inputFilePath, rp);
+		monoCecilMdb.Name.Name = CecilMdbAssemblyName;
+
+		AssemblyNameReference monoCecilRef = monoCecilMdb.MainModule.AssemblyReferences.Single (r => String.Compare ("Mono.Cecil", r.Name, StringComparison.Ordinal) == 0);
+		monoCecilRef.Name = CecilAssemblyName;
+		monoCecilMdb.Write (Path.Combine (outputDirPath, $"{CecilMdbAssemblyName}.dll"), wp);
+
+		return 0;
+	}
+}

build-tools/conjure-xamarin-android-cecil/conjure-xamarin-android-cecil.csproj

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{C876DA71-8573-4CEF-9149-716D72455ED4}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <RootNamespace>Xamarin.Android.Prepare</RootNamespace>
+    <AssemblyName>conjure-xamarin-android-cecil</AssemblyName>
+  </PropertyGroup>
+  <Import Project="..\..\Configuration.props" />
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <Optimize>false</Optimize>
+    <OutputPath>..\..\bin\BuildDebug</OutputPath>
+    <DefineConstants>DEBUG;</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ExternalConsole>true</ExternalConsole>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <Optimize>true</Optimize>
+    <OutputPath>..\..\bin\BuildRelease</OutputPath>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ExternalConsole>true</ExternalConsole>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="Mono.Cecil">
+      <HintPath>$(XamarinAndroidSourcePath)external\mono\sdks\out\android-bcl\monodroid_tools\Mono.Cecil.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="conjure-xamarin-android-cecil.cs" />
+  </ItemGroup>
+  <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
+</Project>

build-tools/remap-assembly-ref/remap-assembly-ref.csproj

@@ -27,10 +27,10 @@
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
-    <ProjectReference Include="..\..\external\Java.Interop\src\Xamarin.Android.Cecil\Xamarin.Android.Cecil.csproj">
-      <Project>{15945D4B-FF56-4BCC-B598-2718D199DD08}</Project>
-      <Name>Xamarin.Android.Cecil</Name>
-    </ProjectReference>  </ItemGroup>
+    <Reference Include="Mono.Cecil">
+      <HintPath>$(XamarinAndroidSourcePath)external\mono\sdks\out\android-bcl\monodroid_tools\Mono.Cecil.dll</HintPath>
+    </Reference>
+  </ItemGroup>
   <ItemGroup>
     <Compile Include="remap-assembly-ref.cs" />
   </ItemGroup>

build-tools/scripts/XAVersionInfo.targets

@@ -6,15 +6,13 @@
   <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll"  TaskName="Xamarin.Android.BuildTools.PrepTasks.GitCommitsInRange" />
   <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll"  TaskName="Xamarin.Android.BuildTools.PrepTasks.GitCommitTime" />
   <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll"  TaskName="Xamarin.Android.BuildTools.PrepTasks.GitCommitInfo" />
+  <Import Project="..\..\bin\Build$(Configuration)\MonoGitHash.props" />
   <Target Name="_GetSubmodulesVersionInfo">
     <ItemGroup>
       <!-- If anything changes in this list, make sure to update:
            src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets (the _GenerateXACommonProps target)
            build-tools/create-vsix/create-vsix.targets (the _CreateDependencies target)
       -->
-      <_SubmoduleBranchInfo Include="external/mono">
-        <OutputPropertyName>_BuildInfo_MonoCommit</OutputPropertyName>
-      </_SubmoduleBranchInfo>
       <_SubmoduleBranchInfo Include="external/Java.Interop">
         <OutputPropertyName>_BuildInfo_JavaInteropCommit</OutputPropertyName>
       </_SubmoduleBranchInfo>

build-tools/xaprepare/xaprepare/Application/BuildInfo.cs

@@ -105,7 +105,13 @@ void DetermineBundleHashes (Context context)
 			LibZipHash = EnsureHash ("LibZip", Utilities.ShortenGitHash (FullLibZipHash));
 
 			Log.StatusLine ($"  {context.Characters.Bullet} Mono commit hash", ConsoleColor.Gray);
-			FullMonoHash = git.GetTopCommitHash (context.Properties.GetRequiredValue (KnownProperties.MonoSourceFullPath), shortHash: false);
+			List<ExternalGitDependency> externalDependencies = ExternalGitDependency.GetDependencies (context, Configurables.Paths.ExternalGitDepsFilePath, quiet: true);
+			ExternalGitDependency mono = externalDependencies?.Where (
+				eg => eg != null &&
+				      String.Compare ("mono", eg.Owner, StringComparison.Ordinal) == 0 &&
+				      String.Compare ("mono", eg.Name, StringComparison.Ordinal) == 0).FirstOrDefault ();
+
+			FullMonoHash = mono?.Commit?.Trim ();
 			MonoHash = EnsureHash ("Mono", Utilities.ShortenGitHash (FullMonoHash));
 
 			if (Configurables.Paths.BundleVersionHashFiles == null || Configurables.Paths.BundleVersionHashFiles.Count == 0) {

build-tools/xaprepare/xaprepare/Application/Context.cs

@@ -329,6 +329,13 @@ public string DebugFileExtension {
 		/// </summary>
 		public RefreshableComponent ComponentsToRefresh { get; set; }
 
+		/// <summary>
+		///   Set by <see cref="Step_DownloadMonoArchive"/> if the archive has been downloaded and validated, so
+		///   that the <see cref="Step_BuildMonoRuntimes"/> step doesn't have to re-download the archive, should
+		///   bundle be absent and <see cref="Step_PrepareBundle"/> fails.
+		/// </summary>
+		public bool MonoAlreadyBuilt { get; set; }
+
 		static Context ()
 		{
 			Instance = new Context ();
@@ -742,8 +749,10 @@ public async Task<bool> Init (string scenarioName = null)
 			Banner ("Updating Git submodules");
 
 			var git = new GitRunner (this);
-			if (!await git.SubmoduleUpdate ())
-				Log.WarningLine ("Failed to update Git submodules");
+			if (!await git.SubmoduleUpdate ()) {
+				Log.ErrorLine ("Failed to update Git submodules");
+				return false;
+			}
 
 			BuildInfo = new BuildInfo ();
 			await BuildInfo.GatherGitInfo (this);

build-tools/xaprepare/xaprepare/Application/ExternalGitDependency.cs

@@ -33,11 +33,13 @@ sealed class ExternalGitDependency : AppObject
 		public string Name   { get; private set; }
 		public string Owner  { get; private set; }
 
-		public static List<ExternalGitDependency> GetDependencies (Context context, string externalFilePath)
+		public static List<ExternalGitDependency> GetDependencies (Context context, string externalFilePath, bool quiet = false)
 		{
-			Log.Instance.StatusLine ($"  {context.Characters.Bullet} Reading external dependencies from {Utilities.GetRelativePath (BuildPaths.XamarinAndroidSourceRoot, externalFilePath)}");
+			if (!quiet)
+				Log.Instance.StatusLine ($"  {context.Characters.Bullet} Reading external dependencies from {Utilities.GetRelativePath (BuildPaths.XamarinAndroidSourceRoot, externalFilePath)}");
 			string[] unparsedExternals = File.ReadAllLines (externalFilePath);
 			var externals = new List<ExternalGitDependency> (unparsedExternals.Length);
+			bool includeCommercial = context.CheckCondition (KnownConditions.IncludeCommercial);
 
 			foreach (string external in unparsedExternals) {
 				Match match = externalRegex.Match (external);
@@ -47,14 +49,22 @@ public static List<ExternalGitDependency> GetDependencies (Context context, stri
 						continue;
 					}
 
+					string owner = match.Groups["owner"].Value;
+					string repo = match.Groups["repo"].Value;
+					if (!includeCommercial && Configurables.Defaults.CommercialExternalDependencies.Contains ($"{owner}/{repo}")) {
+						Log.Instance.DebugLine ($"Ignoring external commercial dependency '{owner}/{repo}'");
+						continue;
+					}
+
 					var e = new ExternalGitDependency {
 						Branch = match.Groups["branch"].Value,
 						Commit = match.Groups["commit"].Value,
-						Name   = match.Groups["repo"].Value,
-						Owner  = match.Groups["owner"].Value,
+						Name   = repo,
+						Owner  = owner,
 					};
 					externals.Add (e);
-					Log.Instance.StatusLine ($"    {context.Characters.Bullet} {e.Owner}/{e.Name} ({e.Commit})");
+					if (!quiet)
+						Log.Instance.StatusLine ($"    {context.Characters.Bullet} {e.Owner}/{e.Name} ({e.Commit})");
 				}
 			}
 

build-tools/xaprepare/xaprepare/Application/KnownConditions.cs

@@ -18,5 +18,11 @@ public enum KnownConditions
 		///   only a single program (e.g. the UpdateMono scenario) but not the rest. Default: unset. <see cref="Scenario_UpdateMono" />
 		/// </summary>
 		IgnoreMissingPrograms,
+
+		/// <summary>
+		///   If set, will cause checkout of the commercial dependencies mentioned in the `.external` file at
+		///   the top of the repository. Default: unset
+		/// </summary>
+		IncludeCommercial,
 	}
 }

build-tools/xaprepare/xaprepare/Application/Utilities.cs

@@ -28,6 +28,84 @@ static partial class Utilities
 
 		public static readonly Encoding UTF8NoBOM = new UTF8Encoding (false);
 
+		public static bool AbiChoiceChanged (Context context)
+		{
+			string cacheFile = Configurables.Paths.MonoRuntimesEnabledAbisCachePath;
+			if (!File.Exists (cacheFile)) {
+				Log.DebugLine ($"Enabled ABI cache file not found at {cacheFile}");
+				return true;
+			}
+
+			var oldAbis = new HashSet<string> (StringComparer.Ordinal);
+			foreach (string l in File.ReadAllLines (cacheFile)) {
+				string line = l?.Trim ();
+				if (String.IsNullOrEmpty (line) || oldAbis.Contains (line))
+					continue;
+				oldAbis.Add (line);
+			}
+
+			HashSet<string> currentAbis = null;
+			FillCurrentAbis (context, ref currentAbis);
+
+			if (oldAbis.Count != currentAbis.Count)
+				return true;
+
+			foreach (string abi in oldAbis) {
+				if (!currentAbis.Contains (abi))
+					return true;
+			}
+
+			return false;
+		}
+
+		public static void SaveAbiChoice (Context context)
+		{
+			HashSet<string> currentAbis = null;
+			FillCurrentAbis (context, ref currentAbis);
+
+			string cacheFile = Configurables.Paths.MonoRuntimesEnabledAbisCachePath;
+			Log.DebugLine ($"Writing ABI cache file {cacheFile}");
+			File.WriteAllLines (cacheFile, currentAbis);
+		}
+
+		static void FillCurrentAbis (Context context, ref HashSet<string> currentAbis)
+		{
+			Utilities.AddAbis (context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedTargetJitAbis).Trim (), ref currentAbis);
+			Utilities.AddAbis (context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedTargetAotAbis).Trim (), ref currentAbis);
+			Utilities.AddAbis (context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedHostJitAbis).Trim (), ref currentAbis);
+		}
+
+		public static void PropagateXamarinAndroidCecil (Context context)
+		{
+			const string CecilAssembly = "Xamarin.Android.Cecil.dll";
+
+			CopyFile (
+				Path.Combine (Configurables.Paths.InstallMSBuildDir, CecilAssembly),
+				Path.Combine (Configurables.Paths.ExternalJavaInteropDir, "bin", context.Configuration, CecilAssembly)
+			);
+		}
+
+		public static async Task<bool> BuildRemapRef (Context context, bool haveManagedRuntime, string managedRuntime, bool quiet = false)
+		{
+			if (!quiet)
+				Log.StatusLine ("Building remap-assembly-ref");
+
+			var msbuild = new MSBuildRunner (context);
+			string projectPath = Path.Combine (Configurables.Paths.BuildToolsDir, "remap-assembly-ref", "remap-assembly-ref.csproj");
+			bool result = await msbuild.Run (
+				projectPath: projectPath,
+				logTag: "remap-assembly-ref",
+				binlogName: "build-remap-assembly-ref"
+			);
+
+			if (!result) {
+				Log.ErrorLine ("Failed to build remap-assembly-ref");
+				return false;
+			}
+
+			return true;
+		}
+
 		public static string ToXamarinAndroidPropertyValue (ICollection<string> coll)
 		{
 			if (coll == null)

build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Unix.cs

@@ -12,6 +12,7 @@ partial class Defaults
 		partial class Paths
 		{
 			static string BundleOSType                  => Context.Instance.OS.Type;
+			static string ArchiveOSType                 => Context.Instance.OS.Type;
 
 			public static string BCLTestsSourceDir      => GetCachedPath (ref bclTestsSourceDir, () => Path.Combine (MonoProfileDir, "tests"));
 			public static string BCLAssembliesSourceDir => MonoProfileDir;

build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs

@@ -34,6 +34,7 @@ partial class Defaults
 		partial class Paths
 		{
 			static string BundleOSType                                            => "Darwin"; // Windows doesn't build the bundle
+			static string ArchiveOSType                                           => "Darwin"; // Windows need sources from there
 
 			// Windows doesn't build the bundle so we need to look in the XA framework dir as installed
 			public static string BCLAssembliesSourceDir                           => InstallBCLFrameworkDir;