From 4354afb7bc5c4f1fb424958c7954e0af45685d33 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Fri, 24 Feb 2017 22:08:11 -0500 Subject: [PATCH 1/7] Bump $(ProductVersion) to 7.2.0. (Commercial) Xamarin.Android v7.2 is being tracked in the [xamarin-android/d15-1][0] branch, which was branched from commit 1a233033. [0]: https://github.com/xamarin/xamarin-android/commits/d15-1 --- Configuration.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Configuration.props b/Configuration.props index 8f73f80b744..d347b6a297e 100644 --- a/Configuration.props +++ b/Configuration.props @@ -12,7 +12,7 @@ Condition=" '$(DoNotLoadOSProperties)' != 'True' " /> - 7.1.99 + 7.2.0 False False Windows From 4ba9439399b240993ddbaa4efe8be3c4e175378b Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Fri, 24 Feb 2017 22:14:43 -0500 Subject: [PATCH 2/7] Track the Java.Interop/d15-1 branch --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 534a05a7873..e0bfe826871 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "external/Java.Interop"] path = external/Java.Interop url = https://github.com/xamarin/java.interop.git - branch = master + branch = d15-1 [submodule "external/mono"] path = external/mono url = https://github.com/mono/mono.git From f9bf1a92cca8dcf5798d229dedbd3b350936e743 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Thu, 16 Mar 2017 10:36:47 -0400 Subject: [PATCH 3/7] [Xamarin.Android.Build.Tasks] Keep ICustomMarshaler.GetInstance (#499) - preserve the static ICustomMarshaler GetInstance (string) method of types implementing ICustomMarshaler interface - it avoids errors like: ReadlinkTest ReadLink [FAIL] : System.ApplicationException : Custom marshaler 'FileNameMarshaler' does not implement a static GetInstance method that takes a single string parameter and returns an ICustomMarshaler. at (wrapper managed-to-native) Mono.Unix.Native.Syscall:readlinkat (int,string,byte[],ulong) at Mono.Unix.Native.Syscall.readlinkat (System.Int32 dirfd, System.String pathname, System.Byte[] buf) [0x0000a] in <75558c6e0e1f444f8a03d1c694a92a08>:0 at MonoTests.Mono.Unix.ReadlinkTest.SetUp () [0x00007] in <9df8009b87984e1cb863e771d21f9f39>:0 at (wrapper managed-to-native) System.Reflection.MonoMethod:InternalInvoke (System.Reflection.MonoMethod,object,object[],System.Exception&) at System.Reflection.MonoMethod.Invoke (System.Object obj, System.Reflection.BindingFlags invokeAttr, System.Reflection.Binder binder, System.Object[] parameters, System.Globalization.CultureInfo culture) [0x00038] in <9f55149ed93f4bad9226ce1d7634fe5b>:0 readlink_byte [FAIL] : System.ApplicationException : Custom marshaler 'FileNameMarshaler' does not implement a static GetInstance method that takes a single string parameter and returns an ICustomMarshaler. TearDown : System.ArgumentException : ** Unknown error code: 9** ----> Mono.Unix.UnixIOException : ** Unknown error code: 9** [EBADF]. at (wrapper managed-to-native) Mono.Unix.Native.Syscall:readlinkat (int,string,byte[],ulong) at Mono.Unix.Native.Syscall.readlinkat (System.Int32 dirfd, System.String pathname, System.Byte[] buf) [0x0000a] in <75558c6e0e1f444f8a03d1c694a92a08>:0 at MonoTests.Mono.Unix.ReadlinkTest.SetUp () [0x00007] in <9df8009b87984e1cb863e771d21f9f39>:0 at (wrapper managed-to-native) System.Reflection.MonoMethod:InternalInvoke (System.Reflection.MonoMethod,object,object[],System.Exception&) at System.Reflection.MonoMethod.Invoke (System.Object obj, System.Reflection.BindingFlags invokeAttr, System.Reflection.Binder binder, System.Object[] parameters, System.Globalization.CultureInfo culture) [0x00038] in <9f55149ed93f4bad9226ce1d7634fe5b>:0 --TearDown at Mono.Unix.UnixMarshal.ThrowExceptionForLastError () [0x00006] in <75558c6e0e1f444f8a03d1c694a92a08>:0 at MonoTests.Mono.Unix.ReadlinkTest.TearDown () [0x00011] in <9df8009b87984e1cb863e771d21f9f39>:0 at (wrapper managed-to-native) System.Reflection.MonoMethod:InternalInvoke (System.Reflection.MonoMethod,object,object[],System.Exception&) at System.Reflection.MonoMethod.Invoke (System.Object obj, System.Reflection.BindingFlags invokeAttr, System.Reflection.Binder binder, System.Object[] parameters, System.Globalization.CultureInfo culture) [0x00038] in <9f55149ed93f4bad9226ce1d7634fe5b>:0 --UnixIOException --- .../Linker/MonoDroid.Tuner/MonoDroidMarkStep.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/MonoDroidMarkStep.cs b/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/MonoDroidMarkStep.cs index b194bf8618d..3bbcf2bbe7f 100644 --- a/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/MonoDroidMarkStep.cs +++ b/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/MonoDroidMarkStep.cs @@ -11,6 +11,7 @@ namespace MonoDroid.Tuner class MonoDroidMarkStep : MarkStep { const string RegisterAttribute = "Android.Runtime.RegisterAttribute"; + const string ICustomMarshalerName = "System.Runtime.InteropServices.ICustomMarshaler"; // If this is one of our infrastructure methods that has [Register], like: // [Register ("hasWindowFocus", "()Z", "GetHasWindowFocusHandler")], @@ -142,6 +143,15 @@ protected override TypeDefinition MarkType (TypeReference reference) if (type.Module.Assembly.Name.Name == "System.Core") ProcessSystemCore (type); + if (type.HasMethods && type.HasInterfaces && type.Implements (ICustomMarshalerName)) { + foreach (MethodDefinition method in type.Methods) { + if (method.Name == "GetInstance" && method.IsStatic && method.HasParameters && method.Parameters.Count == 1 && method.ReturnType.FullName == ICustomMarshalerName && method.Parameters.First ().ParameterType.FullName == "System.String") { + MarkMethod (method); + break; + } + } + } + return type; } From 70cb80f745c2386cb06b1f6af031c5c5c99374b8 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Fri, 17 Mar 2017 11:24:51 -0400 Subject: [PATCH 4/7] Bump to mono/mono-4.8.0-branch/e464ddae (#503) Fixes: https://bugzilla.xamarin.com/show_bug.cgi?id=41133 Fixes: https://bugzilla.xamarin.com/show_bug.cgi?id=46929 Fixes: https://bugzilla.xamarin.com/show_bug.cgi?id=51545 Fixes: https://bugzilla.xamarin.com/show_bug.cgi?id=51562 Fixes: https://bugzilla.xamarin.com/show_bug.cgi?id=52437 Fixes: https://bugzilla.xamarin.com/show_bug.cgi?id=52845 Fixes: https://bugzilla.xamarin.com/show_bug.cgi?id=53066 --- external/mono | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/mono b/external/mono index dd8ecf3a1dc..e464ddae64c 160000 --- a/external/mono +++ b/external/mono @@ -1 +1 @@ -Subproject commit dd8ecf3a1dc09e88fd5c82dddf56d14a2aff65d9 +Subproject commit e464ddae64cd181c360d4cf93d087db79c4fb192 From 33504fd9c45b39cc70c6846726f5697d47dd21ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Laval?= Date: Fri, 17 Mar 2017 15:37:20 -0400 Subject: [PATCH 5/7] [monodroid] On Windows, look for side-by-side libmonosgen (#496) (#504) Fixes: https://bugzilla.xamarin.com/show_bug.cgi?id=53163 A user uncovered an issue using the Xamarin.Forms previewer where libmonosgen failed to load correctly on Windows. The root cause was that the already loaded libmono-android was incompatible with the version of mono being loaded (different bridge versions) causing a forced runtime shutdown with the error message: Invalid bridge callback version. Expected 4 but got 5 After investigating, it turns out the user had an old version of Xamarin installed via MSI (for VS2015) on his machine and was trying to use the newer Xamarin embedded in VS2017 via the Willow distribution mechanism which places binaries such as libmonosgen in a separate directory structure. Thus the problem was that, while libmono-android itself was loaded correctly from that separate path, the code was then trying to fetch libmonosgen from the global system path (now completely obsolete) causing the bridge version mismatch. With this patch, the code will now try first to get libmonosgen from the same directory libmono-android was loaded from which should cover the Willow case. --- src/monodroid/jni/monodroid-glue.c | 27 +++++++++++++++++++++++++++ src/monodroid/monodroid.props | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/monodroid/jni/monodroid-glue.c b/src/monodroid/jni/monodroid-glue.c index e612bdf1cac..7180b6d23b8 100644 --- a/src/monodroid/jni/monodroid-glue.c +++ b/src/monodroid/jni/monodroid-glue.c @@ -45,6 +45,7 @@ #include #include #include +#include #endif #include @@ -535,6 +536,28 @@ get_xamarin_android_msbuild_path (void) return msbuild_folder_path; } +static char *libmonoandroid_directory_path = NULL; + +// Returns the directory in which this library was loaded from +static char* +get_libmonoandroid_directory_path () +{ + wchar_t module_path[MAX_PATH]; + HMODULE module = NULL; + + if (libmonoandroid_directory_path != NULL) + return libmonoandroid_directory_path; + + DWORD flags = GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT; + if (!GetModuleHandleExW (flags, (void*)&libmonoandroid_directory_path, &module)) + return NULL; + + GetModuleFileNameW (module, module_path, sizeof (module_path) / sizeof (module_path[0])); + PathRemoveFileSpecW (module_path); + libmonoandroid_directory_path = utf16_to_utf8 (module_path); + return libmonoandroid_directory_path; +} + static int setenv(const char *name, const char *value, int overwrite) { @@ -656,6 +679,10 @@ get_libmonosgen_path () return libmonoso; free (libmonoso); +#ifdef WINDOWS + TRY_LIBMONOSGEN (get_libmonoandroid_directory_path ()) +#endif + TRY_LIBMONOSGEN (SYSTEM_LIB_PATH) #ifdef RELEASE diff --git a/src/monodroid/monodroid.props b/src/monodroid/monodroid.props index 177c103081b..d2ee096fc28 100644 --- a/src/monodroid/monodroid.props +++ b/src/monodroid/monodroid.props @@ -7,7 +7,7 @@ <_HostUnixCFlags>$(_CommonCFlags) -Wa,--noexecstack <_HostUnixLdFlags>-Wall -lstdc++ -lz -shared -fpic <_HostCommonWinCFlags>$(_CommonCFlags) -DWINDOWS -DNTDDI_VERSION=NTDDI_VISTA -D_WIN32_WINNT=_WIN32_WINNT_VISTA -fomit-frame-pointer - <_HostCommonWinLdFlags>-Wall -lstdc++ -lz -shared -fpic -ldl -lmman -pthread -lwsock32 -lole32 -luuid + <_HostCommonWinLdFlags>-Wall -lstdc++ -lz -shared -fpic -ldl -lmman -pthread -lwsock32 -lole32 -luuid -lshlwapi <_UnixAdditionalSourceFiles>$(MonoSourceFullPath)\support\nl.c jni\debug.c jni\monodroid-networkinfo.c jni\xamarin_getifaddrs.c <_LinuxFlatPakBuild Condition="Exists('/.flatpak-info')" >-DLINUX_FLATPAK From 1f795758b591769e0fbb7f54a66b98f46fc4ced0 Mon Sep 17 00:00:00 2001 From: Dean Ellis Date: Thu, 23 Mar 2017 12:17:48 +0000 Subject: [PATCH 6/7] [Xamarin.Android.Build.Tasks] Hitting "System.IO.IOException: Too many open files" when building a large app with AOT and all supported architectures (#493) (#516) Fixes: https://bugzilla.xamarin.com/show_bug.cgi?id=53122 There is a problem with libzip in that is does not have an API for Flushing the current items in memory to disk. This was why ZipArchiveEx was introduced in commit 9166e036. This commit reworks the BuildApk task to make sure of this new functinality and to periodically flush the zip to disk. This will reduce the chances of running out of memory. It also adds a unit test which generates and builds an app which has a large number of refernces and assets. This app also gets AOT'd so it should be pushing the system to the limit. Also added code to make sure the cross tools and llc have execute permissions. Added code to make sure we dispose of each process in the Aapt and Aot tasks. This is to ensure any files or pipes these processed have are removed as soon as possible. Updated the unix-distribution-setup.targets to ensure that the cross-arm tooling and llc have the correct execute permissions on MacOS and Linux. --- src/Xamarin.Android.Build.Tasks/Tasks/Aapt.cs | 54 ++++++---- src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs | 38 +++++-- .../Tasks/BuildApk.cs | 98 +++++++++++++------ .../Xamarin.Android.Build.Tests/BuildTest.cs | 1 + .../Common/SolutionBuilder.cs | 28 ++++-- .../Utilities/ZipArchiveEx.cs | 20 +++- 6 files changed, 166 insertions(+), 73 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/Aapt.cs b/src/Xamarin.Android.Build.Tasks/Tasks/Aapt.cs index 0187141b582..b3d0b86e98e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/Aapt.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/Aapt.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Threading; using System.Xml; using System.Xml.Linq; using Microsoft.Build.Utilities; @@ -89,6 +90,8 @@ bool ManifestIsUpToDate (string manifestFile) bool RunAapt (string commandLine) { + var stdout_completed = new ManualResetEvent (false); + var stderr_completed = new ManualResetEvent (false); var psi = new ProcessStartInfo () { FileName = GenerateFullPathToTool (), Arguments = commandLine, @@ -99,26 +102,37 @@ bool RunAapt (string commandLine) WindowStyle = ProcessWindowStyle.Hidden, }; - var proc = new Process (); - proc.OutputDataReceived += (sender, e) => { - LogEventsFromTextOutput (e.Data, MessageImportance.Normal); - }; - proc.ErrorDataReceived += (sender, e) => { - LogEventsFromTextOutput (e.Data, MessageImportance.Normal); - }; - proc.StartInfo = psi; - proc.Start (); - proc.BeginOutputReadLine (); - proc.BeginErrorReadLine (); - Token.Register (() => { - try { - proc.Kill (); - } catch (Exception) { - } - }); - LogDebugMessage ("Executing {0}", commandLine); - proc.WaitForExit (); - return proc.ExitCode == 0; + using (var proc = new Process ()) { + proc.OutputDataReceived += (sender, e) => { + if (e.Data != null) + LogEventsFromTextOutput (e.Data, MessageImportance.Normal); + else + stdout_completed.Set (); + }; + proc.ErrorDataReceived += (sender, e) => { + if (e.Data != null) + LogEventsFromTextOutput (e.Data, MessageImportance.Normal); + else + stderr_completed.Set (); + }; + proc.StartInfo = psi; + proc.Start (); + proc.BeginOutputReadLine (); + proc.BeginErrorReadLine (); + Token.Register (() => { + try { + proc.Kill (); + } catch (Exception) { + } + }); + LogDebugMessage ("Executing {0}", commandLine); + proc.WaitForExit (); + if (psi.RedirectStandardError) + stderr_completed.WaitOne (TimeSpan.FromSeconds (30)); + if (psi.RedirectStandardOutput) + stdout_completed.WaitOne (TimeSpan.FromSeconds (30)); + return proc.ExitCode == 0; + } } bool ExecuteForAbi (string cmd, string currentResourceOutputFile) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs b/src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs index 6411dec6b70..4f00d3507cf 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs @@ -434,6 +434,8 @@ IEnumerable GetAotConfigs () bool RunAotCompiler (string assembliesPath, string aotCompiler, string aotOptions, string assembly, CancellationToken token) { + var stdout_completed = new ManualResetEvent (false); + var stderr_completed = new ManualResetEvent (false); var psi = new ProcessStartInfo () { FileName = aotCompiler, Arguments = aotOptions + " \"" + assembly + "\"", @@ -453,16 +455,32 @@ bool RunAotCompiler (string assembliesPath, string aotCompiler, string aotOption LogDebugMessage ("[AOT] MONO_PATH=\"{0}\" MONO_ENV_OPTIONS=\"{1}\" {2} {3}", psi.EnvironmentVariables ["MONO_PATH"], psi.EnvironmentVariables ["MONO_ENV_OPTIONS"], psi.FileName, psi.Arguments); - var proc = new Process (); - proc.OutputDataReceived += OnAotOutputData; - proc.ErrorDataReceived += OnAotErrorData; - proc.StartInfo = psi; - proc.Start (); - proc.BeginOutputReadLine (); - proc.BeginErrorReadLine (); - token.Register (() => { try { proc.Kill (); } catch (Exception) {} }); - proc.WaitForExit (); - return proc.ExitCode == 0; + using (var proc = new Process ()) { + proc.OutputDataReceived += (s, e) => { + if (e.Data != null) + OnAotOutputData (s, e); + else + stdout_completed.Set (); + }; + proc.ErrorDataReceived += (s, e) => { + if (e.Data != null) + OnAotErrorData (s, e); + else + stderr_completed.Set (); + }; + proc.StartInfo = psi; + proc.Start (); + proc.BeginOutputReadLine (); + proc.BeginErrorReadLine (); + token.Register (() => { try { proc.Kill (); } catch (Exception) { } }); + proc.WaitForExit (); + if (psi.RedirectStandardError) + stderr_completed.WaitOne (TimeSpan.FromSeconds (30)); + if (psi.RedirectStandardOutput) + stdout_completed.WaitOne (TimeSpan.FromSeconds (30)); + return proc.ExitCode == 0; + } + GC.Collect (); } void OnAotOutputData (object sender, DataReceivedEventArgs e) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 9d8f1a6de1c..88a7e203df7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -116,34 +116,44 @@ void ExecuteWithAbi (string supportedAbis, string apkInputPath, string apkOutput ArchiveFileList files = new ArchiveFileList (); if (apkInputPath != null) File.Copy (apkInputPath, apkOutputPath + "new", overwrite: true); - using (var apk = ZipArchive.Open (apkOutputPath + "new", apkInputPath != null ? FileMode.Open : FileMode.Create )) { - apk.AddEntry ("NOTICE", + using (var apk = new ZipArchiveEx (apkOutputPath + "new", apkInputPath != null ? FileMode.Open : FileMode.Create )) { + apk.Archive.AddEntry ("NOTICE", Assembly.GetExecutingAssembly ().GetManifestResourceStream ("NOTICE.txt")); // Add classes.dx - apk.AddFiles (DalvikClasses, useFileDirectories: false); + apk.Archive.AddFiles (DalvikClasses, useFileDirectories: false); if (EmbedAssemblies && !BundleAssemblies) AddAssemblies (apk); AddEnvironment (apk); AddRuntimeLibraries (apk, supportedAbis); + apk.Flush(); AddNativeLibraries (files, supportedAbis); + apk.Flush(); AddAdditionalNativeLibraries (files, supportedAbis); + apk.Flush(); AddNativeLibrariesFromAssemblies (apk, supportedAbis); + apk.Flush(); foreach (ITaskItem typemap in TypeMappings) { - apk.AddFile (typemap.ItemSpec, Path.GetFileName(typemap.ItemSpec), compressionMethod: CompressionMethod.Store); + apk.Archive.AddFile (typemap.ItemSpec, Path.GetFileName(typemap.ItemSpec), compressionMethod: CompressionMethod.Store); } + int count = 0; foreach (var file in files) { var item = Path.Combine (file.Item2, Path.GetFileName (file.Item1)) .Replace (Path.DirectorySeparatorChar, '/'); - if (apk.ContainsEntry (item)) { + if (apk.Archive.ContainsEntry (item)) { Log.LogWarning (null, "XA4301", null, file.Item1, 0, 0, 0, 0, "Apk already contains the item {0}; ignoring.", item); continue; } - apk.AddFile (file.Item1, item); + apk.Archive.AddFile (file.Item1, item); + count++; + if (count == ZipArchiveEx.ZipFlushLimit) { + apk.Flush(); + count = 0; + } } if (_Debug) AddGdbservers (apk, files, supportedAbis, debugServer); @@ -160,6 +170,7 @@ void ExecuteWithAbi (string supportedAbis, string apkInputPath, string apkOutput var jarFilePaths = libraryProjectJars.Concat (jarFiles != null ? jarFiles.Select (j => j.ItemSpec) : Enumerable.Empty ()); jarFilePaths = MonoAndroidHelper.DistinctFilesByContent (jarFilePaths); + count = 0; foreach (var jarFile in jarFilePaths) { using (var jar = ZipArchive.Open (File.OpenRead (jarFile))) { foreach (var jarItem in jar.Where (ze => !ze.IsDirectory && !ze.FullName.StartsWith ("META-INF") && !ze.FullName.EndsWith (".class") && !ze.FullName.EndsWith (".java") && !ze.FullName.EndsWith ("MANIFEST.MF"))) { @@ -168,15 +179,20 @@ void ExecuteWithAbi (string supportedAbis, string apkInputPath, string apkOutput jarItem.Extract (d); data = d.ToArray (); } - if (apk.Any (e => e.FullName == jarItem.FullName)) + if (apk.Archive.Any (e => e.FullName == jarItem.FullName)) Log.LogMessage ("Warning: failed to add jar entry {0} from {1}: the same file already exists in the apk", jarItem.FullName, Path.GetFileName (jarFile)); else - apk.AddEntry (data, jarItem.FullName); + apk.Archive.AddEntry (data, jarItem.FullName); } } + count++; + if (count == ZipArchiveEx.ZipFlushLimit) { + apk.Flush(); + count = 0; + } } if (StubApplicationDataFile != null && File.Exists (StubApplicationDataFile)) - apk.AddFile (StubApplicationDataFile, Path.GetFileName (StubApplicationDataFile)); + apk.Archive.AddFile (StubApplicationDataFile, Path.GetFileName (StubApplicationDataFile)); } MonoAndroidHelper.CopyIfZipChanged (apkOutputPath + "new", apkOutputPath); File.Delete (apkOutputPath + "new"); @@ -249,14 +265,15 @@ public override bool Execute () return !Log.HasLoggedErrors; } - private void AddAssemblies (ZipArchive apk) + private void AddAssemblies (ZipArchiveEx apk) { bool debug = _Debug; bool use_shared_runtime = String.Equals (UseSharedRuntime, "true", StringComparison.OrdinalIgnoreCase); + int count = 0; foreach (ITaskItem assembly in ResolvedUserAssemblies) { // Add assembly - apk.AddFile (assembly.ItemSpec, GetTargetDirectory (assembly.ItemSpec) + "/" + Path.GetFileName (assembly.ItemSpec), compressionMethod: CompressionMethod.Store); + apk.Archive.AddFile (assembly.ItemSpec, GetTargetDirectory (assembly.ItemSpec) + "/" + Path.GetFileName (assembly.ItemSpec), compressionMethod: CompressionMethod.Store); // Try to add config if exists var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); @@ -267,16 +284,22 @@ private void AddAssemblies (ZipArchive apk) var symbols = Path.ChangeExtension (assembly.ItemSpec, "dll.mdb"); if (File.Exists (symbols)) - apk.AddFile (symbols, "assemblies/" + Path.GetFileName (symbols), compressionMethod: CompressionMethod.Store); + apk.Archive.AddFile (symbols, "assemblies/" + Path.GetFileName (symbols), compressionMethod: CompressionMethod.Store); + } + count++; + if (count == ZipArchiveEx.ZipFlushLimit) { + apk.Flush(); + count = 0; } } if (use_shared_runtime) return; + count = 0; // Add framework assemblies foreach (ITaskItem assembly in ResolvedFrameworkAssemblies) { - apk.AddFile (assembly.ItemSpec, "assemblies/" + Path.GetFileName (assembly.ItemSpec), compressionMethod: CompressionMethod.Store); + apk.Archive.AddFile (assembly.ItemSpec, "assemblies/" + Path.GetFileName (assembly.ItemSpec), compressionMethod: CompressionMethod.Store); var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); AddAssemblyConfigEntry (apk, config); // Try to add symbols if Debug @@ -284,12 +307,17 @@ private void AddAssemblies (ZipArchive apk) var symbols = Path.ChangeExtension (assembly.ItemSpec, "dll.mdb"); if (File.Exists (symbols)) - apk.AddFile (symbols, "assemblies/" + Path.GetFileName (symbols), compressionMethod: CompressionMethod.Store); + apk.Archive.AddFile (symbols, "assemblies/" + Path.GetFileName (symbols), compressionMethod: CompressionMethod.Store); + } + count++; + if (count == ZipArchiveEx.ZipFlushLimit) { + apk.Flush(); + count = 0; } } } - void AddAssemblyConfigEntry (ZipArchive apk, string configFile) + void AddAssemblyConfigEntry (ZipArchiveEx apk, string configFile) { if (!File.Exists (configFile)) return; @@ -299,7 +327,7 @@ void AddAssemblyConfigEntry (ZipArchive apk, string configFile) source.CopyTo (dest); dest.WriteByte (0); dest.Position = 0; - apk.AddEntry ("assemblies/" + Path.GetFileName (configFile), dest, compressionMethod: CompressionMethod.Store); + apk.Archive.AddEntry ("assemblies/" + Path.GetFileName (configFile), dest, compressionMethod: CompressionMethod.Store); } } @@ -312,7 +340,7 @@ static string GetTargetDirectory (string path) return "assemblies"; } - void AddEnvironment (ZipArchive apk) + void AddEnvironment (ZipArchiveEx apk) { var environment = new StringWriter () { NewLine = "\n", @@ -385,7 +413,7 @@ void AddEnvironment (ZipArchive apk) environment.WriteLine ("MONO_GC_PARAMS=major=marksweep"); } - apk.AddEntry ("environment", environment.ToString (), + apk.Archive.AddEntry ("environment", environment.ToString (), new UTF8Encoding (encoderShouldEmitUTF8Identifier:false)); } @@ -421,13 +449,13 @@ HashSet ParseProfilers (string value) return results; } - void AddNativeLibrary (ZipArchive apk, string abi, string filename) + void AddNativeLibrary (ZipArchiveEx apk, string abi, string filename) { var path = Path.Combine (MSBuildXamarinAndroidDirectory, "lib", abi, filename); - apk.AddEntry (string.Format ("lib/{0}/{1}", abi, filename), File.OpenRead (path)); + apk.Archive.AddEntry (string.Format ("lib/{0}/{1}", abi, filename), File.OpenRead (path)); } - void AddProfilers (ZipArchive apk, string abi) + void AddProfilers (ZipArchiveEx apk, string abi) { if (!string.IsNullOrEmpty (AndroidEmbedProfilers)) { foreach (var profiler in ParseProfilers (AndroidEmbedProfilers)) { @@ -437,7 +465,7 @@ void AddProfilers (ZipArchive apk, string abi) } } - void AddBtlsLibs (ZipArchive apk, string abi) + void AddBtlsLibs (ZipArchiveEx apk, string abi) { if (string.Compare ("btls", TlsProvider, StringComparison.OrdinalIgnoreCase) == 0) { AddNativeLibrary (apk, abi, "libmono-btls-shared.so"); @@ -447,14 +475,14 @@ void AddBtlsLibs (ZipArchive apk, string abi) // * "legacy": } - void AddRuntimeLibraries (ZipArchive apk, string supportedAbis) + void AddRuntimeLibraries (ZipArchiveEx apk, string supportedAbis) { bool use_shared_runtime = String.Equals (UseSharedRuntime, "true", StringComparison.OrdinalIgnoreCase); var abis = supportedAbis.Split (new char[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries); foreach (var abi in abis) { string library = string.Format ("libmono-android.{0}.so", _Debug ? "debug" : "release"); var path = Path.Combine (MSBuildXamarinAndroidDirectory, "lib", abi, library); - apk.AddEntry (string.Format ("lib/{0}/libmonodroid.so", abi), File.OpenRead (path)); + apk.Archive.AddEntry (string.Format ("lib/{0}/libmonodroid.so", abi), File.OpenRead (path)); if (!use_shared_runtime) { // include the sgen AddNativeLibrary (apk, abi, "libmonosgen-2.0.so"); @@ -464,8 +492,9 @@ void AddRuntimeLibraries (ZipArchive apk, string supportedAbis) } } - void AddNativeLibrariesFromAssemblies (ZipArchive apk, string supportedAbis) + void AddNativeLibrariesFromAssemblies (ZipArchiveEx apk, string supportedAbis) { + int count = 0; var abis = supportedAbis.Split (new char[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries); using (var res = new DirectoryAssemblyResolver (Console.WriteLine, loadDebugSymbols: false)) { foreach (var assembly in EmbeddedNativeLibraryAssemblies) @@ -483,19 +512,24 @@ void AddNativeLibrariesFromAssemblies (ZipArchive apk, string supportedAbis) if (e.IsDirectory) continue; var key = e.FullName.Replace ("native_library_imports", "lib"); - if (apk.Any (k => k.FullName == key)) { + if (apk.Archive.Any (k => k.FullName == key)) { Log.LogCodedWarning ("4301", "Apk already contains the item {0}; ignoring.", key); continue; } using (var s = new MemoryStream ()) { e.Extract (s); s.Position = 0; - apk.AddEntry (s.ToArray (), key); + apk.Archive.AddEntry (s.ToArray (), key); } } } } } + count++; + if (count == ZipArchiveEx.ZipFlushLimit) { + apk.Flush(); + count = 0; + } } } } @@ -556,11 +590,12 @@ void AddNativeLibrary (ArchiveFileList files, string path, string abi) files.Add (new Tuple (path, string.Format ("lib/{0}", abi))); } - private void AddGdbservers (ZipArchive apk, ArchiveFileList files, string supportedAbis, AndroidDebugServer debugServer) + private void AddGdbservers (ZipArchiveEx apk, ArchiveFileList files, string supportedAbis, AndroidDebugServer debugServer) { if (string.IsNullOrEmpty (AndroidNdkDirectory)) return; + int count = 0; foreach (var sabi in supportedAbis.Split (new char[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)) { var arch = GdbPaths.GetArchFromAbi (sabi); var abi = GdbPaths.GetAbiFromArch (arch); @@ -575,7 +610,12 @@ private void AddGdbservers (ZipArchive apk, ArchiveFileList files, string suppor if (!File.Exists (debugServerPath)) continue; Log.LogDebugMessage ("Adding {0} debug server '{1}' to the APK as '{2}'", sabi, debugServerPath, entryName); - apk.AddEntry (entryName, File.OpenRead (debugServerPath)); + apk.Archive.AddEntry (entryName, File.OpenRead (debugServerPath)); + count++; + if (count == ZipArchiveEx.ZipFlushLimit) { + apk.Flush(); + count = 0; + } } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index 2e07469d0e8..594f5002fdc 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Linq; +using System.Text; using System.Xml.Linq; using NUnit.Framework; using Xamarin.ProjectTools; diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/SolutionBuilder.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/SolutionBuilder.cs index 22d9556f77e..3abae105911 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/SolutionBuilder.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/SolutionBuilder.cs @@ -10,17 +10,19 @@ public class SolutionBuilder : Builder { public IList Projects { get; } public string SolutionPath { get; set; } + public string SolutionName { get; set; } public bool BuildSucceeded { get; set; } - public SolutionBuilder () : base() + public SolutionBuilder (string solutionName) : base() { + SolutionName = solutionName; Projects = new List (); } public void Save () { foreach (var p in Projects) { - using (var pb = new ProjectBuilder (Path.Combine (Path.GetDirectoryName (SolutionPath), p.ProjectName))) { + using (var pb = new ProjectBuilder (Path.Combine (SolutionPath, p.ProjectName))) { pb.Save (p); } } @@ -47,27 +49,33 @@ public void Save () } sb.Append ("\tEndGlobalSection\n"); sb.Append ("EndGlobal\n"); - File.WriteAllText (SolutionPath, sb.ToString ()); + File.WriteAllText (Path.Combine (SolutionPath, SolutionName), sb.ToString ()); } - public bool Build () + public bool BuildProject(XamarinProject project, string target = "Build") + { + BuildSucceeded = BuildInternal(Path.Combine (SolutionPath, project.ProjectName, project.ProjectFilePath), target); + return BuildSucceeded; + } + + public bool Build (params string[] parameters) { Save (); - BuildSucceeded = BuildInternal (SolutionPath, "Build"); + BuildSucceeded = BuildInternal (Path.Combine (SolutionPath, SolutionName), "Build", parameters); return BuildSucceeded; } - public bool ReBuild () + public bool ReBuild(params string[] parameters) { Save (); - BuildSucceeded = BuildInternal (SolutionPath, "ReBuild"); + BuildSucceeded = BuildInternal(Path.Combine(SolutionPath, SolutionName), "ReBuild", parameters); return BuildSucceeded; } - public bool Clean () + public bool Clean(params string[] parameters) { Save (); - BuildSucceeded = BuildInternal (SolutionPath, "Clean"); + BuildSucceeded = BuildInternal(Path.Combine(SolutionPath, SolutionName), "Clean", parameters); return BuildSucceeded; } @@ -75,7 +83,7 @@ protected override void Dispose (bool disposing) { if (disposing) if (BuildSucceeded) - Directory.Delete (Path.GetDirectoryName (SolutionPath), recursive: true); + Directory.Delete (SolutionPath, recursive: true); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveEx.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveEx.cs index 80aa16f051e..b9c7d3571b2 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveEx.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveEx.cs @@ -1,21 +1,33 @@ using System; using System.IO; +using System.Linq; using Xamarin.Tools.Zip; namespace Xamarin.Android.Tasks { public class ZipArchiveEx : IDisposable { + + public static int ZipFlushLimit = 50; + ZipArchive zip; string archive; - public ZipArchiveEx (string archive) + public ZipArchive Archive { + get { return zip; } + } + + public ZipArchiveEx (string archive) : this (archive, FileMode.CreateNew) + { + } + + public ZipArchiveEx(string archive, FileMode filemode) { this.archive = archive; - zip = ZipArchive.Open (archive, FileMode.CreateNew); + zip = ZipArchive.Open(archive, filemode); } - void Flush () + public void Flush () { if (zip != null) { zip.Close (); @@ -51,7 +63,7 @@ void AddFiles (string folder, string folderInArchive) continue; zip.AddFile (fileName, ArchiveNameForFile (fileName, folderInArchive)); count++; - if (count == 50) { + if (count == ZipArchiveEx.ZipFlushLimit) { Flush (); count = 0; } From b3fb39f1a7c4d3a890c27d652f03ef5adfd740b9 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 9 Mar 2017 03:20:34 +0100 Subject: [PATCH 7/7] [Mono.Android] Make HTTP connection attempt cancellable (#472) The `HttpURLConnection.ConnectAsync` API doesn't accept a `CancellationToken` instance and, thus, cannot be easily cancelled in a graceful manner. This limitation resulted in using the `Task.WhenAny` to make sure the connection attempt is aborted whenever the calling task is cancelled. This, however, led to a problem of unobserved exceptions should the cancellation occur before the connection attempt was successful. This commit wraps the synchronous `HttpURLConnection.Connect` call in a task that is passed the cancellation token, so that it can be gracefully cancelled when needed. The cancellation is done by registering a handler for when token is about to expire as Task.Run will check if cancellation was requested only before it runs the passed code, the rest is the code's responsibility. Since `URLConnection.Connect()` is asynchronous we abort it by calling `Disconnect()` on the instance which is a brutal but effective way to interrupt the connection attempt. Additionally, the diff makes sure to properly configure a few tasks for `await` Fixes https://bugzilla.xamarin.com/show_bug.cgi?id=51804 --- .../AndroidClientHandler.cs | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/Mono.Android/Xamarin.Android.Net/AndroidClientHandler.cs b/src/Mono.Android/Xamarin.Android.Net/AndroidClientHandler.cs index 3d4d6fb14de..ad577a49673 100644 --- a/src/Mono.Android/Xamarin.Android.Net/AndroidClientHandler.cs +++ b/src/Mono.Android/Xamarin.Android.Net/AndroidClientHandler.cs @@ -209,8 +209,8 @@ string EncodeUrl (Uri url) while (true) { URL java_url = new URL (EncodeUrl (redirectState.NewUrl)); URLConnection java_connection = java_url.OpenConnection (); - HttpURLConnection httpConnection = await SetupRequestInternal (request, java_connection); - HttpResponseMessage response = await ProcessRequest (request, java_url, httpConnection, cancellationToken, redirectState); + HttpURLConnection httpConnection = await SetupRequestInternal (request, java_connection).ConfigureAwait (continueOnCapturedContext: false);; + HttpResponseMessage response = await ProcessRequest (request, java_url, httpConnection, cancellationToken, redirectState).ConfigureAwait (continueOnCapturedContext: false);; if (response != null) return response; @@ -235,6 +235,19 @@ Task DisconnectAsync (HttpURLConnection httpConnection) return Task.Run (() => httpConnection?.Disconnect ()); } + Task ConnectAsync (HttpURLConnection httpConnection, CancellationToken ct) + { + return Task.Run (() => { + try { + using (ct.Register (() => httpConnection?.Disconnect ())) + httpConnection?.Connect (); + } catch { + ct.ThrowIfCancellationRequested (); + throw; + } + }, ct); + } + async Task DoProcessRequest (HttpRequestMessage request, URL javaUrl, HttpURLConnection httpConnection, CancellationToken cancellationToken, RequestRedirectionState redirectState) { if (Logger.LogNet) @@ -247,20 +260,11 @@ Task DisconnectAsync (HttpURLConnection httpConnection) cancellationToken.ThrowIfCancellationRequested (); } - CancellationTokenSource waitHandleSource = new CancellationTokenSource(); try { if (Logger.LogNet) Logger.Log (LogLevel.Info, LOG_APP, $" connecting"); - CancellationToken linkedToken = CancellationTokenSource.CreateLinkedTokenSource(waitHandleSource.Token, cancellationToken).Token; - await Task.WhenAny ( - httpConnection.ConnectAsync (), - Task.Run (()=> { - linkedToken.WaitHandle.WaitOne(); - if (Logger.LogNet) - Logger.Log(LogLevel.Info, LOG_APP, $"Wait handle task finished"); - })) - .ConfigureAwait(false); + await ConnectAsync (httpConnection, cancellationToken).ConfigureAwait (continueOnCapturedContext: false); if (Logger.LogNet) Logger.Log (LogLevel.Info, LOG_APP, $" connected"); } catch (Java.Net.ConnectException ex) { @@ -268,9 +272,6 @@ await Task.WhenAny ( Logger.Log (LogLevel.Info, LOG_APP, $"Connection exception {ex}"); // Wrap it nicely in a "standard" exception so that it's compatible with HttpClientHandler throw new WebException (ex.Message, ex, WebExceptionStatus.ConnectFailure, null); - } finally{ - //If not already cancelled, cancel the WaitOne through the waitHandleSource to prevent an orphaned thread - waitHandleSource.Cancel(); } if (cancellationToken.IsCancellationRequested) { @@ -541,7 +542,7 @@ void CopyHeaders (HttpURLConnection httpConnection, HttpResponseMessage response /// Pre-configured connection instance protected virtual Task SetupRequest (HttpRequestMessage request, HttpURLConnection conn) { - return Task.Factory.StartNew (AssertSelf); + return Task.Run (AssertSelf); } /// @@ -645,9 +646,9 @@ void AppendEncoding (string encoding, ref List list) } HandlePreAuthentication (httpConnection); - await SetupRequest (request, httpConnection); + await SetupRequest (request, httpConnection).ConfigureAwait (continueOnCapturedContext: false);; SetupRequestBody (httpConnection, request); - + return httpConnection; }