Support HTML comments & embedded images

Mail2Bug/Config.cs

@@ -130,6 +130,7 @@ public enum ProcessingStrategyType
             public bool ApplyOverridesDuringUpdate { get; set; }
             public bool AttachOriginalMessage { get; set; }
             public bool AttachUpdateMessages { get; set; }
+            public bool EnableExperimentalHtmlFeatures { get; set; }
 
             public ProcessingStrategyType ProcessingStrategy = ProcessingStrategyType.SimpleBugStrategy;
         }

Mail2Bug/Email/EWS/EWSIncomingFileAttachment.cs

@@ -28,6 +28,12 @@ public string SaveAttachmentToFile(string filename)
             return filename;
         }
 
+        public string ContentId
+        {
+            get => _attachment.ContentId;
+            set => _attachment.ContentId = value;
+        }
+
         private static readonly ILog Logger = LogManager.GetLogger(typeof(EWSIncomingFileAttachment));
     }
 }

Mail2Bug/Email/EWS/EWSIncomingItemAttachment.cs

@@ -38,6 +38,12 @@ public string SaveAttachmentToFile(string filename)
             return filename;
         }
 
+        public string ContentId
+        {
+            get => _attachment.ContentId;
+            set => _attachment.ContentId = value;
+        }
+
         private static readonly ILog Logger = LogManager.GetLogger(typeof (EWSIncomingItemAttachment));
     }
 }

Mail2Bug/Email/EWS/EWSIncomingMessage.cs

@@ -116,9 +116,9 @@ public string SaveToFile(string filename)
             return filename;
         }
 
-        public string GetLastMessageText()
+        public string GetLastMessageText(bool enableExperimentalHtmlFeatures)
         {
-            return EmailBodyProcessingUtils.GetLastMessageText(this);
+            return EmailBodyProcessingUtils.GetLastMessageText(this, enableExperimentalHtmlFeatures);
         }
 
         public void Delete()

Mail2Bug/Email/EmailBodyProcessingUtils.cs

@@ -3,12 +3,18 @@
 using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
+using CsQuery;
 
 namespace Mail2Bug.Email
 {
     public class EmailBodyProcessingUtils
     {
-        public static string GetLastMessageText(IIncomingEmailMessage message)
+        public static string GetLastMessageText(IIncomingEmailMessage message, bool enableExperimentalHtmlFeatures)
+        {
+            return enableExperimentalHtmlFeatures && message.IsHtmlBody ? GetLastMessageText_Html(message.RawBody) : GetLastMessageText_PlainText(message);
+        }
+
+        private static string GetLastMessageText_PlainText(IIncomingEmailMessage message)
         {
             var lastMessage = new StringBuilder();
             lastMessage.Append(message.PlainTextBody);
@@ -23,27 +29,81 @@ public static string GetLastMessageText(IIncomingEmailMessage message)
             return lastMessage.ToString();
         }
 
+        public static string GetLastMessageText_Html(string rawBody)
+        {
+            CQ dom = rawBody;
+
+            const string messageSeparatorStyle = "border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in";
+
+            foreach (IDomObject element in dom["*"])
+            {
+                // Lots of email clients insert html elements as message delimiters which have styling but no inner text
+                // This block checks for some of these patterns
+                if (element.NodeName == "DIV")
+                {
+                    if (element.Id == "divRplyFwdMsg" || element.Id == "x_divRplyFwdMsg" || messageSeparatorStyle.Equals(element.GetAttribute("style")))
+                    {
+                        IDomContainer parent = element.ParentNode;
+                        element.Remove();
+                        RemoveSubsequent(parent);
+                        break;
+                    }
+                }
+
+                if (!element.ChildElements.Any() && !string.IsNullOrWhiteSpace(element.InnerText))
+                {
+                    var separatorIndex = IndexOfAny(element.InnerText, MessageBorderMarkers);
+                    if (separatorIndex.HasValue)
+                    {
+                        element.InnerText = element.InnerText.Substring(0, separatorIndex.Value);
+                        RemoveSubsequent(element);
+                        break;
+                    }
+                }
+            }
+
+            return dom.Render();
+        }
+
+        private static void RemoveSubsequent(IDomObject element)
+        {
+            IDomObject nextNode;
+            while ((nextNode = element.NextSibling) != null)
+            {
+                nextNode.Remove();
+            }
+
+            if (element.ParentNode is IDomObject parentElement)
+            {
+                RemoveSubsequent(parentElement);
+            }
+        }
+
+        private static readonly string[] MessageBorderMarkers =
+        {
+            "_____",
+            "-----Original Message",
+            "From:",
+        };
+
         /// <summary>
         /// Get the seperate denotation between reply content and original content.
         /// </summary>
         /// <param name="description"></param>
         /// <returns></returns>
         private static int GetReplySeperatorIndex(string description)
         {
-            var indices = new List<int> 
-            {
-                description.IndexOf("_____", StringComparison.Ordinal),
-                description.IndexOf("-----Original Message", StringComparison.Ordinal),
-                description.IndexOf("From:", StringComparison.Ordinal)
-            };
-
-            indices.RemoveAll(x => x < 0);
-            if (indices.Count == 0)
-            {
-                return description.Length;
-            }
+            return IndexOfAny(description, MessageBorderMarkers) ?? description.Length;
+        }
 
-            return indices.Min();
+        private static int? IndexOfAny(string text, IEnumerable<string> searchTerms)
+        {
+            return searchTerms
+                .Select(search => text.IndexOf(search, StringComparison.Ordinal))
+                .Where(index => index >= 0)
+                .OrderBy(index => index)
+                .Cast<int?>()
+                .FirstOrDefault();
         }
 
         public static string ConvertHtmlMessageToPlainText(string text)
@@ -78,5 +138,35 @@ private static string ResolveOridnalToChar(Match ordinalMatch)
             var unicodeChar = (char)int.Parse(ordinalMatch.Groups["ordinal"].Value);
             return new string(new[] {unicodeChar});
         }
+
+        /// <summary>
+        /// If an email embeds an email inline in its html, that embedded image won't display correctly unless we modify the html.
+        /// This method does that, given information about email's known attachments
+        /// </summary>
+        public static string UpdateEmbeddedImageLinks(string originalHtml, System.Collections.Generic.IReadOnlyCollection<MessageAttachmentInfo> attachments)
+        {
+            if (attachments == null || attachments.Count == 0)
+            {
+                return originalHtml;
+            }
+
+            CQ dom = originalHtml;
+            foreach (var attachment in attachments)
+            {
+                string originalImgSrc = $"cid:{attachment.ContentId}";
+                var matchingImgLinks = dom[$"img[src$='{originalImgSrc}']"];
+
+                // This may point to the file on the local file-system if we haven't yet uploaded the attachment
+                // However, the work item APIs seem to magically 'just work' with this and update the URI to point to the uploaded location
+                // If for some reason that stops working, we'd need to either infer the uploaded URI or upload first and mutate the html afterward
+                var newSrc = new Uri(attachment.FilePath);
+                foreach (IDomObject img in matchingImgLinks)
+                {
+                    img.SetAttribute("src", newSrc.AbsoluteUri);
+                }
+            }
+
+            return dom.Render();
+        }
     }
 }

Mail2Bug/Email/IIncomingEmailAttachment.cs

@@ -7,5 +7,6 @@ public interface IIncomingEmailAttachment
     {
         string SaveAttachmentToFile();
         string SaveAttachmentToFile(string filename);
+        string ContentId { get; set; }
     }
 }

Mail2Bug/Email/IIncomingEmailMessage.cs

@@ -43,7 +43,7 @@ public interface IIncomingEmailMessage
         /// messages dropped, whereas the body includes all of that "history".
         /// </summary>
         /// <returns></returns>
-        string GetLastMessageText();
+        string GetLastMessageText(bool enableExperimentalHtmlFeatures);
 
         /// <summary>
         /// Deletes the message

Mail2Bug/Email/MessageAttachmentCollection.cs

@@ -0,0 +1,41 @@
+using System;
+using System.CodeDom.Compiler;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Mail2Bug.Email
+{
+    /// <summary>
+    /// Collection of Exchange email attachments that have been downloaded locally
+    /// </summary>
+    public class MessageAttachmentCollection : IDisposable
+    {
+        private readonly List<MessageAttachmentInfo> _attachments;
+        private readonly TempFileCollection _tempFileCollection;
+
+        public IReadOnlyCollection<MessageAttachmentInfo> Attachments => _attachments;
+        public IEnumerable<string> LocalFilePaths => _attachments.Select(a => a.FilePath);
+
+        public MessageAttachmentCollection()
+        {
+            _attachments = new List<MessageAttachmentInfo>();
+            _tempFileCollection = new TempFileCollection();
+        }
+
+        public void Add(string localFilePath, string contentId)
+        {
+            _attachments.Add(new MessageAttachmentInfo(localFilePath, contentId));
+            _tempFileCollection.AddFile(localFilePath, keepFile: false);
+        }
+
+        public void DeleteLocalFiles()
+        {
+            _tempFileCollection.Delete();
+        }
+
+        public void Dispose()
+        {
+            DeleteLocalFiles();
+        }
+    }
+}

Mail2Bug/Email/MessageAttachmentInfo.cs

@@ -0,0 +1,18 @@
+namespace Mail2Bug.Email
+{
+    /// <summary>
+    /// Represents basic information about an Exchange email attachment that has been downloaded locally and has a known exchange content id
+    /// </summary>
+    public class MessageAttachmentInfo
+    {
+        public MessageAttachmentInfo(string filePath, string contentId)
+        {
+            FilePath = filePath;
+            ContentId = contentId;
+        }
+
+        public string FilePath { get; }
+
+        public string ContentId { get; }
+    }
+}

Mail2Bug/Mail2Bug.csproj

@@ -36,6 +36,9 @@
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="CsQuery, Version=1.3.3.249, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\CsQuery.1.3.4\lib\net40\CsQuery.dll</HintPath>
+    </Reference>
     <Reference Include="Hyak.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Hyak.Common.1.0.2\lib\net45\Hyak.Common.dll</HintPath>
       <Private>True</Private>
@@ -320,6 +323,8 @@
     <Compile Include="MessageProcessingStrategies\DateBasedValueResolver.cs" />
     <Compile Include="MessageProcessingStrategies\IMessageProcessingStrategy.cs" />
     <Compile Include="MessageProcessingStrategies\INameResolver.cs" />
+    <Compile Include="Email\MessageAttachmentCollection.cs" />
+    <Compile Include="Email\MessageAttachmentInfo.cs" />
     <Compile Include="MessageProcessingStrategies\NameResolver.cs" />
     <Compile Include="MessageProcessingStrategies\NameResolverMock.cs" />
     <Compile Include="MessageProcessingStrategies\OverridesExtractor.cs" />