Fix rendering issues when built against the 10.14 SDK.

Before this commit, the Core Text renderer relied on a legacy behavior
of NSView to keep its content across draws. setNeedsDisplayInRect: could
be used to draw over parts of the existing content as drawing commands
were received without needing to redraw old content.

Layer-backed views lose this behavior and may be asked to redraw any or
all of their content at any time, and layer backing becomes the default
for apps built against the macOS 10.14 SDK.

This change adds a way to draw to an intermediate NSImage and then
create a layer contents from that image. It's similar to the CGLayer
path, but I wasn't able to get the CGLayer path to work without hanging
or crashing when I scrolled. My best guess from looking at stack traces
is that using CGContextDrawLayerInRect to draw a layer into itself
doesn't actually copy pixels, but adds the self-draw as an action to be
performed when the CGLayer is drawn into a bitmap context. After a bunch
of scrolling, these actions stack deeper and deeper and either hang or
overflow the stack when drawn.

When on, the new code skips -drawRect: by returning YES from
-wantsUpdateLayer. -updateLayer draws into an NSImage and then creates a
layer contents from the NSImage. On my machine, scrolling a large
document seems to be faster and use less CPU vs drawRect, which is
promising.

The new code is controlled by the MMUseCALayer user default, which is on
by default in this change. Fixes #751.

Author: s4y

src/MacVim/MMAppController.m

@@ -234,6 +234,7 @@ + (void)initialize
         [NSNumber numberWithBool:YES],    MMNativeFullScreenKey,
         [NSNumber numberWithDouble:0.25], MMFullScreenFadeTimeKey,
         [NSNumber numberWithBool:NO],     MMUseCGLayerAlwaysKey,
+        [NSNumber numberWithBool:YES],    MMUseCALayerKey,
         [NSNumber numberWithBool:YES],    MMShareFindPboardKey,
         nil];
 

src/MacVim/MMCoreTextView.h

@@ -46,6 +46,8 @@
     CGContextRef                cgLayerContext;
     NSLock                      *cgLayerLock;
 
+    NSImage                     *contentsImage;
+
     // These are used in MMCoreTextView+ToolTip.m
     id trackingRectOwner_;              // (not retained)
     void *trackingRectUserData_;

src/MacVim/MMCoreTextView.m

@@ -137,6 +137,12 @@ - (id)initWithFrame:(NSRect)frame
             boolForKey:MMUseCGLayerAlwaysKey];
     cgLayerLock = [NSLock new];
 
+    if ([[NSUserDefaults standardUserDefaults]
+         boolForKey:MMUseCALayerKey]) {
+        self.wantsLayer = YES;
+        contentsImage = [[NSImage alloc] initWithSize:frame.size];
+    }
+
     // NOTE!  It does not matter which font is set here, Vim will set its
     // own font on startup anyway.  Just set some bogus values.
     font = [[NSFont userFixedPitchFontOfSize:0] retain];
@@ -448,6 +454,7 @@ - (void)setFrameSize:(NSSize)newSize {
         [NSAnimationContext beginGrouping];
         drawPending = YES;
     }
+    [contentsImage setSize:newSize];
     [super setFrameSize:newSize];
 }
 
@@ -606,6 +613,19 @@ - (BOOL)isFlipped
     return NO;
 }
 
+- (BOOL)wantsUpdateLayer {
+    return contentsImage != nil;
+}
+
+- (void)updateLayer {
+    [contentsImage lockFocus];
+    for (NSData* data in drawData)
+        [self batchDrawData:data];
+    [drawData removeAllObjects];
+    [contentsImage unlockFocus];
+    self.layer.contents = [contentsImage layerContentsForContentsScale:self.layer.contentsScale];
+}
+
 - (void)drawRect:(NSRect)rect
 {
     NSGraphicsContext *context = [NSGraphicsContext currentContext];
@@ -669,6 +689,9 @@ - (void)performBatchDrawWithData:(NSData *)data
 
 - (void)setCGLayerEnabled:(BOOL)enabled
 {
+    if (contentsImage)
+        return;
+
     cgLayerEnabled = enabled;
 
     if (!cgLayerEnabled)
@@ -1491,7 +1514,15 @@ - (void)drawString:(const UniChar *)chars length:(UniCharCount)length
 
 - (void)scrollRect:(NSRect)rect lineCount:(int)count
 {
-    if (cgLayerEnabled) {
+    if (contentsImage) {
+        // Toggle focus to commit any fresh drawing before drawing the image onto itself.
+        [contentsImage unlockFocus];
+        [contentsImage lockFocus];
+        [contentsImage drawInRect:NSOffsetRect(rect, 0, -count * cellSize.height)
+                         fromRect:rect
+                        operation:NSCompositingOperationCopy
+                         fraction:1.0];
+    } else if (cgLayerEnabled) {
         CGContextRef context = [self getCGContext];
         int yOffset = count * cellSize.height;
         NSRect clipRect = rect;

src/MacVim/Miscellaneous.h

@@ -53,6 +53,7 @@ extern NSString *MMNativeFullScreenKey;
 extern NSString *MMUseMouseTimeKey;
 extern NSString *MMFullScreenFadeTimeKey;
 extern NSString *MMUseCGLayerAlwaysKey;
+extern NSString *MMUseCALayerKey;
 
 
 // Enum for MMUntitledWindowKey

src/MacVim/Miscellaneous.m

@@ -49,6 +49,7 @@
 NSString *MMUseMouseTimeKey             = @"MMUseMouseTime";
 NSString *MMFullScreenFadeTimeKey       = @"MMFullScreenFadeTime";
 NSString *MMUseCGLayerAlwaysKey         = @"MMUseCGLayerAlways";
+NSString *MMUseCALayerKey               = @"MMUseCALayer";