Fix CoreText clipping issues with tall texts

This fixes the issue where particularly tall characters will get clipped
at the row boundary. This happens because even though a font describes
the line height with font metrics, individual glyphs do not have to
respect them, and we can see with emoji rendering sometimes they can
poke upwards past the line height. Also, it's trivially easy to
construct composing characters that become really tall, e.g. "x゙̂⃗", or
Tibetan scripts like "ཧཱུྃ".

To fix this, we do the following:

1. Remove the explicit clipping call at rendering.

2. Fix partial redraw to not lead to clipping / corruption. This is
   quite tricky, because let's say we have a character that is tall
   enough to touch other lines, if those lines are redraw but not the
   line with the tall char, the redraw will paint over the parts of the
   glyphs poking through. Alternatively if we redraw the line with the
   tall chars we also need to expand the redraw region to make sure
   other lines get repainted as well. To fix this properly, we should do
   a proper glyph calculation when we receive the draw command before we
   issue before we call `setNeedsDisplayInRect`, but since right now we
   only extract glyph info later (during drawRect call), it's too late.
   We just do the hacky solution by storing a variable
   `redrawExpandRows` that tracks how many lines to expand for all
   lines. It's a little hacky since this will affect all lines
   permanently regardless if they have tall characters or not. The
   proper fix may come later as an optimization (or when we do hardware
   rendering via Metal).

3. Re-order the rendering so we have a two pass system, where we first
   draw the background fill color for all rows, then the text. This
   helps prevent things like Vim's window split or cursorline from
   obscuring the text.

4. Add a preference to turn on clipping (old behavior). This helps
   prevent odd issues with really tall texts (e.g. Zalgo text) making it
   hard to see what's going on. The preference `MMRendererClipToRow` is
   not exposed in UI for now as it's a relatively niche. It will be
   exposed later when we have a dedicated render tab in settings.

Note that a lot of these characters only show their full height by doing
`set maxcombine=8` because the default (2) is quite low.

Part of the fix for #995

Author: ychin

runtime/doc/gui_mac.txt

@@ -290,6 +290,7 @@ KEY				VALUE ~
 *MMNoTitleBarWindow*		hide title bar [bool]
 *MMTitlebarAppearsTransparent*	enable a transparent titlebar [bool]
 *MMAppearanceModeSelection*	dark mode selection (|macvim-dark-mode|)[bool]
+*MMRendererClipToRow*		clip tall characters to the row they are on [bool]
 *MMSmoothResize*		allow smooth resizing of MacVim window [bool]
 *MMShareFindPboard*		share search text to Find Pasteboard [bool]
 *MMShowAddTabButton*		enable "add tab" button on tabline [bool]

runtime/doc/tags

@@ -5498,6 +5498,7 @@ MMNoFontSubstitution	gui_mac.txt	/*MMNoFontSubstitution*
 MMNoTitleBarWindow	gui_mac.txt	/*MMNoTitleBarWindow*
 MMNonNativeFullScreenSafeAreaBehavior	gui_mac.txt	/*MMNonNativeFullScreenSafeAreaBehavior*
 MMNonNativeFullScreenShowMenu	gui_mac.txt	/*MMNonNativeFullScreenShowMenu*
+MMRendererClipToRow	gui_mac.txt	/*MMRendererClipToRow*
 MMShareFindPboard	gui_mac.txt	/*MMShareFindPboard*
 MMShowAddTabButton	gui_mac.txt	/*MMShowAddTabButton*
 MMSmoothResize	gui_mac.txt	/*MMSmoothResize*

src/MacVim/MMAppController.m

@@ -260,6 +260,7 @@ + (void)initialize
         [NSNumber numberWithBool:YES],    MMShareFindPboardKey,
         [NSNumber numberWithBool:NO],     MMSmoothResizeKey,
         [NSNumber numberWithBool:NO],     MMCmdLineAlignBottomKey,
+        [NSNumber numberWithBool:NO],     MMRendererClipToRowKey,
         [NSNumber numberWithBool:YES],    MMAllowForceClickLookUpKey,
         [NSNumber numberWithBool:NO],     MMUpdaterPrereleaseChannelKey,
         nil];

src/MacVim/MMCoreTextView.h

@@ -12,6 +12,8 @@
 
 @class MMTextViewHelper;
 
+NS_ASSUME_NONNULL_BEGIN
+
 
 /// The main text view that manages drawing Vim's content using Core Text, and
 /// handles input. We are using this instead of NSTextView because of the
@@ -84,7 +86,7 @@
 //
 // NSFontChanging methods
 //
-- (void)changeFont:(id)sender;
+- (void)changeFont:(nullable id)sender;
 
 //
 // NSMenuItemValidation
@@ -100,7 +102,7 @@
 - (IBAction)paste:(id)sender;
 - (IBAction)undo:(id)sender;
 - (IBAction)redo:(id)sender;
-- (IBAction)selectAll:(id)sender;
+- (IBAction)selectAll:(nullable id)sender;
 
 //
 // MMTextStorage methods
@@ -186,3 +188,5 @@
 @interface MMCoreTextView (ToolTip)
 - (void)setToolTipAtMousePoint:(NSString *)string;
 @end
+
+NS_ASSUME_NONNULL_END

src/MacVim/MMCoreTextView.m

@@ -225,6 +225,16 @@ @implementation MMCoreTextView {
 
     BOOL alignCmdLineToBottom; ///< Whether to pin the Vim command-line to the bottom of the window
     int cmdlineRow; ///< Row number (0-indexed) where the cmdline starts. Used for pinning it to the bottom if desired.
+
+    /// Number of rows to expand when redrawing to make sure we don't clip tall
+    /// characters whose glyphs extend beyond the bottom/top of the cell.
+    ///
+    /// Note: This is a short-term hacky solution as it permanently increases
+    /// the number of rows to expand every time we redraw. Eventually we should
+    /// calculate each line's glyphs' bounds before issuing a redraw and use
+    /// that to determine the accurate redraw bounds instead. Currently we
+    /// calculate the glyph run too late (inside the draw call itself).
+    unsigned int redrawExpandRows;
 }
 
 - (instancetype)initWithFrame:(NSRect)frame
@@ -255,6 +265,7 @@ - (instancetype)initWithFrame:(NSRect)frame
 
     alignCmdLineToBottom = NO; // this would be updated to the user preferences later
     cmdlineRow = -1; // this would be updated by Vim
+    redrawExpandRows = 0; // start at 0, until we see a tall character. and then we expand it.
 
     return self;
 }
@@ -739,11 +750,16 @@ - (BOOL)isFlipped
 
 - (void)setNeedsDisplayFromRow:(int)row column:(int)col toRow:(int)row2
                         column:(int)col2 {
+    row -= redrawExpandRows;
+    row2 += redrawExpandRows;
     [self setNeedsDisplayInRect:[self rectForRow:row column:0 numRows:row2-row+1 numColumns:maxColumns]];
 }
 
 - (void)drawRect:(NSRect)rect
 {
+    NSUserDefaults *ud = [NSUserDefaults standardUserDefaults];
+    const BOOL clipTextToRow = [ud boolForKey:MMRendererClipToRowKey]; // Specify whether to clip tall characters by the row boundary.
+
     NSGraphicsContext *context = [NSGraphicsContext currentContext];
 #if MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_10
     CGContextRef ctx = context.CGContext;
@@ -777,14 +793,101 @@ - (void)drawRect:(NSRect)rect
 
     CGContextFillRect(ctx, rect);
 
-    for (size_t r = 0; r < grid.rows; r++) {
-        const CGRect rowRect = [self rectForRow:r column:0 numRows:1 numColumns:grid.cols];
-        const CGRect rowClipRect = CGRectIntersection(rowRect, rect);
-        if (CGRectIsNull(rowClipRect))
-            continue;
-        CGContextSaveGState(ctx);
-        CGContextClipToRect(ctx, rowClipRect);
-        
+    // Function to draw all rows
+    void (^drawAllRows)(void (^)(CGContextRef,CGRect,int)) = ^(void (^drawFunc)(CGContextRef,CGRect,int)){
+        for (size_t r = 0; r < grid.rows; r++) {
+            const CGRect rowRect = [self rectForRow:(int)r
+                                             column:0
+                                            numRows:1
+                                         numColumns:grid.cols];
+
+            // Expand the clip rect to include some above/below rows in case we have tall characters.
+            const CGRect rowExpandedRect = [self rectForRow:(int)(r-redrawExpandRows)
+                                                     column:0
+                                                    numRows:(1+redrawExpandRows*2)
+                                                 numColumns:grid.cols];
+
+            const CGRect rowClipRect = CGRectIntersection(rowExpandedRect, rect);
+            if (CGRectIsNull(rowClipRect))
+                continue;
+            CGContextSaveGState(ctx);
+            if (clipTextToRow)
+                CGContextClipToRect(ctx, rowClipRect);
+
+            drawFunc(ctx, rowRect, (int)r);
+
+            CGContextRestoreGState(ctx);
+        }
+    };
+
+    // Function to draw a row of background colors, signs, and cursor rect. These should go below
+    // any text.
+    void (^drawBackgroundAndCursorFunc)(CGContextRef,CGRect,int) = ^(CGContextRef ctx, CGRect rowRect, int r){
+        for (int c = 0; c < grid.cols; c++) {
+            GridCell cell = *grid_cell(&grid, r, c);
+            CGRect cellRect = {{rowRect.origin.x + cellSize.width * c, rowRect.origin.y}, cellSize};
+            if (cell.textFlags & DRAW_WIDE)
+                cellRect.size.width *= 2;
+            if (cell.inverted) {
+                cell.bg ^= 0xFFFFFF;
+                cell.fg ^= 0xFFFFFF;
+                cell.sp ^= 0xFFFFFF;
+            }
+
+            // Fill background
+            if (cell.bg != defaultBg && ALPHA(cell.bg) > 0) {
+                CGRect fillCellRect = cellRect;
+
+                if (c == grid.cols - 1 || (c == grid.cols - 2 && (cell.textFlags & DRAW_WIDE))) {
+                    // Fill a little extra to the right if this is the last
+                    // column, and the frame size isn't exactly the same size
+                    // as the grid (due to smooth resizing, etc). This makes it
+                    // look less ugly and more consisten. See rectForRow:'s
+                    // implementation for extra comments.
+                    CGFloat extraWidth = rowRect.origin.x + rowRect.size.width - (cellRect.size.width + cellRect.origin.x);
+                    fillCellRect.size.width += extraWidth;
+                }
+
+                CGContextSetFillColor(ctx, COMPONENTS(cell.bg));
+                CGContextFillRect(ctx, fillCellRect);
+            }
+
+            // Handle signs
+            if (cell.sign) {
+                CGRect signRect = cellRect;
+                signRect.size.width *= 2;
+                [cell.sign drawInRect:signRect
+                             fromRect:(NSRect){{0, 0}, cell.sign.size}
+                            operation:(cell.inverted ? NSCompositingOperationDifference : NSCompositingOperationSourceOver)
+                             fraction:1.0];
+            }
+
+            // Insertion point (cursor)
+            if (cell.insertionPoint.color && cell.insertionPoint.fraction) {
+                float frac = cell.insertionPoint.fraction / 100.0;
+                NSRect rect = cellRect;
+                if (MMInsertionPointHorizontal == cell.insertionPoint.shape) {
+                    rect.size.height = cellSize.height * frac;
+                } else if (MMInsertionPointVertical == cell.insertionPoint.shape) {
+                    rect.size.width = cellSize.width * frac;
+                } else if (MMInsertionPointVerticalRight == cell.insertionPoint.shape) {
+                    rect.size.width = cellSize.width * frac;
+                    rect.origin.x += cellRect.size.width - rect.size.width;
+                }
+                rect = [self backingAlignedRect:rect options:NSAlignAllEdgesInward];
+                
+                [[NSColor colorWithArgbInt:cell.insertionPoint.color] set];
+                if (MMInsertionPointHollow == cell.insertionPoint.shape) {
+                    [NSBezierPath strokeRect:NSInsetRect(rect, 0.5, 0.5)];
+                } else {
+                    NSRectFill(rect);
+                }
+            }
+        }
+    };
+
+    // Function to draw a row of text with their corresponding text styles.
+    void (^drawTextFunc)(CGContextRef,CGRect,int) = ^(CGContextRef ctx, CGRect rowRect, int r){
         __block NSMutableString *lineString = nil;
         __block CGFloat lineStringStart = 0;
         __block CFRange lineStringRange = {};
@@ -797,7 +900,7 @@ - (void)drawRect:(NSRect)rect
             if (!lineString.length)
                 return;
             size_t cellOffsetByIndex[lineString.length];
-            for (size_t i = 0, stringIndex = 0; i < lineStringRange.length; i++) {
+            for (int i = 0, stringIndex = 0; i < lineStringRange.length; i++) {
                 GridCell cell = *grid_cell(&grid, r, lineStringRange.location + i);
                 size_t cell_length = cell.string.length;
                 for (size_t j = 0; j < cell_length; j++) {
@@ -860,6 +963,38 @@ - (void)drawRect:(NSRect)rect
                     layoutPositions = layoutPositions_storage;
                 }
 
+                const int maxRedrawExpandRows = clipTextToRow ? 0 : 3; // Hard-code a sane maximum for now to prevent degenerate edge cases
+                if (redrawExpandRows < maxRedrawExpandRows) {
+                    // Check if we encounter any glyphs in this line that are too tall and would be
+                    // clipped / not redrawn properly. If we encounter that, increase
+                    // redrawExpandRows and redraw.
+                    // Note: This is kind of a hacky solution. See comments for redrawExpandRows.
+                    CGRect lineBounds = CTRunGetImageBounds(run, ctx, CFRangeMake(0,0));
+                    if (!CGRectIsNull(lineBounds)) {
+                        unsigned int newRedrawExpandRows = 0;
+                        if (lineBounds.origin.y < rowRect.origin.y) {
+                            newRedrawExpandRows = (int)ceil((rowRect.origin.y - lineBounds.origin.y) / cellSize.height);
+                        }
+                        if (lineBounds.origin.y + lineBounds.size.height > rowRect.origin.y + cellSize.height) {
+                            int rowsAbove = (int)ceil(((lineBounds.origin.y + lineBounds.size.height) - (rowRect.origin.y + cellSize.height)) / cellSize.height);
+                            if (rowsAbove > newRedrawExpandRows) {
+                                newRedrawExpandRows = rowsAbove;
+                            }
+                        }
+
+                        if (newRedrawExpandRows > redrawExpandRows) {
+                            redrawExpandRows = newRedrawExpandRows;
+                            if (redrawExpandRows > maxRedrawExpandRows) {
+                                redrawExpandRows = maxRedrawExpandRows;
+                            }
+                            [self setNeedsDisplay:YES];
+                        }
+                    }
+                }
+                else {
+                    redrawExpandRows = maxRedrawExpandRows;
+                }
+
                 for (CFIndex i = 0; i < glyphCount; i++) {
                     if (indices[i] >= lineStringLength) {
                         ASLogDebug(@"Invalid glyph pos index: %ld, len: %lu", (long)indices[i], (unsigned long)lineStringLength);
@@ -916,7 +1051,7 @@ - (void)drawRect:(NSRect)rect
 
         BOOL hasStrikeThrough = NO;
 
-        for (size_t c = 0; c < grid.cols; c++) {
+        for (int c = 0; c < grid.cols; c++) {
             GridCell cell = *grid_cell(&grid, r, c);
             CGRect cellRect = {{rowRect.origin.x + cellSize.width * c, rowRect.origin.y}, cellSize};
             if (cell.textFlags & DRAW_WIDE)
@@ -927,56 +1062,6 @@ - (void)drawRect:(NSRect)rect
                 cell.sp ^= 0xFFFFFF;
             }
 
-            // Fill background
-            if (cell.bg != defaultBg && ALPHA(cell.bg) > 0) {
-                CGRect fillCellRect = cellRect;
-
-                if (c == grid.cols - 1 || (c == grid.cols - 2 && (cell.textFlags & DRAW_WIDE))) {
-                    // Fill a little extra to the right if this is the last
-                    // column, and the frame size isn't exactly the same size
-                    // as the grid (due to smooth resizing, etc). This makes it
-                    // look less ugly and more consisten. See rectForRow:'s
-                    // implementation for extra comments.
-                    CGFloat extraWidth = rowRect.origin.x + rowRect.size.width - (cellRect.size.width + cellRect.origin.x);
-                    fillCellRect.size.width += extraWidth;
-                }
-
-                CGContextSetFillColor(ctx, COMPONENTS(cell.bg));
-                CGContextFillRect(ctx, fillCellRect);
-            }
-
-            // Handle signs
-            if (cell.sign) {
-                CGRect signRect = cellRect;
-                signRect.size.width *= 2;
-                [cell.sign drawInRect:signRect
-                             fromRect:(NSRect){{0, 0}, cell.sign.size}
-                            operation:(cell.inverted ? NSCompositingOperationDifference : NSCompositingOperationSourceOver)
-                             fraction:1.0];
-            }
-
-            // Insertion point (cursor)
-            if (cell.insertionPoint.color && cell.insertionPoint.fraction) {
-                float frac = cell.insertionPoint.fraction / 100.0;
-                NSRect rect = cellRect;
-                if (MMInsertionPointHorizontal == cell.insertionPoint.shape) {
-                    rect.size.height = cellSize.height * frac;
-                } else if (MMInsertionPointVertical == cell.insertionPoint.shape) {
-                    rect.size.width = cellSize.width * frac;
-                } else if (MMInsertionPointVerticalRight == cell.insertionPoint.shape) {
-                    rect.size.width = cellSize.width * frac;
-                    rect.origin.x += cellRect.size.width - rect.size.width;
-                }
-                rect = [self backingAlignedRect:rect options:NSAlignAllEdgesInward];
-                
-                [[NSColor colorWithArgbInt:cell.insertionPoint.color] set];
-                if (MMInsertionPointHollow == cell.insertionPoint.shape) {
-                    [NSBezierPath strokeRect:NSInsetRect(rect, 0.5, 0.5)];
-                } else {
-                    NSRectFill(rect);
-                }
-            }
-
             // Text underline styles. We only allow one of them to be active.
             // Note: We are not currently using underlineThickness or underlinePosition. Should fix to use them.
             const CGFloat underlineY = 0.4*fontDescent; // Just a hard-coded value for now. Should fix to use underlinePosition.
@@ -1089,7 +1174,7 @@ - (void)drawRect:(NSRect)rect
         if (hasStrikeThrough) {
             // Second pass to render strikethrough. Unfortunately have to duplicate a little bit of code here to loop
             // through the cells.
-            for (size_t c = 0; c < grid.cols; c++) {
+            for (int c = 0; c < grid.cols; c++) {
                 GridCell cell = *grid_cell(&grid, r, c);
                 CGRect cellRect = {{rowRect.origin.x + cellSize.width * c, rowRect.origin.y}, cellSize};
                 if (cell.textFlags & DRAW_WIDE)
@@ -1109,9 +1194,21 @@ - (void)drawRect:(NSRect)rect
 
             }
         }
+    };
+
+    // Render passes:
+
+    // 1. Draw background color and cursor rect.
+    drawAllRows(drawBackgroundAndCursorFunc);
+
+    // 2. Draw text.
+    // We need to do this in a separate pass in case some characters are taller than a cell. This
+    // could easily happen when we have composed characters (e.g. T゙̂⃗) that either goes below or above
+    // the cell boundary. We draw the background colors in 1st pass to make sure all the texts will
+    // be drawn on top of them. Also see redrawExpandRows which handles making such tall characters
+    // redraw/clip correctly.
+    drawAllRows(drawTextFunc);
 
-        CGContextRestoreGState(ctx);
-    }
     if (thinStrokes) {
         CGContextSetFontSmoothingStyle(ctx, originalSmoothingStyle);
     }

src/MacVim/Miscellaneous.h

@@ -61,6 +61,7 @@ extern NSString *MMNonNativeFullScreenShowMenuKey;
 extern NSString *MMNonNativeFullScreenSafeAreaBehaviorKey;
 extern NSString *MMSmoothResizeKey;
 extern NSString *MMCmdLineAlignBottomKey;
+extern NSString *MMRendererClipToRowKey;
 extern NSString *MMAllowForceClickLookUpKey;
 extern NSString *MMUpdaterPrereleaseChannelKey;
 

src/MacVim/Miscellaneous.m

@@ -57,6 +57,7 @@
 NSString *MMNonNativeFullScreenSafeAreaBehaviorKey = @"MMNonNativeFullScreenSafeAreaBehavior";
 NSString *MMSmoothResizeKey               = @"MMSmoothResize";
 NSString *MMCmdLineAlignBottomKey         = @"MMCmdLineAlignBottom";
+NSString *MMRendererClipToRowKey      = @"MMRendererClipToRow";
 NSString *MMAllowForceClickLookUpKey      = @"MMAllowForceClickLookUp";
 NSString *MMUpdaterPrereleaseChannelKey   = @"MMUpdaterPrereleaseChannel";