From 7d09940d96bc738ac78c95b2dc5418ec5a262e31 Mon Sep 17 00:00:00 2001 From: Huan Lin Date: Wed, 16 Aug 2023 13:09:26 -0700 Subject: [PATCH 1/4] [ios][ios17]fix auto correction highlight on top left corner --- .../Source/FlutterTextInputPlugin.mm | 34 ++++++++-- .../Source/FlutterTextInputPluginTest.mm | 67 +++++++++++++++++++ 2 files changed, 94 insertions(+), 7 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm index 0d6ecbe9b63f6..5560d399cd7b6 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm @@ -2449,18 +2449,27 @@ - (void)takeKeyboardScreenshotAndDisplay { } - (void)setEditableSizeAndTransform:(NSDictionary*)dictionary { - [_activeView setEditableTransform:dictionary[@"transform"]]; + NSArray* transform = dictionary[@"transform"]; + [_activeView setEditableTransform:transform]; + int leftIndex = 12; + int topIndex = 13; if ([_activeView isScribbleAvailable]) { // This is necessary to set up where the scribble interactable element will be. - int leftIndex = 12; - int topIndex = 13; _inputHider.frame = - CGRectMake([dictionary[@"transform"][leftIndex] intValue], - [dictionary[@"transform"][topIndex] intValue], [dictionary[@"width"] intValue], - [dictionary[@"height"] intValue]); + CGRectMake([transform[leftIndex] intValue], [transform[topIndex] intValue], + [dictionary[@"width"] intValue], [dictionary[@"height"] intValue]); _activeView.frame = CGRectMake(0, 0, [dictionary[@"width"] intValue], [dictionary[@"height"] intValue]); _activeView.tintColor = [UIColor clearColor]; + } else { + if (@available(iOS 17, *)) { + // Move auto-correction highlight to overlap with the actual text. + // This is to fix an issue where the system auto-correction highlight is displayed at + // the top left corner of the screen on iOS 17+. + // See https://github.com/flutter/flutter/issues/131695 + _inputHider.frame = + CGRectMake([transform[leftIndex] intValue], [transform[topIndex] intValue], 0, 0); + } } } @@ -2488,7 +2497,18 @@ - (void)setSelectionRects:(NSArray*)encodedRects { ? NSWritingDirectionLeftToRight : NSWritingDirectionRightToLeft]]; } - _activeView.selectionRects = rectsAsRect; + + if (@available(iOS 17, *)) { + // Force UIKit to query the selectionRects again on iOS 17+ + // This is to fix a bug on iOS 17+ where UIKit queries the outdated selectionRects after + // entering a character, resulting in auto-correction highlight region missing the last + // character. + [_activeView.inputDelegate textWillChange:_activeView]; + _activeView.selectionRects = rectsAsRect; + [_activeView.inputDelegate textDidChange:_activeView]; + } else { + _activeView.selectionRects = rectsAsRect; + } } - (void)startLiveTextInput { diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.mm b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.mm index 5a214281eb573..b6b1967ceb41d 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.mm @@ -61,6 +61,7 @@ @interface FlutterSecureTextInputView : FlutterTextInputView @interface FlutterTextInputPlugin () @property(nonatomic, assign) FlutterTextInputView* activeView; +@property(nonatomic, readonly) UIView* inputHider; @property(nonatomic, readonly) UIView* keyboardViewContainer; @property(nonatomic, readonly) UIView* keyboardView; @property(nonatomic, assign) UIView* cachedFirstResponder; @@ -422,6 +423,72 @@ - (void)testAutocorrectionPromptRectDoesNotAppearDuringScribble { } } +- (void)testInputHiderOverlapWithTextWhenScribbleIsDisabledAfterIOS17AndDoesNotOverlapBeforeIOS17 { + FlutterTextInputPlugin* myInputPlugin = + [[FlutterTextInputPlugin alloc] initWithDelegate:OCMClassMock([FlutterEngine class])]; + + FlutterMethodCall* setClientCall = + [FlutterMethodCall methodCallWithMethodName:@"TextInput.setClient" + arguments:@[ @(123), self.mutableTemplateCopy ]]; + [myInputPlugin handleMethodCall:setClientCall + result:^(id _Nullable result){ + }]; + + FlutterTextInputView* mockInputView = OCMPartialMock(myInputPlugin.activeView); + OCMStub([mockInputView isScribbleAvailable]).andReturn(NO); + + // yOffset = 200. + NSArray* yOffsetMatrix = @[ @1, @0, @0, @0, @0, @1, @0, @0, @0, @0, @1, @0, @0, @200, @0, @1 ]; + + FlutterMethodCall* setPlatformViewClientCall = + [FlutterMethodCall methodCallWithMethodName:@"TextInput.setEditableSizeAndTransform" + arguments:@{@"transform" : yOffsetMatrix}]; + [myInputPlugin handleMethodCall:setPlatformViewClientCall + result:^(id _Nullable result){ + }]; + + if (@available(iOS 17, *)) { + XCTAssert(CGRectEqualToRect(myInputPlugin.inputHider.frame, CGRectMake(0, 200, 0, 0)), + @"The input hider should overlap with the text on and after iOS 17"); + + } else { + XCTAssert(CGRectEqualToRect(myInputPlugin.inputHider.frame, CGRectZero), + @"The input hider should be on the origin of screen on and before iOS 16."); + } +} + +- (void)testSetSelectionRectsNotifiesTextChangeAfterIOS17AndDoesNotNotifyBeforeIOS17 { + FlutterTextInputPlugin* myInputPlugin = + [[FlutterTextInputPlugin alloc] initWithDelegate:OCMClassMock([FlutterEngine class])]; + + FlutterMethodCall* setClientCall = + [FlutterMethodCall methodCallWithMethodName:@"TextInput.setClient" + arguments:@[ @(123), self.mutableTemplateCopy ]]; + [myInputPlugin handleMethodCall:setClientCall + result:^(id _Nullable result){ + }]; + + id mockInputDelegate = OCMProtocolMock(@protocol(UITextInputDelegate)); + myInputPlugin.activeView.inputDelegate = mockInputDelegate; + + NSArray* selectionRect = [NSArray arrayWithObjects:@0, @0, @100, @100, @0, @1, nil]; + NSArray* selectionRects = [NSArray arrayWithObjects:selectionRect, nil]; + FlutterMethodCall* methodCall = + [FlutterMethodCall methodCallWithMethodName:@"Scribble.setSelectionRects" + arguments:selectionRects]; + [myInputPlugin handleMethodCall:methodCall + result:^(id _Nullable result){ + }]; + + if (@available(iOS 17.0, *)) { + OCMVerify([mockInputDelegate textWillChange:myInputPlugin.activeView]); + OCMVerify([mockInputDelegate textDidChange:myInputPlugin.activeView]); + } else { + OCMVerify(never(), [mockInputDelegate textWillChange:myInputPlugin.activeView]); + OCMVerify(never(), [mockInputDelegate textDidChange:myInputPlugin.activeView]); + } +} + - (void)testTextRangeFromPositionMatchesUITextViewBehavior { FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin]; FlutterTextPosition* fromPosition = [FlutterTextPosition positionWithIndex:2]; From 56edf770672cffb3934e7cb544d647b892785327 Mon Sep 17 00:00:00 2001 From: Huan Lin Date: Wed, 16 Aug 2023 14:16:17 -0700 Subject: [PATCH 2/4] address some comments and nits --- .../framework/Source/FlutterTextInputPlugin.mm | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm index 5560d399cd7b6..90930ccd52e2a 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm @@ -2451,8 +2451,8 @@ - (void)takeKeyboardScreenshotAndDisplay { - (void)setEditableSizeAndTransform:(NSDictionary*)dictionary { NSArray* transform = dictionary[@"transform"]; [_activeView setEditableTransform:transform]; - int leftIndex = 12; - int topIndex = 13; + const int leftIndex = 12; + const int topIndex = 13; if ([_activeView isScribbleAvailable]) { // This is necessary to set up where the scribble interactable element will be. _inputHider.frame = @@ -2462,6 +2462,9 @@ - (void)setEditableSizeAndTransform:(NSDictionary*)dictionary { CGRectMake(0, 0, [dictionary[@"width"] intValue], [dictionary[@"height"] intValue]); _activeView.tintColor = [UIColor clearColor]; } else { + // TODO: Also need to handle iOS 16 case, where the auto-correction highlight does + // not match the size of text. + // See https://github.com/flutter/flutter/issues/131695 if (@available(iOS 17, *)) { // Move auto-correction highlight to overlap with the actual text. // This is to fix an issue where the system auto-correction highlight is displayed at @@ -2498,16 +2501,20 @@ - (void)setSelectionRects:(NSArray*)encodedRects { : NSWritingDirectionRightToLeft]]; } + BOOL shouldNotifyTextChange = NO; if (@available(iOS 17, *)) { // Force UIKit to query the selectionRects again on iOS 17+ // This is to fix a bug on iOS 17+ where UIKit queries the outdated selectionRects after // entering a character, resulting in auto-correction highlight region missing the last // character. + shouldNotifyTextChange = YES; + } + if (shouldNotifyTextChange) { [_activeView.inputDelegate textWillChange:_activeView]; - _activeView.selectionRects = rectsAsRect; + } + _activeView.selectionRects = rectsAsRect; + if (shouldNotifyTextChange) { [_activeView.inputDelegate textDidChange:_activeView]; - } else { - _activeView.selectionRects = rectsAsRect; } } From 6ab4faa3e96049f264db83613751b5b4c19422d6 Mon Sep 17 00:00:00 2001 From: Huan Lin Date: Wed, 16 Aug 2023 15:16:01 -0700 Subject: [PATCH 3/4] fix todo --- .../darwin/ios/framework/Source/FlutterTextInputPlugin.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm index 90930ccd52e2a..0b8e0c8fdb585 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm @@ -2462,7 +2462,7 @@ - (void)setEditableSizeAndTransform:(NSDictionary*)dictionary { CGRectMake(0, 0, [dictionary[@"width"] intValue], [dictionary[@"height"] intValue]); _activeView.tintColor = [UIColor clearColor]; } else { - // TODO: Also need to handle iOS 16 case, where the auto-correction highlight does + // TODO(hellohuanlin): Also need to handle iOS 16 case, where the auto-correction highlight does // not match the size of text. // See https://github.com/flutter/flutter/issues/131695 if (@available(iOS 17, *)) { From 18c82694aaaceb046eb8d7c1b373047d9a4f6455 Mon Sep 17 00:00:00 2001 From: Huan Lin Date: Thu, 17 Aug 2023 09:11:28 -0700 Subject: [PATCH 4/4] updated some comments --- .../darwin/ios/framework/Source/FlutterTextInputPlugin.mm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm index 0b8e0c8fdb585..d8fc121a5d839 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm @@ -2469,7 +2469,9 @@ - (void)setEditableSizeAndTransform:(NSDictionary*)dictionary { // Move auto-correction highlight to overlap with the actual text. // This is to fix an issue where the system auto-correction highlight is displayed at // the top left corner of the screen on iOS 17+. + // This problem also happens on iOS 16, but the size of highlight does not match the text. // See https://github.com/flutter/flutter/issues/131695 + // TODO(hellohuanlin): Investigate if we can use non-zero size. _inputHider.frame = CGRectMake([transform[leftIndex] intValue], [transform[topIndex] intValue], 0, 0); }