From 9e3b31b8c24941087d68ea38592cbd90f46ceb80 Mon Sep 17 00:00:00 2001 From: George Wright Date: Wed, 22 Jul 2020 12:09:20 -0700 Subject: [PATCH 1/8] Add support for mice on iOS 13.4+ --- .../framework/Headers/FlutterViewController.h | 5 + .../framework/Source/FlutterViewController.mm | 106 +++++++++++++++++- 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h b/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h index 8491708fa372f..154e63d8b4068 100644 --- a/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h +++ b/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h @@ -47,8 +47,13 @@ extern NSNotificationName const FlutterSemanticsUpdateNotification; * Dart-related state and asynchronous tasks when navigating back and forth between a * FlutterViewController and other `UIViewController`s. */ +#ifdef __IPHONE_13_4 FLUTTER_EXPORT +@interface FlutterViewController + : UIViewController +#else @interface FlutterViewController : UIViewController +#endif /** * Initializes this FlutterViewController with the specified `FlutterEngine`. diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm index 7f593583a54c8..93d5bb106707e 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm @@ -34,6 +34,28 @@ NSNotificationName const FlutterViewControllerShowHomeIndicator = @"FlutterViewControllerShowHomeIndicator"; +// Struct holding the mouse state. The engine doesn't keep track of which +// mouse buttons have been pressed, so it's the embedding's responsibility. +typedef struct MouseState { + // True if the last event sent to Flutter had at least one mouse button. + // pressed. + bool flutter_state_is_down = false; + + // True if kAdd has been sent to Flutter. Used to determine whether + // to send a kAdd event before sending an incoming mouse event, since + // Flutter expects pointers to be added before events are sent for them. + bool flutter_state_is_added = false; + + // Current coordinate of the mouse cursor + CGPoint location = CGPointZero; + + // Last reported translation for an in-flight pan gesture + CGPoint last_translation = CGPointZero; + + // The currently pressed buttons, as represented in FlutterPointerEvent. + uint64_t buttons = 0; +} MouseState; + // This is left a FlutterBinaryMessenger privately for now to give people a chance to notice the // change. Unfortunately unless you have Werror turned on, incompatible pointers as arguments are // just a warning. @@ -78,6 +100,11 @@ @implementation FlutterViewController { // UIScrollView with height zero and a content offset so we can get those events. See also: // https://github.com/flutter/flutter/issues/35050 fml::scoped_nsobject _scrollView; +#ifdef __IPHONE_13_4 + fml::scoped_nsobject _pointerInteraction API_AVAILABLE(ios(13.4)); + fml::scoped_nsobject _panGestureRecognizer API_AVAILABLE(ios(13.4)); + MouseState _mouseState; +#endif } @synthesize displayingFlutterUI = _displayingFlutterUI; @@ -603,6 +630,19 @@ - (void)viewDidLoad { [_engine.get() attachView]; +#ifdef __IPHONE_13_4 + if (@available(iOS 13.4, *)) { + _pointerInteraction.reset([[UIPointerInteraction alloc] initWithDelegate:self]); + [self.view addInteraction:_pointerInteraction]; + + _panGestureRecognizer.reset( + [[UIPanGestureRecognizer alloc] initWithTarget:self action:@selector(scrollEvent:)]); + _panGestureRecognizer.get().allowedScrollTypesMask = UIScrollTypeMaskAll; + _panGestureRecognizer.get().allowedTouchTypes = @[ @(UITouchTypeIndirectPointer) ]; + [_flutterView.get() addGestureRecognizer:_panGestureRecognizer.get()]; + } +#endif + [super viewDidLoad]; } @@ -759,8 +799,9 @@ - (void)goToApplicationLifecycle:(nonnull NSString*)state { return flutter::PointerData::DeviceKind::kTouch; case UITouchTypeStylus: return flutter::PointerData::DeviceKind::kStylus; + case UITouchTypeIndirectPointer: + return flutter::PointerData::DeviceKind::kMouse; default: - // TODO(53696): Handle the UITouchTypeIndirectPointer enum value. FML_DLOG(INFO) << "Unhandled touch type: " << touch.type; break; } @@ -1423,4 +1464,67 @@ - (BOOL)isPresentingViewController { return self.presentedViewController != nil || self.isPresentingViewControllerAnimating; } +#ifdef __IPHONE_13_4 +- (flutter::PointerData)generatePointerDataForMouse { + const CGFloat scale = [UIScreen mainScreen].scale; + + flutter::PointerData pointer_data; + + pointer_data.Clear(); + + pointer_data.kind = flutter::PointerData::DeviceKind::kMouse; + pointer_data.change = _mouseState.flutter_state_is_added ? flutter::PointerData::Change::kAdd + : flutter::PointerData::Change::kHover; + pointer_data.pointer_identifier = reinterpret_cast(_pointerInteraction.get()); + + pointer_data.physical_x = _mouseState.location.x * scale; + pointer_data.physical_y = _mouseState.location.y * scale; + + return pointer_data; +} + +- (UIPointerRegion*)pointerInteraction:(UIPointerInteraction*)interaction + regionForRequest:(UIPointerRegionRequest*)request + defaultRegion:(UIPointerRegion*)defaultRegion API_AVAILABLE(ios(13.4)) { + if (request != nil) { + auto packet = std::make_unique(1); + _mouseState.location = request.location; + + flutter::PointerData pointer_data = [self generatePointerDataForMouse]; + + pointer_data.signal_kind = flutter::PointerData::SignalKind::kNone; + packet->SetPointerData(0, pointer_data); + + [_engine.get() dispatchPointerDataPacket:std::move(packet)]; + } + return nil; +} + +- (void)scrollEvent:(UIPanGestureRecognizer*)recognizer { + CGPoint translation = [recognizer translationInView:self.view]; + const CGFloat scale = [UIScreen mainScreen].scale; + + auto packet = std::make_unique(1); + + flutter::PointerData pointer_data = [self generatePointerDataForMouse]; + pointer_data.signal_kind = flutter::PointerData::SignalKind::kScroll; + pointer_data.scroll_delta_x = (translation.x - _mouseState.last_translation.x) * scale; + pointer_data.scroll_delta_y = -(translation.y - _mouseState.last_translation.y) * scale; + + // The translation reported by UIPanGestureRecognizer is the total translation + // generated by the pan gesture since the gesture began. We need to be able + // to keep track of the last translation value in order to generate the deltaX + // and deltaY coordinates for each subsequent scroll event. + if (recognizer.state != UIGestureRecognizerStateEnded) { + _mouseState.last_translation = translation; + } else { + _mouseState.last_translation = CGPointZero; + } + + packet->SetPointerData(0, pointer_data); + + [_engine.get() dispatchPointerDataPacket:std::move(packet)]; +} +#endif + @end From 85c33ba0f4dd7f2607c4c4e2b398de19d525a3d1 Mon Sep 17 00:00:00 2001 From: George Wright Date: Wed, 30 Dec 2020 17:40:59 -0800 Subject: [PATCH 2/8] Review updates --- .../framework/Source/FlutterViewController.mm | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm index 93d5bb106707e..73c94adebedb8 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm @@ -46,10 +46,10 @@ // Flutter expects pointers to be added before events are sent for them. bool flutter_state_is_added = false; - // Current coordinate of the mouse cursor + // Current coordinate of the mouse cursor in physical device pixels. CGPoint location = CGPointZero; - // Last reported translation for an in-flight pan gesture + // Last reported translation for an in-flight pan gesture in physical device pixels. CGPoint last_translation = CGPointZero; // The currently pressed buttons, as represented in FlutterPointerEvent. @@ -100,11 +100,9 @@ @implementation FlutterViewController { // UIScrollView with height zero and a content offset so we can get those events. See also: // https://github.com/flutter/flutter/issues/35050 fml::scoped_nsobject _scrollView; -#ifdef __IPHONE_13_4 fml::scoped_nsobject _pointerInteraction API_AVAILABLE(ios(13.4)); fml::scoped_nsobject _panGestureRecognizer API_AVAILABLE(ios(13.4)); MouseState _mouseState; -#endif } @synthesize displayingFlutterUI = _displayingFlutterUI; @@ -630,7 +628,6 @@ - (void)viewDidLoad { [_engine.get() attachView]; -#ifdef __IPHONE_13_4 if (@available(iOS 13.4, *)) { _pointerInteraction.reset([[UIPointerInteraction alloc] initWithDelegate:self]); [self.view addInteraction:_pointerInteraction]; @@ -641,7 +638,6 @@ - (void)viewDidLoad { _panGestureRecognizer.get().allowedTouchTypes = @[ @(UITouchTypeIndirectPointer) ]; [_flutterView.get() addGestureRecognizer:_panGestureRecognizer.get()]; } -#endif [super viewDidLoad]; } @@ -1464,10 +1460,7 @@ - (BOOL)isPresentingViewController { return self.presentedViewController != nil || self.isPresentingViewControllerAnimating; } -#ifdef __IPHONE_13_4 -- (flutter::PointerData)generatePointerDataForMouse { - const CGFloat scale = [UIScreen mainScreen].scale; - +- (flutter::PointerData)generatePointerDataForMouse API_AVAILABLE(ios(13.4)) { flutter::PointerData pointer_data; pointer_data.Clear(); @@ -1477,8 +1470,8 @@ - (BOOL)isPresentingViewController { : flutter::PointerData::Change::kHover; pointer_data.pointer_identifier = reinterpret_cast(_pointerInteraction.get()); - pointer_data.physical_x = _mouseState.location.x * scale; - pointer_data.physical_y = _mouseState.location.y * scale; + pointer_data.physical_x = _mouseState.location.x; + pointer_data.physical_y = _mouseState.location.y; return pointer_data; } @@ -1488,28 +1481,32 @@ - (UIPointerRegion*)pointerInteraction:(UIPointerInteraction*)interaction defaultRegion:(UIPointerRegion*)defaultRegion API_AVAILABLE(ios(13.4)) { if (request != nil) { auto packet = std::make_unique(1); - _mouseState.location = request.location; + const CGFloat scale = [UIScreen mainScreen].scale; + _mouseState.location = {request.location.x * scale, request.location.y * scale}; flutter::PointerData pointer_data = [self generatePointerDataForMouse]; pointer_data.signal_kind = flutter::PointerData::SignalKind::kNone; - packet->SetPointerData(0, pointer_data); + packet->SetPointerData(0 /* index */, pointer_data); [_engine.get() dispatchPointerDataPacket:std::move(packet)]; } return nil; } -- (void)scrollEvent:(UIPanGestureRecognizer*)recognizer { +- (void)scrollEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(ios(13.4)) { CGPoint translation = [recognizer translationInView:self.view]; const CGFloat scale = [UIScreen mainScreen].scale; + translation.x *= scale; + translation.y *= scale; + auto packet = std::make_unique(1); flutter::PointerData pointer_data = [self generatePointerDataForMouse]; pointer_data.signal_kind = flutter::PointerData::SignalKind::kScroll; - pointer_data.scroll_delta_x = (translation.x - _mouseState.last_translation.x) * scale; - pointer_data.scroll_delta_y = -(translation.y - _mouseState.last_translation.y) * scale; + pointer_data.scroll_delta_x = (translation.x - _mouseState.last_translation.x); + pointer_data.scroll_delta_y = -(translation.y - _mouseState.last_translation.y); // The translation reported by UIPanGestureRecognizer is the total translation // generated by the pan gesture since the gesture began. We need to be able @@ -1521,10 +1518,9 @@ - (void)scrollEvent:(UIPanGestureRecognizer*)recognizer { _mouseState.last_translation = CGPointZero; } - packet->SetPointerData(0, pointer_data); + packet->SetPointerData(0 /* index */, pointer_data); [_engine.get() dispatchPointerDataPacket:std::move(packet)]; } -#endif @end From 95a45b83aec0e02a6f8c250e8cd763e1c7cf4d18 Mon Sep 17 00:00:00 2001 From: George Wright Date: Tue, 5 Jan 2021 14:53:44 -0800 Subject: [PATCH 3/8] Review updates --- .../darwin/ios/framework/Headers/FlutterViewController.h | 4 ---- .../darwin/ios/framework/Source/FlutterViewController.mm | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h b/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h index 154e63d8b4068..37e144d8e2590 100644 --- a/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h +++ b/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h @@ -47,13 +47,9 @@ extern NSNotificationName const FlutterSemanticsUpdateNotification; * Dart-related state and asynchronous tasks when navigating back and forth between a * FlutterViewController and other `UIViewController`s. */ -#ifdef __IPHONE_13_4 FLUTTER_EXPORT @interface FlutterViewController : UIViewController -#else -@interface FlutterViewController : UIViewController -#endif /** * Initializes this FlutterViewController with the specified `FlutterEngine`. diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm index 73c94adebedb8..bcc6fab7be818 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm @@ -1487,7 +1487,7 @@ - (UIPointerRegion*)pointerInteraction:(UIPointerInteraction*)interaction flutter::PointerData pointer_data = [self generatePointerDataForMouse]; pointer_data.signal_kind = flutter::PointerData::SignalKind::kNone; - packet->SetPointerData(0 /* index */, pointer_data); + packet->SetPointerData(/*index=*/0, pointer_data); [_engine.get() dispatchPointerDataPacket:std::move(packet)]; } From c285dbf56704c5fd59a99a024b33fa1961c20f84 Mon Sep 17 00:00:00 2001 From: George Wright Date: Thu, 7 Jan 2021 11:29:05 -0800 Subject: [PATCH 4/8] Add unit tests --- .../Source/FlutterViewControllerTest.mm | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm index 48a1c1ee2cbd1..eda86f16605c5 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm @@ -12,10 +12,15 @@ FLUTTER_ASSERT_ARC +namespace flutter { + class PointerDataPacket {}; +} + @interface FlutterEngine () - (BOOL)createShell:(NSString*)entrypoint libraryURI:(NSString*)libraryURI initialRoute:(NSString*)initialRoute; +- (void)dispatchPointerDataPacket:(std::unique_ptr)packet; @end @interface FlutterEngine (TestLowMemory) @@ -64,6 +69,7 @@ @interface FlutterViewController (Tests) - (void)surfaceUpdated:(BOOL)appeared; - (void)performOrientationUpdate:(UIInterfaceOrientationMask)new_preferences; - (void)dispatchPresses:(NSSet*)presses; +- (void)scrollEvent:(UIPanGestureRecognizer*)recognizer; @end @implementation FlutterViewControllerTest @@ -645,6 +651,52 @@ - (void)testIgnoredKeyEvents API_AVAILABLE(ios(13.4)) { [keyEventChannel stopMocking]; } +- (void)testPanGestureRecognizer API_AVAILABLE(ios(13.4)) { + if (@available(iOS 13.4, *)) { + // noop + } else { + return; + } + + FlutterViewController* vc = [[FlutterViewController alloc] initWithEngine:self.mockEngine + nibName:nil + bundle:nil]; + XCTAssertNotNil(vc); + UIView* view = vc.view; + XCTAssertNotNil(view); + NSArray* gestureRecognizers = view.gestureRecognizers; + XCTAssertNotNil(gestureRecognizers); + + BOOL found = NO; + for (id gesture in gestureRecognizers) { + if ([gesture isKindOfClass:[UIPanGestureRecognizer class]]) { + found = YES; + break; + } + } + XCTAssertTrue(found); +} + +- (void)testMouseSupport API_AVAILABLE(ios(13.4)) { + if (@available(iOS 13.4, *)) { + // noop + } else { + return; + } + + FlutterViewController* vc = [[FlutterViewController alloc] initWithEngine:self.mockEngine + nibName:nil + bundle:nil]; + XCTAssertNotNil(vc); + + id mockPanGestureRecognizer = OCMClassMock([UIPanGestureRecognizer class]); + XCTAssertNotNil(mockPanGestureRecognizer); + + [vc scrollEvent:mockPanGestureRecognizer]; + + [[[self.mockEngine verify] ignoringNonObjectArgs] dispatchPointerDataPacket:std::make_unique()]; +} + - (NSSet*)fakeUiPressSetForPhase:(UIPressPhase)phase keyCode:(UIKeyboardHIDUsage)keyCode modifierFlags:(UIKeyModifierFlags)modifierFlags From 96fee3b8f3a5c06809990fd82f229f575b76415d Mon Sep 17 00:00:00 2001 From: George Wright Date: Thu, 7 Jan 2021 11:29:20 -0800 Subject: [PATCH 5/8] Review update --- .../darwin/ios/framework/Source/FlutterViewController.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm index bcc6fab7be818..a0cffaace53b3 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm @@ -1518,7 +1518,7 @@ - (void)scrollEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(ios(13.4)) _mouseState.last_translation = CGPointZero; } - packet->SetPointerData(0 /* index */, pointer_data); + packet->SetPointerData(/*index=*/0, pointer_data); [_engine.get() dispatchPointerDataPacket:std::move(packet)]; } From 7b2ef94a7343510188f43dc22e59a9b868d0eff9 Mon Sep 17 00:00:00 2001 From: George Wright Date: Thu, 7 Jan 2021 11:34:12 -0800 Subject: [PATCH 6/8] Formatting --- .../darwin/ios/framework/Source/FlutterViewControllerTest.mm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm index eda86f16605c5..a70387182e185 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewControllerTest.mm @@ -13,7 +13,7 @@ FLUTTER_ASSERT_ARC namespace flutter { - class PointerDataPacket {}; +class PointerDataPacket {}; } @interface FlutterEngine () @@ -694,7 +694,8 @@ - (void)testMouseSupport API_AVAILABLE(ios(13.4)) { [vc scrollEvent:mockPanGestureRecognizer]; - [[[self.mockEngine verify] ignoringNonObjectArgs] dispatchPointerDataPacket:std::make_unique()]; + [[[self.mockEngine verify] ignoringNonObjectArgs] + dispatchPointerDataPacket:std::make_unique()]; } - (NSSet*)fakeUiPressSetForPhase:(UIPressPhase)phase From b692d7fc7e88de4ae77fc32dfdb2c99e02399c8d Mon Sep 17 00:00:00 2001 From: George Wright Date: Tue, 12 Jan 2021 14:24:03 -0800 Subject: [PATCH 7/8] Guard header --- .../darwin/ios/framework/Headers/FlutterViewController.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h b/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h index 37e144d8e2590..154e63d8b4068 100644 --- a/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h +++ b/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h @@ -47,9 +47,13 @@ extern NSNotificationName const FlutterSemanticsUpdateNotification; * Dart-related state and asynchronous tasks when navigating back and forth between a * FlutterViewController and other `UIViewController`s. */ +#ifdef __IPHONE_13_4 FLUTTER_EXPORT @interface FlutterViewController : UIViewController +#else +@interface FlutterViewController : UIViewController +#endif /** * Initializes this FlutterViewController with the specified `FlutterEngine`. From fdff6a421e27aba493e74414a7783c3eeced9eee Mon Sep 17 00:00:00 2001 From: George Wright Date: Wed, 13 Jan 2021 13:31:45 -0800 Subject: [PATCH 8/8] Fix EXPORT --- .../darwin/ios/framework/Headers/FlutterViewController.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h b/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h index 154e63d8b4068..7cffc195d7f15 100644 --- a/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h +++ b/shell/platform/darwin/ios/framework/Headers/FlutterViewController.h @@ -47,8 +47,8 @@ extern NSNotificationName const FlutterSemanticsUpdateNotification; * Dart-related state and asynchronous tasks when navigating back and forth between a * FlutterViewController and other `UIViewController`s. */ -#ifdef __IPHONE_13_4 FLUTTER_EXPORT +#ifdef __IPHONE_13_4 @interface FlutterViewController : UIViewController #else