Remove iPadOS mouse pointer if no longer connected

shell/platform/darwin/ios/framework/Headers/FlutterViewController.h

@@ -50,7 +50,7 @@ extern NSNotificationName const FlutterSemanticsUpdateNotification;
 FLUTTER_DARWIN_EXPORT
 #ifdef __IPHONE_13_4
 @interface FlutterViewController
-    : UIViewController <FlutterTextureRegistry, FlutterPluginRegistry, UIPointerInteractionDelegate>
+    : UIViewController <FlutterTextureRegistry, FlutterPluginRegistry, UIGestureRecognizerDelegate>
 #else
 @interface FlutterViewController : UIViewController <FlutterTextureRegistry, FlutterPluginRegistry>
 #endif

shell/platform/darwin/ios/framework/Source/FlutterViewController.mm

@@ -39,26 +39,13 @@
 NSNotificationName const FlutterViewControllerShowHomeIndicator =
     @"FlutterViewControllerShowHomeIndicator";
 
-// Struct holding the mouse state. The engine doesn't keep track of which
-// mouse buttons have been pressed, so it's the embedding's responsibility.
+// Struct holding the mouse state.
 typedef struct MouseState {
-  // True if the last event sent to Flutter had at least one mouse button.
-  // pressed.
-  bool flutter_state_is_down = false;
-
-  // True if kAdd has been sent to Flutter. Used to determine whether
-  // to send a kAdd event before sending an incoming mouse event, since
-  // Flutter expects pointers to be added before events are sent for them.
-  bool flutter_state_is_added = false;
-
   // Current coordinate of the mouse cursor in physical device pixels.
   CGPoint location = CGPointZero;
 
   // Last reported translation for an in-flight pan gesture in physical device pixels.
   CGPoint last_translation = CGPointZero;
-
-  // The currently pressed buttons, as represented in FlutterPointerEvent.
-  uint64_t buttons = 0;
 } MouseState;
 
 // This is left a FlutterBinaryMessenger privately for now to give people a chance to notice the
@@ -117,7 +104,7 @@ @implementation FlutterViewController {
   // UIScrollView with height zero and a content offset so we can get those events. See also:
   // https://github.com/flutter/flutter/issues/35050
   fml::scoped_nsobject<UIScrollView> _scrollView;
-  fml::scoped_nsobject<UIPointerInteraction> _pointerInteraction API_AVAILABLE(ios(13.4));
+  fml::scoped_nsobject<UIHoverGestureRecognizer> _hoverGestureRecognizer API_AVAILABLE(ios(13.4));
   fml::scoped_nsobject<UIPanGestureRecognizer> _panGestureRecognizer API_AVAILABLE(ios(13.4));
   fml::scoped_nsobject<UIView> _keyboardAnimationView;
   MouseState _mouseState;
@@ -658,11 +645,14 @@ - (void)viewDidLoad {
   }
 
   if (@available(iOS 13.4, *)) {
-    _pointerInteraction.reset([[UIPointerInteraction alloc] initWithDelegate:self]);
-    [self.view addInteraction:_pointerInteraction];
+    _hoverGestureRecognizer.reset(
+        [[UIHoverGestureRecognizer alloc] initWithTarget:self action:@selector(hoverEvent:)]);
+    _hoverGestureRecognizer.get().delegate = self;
+    [_flutterView.get() addGestureRecognizer:_hoverGestureRecognizer.get()];
 
     _panGestureRecognizer.reset(
         [[UIPanGestureRecognizer alloc] initWithTarget:self action:@selector(scrollEvent:)]);
+    _panGestureRecognizer.get().delegate = self;
     _panGestureRecognizer.get().allowedScrollTypesMask = UIScrollTypeMaskAll;
     _panGestureRecognizer.get().allowedTouchTypes = @[ @(UITouchTypeIndirectPointer) ];
     [_flutterView.get() addGestureRecognizer:_panGestureRecognizer.get()];
@@ -796,6 +786,9 @@ - (void)dealloc {
   [self deregisterNotifications];
 
   [_displayLink release];
+  _scrollView.get().delegate = nil;
+  _hoverGestureRecognizer.get().delegate = nil;
+  _panGestureRecognizer.get().delegate = nil;
   [super dealloc];
 }
 
@@ -1680,32 +1673,50 @@ - (BOOL)isPresentingViewController {
   pointer_data.Clear();
 
   pointer_data.kind = flutter::PointerData::DeviceKind::kMouse;
-  pointer_data.change = _mouseState.flutter_state_is_added ? flutter::PointerData::Change::kAdd
-                                                           : flutter::PointerData::Change::kHover;
-  pointer_data.pointer_identifier = reinterpret_cast<int64_t>(_pointerInteraction.get());
+  switch (_hoverGestureRecognizer.get().state) {
+    case UIGestureRecognizerStateBegan:
+      pointer_data.change = flutter::PointerData::Change::kAdd;
+      break;
+    case UIGestureRecognizerStateChanged:
+      pointer_data.change = flutter::PointerData::Change::kHover;
+      break;
+    case UIGestureRecognizerStateEnded:
+    case UIGestureRecognizerStateCancelled:
+      pointer_data.change = flutter::PointerData::Change::kRemove;
+      break;
+    default:
+      // Sending kHover is the least harmful thing to do here
+      // But this state is not expected to ever be reached.
+      pointer_data.change = flutter::PointerData::Change::kHover;
+      break;
+  }
+  pointer_data.time_stamp = [[NSProcessInfo processInfo] systemUptime] * kMicrosecondsPerSecond;
+  pointer_data.device = reinterpret_cast<int64_t>(_hoverGestureRecognizer.get());
 
   pointer_data.physical_x = _mouseState.location.x;
   pointer_data.physical_y = _mouseState.location.y;
 
   return pointer_data;
 }
 
-- (UIPointerRegion*)pointerInteraction:(UIPointerInteraction*)interaction
-                      regionForRequest:(UIPointerRegionRequest*)request
-                         defaultRegion:(UIPointerRegion*)defaultRegion API_AVAILABLE(ios(13.4)) {
-  if (request != nil) {
-    auto packet = std::make_unique<flutter::PointerDataPacket>(1);
-    const CGFloat scale = [UIScreen mainScreen].scale;
-    _mouseState.location = {request.location.x * scale, request.location.y * scale};
+- (BOOL)gestureRecognizer:(UIGestureRecognizer*)gestureRecognizer
+    shouldRecognizeSimultaneouslyWithGestureRecognizer:(UIGestureRecognizer*)otherGestureRecognizer
+    API_AVAILABLE(ios(13.4)) {
+  return YES;
+}
+
+- (void)hoverEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(ios(13.4)) {
+  auto packet = std::make_unique<flutter::PointerDataPacket>(1);
+  CGPoint location = [recognizer locationInView:self.view];
+  CGFloat scale = [UIScreen mainScreen].scale;
+  _mouseState.location = {location.x * scale, location.y * scale};
 
-    flutter::PointerData pointer_data = [self generatePointerDataForMouse];
+  flutter::PointerData pointer_data = [self generatePointerDataForMouse];
 
-    pointer_data.signal_kind = flutter::PointerData::SignalKind::kNone;
-    packet->SetPointerData(/*index=*/0, pointer_data);
+  pointer_data.signal_kind = flutter::PointerData::SignalKind::kNone;
+  packet->SetPointerData(/*index=*/0, pointer_data);
 
-    [_engine.get() dispatchPointerDataPacket:std::move(packet)];
-  }
-  return nil;
+  [_engine.get() dispatchPointerDataPacket:std::move(packet)];
 }
 
 - (void)scrollEvent:(UIPanGestureRecognizer*)recognizer API_AVAILABLE(ios(13.4)) {

testing/scenario_app/ios/Scenarios/ScenariosUITests/StatusBarTest.m

@@ -30,13 +30,15 @@ - (void)testTapStatusBar {
     [[self.application.statusBars firstMatch] tap];
   }
 
-  XCUIElement* addTextField = self.application.textFields[@"PointerChange.add:0"];
+  XCUIElement* addTextField =
+      self.application.textFields[@"0,PointerChange.add,device=0,buttons=0"];
   BOOL exists = [addTextField waitForExistenceWithTimeout:1];
   XCTAssertTrue(exists, @"");
-  XCUIElement* downTextField = self.application.textFields[@"PointerChange.down:0"];
+  XCUIElement* downTextField =
+      self.application.textFields[@"1,PointerChange.down,device=0,buttons=0"];
   exists = [downTextField waitForExistenceWithTimeout:1];
   XCTAssertTrue(exists, @"");
-  XCUIElement* upTextField = self.application.textFields[@"PointerChange.up:0"];
+  XCUIElement* upTextField = self.application.textFields[@"2,PointerChange.up,device=0,buttons=0"];
   exists = [upTextField waitForExistenceWithTimeout:1];
   XCTAssertTrue(exists, @"");
 }

testing/scenario_app/ios/Scenarios/ScenariosUITests/iPadGestureTests.m

@@ -28,6 +28,13 @@ static BOOL performBoolSelector(id target, SEL selector) {
   return returnValue;
 }
 
+static int assertOneMessageAndGetSequenceNumber(NSMutableDictionary* messages, NSString* message) {
+  NSMutableArray<NSNumber*>* matchingMessages = messages[message];
+  XCTAssertNotNil(matchingMessages, @"Did not receive \"%@\" message", message);
+  XCTAssertEqual(matchingMessages.count, 1, @"More than one \"%@\" message", message);
+  return matchingMessages.firstObject.intValue;
+}
+
 // TODO(85810): Remove reflection in this test when Xcode version is upgraded to 13.
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wundeclared-selector"
@@ -45,11 +52,7 @@ - (void)testPointerButtons {
   [app launch];
 
   NSPredicate* predicateToFindFlutterView =
-      [NSPredicate predicateWithBlock:^BOOL(id _Nullable evaluatedObject,
-                                            NSDictionary<NSString*, id>* _Nullable bindings) {
-        XCUIElement* element = evaluatedObject;
-        return [element.identifier hasPrefix:@"flutter_view"];
-      }];
+      [NSPredicate predicateWithFormat:@"identifier BEGINSWITH 'flutter_view'"];
   XCUIElement* flutterView = [[app descendantsMatchingType:XCUIElementTypeAny]
       elementMatchingPredicate:predicateToFindFlutterView];
   if (![flutterView waitForExistenceWithTimeout:kSecondsToWaitForFlutterView]) {
@@ -62,23 +65,136 @@ - (void)testPointerButtons {
 
   [flutterView tap];
   // Initial add event should have buttons = 0
-  XCTAssertTrue([app.textFields[@"PointerChange.add:0"] waitForExistenceWithTimeout:1],
-                @"PointerChange.add event did not occur");
+  XCTAssertTrue(
+      [app.textFields[@"0,PointerChange.add,device=0,buttons=0"] waitForExistenceWithTimeout:1],
+      @"PointerChange.add event did not occur for a normal tap");
   // Normal tap should have buttons = 0, the flutter framework will ensure it has buttons = 1
-  XCTAssertTrue([app.textFields[@"PointerChange.down:0"] waitForExistenceWithTimeout:1],
-                @"PointerChange.down event did not occur for a normal tap");
-  XCTAssertTrue([app.textFields[@"PointerChange.up:0"] waitForExistenceWithTimeout:1],
-                @"PointerChange.up event did not occur for a normal tap");
+  XCTAssertTrue(
+      [app.textFields[@"1,PointerChange.down,device=0,buttons=0"] waitForExistenceWithTimeout:1],
+      @"PointerChange.down event did not occur for a normal tap");
+  XCTAssertTrue(
+      [app.textFields[@"2,PointerChange.up,device=0,buttons=0"] waitForExistenceWithTimeout:1],
+      @"PointerChange.up event did not occur for a normal tap");
   SEL rightClick = @selector(rightClick);
   XCTAssertTrue([flutterView respondsToSelector:rightClick],
                 @"If supportsPointerInteraction is true, this should be true too.");
   [flutterView performSelector:rightClick];
-  // Since each touch is its own device, we can't distinguish the other add event(s)
+  // On simulated right click, a hover also occurs, so the hover pointer is added
+  XCTAssertTrue(
+      [app.textFields[@"3,PointerChange.add,device=1,buttons=0"] waitForExistenceWithTimeout:1],
+      @"PointerChange.add event did not occur for a right-click's hover pointer");
+
+  // The hover pointer is removed after ~3.5 seconds, this ensures that all events are received
+  XCTestExpectation* sleepExpectation = [self expectationWithDescription:@"never fires"];
+  sleepExpectation.inverted = true;
+  [self waitForExpectations:@[ sleepExpectation ] timeout:5.0];
+
+  // The hover events are interspersed with the right-click events in a varying order
+  // Ensure the individual orderings are respected without hardcoding the absolute sequence
+  NSMutableDictionary<NSString*, NSMutableArray<NSNumber*>*>* messages =
+      [[NSMutableDictionary alloc] init];
+  for (XCUIElement* element in [app.textFields allElementsBoundByIndex]) {
+    NSString* rawMessage = element.value;
+    // Parse out the sequence number
+    NSUInteger commaIndex = [rawMessage rangeOfString:@","].location;
+    NSInteger messageSequenceNumber =
+        [rawMessage substringWithRange:NSMakeRange(0, commaIndex)].integerValue;
+    // Parse out the rest of the message
+    NSString* message = [rawMessage
+        substringWithRange:NSMakeRange(commaIndex + 1, rawMessage.length - (commaIndex + 1))];
+    NSMutableArray<NSNumber*>* messageSequenceNumberList = messages[message];
+    if (messageSequenceNumberList == nil) {
+      messageSequenceNumberList = [[NSMutableArray alloc] init];
+      messages[message] = messageSequenceNumberList;
+    }
+    [messageSequenceNumberList addObject:@(messageSequenceNumber)];
+  }
+  // The number of hover events is not consistent, there could be one or many
+  NSMutableArray<NSNumber*>* hoverSequenceNumbers =
+      messages[@"PointerChange.hover,device=1,buttons=0"];
+  int hoverRemovedSequenceNumber =
+      assertOneMessageAndGetSequenceNumber(messages, @"PointerChange.remove,device=1,buttons=0");
   // Right click should have buttons = 2
-  XCTAssertTrue([app.textFields[@"PointerChange.down:2"] waitForExistenceWithTimeout:1],
-                @"PointerChange.down event did not occur for a right-click");
-  XCTAssertTrue([app.textFields[@"PointerChange.up:2"] waitForExistenceWithTimeout:1],
-                @"PointerChange.up event did not occur for a right-click");
+  int rightClickAddedSequenceNumber;
+  int rightClickDownSequenceNumber;
+  int rightClickUpSequenceNumber;
+  if (messages[@"PointerChange.add,device=2,buttons=0"] == nil) {
+    // Sometimes the tap pointer has the same device as the right-click (the UITouch is reused)
+    rightClickAddedSequenceNumber = 0;
+    rightClickDownSequenceNumber =
+        assertOneMessageAndGetSequenceNumber(messages, @"PointerChange.down,device=0,buttons=2");
+    rightClickUpSequenceNumber =
+        assertOneMessageAndGetSequenceNumber(messages, @"PointerChange.up,device=0,buttons=2");
+  } else {
+    rightClickAddedSequenceNumber =
+        assertOneMessageAndGetSequenceNumber(messages, @"PointerChange.add,device=2,buttons=0");
+    rightClickDownSequenceNumber =
+        assertOneMessageAndGetSequenceNumber(messages, @"PointerChange.down,device=2,buttons=2");
+    rightClickUpSequenceNumber =
+        assertOneMessageAndGetSequenceNumber(messages, @"PointerChange.up,device=2,buttons=2");
+  }
+  XCTAssertGreaterThan(rightClickDownSequenceNumber, rightClickAddedSequenceNumber,
+                       @"Right-click pointer was pressed before it was added");
+  XCTAssertGreaterThan(rightClickUpSequenceNumber, rightClickDownSequenceNumber,
+                       @"Right-click pointer was released before it was pressed");
+  XCTAssertGreaterThan([[hoverSequenceNumbers firstObject] intValue], 3,
+                       @"Hover occured before hover pointer was added");
+  XCTAssertGreaterThan(hoverRemovedSequenceNumber, [[hoverSequenceNumbers lastObject] intValue],
+                       @"Hover occured after hover pointer was removed");
+}
+
+- (void)testPointerHover {
+  BOOL supportsPointerInteraction = NO;
+  SEL supportsPointerInteractionSelector = @selector(supportsPointerInteraction);
+  if ([XCUIDevice.sharedDevice respondsToSelector:supportsPointerInteractionSelector]) {
+    supportsPointerInteraction =
+        performBoolSelector(XCUIDevice.sharedDevice, supportsPointerInteractionSelector);
+  }
+  XCTSkipUnless(supportsPointerInteraction, "Device does not support pointer interaction.");
+  XCUIApplication* app = [[XCUIApplication alloc] init];
+  app.launchArguments = @[ @"--pointer-events" ];
+  [app launch];
+
+  NSPredicate* predicateToFindFlutterView =
+      [NSPredicate predicateWithFormat:@"identifier BEGINSWITH 'flutter_view'"];
+  XCUIElement* flutterView = [[app descendantsMatchingType:XCUIElementTypeAny]
+      elementMatchingPredicate:predicateToFindFlutterView];
+  if (![flutterView waitForExistenceWithTimeout:kSecondsToWaitForFlutterView]) {
+    NSLog(@"%@", app.debugDescription);
+    XCTFail(@"Failed due to not able to find any flutterView with %@ seconds",
+            @(kSecondsToWaitForFlutterView));
+  }
+
+  XCTAssertNotNil(flutterView);
+
+  SEL hover = @selector(hover);
+  XCTAssertTrue([flutterView respondsToSelector:hover],
+                @"If supportsPointerInteraction is true, this should be true too.");
+  [flutterView performSelector:hover];
+  XCTAssertTrue(
+      [app.textFields[@"0,PointerChange.add,device=0,buttons=0"] waitForExistenceWithTimeout:1],
+      @"PointerChange.add event did not occur for a hover");
+  XCTAssertTrue(
+      [app.textFields[@"1,PointerChange.hover,device=0,buttons=0"] waitForExistenceWithTimeout:1],
+      @"PointerChange.hover event did not occur for a hover");
+  // The number of hover events fired is not always the same
+  NSInteger lastHoverSequenceNumber = -1;
+  NSPredicate* predicateToFindHoverEvents =
+      [NSPredicate predicateWithFormat:@"value ENDSWITH ',PointerChange.hover,device=0,buttons=0'"];
+  for (XCUIElement* textField in
+       [[app.textFields matchingPredicate:predicateToFindHoverEvents] allElementsBoundByIndex]) {
+    NSInteger messageSequenceNumber =
+        [[textField.value componentsSeparatedByString:@","] firstObject].integerValue;
+    if (messageSequenceNumber > lastHoverSequenceNumber) {
+      lastHoverSequenceNumber = messageSequenceNumber;
+    }
+  }
+  XCTAssertNotEqual(lastHoverSequenceNumber, -1,
+                    @"PointerChange.hover event did not occur for a hover");
+  NSString* removeMessage = [NSString
+      stringWithFormat:@"%d,PointerChange.remove,device=0,buttons=0", lastHoverSequenceNumber + 1];
+  XCTAssertTrue([app.textFields[removeMessage] waitForExistenceWithTimeout:1],
+                @"PointerChange.remove event did not occur for a hover");
 }
 #pragma clang diagnostic pop
 

testing/scenario_app/lib/src/touches_scenario.dart

@@ -9,6 +9,9 @@ import 'scenario.dart';
 
 /// A scenario that sends back messages when touches are received.
 class TouchesScenario extends Scenario {
+  final Map<int, int> _knownDevices = <int, int>{};
+  int _sequenceNo = 0;
+
   /// Constructor for `TouchesScenario`.
   TouchesScenario(PlatformDispatcher dispatcher) : super(dispatcher);
 
@@ -23,13 +26,22 @@ class TouchesScenario extends Scenario {
   @override
   void onPointerDataPacket(PointerDataPacket packet) {
     for (final PointerData datum in packet.data) {
+      final int deviceId =
+          _knownDevices.putIfAbsent(datum.device, () => _knownDevices.length);
       sendJsonMessage(
         dispatcher: dispatcher,
         channel: 'display_data',
         json: <String, dynamic>{
-          'data': datum.change.toString() + ':' + datum.buttons.toString(),
+          'data': _sequenceNo.toString() +
+              ',' +
+              datum.change.toString() +
+              ',device=' +
+              deviceId.toString() +
+              ',buttons=' +
+              datum.buttons.toString(),
         },
       );
+      _sequenceNo++;
     }
   }
 }