From dbb14f067913bc11cfb05d00ac327675b76500f7 Mon Sep 17 00:00:00 2001 From: lapfelix Date: Wed, 11 Feb 2026 23:43:12 -0500 Subject: [PATCH] Fix describe remote web content discovery and add regression tests --- CompanionLib/FBIDBCommandExecutor.m | 2 + .../FBSimulatorAccessibilityCommands.m | 51 +++++-- .../FBSimulatorAccessibilityCommandsTests.m | 130 +++++++++++++++++- .../Utilities/AccessibilityDoubles.h | 3 + .../Utilities/AccessibilityDoubles.m | 25 +++- 5 files changed, 198 insertions(+), 13 deletions(-) diff --git a/CompanionLib/FBIDBCommandExecutor.m b/CompanionLib/FBIDBCommandExecutor.m index 4a1addefd..ac79e7dfc 100644 --- a/CompanionLib/FBIDBCommandExecutor.m +++ b/CompanionLib/FBIDBCommandExecutor.m @@ -155,6 +155,8 @@ - (instancetype)initWithTarget:(id)target storageManager:(FBIDBStor FBAccessibilityRequestOptions *options = [FBAccessibilityRequestOptions defaultOptions]; options.nestedFormat = nestedFormat; options.enableLogging = YES; + options.collectFrameCoverage = YES; + options.remoteContentOptions = [FBAccessibilityRemoteContentOptions defaultOptions]; if (value) { return [commands accessibilityElementAtPoint:value.pointValue options:options]; diff --git a/FBSimulatorControl/Commands/FBSimulatorAccessibilityCommands.m b/FBSimulatorControl/Commands/FBSimulatorAccessibilityCommands.m index e6fc110cc..987f8fab3 100644 --- a/FBSimulatorControl/Commands/FBSimulatorAccessibilityCommands.m +++ b/FBSimulatorControl/Commands/FBSimulatorAccessibilityCommands.m @@ -313,8 +313,10 @@ - (void)markFilledWithFrame:(CGRect)frame // Fill cells row by row using memset for efficiency NSUInteger fillWidth = (NSUInteger)(maxX - minX + 1); + NSUInteger startX = (NSUInteger)minX; for (NSInteger y = minY; y <= maxY; y++) { - memset(&_grid[y * _width + minX], 1, fillWidth); + NSUInteger row = (NSUInteger)y; + memset(&_grid[row * _width + startX], 1, fillWidth); } } @@ -339,7 +341,9 @@ - (BOOL)isFilledAtPoint:(CGPoint)point } // O(1) lookup in the grid array - return _grid[cellY * _width + cellX] != 0; + NSUInteger x = (NSUInteger)cellX; + NSUInteger y = (NSUInteger)cellY; + return _grid[y * _width + x] != 0; } - (CGFloat)coverageRatio @@ -370,6 +374,14 @@ @implementation FBSimulatorAccessibilitySerializer static NSString *const AXPrefix = @"AX"; +static BOOL FBIsWebContentRole(NSString *rawRole) +{ + if (rawRole.length == 0) { + return NO; + } + return [rawRole hasPrefix:@"AXWeb"] || [rawRole hasPrefix:@"Web"]; +} + + (NSArray *)customActionsFromElement:(AXPMacPlatformElement *)element { NSMutableArray *customActionsTemp = [[NSMutableArray alloc] init]; @@ -487,7 +499,8 @@ @implementation FBSimulatorAccessibilitySerializer } // Skip Application elements when calculating coverage BOOL isApplication = [rawRole isEqualToString:@"AXApplication"] || [rawRole isEqualToString:@"Application"]; - if (!isApplication) { + BOOL isWebContentContainer = FBIsWebContentRole(rawRole); + if (!isApplication && !isWebContentContainer) { [coverageGrid markFilledWithFrame:frame]; } } @@ -656,6 +669,14 @@ - (AXPTranslationObject *)performWithTranslator:(AXPTranslator *)translator CGRect region = CGRectIsNull(remoteOptions.region) ? screenBounds : remoteOptions.region; NSUInteger maxPoints = remoteOptions.maxPoints; NSUInteger pointCount = 0; + // If native traversal claims near-total coverage, we still probe covered points. + // This handles full-screen proxy/container elements (including some WebView wrappers) + // that can mask remote-process accessibility content. + BOOL allowCoveredPointProbing = NO; + if (coverageGrid) { + CGFloat coverageRatio = [coverageGrid coverageRatio]; + allowCoveredPointProbing = coverageRatio >= 0.98; + } for (CGFloat y = stepSize; y < region.size.height - stepSize; y += stepSize) { for (CGFloat x = stepSize; x < region.size.width - stepSize; x += stepSize) { @@ -667,7 +688,7 @@ - (AXPTranslationObject *)performWithTranslator:(AXPTranslator *)translator // Skip points already covered by native accessibility elements. // This dynamically excludes toolbars, nav bars, and other covered regions. - if (coverageGrid && [coverageGrid isFilledAtPoint:point]) { + if (coverageGrid && [coverageGrid isFilledAtPoint:point] && !allowCoveredPointProbing) { continue; } @@ -681,12 +702,11 @@ - (AXPTranslationObject *)performWithTranslator:(AXPTranslator *)translator hitTranslation.bridgeDelegateToken = self.token; pid_t hitPid = hitTranslation.pid; - // Skip if PID was already seen in main traversal - if ([seenPids containsObject:@(hitPid)]) { - continue; - } - - if (hitPid <= 0 || hitPid == frontmostPid) { + BOOL seenInMainTraversal = [seenPids containsObject:@(hitPid)]; + // In near-full-coverage mode, allow seen/frontmost/unknown PID hits to recover + // hidden content that isn't surfaced in the recursive tree. + BOOL potentiallyHiddenContent = allowCoveredPointProbing && (seenInMainTraversal || hitPid <= 0 || hitPid == frontmostPid); + if (seenInMainTraversal && !allowCoveredPointProbing) { continue; } @@ -696,6 +716,17 @@ - (AXPTranslationObject *)performWithTranslator:(AXPTranslator *)translator } CGRect hitFrame = hitElement.accessibilityFrame; + if (potentiallyHiddenContent) { + // Avoid duplicating top-level full-screen proxy elements. + CGFloat screenArea = CGRectGetWidth(screenBounds) * CGRectGetHeight(screenBounds); + CGFloat hitArea = CGRectGetWidth(hitFrame) * CGRectGetHeight(hitFrame); + if (screenArea <= 0 || hitArea >= (screenArea * 0.98)) { + continue; + } + } else if (hitPid <= 0 || hitPid == frontmostPid) { + continue; + } + NSValue *hitFrameValue = [NSValue valueWithRect:hitFrame]; if ([discoveredFrames containsObject:hitFrameValue]) { diff --git a/FBSimulatorControlTests/Tests/Unit/FBSimulatorAccessibilityCommandsTests.m b/FBSimulatorControlTests/Tests/Unit/FBSimulatorAccessibilityCommandsTests.m index 0ad1f57ce..f345ff4fa 100644 --- a/FBSimulatorControlTests/Tests/Unit/FBSimulatorAccessibilityCommandsTests.m +++ b/FBSimulatorControlTests/Tests/Unit/FBSimulatorAccessibilityCommandsTests.m @@ -124,8 +124,8 @@ - (void)assertProfilingData:(FBAccessibilityProfilingData *)profilingData expectedAttributeFetches:(NSUInteger)expectedAttributeFetchCount { XCTAssertNotNil(profilingData, @"Profiling data should be present"); - XCTAssertEqual(profilingData.elementCount, expectedElementCount, @"Element count mismatch"); - XCTAssertEqual(profilingData.attributeFetchCount, expectedAttributeFetchCount, @"Attribute fetch count mismatch"); + XCTAssertEqual(profilingData.elementCount, (int64_t)expectedElementCount, @"Element count mismatch"); + XCTAssertEqual(profilingData.attributeFetchCount, (int64_t)expectedAttributeFetchCount, @"Attribute fetch count mismatch"); XCTAssertGreaterThanOrEqual(profilingData.xpcCallCount, 0, @"XPC call count should be non-negative"); XCTAssertGreaterThanOrEqual(profilingData.translationDuration, 0, @"Translation duration should be non-negative"); XCTAssertGreaterThanOrEqual(profilingData.elementConversionDuration, 0, @"Element conversion duration should be non-negative"); @@ -545,6 +545,23 @@ - (FBSimulatorControlTests_AXPMacPlatformElement_Double *)defaultElementTree return [self defaultRootWithChildren:@[[self defaultTitleLabel], [self defaultOkButton], [self defaultCancelButton]]]; } +- (BOOL)elements:(NSArray *> *)elements containsLabel:(NSString *)label +{ + for (NSDictionary *element in elements) { + NSString *elementLabel = element[@"AXLabel"]; + if ([elementLabel isKindOfClass:NSString.class] && [elementLabel isEqualToString:label]) { + return YES; + } + } + return NO; +} + +- (NSUInteger)objectAtPointCallCount +{ + NSPredicate *predicate = [NSPredicate predicateWithFormat:@"SELF BEGINSWITH %@", @"objectAtPoint:"]; + return [[self.fixture.translator.methodCalls filteredArrayUsingPredicate:predicate] count]; +} + - (void)testAccessibilityCommandsProducesCorrectFlatOutput { NSArray *children = @[[self defaultTitleLabel], [self defaultOkButton], [self defaultCancelButton]]; @@ -920,4 +937,113 @@ - (void)testAdditionalFrameCoverageIsNilWithoutRemoteContentOptions XCTAssertNil(response.additionalFrameCoverage, @"additionalFrameCoverage should be nil without remoteContentOptions"); } +- (void)testCoverageCalculationSkipsWebContentRoleElements +{ + FBSimulatorControlTests_AXPMacPlatformElement_Double *webArea = + [[FBSimulatorControlTests_AXPMacPlatformElement_Double alloc] + initWithLabel:@"Embedded Web Content" + identifier:nil + role:@"AXWebArea" + frame:NSMakeRect(0, 0, 390, 844) + enabled:YES + actionNames:nil + children:nil]; + + FBSimulatorControlTests_AXPMacPlatformElement_Double *root = + [FBAccessibilityTestElementBuilder applicationWithLabel:@"App Window" + frame:NSMakeRect(0, 0, 390, 844) + children:@[webArea]]; + [self setUpWithRootElement:root]; + + FBSimulatorAccessibilityCommands *commands = [self commands]; + FBAccessibilityRequestOptions *options = [FBAccessibilityRequestOptions defaultOptions]; + options.collectFrameCoverage = YES; + + NSError *error = nil; + FBAccessibilityElementsResponse *response = [[commands accessibilityElementsWithOptions:options] awaitWithTimeout:5.0 error:&error]; + + XCTAssertNil(error, @"Should not have error: %@", error); + XCTAssertNotNil(response); + XCTAssertNotNil(response.frameCoverage); + XCTAssertEqualWithAccuracy([response.frameCoverage doubleValue], 0.0, 0.001, @"AXWeb* container elements should not count as coverage"); +} + +- (void)testRemoteDiscoveryFindsHiddenContentWhenCoverageIsNearFull +{ + FBSimulatorControlTests_AXPMacPlatformElement_Double *fullScreenProxy = + [FBAccessibilityTestElementBuilder staticTextWithLabel:@"Full-Screen Proxy" + frame:NSMakeRect(0, 0, 390, 844)]; + FBSimulatorControlTests_AXPMacPlatformElement_Double *root = + [FBAccessibilityTestElementBuilder applicationWithLabel:@"App Window" + frame:NSMakeRect(0, 0, 390, 844) + children:@[fullScreenProxy]]; + [self setUpWithRootElement:root]; + + FBSimulatorControlTests_AXPMacPlatformElement_Double *hiddenRemoteElement = + [FBAccessibilityTestElementBuilder staticTextWithLabel:@"Hidden Web CTA" + frame:NSMakeRect(24, 520, 220, 44)]; + FBSimulatorControlTests_AXPTranslationObject_Double *remoteHitTranslation = + [[FBSimulatorControlTests_AXPTranslationObject_Double alloc] init]; + // Same pid as frontmost app, which is normally filtered. + remoteHitTranslation.pid = 12345; + [self.fixture.translator setMacPlatformElement:hiddenRemoteElement forTranslation:remoteHitTranslation]; + self.fixture.translator.objectAtPointResult = remoteHitTranslation; + + FBSimulatorAccessibilityCommands *commands = [self commands]; + FBAccessibilityRequestOptions *options = [FBAccessibilityRequestOptions defaultOptions]; + options.collectFrameCoverage = YES; + options.remoteContentOptions = [FBAccessibilityRemoteContentOptions defaultOptions]; + options.remoteContentOptions.gridStepSize = 150; + options.remoteContentOptions.maxPoints = 4; + + NSError *error = nil; + FBAccessibilityElementsResponse *response = [[commands accessibilityElementsWithOptions:options] awaitWithTimeout:5.0 error:&error]; + + XCTAssertNil(error, @"Should not have error: %@", error); + XCTAssertNotNil(response); + + NSArray *> *elements = (NSArray *> *)response.elements; + XCTAssertTrue([self elements:elements containsLabel:@"Hidden Web CTA"], @"Expected hidden remote element to be discovered"); + XCTAssertGreaterThan([self objectAtPointCallCount], 0U, @"Near-full coverage path should still probe covered points"); +} + +- (void)testRemoteDiscoverySkipsFullscreenProxyInNearFullCoverageMode +{ + FBSimulatorControlTests_AXPMacPlatformElement_Double *fullScreenProxy = + [FBAccessibilityTestElementBuilder staticTextWithLabel:@"Native Full-Screen Proxy" + frame:NSMakeRect(0, 0, 390, 844)]; + FBSimulatorControlTests_AXPMacPlatformElement_Double *root = + [FBAccessibilityTestElementBuilder applicationWithLabel:@"App Window" + frame:NSMakeRect(0, 0, 390, 844) + children:@[fullScreenProxy]]; + [self setUpWithRootElement:root]; + + FBSimulatorControlTests_AXPMacPlatformElement_Double *remoteProxyElement = + [FBAccessibilityTestElementBuilder staticTextWithLabel:@"Remote Full-Screen Proxy" + frame:NSMakeRect(0, 0, 390, 844)]; + FBSimulatorControlTests_AXPTranslationObject_Double *remoteHitTranslation = + [[FBSimulatorControlTests_AXPTranslationObject_Double alloc] init]; + remoteHitTranslation.pid = 12345; + [self.fixture.translator setMacPlatformElement:remoteProxyElement forTranslation:remoteHitTranslation]; + self.fixture.translator.objectAtPointResult = remoteHitTranslation; + + FBSimulatorAccessibilityCommands *commands = [self commands]; + FBAccessibilityRequestOptions *options = [FBAccessibilityRequestOptions defaultOptions]; + options.collectFrameCoverage = YES; + options.remoteContentOptions = [FBAccessibilityRemoteContentOptions defaultOptions]; + options.remoteContentOptions.gridStepSize = 150; + options.remoteContentOptions.maxPoints = 4; + + NSError *error = nil; + FBAccessibilityElementsResponse *response = [[commands accessibilityElementsWithOptions:options] awaitWithTimeout:5.0 error:&error]; + + XCTAssertNil(error, @"Should not have error: %@", error); + XCTAssertNotNil(response); + + NSArray *> *elements = (NSArray *> *)response.elements; + XCTAssertFalse([self elements:elements containsLabel:@"Remote Full-Screen Proxy"], @"Expected full-screen proxy hit to be skipped"); + XCTAssertEqual(elements.count, 2, @"Expected only main traversal elements when remote hit is a full-screen proxy"); + XCTAssertGreaterThan([self objectAtPointCallCount], 0U, @"Expected point probing to run in near-full coverage mode"); +} + @end diff --git a/FBSimulatorControlTests/Utilities/AccessibilityDoubles.h b/FBSimulatorControlTests/Utilities/AccessibilityDoubles.h index 74067a939..a062397cc 100644 --- a/FBSimulatorControlTests/Utilities/AccessibilityDoubles.h +++ b/FBSimulatorControlTests/Utilities/AccessibilityDoubles.h @@ -82,6 +82,9 @@ NS_ASSUME_NONNULL_BEGIN - (FBSimulatorControlTests_AXPTranslationObject_Double *)frontmostApplicationWithDisplayId:(int)displayId bridgeDelegateToken:(NSString *)token; - (FBSimulatorControlTests_AXPTranslationObject_Double *)objectAtPoint:(CGPoint)point displayId:(int)displayId bridgeDelegateToken:(NSString *)token; - (FBSimulatorControlTests_AXPMacPlatformElement_Double *)macPlatformElementFromTranslation:(FBSimulatorControlTests_AXPTranslationObject_Double *)translation; +- (void)setMacPlatformElement:(nullable FBSimulatorControlTests_AXPMacPlatformElement_Double *)element + forTranslation:(FBSimulatorControlTests_AXPTranslationObject_Double *)translation; +- (void)clearMacPlatformElementMappings; - (void)resetTracking; diff --git a/FBSimulatorControlTests/Utilities/AccessibilityDoubles.m b/FBSimulatorControlTests/Utilities/AccessibilityDoubles.m index 0ca49aa4d..17a927756 100644 --- a/FBSimulatorControlTests/Utilities/AccessibilityDoubles.m +++ b/FBSimulatorControlTests/Utilities/AccessibilityDoubles.m @@ -187,6 +187,9 @@ - (BOOL)accessibilityPerformPress @end @implementation FBSimulatorControlTests_AXPTranslator_Double +{ + NSMapTable *_macPlatformElementByTranslation; +} - (instancetype)init { @@ -196,6 +199,7 @@ - (instancetype)init } _methodCalls = [NSMutableArray array]; + _macPlatformElementByTranslation = [NSMapTable strongToStrongObjectsMapTable]; return self; } @@ -219,14 +223,33 @@ - (FBSimulatorControlTests_AXPTranslationObject_Double *)objectAtPoint:(CGPoint) - (FBSimulatorControlTests_AXPMacPlatformElement_Double *)macPlatformElementFromTranslation:(FBSimulatorControlTests_AXPTranslationObject_Double *)translation { [_methodCalls addObject:@"macPlatformElementFromTranslation"]; - FBSimulatorControlTests_AXPMacPlatformElement_Double *result = self.macPlatformElementResult; + FBSimulatorControlTests_AXPMacPlatformElement_Double *result = [_macPlatformElementByTranslation objectForKey:translation] ?: self.macPlatformElementResult; result.translation = translation; return result; } +- (void)setMacPlatformElement:(FBSimulatorControlTests_AXPMacPlatformElement_Double *)element + forTranslation:(FBSimulatorControlTests_AXPTranslationObject_Double *)translation +{ + if (!translation) { + return; + } + if (element) { + [_macPlatformElementByTranslation setObject:element forKey:translation]; + } else { + [_macPlatformElementByTranslation removeObjectForKey:translation]; + } +} + +- (void)clearMacPlatformElementMappings +{ + [_macPlatformElementByTranslation removeAllObjects]; +} + - (void)resetTracking { [_methodCalls removeAllObjects]; + [_macPlatformElementByTranslation removeAllObjects]; } @end