From 819763c170d751a8d8108c94d016c08936789167 Mon Sep 17 00:00:00 2001 From: Chris Yang Date: Tue, 19 Sep 2023 11:05:49 -0700 Subject: [PATCH 1/2] conditionally adjust golden test threshold for TwoPlatformViewsWithOtherBackDropFilterTests based on current macOS version --- .../ios/Scenarios/Scenarios/AppDelegate.m | 14 +++++++++++++- .../Scenarios/ScenariosUITests/BogusFontTextTest.m | 2 +- .../ios/Scenarios/ScenariosUITests/GoldenImage.h | 2 +- .../ios/Scenarios/ScenariosUITests/GoldenImage.m | 8 +++----- .../ScenariosUITests/GoldenPlatformViewTests.h | 1 + .../ScenariosUITests/GoldenPlatformViewTests.m | 3 ++- .../Scenarios/ScenariosUITests/GoldenTestManager.h | 3 ++- .../Scenarios/ScenariosUITests/GoldenTestManager.m | 5 +++-- .../ScenariosUITests/PlatformViewUITests.m | 5 +++++ .../Scenarios/ScenariosUITests/SpawnEngineTest.m | 2 +- 10 files changed, 32 insertions(+), 13 deletions(-) diff --git a/testing/scenario_app/ios/Scenarios/Scenarios/AppDelegate.m b/testing/scenario_app/ios/Scenarios/Scenarios/AppDelegate.m index d938a895373d0..f701f720fb1ff 100644 --- a/testing/scenario_app/ios/Scenarios/Scenarios/AppDelegate.m +++ b/testing/scenario_app/ios/Scenarios/Scenarios/AppDelegate.m @@ -9,6 +9,18 @@ #import "ScreenBeforeFlutter.h" #import "TextPlatformView.h" +// A UIViewController that sets YES for its preferedStatusBarHidden property. +@interface NoStatusBarViewController : UIViewController + +@end + +@implementation NoStatusBarViewController +- (BOOL)prefersStatusBarHidden { + return YES; +} +@end + +// The FlutterViewController version of NoStatusBarViewController @interface NoStatusBarFlutterViewController : FlutterViewController @end @@ -166,7 +178,7 @@ - (void)setupFlutterViewControllerTest:(NSString*)scenarioIdentifier { UIViewController* rootViewController = flutterViewController; // Make Flutter View's origin x/y not 0. if ([scenarioIdentifier isEqualToString:@"non_full_screen_flutter_view_platform_view"]) { - rootViewController = [UIViewController new]; + rootViewController = [[NoStatusBarViewController alloc] init]; [rootViewController.view addSubview:flutterViewController.view]; flutterViewController.view.frame = CGRectMake(150, 150, 500, 500); } diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/BogusFontTextTest.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/BogusFontTextTest.m index 14c9e0bb93365..dc554ae2e5986 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/BogusFontTextTest.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/BogusFontTextTest.m @@ -25,7 +25,7 @@ - (void)testFontRenderingWhenSuppliedWithBogusFont { XCTAssertTrue([addTextField waitForExistenceWithTimeout:30]); GoldenTestManager* manager = [[GoldenTestManager alloc] initWithLaunchArg:@"--bogus-font-text"]; - [manager checkGoldenForTest:self]; + [manager checkGoldenForTest:self rmesThreshold:kDefaultRmseThreshold]; } @end diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.h b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.h index dead0c530e126..4c0b9306fe14b 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.h +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.h @@ -19,7 +19,7 @@ NS_ASSUME_NONNULL_BEGIN // Compare this GoldenImage to `image`. // // Return YES if the `image` of this GoldenImage have the same pixels of provided `image`. -- (BOOL)compareGoldenToImage:(UIImage*)image; +- (BOOL)compareGoldenToImage:(UIImage*)image rmesThreshold:(double)rmesThreshold; @end diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m index 698207aa681d4..37ae7b70fbb1d 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m @@ -8,8 +8,6 @@ #import #include -static const double kRmseThreshold = 0.5; - @interface GoldenImage () @end @@ -28,7 +26,7 @@ - (instancetype)initWithGoldenNamePrefix:(NSString*)prefix { return self; } -- (BOOL)compareGoldenToImage:(UIImage*)image { +- (BOOL)compareGoldenToImage:(UIImage*)image rmesThreshold:(double)rmesThreshold { if (!self.image || !image) { os_log_error(OS_LOG_DEFAULT, "GOLDEN DIFF FAILED: image does not exists."); return NO; @@ -91,11 +89,11 @@ - (BOOL)compareGoldenToImage:(UIImage*)image { } } double rmse = sqrt(sum / size); - if (rmse > kRmseThreshold) { + if (rmse > rmesThreshold) { os_log_error( OS_LOG_DEFAULT, "GOLDEN DIFF FAILED: image diff greater than threshold. Current diff: %@, threshold: %@", - @(rmse), @(kRmseThreshold)); + @(rmse), @(rmesThreshold)); return NO; } return YES; diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.h b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.h index fd1b05ef0b009..625fbf8b31e3c 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.h +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.h @@ -18,6 +18,7 @@ NS_ASSUME_NONNULL_BEGIN @interface GoldenPlatformViewTests : XCTestCase @property(nonatomic, strong) XCUIApplication* application; +@property(nonatomic, assign) double rmseThreadhold; // Initialize with a `GoldenTestManager`. - (instancetype)initWithManager:(GoldenTestManager*)manager invocation:(NSInvocation*)invocation; diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.m index 7e4bb368fc6eb..7199fa2600e86 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.m @@ -22,6 +22,7 @@ @implementation GoldenPlatformViewTests - (instancetype)initWithManager:(GoldenTestManager*)manager invocation:(NSInvocation*)invocation { self = [super initWithInvocation:invocation]; _manager = manager; + _rmseThreadhold = kDefaultRmseThreshold; return self; } @@ -45,6 +46,6 @@ - (void)checkPlatformViewGolden { @(kSecondsToWaitForPlatformView)); } - [self.manager checkGoldenForTest:self]; + [self.manager checkGoldenForTest:self rmesThreshold:self.rmseThreadhold]; } @end diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.h b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.h index 4bb0b275a77e9..7026672af7c46 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.h +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.h @@ -9,6 +9,7 @@ NS_ASSUME_NONNULL_BEGIN extern NSDictionary* launchArgsMap; +const extern double kDefaultRmseThreshold; // Manages a `GoldenPlatformViewTests`. // @@ -27,7 +28,7 @@ extern NSDictionary* launchArgsMap; // Take a sceenshot of the test app and check it has the same pixels with // goldenImage inside the `GoldenTestManager`. -- (void)checkGoldenForTest:(XCTestCase*)test; +- (void)checkGoldenForTest:(XCTestCase*)test rmesThreshold:(double)rmesThreshold; @end diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.m index 76c95a06501fe..a8d966379a5aa 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.m @@ -14,6 +14,7 @@ @interface GoldenTestManager () @implementation GoldenTestManager NSDictionary* launchArgsMap; +const double kDefaultRmseThreshold = 0.5; - (instancetype)initWithLaunchArg:(NSString*)launchArg { self = [super init]; @@ -73,7 +74,7 @@ - (instancetype)initWithLaunchArg:(NSString*)launchArg { return self; } -- (void)checkGoldenForTest:(XCTestCase*)test { +- (void)checkGoldenForTest:(XCTestCase*)test rmesThreshold:(double)rmesThreshold { XCUIScreenshot* screenshot = [[XCUIScreen mainScreen] screenshot]; if (!_goldenImage.image) { XCTAttachment* attachment = [XCTAttachment attachmentWithScreenshot:screenshot]; @@ -88,7 +89,7 @@ - (void)checkGoldenForTest:(XCTestCase*)test { _goldenImage.goldenName); } - if (![_goldenImage compareGoldenToImage:screenshot.image]) { + if (![_goldenImage compareGoldenToImage:screenshot.image rmesThreshold:rmesThreshold]) { XCTAttachment* screenshotAttachment = [XCTAttachment attachmentWithImage:screenshot.image]; screenshotAttachment.name = [_goldenImage.goldenName stringByAppendingString:@"_actual.png"]; screenshotAttachment.lifetime = XCTAttachmentLifetimeKeepAlways; diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m index 03984f27fda42..7415a51717ddb 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m @@ -375,6 +375,11 @@ - (instancetype)initWithInvocation:(NSInvocation*)invocation { } - (void)testPlatformView { + // (TODO)cyanglaz: remove the threshold adjustment after all the ci migrates to macOS13. + // https://github.com/flutter/flutter/issues/133207 + if ([NSProcessInfo processInfo].operatingSystemVersion.majorVersion >= 13) { + self.rmseThreadhold = 0.7; + } [self checkPlatformViewGolden]; } diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/SpawnEngineTest.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/SpawnEngineTest.m index f00a8c988bdfd..cc5810eeac608 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/SpawnEngineTest.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/SpawnEngineTest.m @@ -21,7 +21,7 @@ - (void)testSpawnEngineWorks { GoldenTestManager* manager = [[GoldenTestManager alloc] initWithLaunchArg:@"--spawn-engine-works"]; - [manager checkGoldenForTest:self]; + [manager checkGoldenForTest:self rmesThreshold:kDefaultRmseThreshold]; } @end From 940f00186cc6c86dc5c10da6e34464e44a6cc084 Mon Sep 17 00:00:00 2001 From: Chris Yang Date: Tue, 19 Sep 2023 11:33:54 -0700 Subject: [PATCH 2/2] revert status bar change --- .../ios/Scenarios/Scenarios/AppDelegate.m | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/testing/scenario_app/ios/Scenarios/Scenarios/AppDelegate.m b/testing/scenario_app/ios/Scenarios/Scenarios/AppDelegate.m index f701f720fb1ff..d938a895373d0 100644 --- a/testing/scenario_app/ios/Scenarios/Scenarios/AppDelegate.m +++ b/testing/scenario_app/ios/Scenarios/Scenarios/AppDelegate.m @@ -9,18 +9,6 @@ #import "ScreenBeforeFlutter.h" #import "TextPlatformView.h" -// A UIViewController that sets YES for its preferedStatusBarHidden property. -@interface NoStatusBarViewController : UIViewController - -@end - -@implementation NoStatusBarViewController -- (BOOL)prefersStatusBarHidden { - return YES; -} -@end - -// The FlutterViewController version of NoStatusBarViewController @interface NoStatusBarFlutterViewController : FlutterViewController @end @@ -178,7 +166,7 @@ - (void)setupFlutterViewControllerTest:(NSString*)scenarioIdentifier { UIViewController* rootViewController = flutterViewController; // Make Flutter View's origin x/y not 0. if ([scenarioIdentifier isEqualToString:@"non_full_screen_flutter_view_platform_view"]) { - rootViewController = [[NoStatusBarViewController alloc] init]; + rootViewController = [UIViewController new]; [rootViewController.view addSubview:flutterViewController.view]; flutterViewController.view.frame = CGRectMake(150, 150, 500, 500); }