diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/BogusFontTextTest.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/BogusFontTextTest.m index 14c9e0bb93365..dc554ae2e5986 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/BogusFontTextTest.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/BogusFontTextTest.m @@ -25,7 +25,7 @@ - (void)testFontRenderingWhenSuppliedWithBogusFont { XCTAssertTrue([addTextField waitForExistenceWithTimeout:30]); GoldenTestManager* manager = [[GoldenTestManager alloc] initWithLaunchArg:@"--bogus-font-text"]; - [manager checkGoldenForTest:self]; + [manager checkGoldenForTest:self rmesThreshold:kDefaultRmseThreshold]; } @end diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.h b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.h index dead0c530e126..4c0b9306fe14b 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.h +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.h @@ -19,7 +19,7 @@ NS_ASSUME_NONNULL_BEGIN // Compare this GoldenImage to `image`. // // Return YES if the `image` of this GoldenImage have the same pixels of provided `image`. -- (BOOL)compareGoldenToImage:(UIImage*)image; +- (BOOL)compareGoldenToImage:(UIImage*)image rmesThreshold:(double)rmesThreshold; @end diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m index 698207aa681d4..37ae7b70fbb1d 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m @@ -8,8 +8,6 @@ #import #include -static const double kRmseThreshold = 0.5; - @interface GoldenImage () @end @@ -28,7 +26,7 @@ - (instancetype)initWithGoldenNamePrefix:(NSString*)prefix { return self; } -- (BOOL)compareGoldenToImage:(UIImage*)image { +- (BOOL)compareGoldenToImage:(UIImage*)image rmesThreshold:(double)rmesThreshold { if (!self.image || !image) { os_log_error(OS_LOG_DEFAULT, "GOLDEN DIFF FAILED: image does not exists."); return NO; @@ -91,11 +89,11 @@ - (BOOL)compareGoldenToImage:(UIImage*)image { } } double rmse = sqrt(sum / size); - if (rmse > kRmseThreshold) { + if (rmse > rmesThreshold) { os_log_error( OS_LOG_DEFAULT, "GOLDEN DIFF FAILED: image diff greater than threshold. Current diff: %@, threshold: %@", - @(rmse), @(kRmseThreshold)); + @(rmse), @(rmesThreshold)); return NO; } return YES; diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.h b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.h index fd1b05ef0b009..625fbf8b31e3c 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.h +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.h @@ -18,6 +18,7 @@ NS_ASSUME_NONNULL_BEGIN @interface GoldenPlatformViewTests : XCTestCase @property(nonatomic, strong) XCUIApplication* application; +@property(nonatomic, assign) double rmseThreadhold; // Initialize with a `GoldenTestManager`. - (instancetype)initWithManager:(GoldenTestManager*)manager invocation:(NSInvocation*)invocation; diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.m index 7e4bb368fc6eb..7199fa2600e86 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenPlatformViewTests.m @@ -22,6 +22,7 @@ @implementation GoldenPlatformViewTests - (instancetype)initWithManager:(GoldenTestManager*)manager invocation:(NSInvocation*)invocation { self = [super initWithInvocation:invocation]; _manager = manager; + _rmseThreadhold = kDefaultRmseThreshold; return self; } @@ -45,6 +46,6 @@ - (void)checkPlatformViewGolden { @(kSecondsToWaitForPlatformView)); } - [self.manager checkGoldenForTest:self]; + [self.manager checkGoldenForTest:self rmesThreshold:self.rmseThreadhold]; } @end diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.h b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.h index 4bb0b275a77e9..7026672af7c46 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.h +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.h @@ -9,6 +9,7 @@ NS_ASSUME_NONNULL_BEGIN extern NSDictionary* launchArgsMap; +const extern double kDefaultRmseThreshold; // Manages a `GoldenPlatformViewTests`. // @@ -27,7 +28,7 @@ extern NSDictionary* launchArgsMap; // Take a sceenshot of the test app and check it has the same pixels with // goldenImage inside the `GoldenTestManager`. -- (void)checkGoldenForTest:(XCTestCase*)test; +- (void)checkGoldenForTest:(XCTestCase*)test rmesThreshold:(double)rmesThreshold; @end diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.m index 76c95a06501fe..a8d966379a5aa 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenTestManager.m @@ -14,6 +14,7 @@ @interface GoldenTestManager () @implementation GoldenTestManager NSDictionary* launchArgsMap; +const double kDefaultRmseThreshold = 0.5; - (instancetype)initWithLaunchArg:(NSString*)launchArg { self = [super init]; @@ -73,7 +74,7 @@ - (instancetype)initWithLaunchArg:(NSString*)launchArg { return self; } -- (void)checkGoldenForTest:(XCTestCase*)test { +- (void)checkGoldenForTest:(XCTestCase*)test rmesThreshold:(double)rmesThreshold { XCUIScreenshot* screenshot = [[XCUIScreen mainScreen] screenshot]; if (!_goldenImage.image) { XCTAttachment* attachment = [XCTAttachment attachmentWithScreenshot:screenshot]; @@ -88,7 +89,7 @@ - (void)checkGoldenForTest:(XCTestCase*)test { _goldenImage.goldenName); } - if (![_goldenImage compareGoldenToImage:screenshot.image]) { + if (![_goldenImage compareGoldenToImage:screenshot.image rmesThreshold:rmesThreshold]) { XCTAttachment* screenshotAttachment = [XCTAttachment attachmentWithImage:screenshot.image]; screenshotAttachment.name = [_goldenImage.goldenName stringByAppendingString:@"_actual.png"]; screenshotAttachment.lifetime = XCTAttachmentLifetimeKeepAlways; diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m index 03984f27fda42..7415a51717ddb 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m @@ -375,6 +375,11 @@ - (instancetype)initWithInvocation:(NSInvocation*)invocation { } - (void)testPlatformView { + // (TODO)cyanglaz: remove the threshold adjustment after all the ci migrates to macOS13. + // https://github.com/flutter/flutter/issues/133207 + if ([NSProcessInfo processInfo].operatingSystemVersion.majorVersion >= 13) { + self.rmseThreadhold = 0.7; + } [self checkPlatformViewGolden]; } diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/SpawnEngineTest.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/SpawnEngineTest.m index f00a8c988bdfd..cc5810eeac608 100644 --- a/testing/scenario_app/ios/Scenarios/ScenariosUITests/SpawnEngineTest.m +++ b/testing/scenario_app/ios/Scenarios/ScenariosUITests/SpawnEngineTest.m @@ -21,7 +21,7 @@ - (void)testSpawnEngineWorks { GoldenTestManager* manager = [[GoldenTestManager alloc] initWithLaunchArg:@"--spawn-engine-works"]; - [manager checkGoldenForTest:self]; + [manager checkGoldenForTest:self rmesThreshold:kDefaultRmseThreshold]; } @end