From 190c3d4b79ecde3348a78d766f23962b66a85738 Mon Sep 17 00:00:00 2001 From: Jason Simmons Date: Wed, 10 Apr 2024 01:20:28 +0000 Subject: [PATCH 1/3] Fix CkBrowserImageDecoder conversion of images to ImageByteFormat.rawRgba and rawStraightRgba VideoFrames of decoded images typically contain BGRA format. The CkBrowserImageDecoder implementation of toByteData needs to convert that into RGBA format. If the output format is rawRgba then it also needs to apply premultipled alpha. Also fixes an issue where _bgrToRgba was not converting all pixels in the image. Fixes https://github.com/flutter/flutter/issues/135409 Fixes https://github.com/flutter/flutter/issues/144770 --- .../engine/canvaskit/image_web_codecs.dart | 49 ++++++++++++++++--- .../test/canvaskit/image_golden_test.dart | 18 +++++++ lib/web_ui/test/canvaskit/test_data.dart | 18 +++++++ 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart index 054e7640ef9bf..77dcece425a1d 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart @@ -96,10 +96,16 @@ Future readPixelsFromVideoFrame(VideoFrame videoFrame, ui.ImageByteFor // At this point we know we want to read unencoded pixels, and that the video // frame is _not_ using the same format as the requested one. - final bool isBgrFrame = videoFrame.format == 'BGRA' || videoFrame.format == 'BGRX'; - if (format == ui.ImageByteFormat.rawRgba && isBgrFrame) { - _bgrToRgba(pixels); - return pixels.asByteData(); + final bool isBgrx = videoFrame.format == 'BGRX'; + final bool isBgrFrame = videoFrame.format == 'BGRA' || isBgrx; + if (isBgrFrame) { + if (format == ui.ImageByteFormat.rawStraightRgba || isBgrx) { + _bgrToStraightRgba(pixels, isBgrx); + return pixels.asByteData(); + } else if (format == ui.ImageByteFormat.rawRgba) { + _bgrToRawRgba(pixels); + return pixels.asByteData(); + } } // Last resort, just return the original pixels. @@ -107,10 +113,9 @@ Future readPixelsFromVideoFrame(VideoFrame videoFrame, ui.ImageByteFor } /// Mutates the [pixels], converting them from BGRX/BGRA to RGBA. -void _bgrToRgba(ByteBuffer pixels) { - final int pixelCount = pixels.lengthInBytes ~/ 4; +void _bgrToStraightRgba(ByteBuffer pixels, bool isBgrx) { final Uint8List pixelBytes = pixels.asUint8List(); - for (int i = 0; i < pixelCount; i += 4) { + for (int i = 0; i < pixelBytes.length; i += 4) { // It seems even in little-endian machines the BGR_ pixels are encoded as // big-endian, i.e. the blue byte is written into the lowest byte in the // memory address space. @@ -122,6 +127,34 @@ void _bgrToRgba(ByteBuffer pixels) { // codecs that do something different. pixelBytes[i] = r; pixelBytes[i + 2] = b; + if (isBgrx) { + pixelBytes[i + 3] = 255; + } + } +} + +/// Based on Chromium's SetRGBAPremultiply. +int _premultiply(int value, int alpha) { + if (alpha == 255) { + return value; + } + const int kRoundFractionControl = 257 * 128; + return (value * alpha * 257 + kRoundFractionControl) >> 16; +} + +/// Mutates the [pixels], converting them from BGRX/BGRA to RGBA with +/// premultiplied alpha. +void _bgrToRawRgba(ByteBuffer pixels) { + final Uint8List pixelBytes = pixels.asUint8List(); + for (int i = 0; i < pixelBytes.length; i += 4) { + final int a = pixelBytes[i + 3]; + final int r = _premultiply(pixelBytes[i + 2], a); + final int g = _premultiply(pixelBytes[i + 1], a); + final int b = _premultiply(pixelBytes[i], a); + + pixelBytes[i] = r; + pixelBytes[i + 1] = g; + pixelBytes[i + 2] = b; } } @@ -133,7 +166,7 @@ bool _shouldReadPixelsUnmodified(VideoFrame videoFrame, ui.ImageByteFormat forma // Do not convert if the requested format is RGBA and the video frame is // encoded as either RGBA or RGBX. final bool isRgbFrame = videoFrame.format == 'RGBA' || videoFrame.format == 'RGBX'; - return format == ui.ImageByteFormat.rawRgba && isRgbFrame; + return format == ui.ImageByteFormat.rawStraightRgba && isRgbFrame; } Future readVideoFramePixelsUnmodified(VideoFrame videoFrame) async { diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index 824eeca9d7c19..475e174f37c4c 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -895,6 +895,24 @@ void _testCkBrowserImageDecoder() { debugRestoreWebDecoderExpireDuration(); }); + + test('ImageDecoder toByteData(translucent PNG)', () async { + final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( + data: kTranslucentPng, + debugSource: 'test', + ); + final ui.FrameInfo frame = await image.getNextFrame(); + + ByteData? data = await frame.image.toByteData(format: ui.ImageByteFormat.rawStraightRgba); + expect(data!.buffer.asUint8List(), + [0x22, 0x44, 0x66, 0x80, 0x22, 0x44, 0x66, 0x80, + 0x22, 0x44, 0x66, 0x80, 0x22, 0x44, 0x66, 0x80]); + + data = await frame.image.toByteData(format: ui.ImageByteFormat.rawRgba); + expect(data!.buffer.asUint8List(), + [0x11, 0x22, 0x33, 0x80, 0x11, 0x22, 0x33, 0x80, + 0x11, 0x22, 0x33, 0x80, 0x11, 0x22, 0x33, 0x80]); + }); } Future expectFrameData(ui.FrameInfo frame, List data) async { diff --git a/lib/web_ui/test/canvaskit/test_data.dart b/lib/web_ui/test/canvaskit/test_data.dart index 2e53c7366c16d..8a131cde1dd51 100644 --- a/lib/web_ui/test/canvaskit/test_data.dart +++ b/lib/web_ui/test/canvaskit/test_data.dart @@ -35,3 +35,21 @@ final Uint8List kAnimatedGif = Uint8List.fromList( [ 0xf9, 0x04, 0x00, 0x0a, 0x00, 0xff, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x02, 0x02, 0x44, 0x01, 0x00, 0x3b, ]); + +/// A 2x2 translucent PNG. +final Uint8List kTranslucentPng = Uint8List.fromList( [ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, + 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x01, 0x03, + 0x00, 0x00, 0x00, 0x48, 0x78, 0x9f, 0x67, 0x00, 0x00, 0x00, 0x20, 0x63, 0x48, + 0x52, 0x4d, 0x00, 0x00, 0x7a, 0x26, 0x00, 0x00, 0x80, 0x84, 0x00, 0x00, 0xfa, + 0x00, 0x00, 0x00, 0x80, 0xe8, 0x00, 0x00, 0x75, 0x30, 0x00, 0x00, 0xea, 0x60, + 0x00, 0x00, 0x3a, 0x98, 0x00, 0x00, 0x17, 0x70, 0x9c, 0xba, 0x51, 0x3c, 0x00, + 0x00, 0x00, 0x06, 0x50, 0x4c, 0x54, 0x45, 0x22, 0x44, 0x66, 0xff, 0xff, 0xff, + 0x5c, 0x83, 0x6d, 0xb6, 0x00, 0x00, 0x00, 0x01, 0x74, 0x52, 0x4e, 0x53, 0x80, + 0xad, 0x5e, 0x5b, 0x46, 0x00, 0x00, 0x00, 0x01, 0x62, 0x4b, 0x47, 0x44, 0x01, + 0xff, 0x02, 0x2d, 0xde, 0x00, 0x00, 0x00, 0x07, 0x74, 0x49, 0x4d, 0x45, 0x07, + 0xe8, 0x04, 0x0c, 0x15, 0x16, 0x21, 0xc3, 0x89, 0xee, 0x25, 0x00, 0x00, 0x00, + 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0x60, 0x60, 0x60, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x01, 0x27, 0x34, 0x27, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x49, + 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82 +]); From 41eedec3f6f0a50cf34a1be0e26c8b6794a52d43 Mon Sep 17 00:00:00 2001 From: Jason Simmons Date: Fri, 12 Apr 2024 23:20:33 +0000 Subject: [PATCH 2/3] inlining --- lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart index 77dcece425a1d..f2b213c731cd2 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart @@ -134,6 +134,7 @@ void _bgrToStraightRgba(ByteBuffer pixels, bool isBgrx) { } /// Based on Chromium's SetRGBAPremultiply. +@pragma('dart2js:tryInline') int _premultiply(int value, int alpha) { if (alpha == 255) { return value; From 91ee45560d0d497ead31ee05ceb23663a25082c4 Mon Sep 17 00:00:00 2001 From: Jason Simmons Date: Wed, 17 Apr 2024 15:51:43 +0000 Subject: [PATCH 3/3] analyzer --- lib/web_ui/test/canvaskit/image_golden_test.dart | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index 475e174f37c4c..6d3ee7e298859 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -905,13 +905,13 @@ void _testCkBrowserImageDecoder() { ByteData? data = await frame.image.toByteData(format: ui.ImageByteFormat.rawStraightRgba); expect(data!.buffer.asUint8List(), - [0x22, 0x44, 0x66, 0x80, 0x22, 0x44, 0x66, 0x80, - 0x22, 0x44, 0x66, 0x80, 0x22, 0x44, 0x66, 0x80]); + [0x22, 0x44, 0x66, 0x80, 0x22, 0x44, 0x66, 0x80, + 0x22, 0x44, 0x66, 0x80, 0x22, 0x44, 0x66, 0x80]); - data = await frame.image.toByteData(format: ui.ImageByteFormat.rawRgba); + data = await frame.image.toByteData(); expect(data!.buffer.asUint8List(), - [0x11, 0x22, 0x33, 0x80, 0x11, 0x22, 0x33, 0x80, - 0x11, 0x22, 0x33, 0x80, 0x11, 0x22, 0x33, 0x80]); + [0x11, 0x22, 0x33, 0x80, 0x11, 0x22, 0x33, 0x80, + 0x11, 0x22, 0x33, 0x80, 0x11, 0x22, 0x33, 0x80]); }); }