From 49ed8d4a4f1d643033919c6fb8c08613f61c7ea8 Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Fri, 24 May 2024 16:18:09 -0700 Subject: [PATCH 01/15] Use HtmlImageElement for image decoding in skia --- .../lib/src/engine/canvaskit/image.dart | 164 +++++++++++++----- 1 file changed, 119 insertions(+), 45 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart index 23c585f7f8a48..f21047ff5a8f5 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -11,16 +11,81 @@ import 'package:ui/ui.dart' as ui; import 'package:ui/ui_web/src/ui_web.dart' as ui_web; /// Instantiates a [ui.Codec] backed by an `SkAnimatedImage` from Skia. -FutureOr skiaInstantiateImageCodec(Uint8List list, - [int? targetWidth, int? targetHeight]) { +Future skiaInstantiateImageCodec(Uint8List list, + [int? targetWidth, int? targetHeight]) async { // If we have either a target width or target height, use canvaskit to decode. - if (browserSupportsImageDecoder && (targetWidth == null && targetHeight == null)) { - return CkBrowserImageDecoder.create( + ui.Codec codec; + if (browserSupportsImageDecoder) { + codec = await CkBrowserImageDecoder.create( data: list, debugSource: 'encoded image bytes', ); } else { - return CkAnimatedImage.decodeFromBytes(list, 'encoded image bytes', targetWidth: targetWidth, targetHeight: targetHeight); + final DomBlob blob = createDomBlob([list.buffer]); + codec = CkImageBlobCodec(blob); + } + return ResizingCodec( + codec, + targetWidth: targetWidth, + targetHeight: targetHeight, + ); +} + +class CkHtmlImageElementCodec extends HtmlImageElementCodec { + CkHtmlImageElementCodec(super.src, {super.chunkCallback}); + + @override + ui.Image createImageFromHTMLImageElement( + DomHTMLImageElement image, + int naturalWidth, + int naturalHeight, + ) { + final SkImage? skImage = canvasKit.MakeLazyImageFromTextureSourceWithInfo( + image, + SkPartialImageInfo( + alphaType: canvasKit.AlphaType.Premul, + colorType: canvasKit.ColorType.RGBA_8888, + colorSpace: SkColorSpaceSRGB, + width: naturalWidth.toDouble(), + height: naturalHeight.toDouble(), + ), + ); + if (skImage == null) { + throw ImageCodecException( + 'Failed to create image from Image.decode', + ); + } + + return CkImage(skImage); + } +} + +class CkImageBlobCodec extends HtmlBlobCodec { + CkImageBlobCodec(super.blob); + + @override + ui.Image createImageFromHTMLImageElement( + DomHTMLImageElement image, + int naturalWidth, + int naturalHeight, + ) { + final SkImage? skImage = canvasKit.MakeLazyImageFromTextureSourceWithInfo( + image, + SkPartialImageInfo( + alphaType: canvasKit.AlphaType.Premul, + colorType: canvasKit.ColorType.RGBA_8888, + colorSpace: SkColorSpaceSRGB, + width: naturalWidth.toDouble(), + height: naturalHeight.toDouble(), + ), + ); + if (skImage == null) { + throw ImageCodecException( + 'Failed to create image from Image.decode', + ); + } + + return CkImage(skImage); } } @@ -49,7 +114,9 @@ void skiaDecodeImageFromPixels( SkImageInfo( width: width.toDouble(), height: height.toDouble(), - colorType: format == ui.PixelFormat.rgba8888 ? canvasKit.ColorType.RGBA_8888 : canvasKit.ColorType.BGRA_8888, + colorType: format == ui.PixelFormat.rgba8888 + ? canvasKit.ColorType.RGBA_8888 + : canvasKit.ColorType.BGRA_8888, alphaType: canvasKit.AlphaType.Premul, colorSpace: SkColorSpaceSRGB, ), @@ -63,7 +130,8 @@ void skiaDecodeImageFromPixels( } if (targetWidth != null || targetHeight != null) { - if (validUpscale(allowUpscaling, targetWidth, targetHeight, width, height)) { + if (validUpscale( + allowUpscaling, targetWidth, targetHeight, width, height)) { return callback(scaleImage(skImage, targetWidth, targetHeight)); } } @@ -73,7 +141,8 @@ void skiaDecodeImageFromPixels( // An invalid upscale happens when allowUpscaling is false AND either the given // targetWidth is larger than the originalWidth OR the targetHeight is larger than originalHeight. -bool validUpscale(bool allowUpscaling, int? targetWidth, int? targetHeight, int originalWidth, int originalHeight) { +bool validUpscale(bool allowUpscaling, int? targetWidth, int? targetHeight, + int originalWidth, int originalHeight) { if (allowUpscaling) { return true; } @@ -104,42 +173,39 @@ bool validUpscale(bool allowUpscaling, int? targetWidth, int? targetHeight, int /// If either targetWidth or targetHeight is less than or equal to zero, it /// will be treated as if it is null. CkImage scaleImage(SkImage image, int? targetWidth, int? targetHeight) { - assert(targetWidth != null || targetHeight != null); - if (targetWidth != null && targetWidth <= 0) { - targetWidth = null; - } - if (targetHeight != null && targetHeight <= 0) { - targetHeight = null; - } - if (targetWidth == null && targetHeight != null) { - targetWidth = (targetHeight * (image.width() / image.height())).round(); - } else if (targetHeight == null && targetWidth != null) { - targetHeight = targetWidth ~/ (image.width() / image.height()); - } + assert(targetWidth != null || targetHeight != null); + if (targetWidth != null && targetWidth <= 0) { + targetWidth = null; + } + if (targetHeight != null && targetHeight <= 0) { + targetHeight = null; + } + if (targetWidth == null && targetHeight != null) { + targetWidth = (targetHeight * (image.width() / image.height())).round(); + } else if (targetHeight == null && targetWidth != null) { + targetHeight = targetWidth ~/ (image.width() / image.height()); + } - assert(targetWidth != null); - assert(targetHeight != null); + assert(targetWidth != null); + assert(targetHeight != null); - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); + final CkPictureRecorder recorder = CkPictureRecorder(); + final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - final CkPaint paint = CkPaint(); - canvas.drawImageRect( - CkImage(image), - ui.Rect.fromLTWH(0, 0, image.width(), image.height()), - ui.Rect.fromLTWH(0, 0, targetWidth!.toDouble(), targetHeight!.toDouble()), - paint, - ); - paint.dispose(); + final CkPaint paint = CkPaint(); + canvas.drawImageRect( + CkImage(image), + ui.Rect.fromLTWH(0, 0, image.width(), image.height()), + ui.Rect.fromLTWH(0, 0, targetWidth!.toDouble(), targetHeight!.toDouble()), + paint, + ); + paint.dispose(); - final CkPicture picture = recorder.endRecording(); - final ui.Image finalImage = picture.toImageSync( - targetWidth, - targetHeight - ); + final CkPicture picture = recorder.endRecording(); + final ui.Image finalImage = picture.toImageSync(targetWidth, targetHeight); - final CkImage ckImage = finalImage as CkImage; - return ckImage; + final CkImage ckImage = finalImage as CkImage; + return ckImage; } /// Thrown when the web engine fails to decode an image, either due to a @@ -168,7 +234,8 @@ Future skiaInstantiateWebImageCodec( } /// Sends a request to fetch image data. -Future fetchImage(String url, ui_web.ImageCodecChunkCallback? chunkCallback) async { +Future fetchImage( + String url, ui_web.ImageCodecChunkCallback? chunkCallback) async { try { final HttpFetchResponse response = await httpFetch(url); final int? contentLength = response.contentLength; @@ -199,7 +266,8 @@ Future fetchImage(String url, ui_web.ImageCodecChunkCallback? chunkCa /// Reads the [payload] in chunks using the browser's Streams API /// /// See: https://developer.mozilla.org/en-US/docs/Web/API/Streams_API -Future readChunked(HttpFetchPayload payload, int contentLength, ui_web.ImageCodecChunkCallback chunkCallback) async { +Future readChunked(HttpFetchPayload payload, int contentLength, + ui_web.ImageCodecChunkCallback chunkCallback) async { final JSUint8Array result = createUint8ArrayFromLength(contentLength); int position = 0; int cumulativeBytesLoaded = 0; @@ -214,7 +282,7 @@ Future readChunked(HttpFetchPayload payload, int contentLength, ui_we /// A [ui.Image] backed by an `SkImage` from Skia. class CkImage implements ui.Image, StackTraceDebugger { - CkImage(SkImage skImage, { this.videoFrame }) { + CkImage(SkImage skImage, {this.videoFrame}) { box = CountedRef(skImage, this, 'SkImage'); _init(); } @@ -323,7 +391,10 @@ class CkImage implements ui.Image, StackTraceDebugger { assert(_debugCheckIsNotDisposed()); // readPixelsFromVideoFrame currently does not convert I420, I444, I422 // videoFrame formats to RGBA - if (videoFrame != null && videoFrame!.format != 'I420' && videoFrame!.format != 'I444' && videoFrame!.format != 'I422') { + if (videoFrame != null && + videoFrame!.format != 'I420' && + videoFrame!.format != 'I444' && + videoFrame!.format != 'I422') { return readPixelsFromVideoFrame(videoFrame!, format); } else { return _readPixelsFromSkImage(format); @@ -334,7 +405,9 @@ class CkImage implements ui.Image, StackTraceDebugger { ui.ColorSpace get colorSpace => ui.ColorSpace.sRGB; Future _readPixelsFromSkImage(ui.ImageByteFormat format) { - final SkAlphaType alphaType = format == ui.ImageByteFormat.rawStraightRgba ? canvasKit.AlphaType.Unpremul : canvasKit.AlphaType.Premul; + final SkAlphaType alphaType = format == ui.ImageByteFormat.rawStraightRgba + ? canvasKit.AlphaType.Unpremul + : canvasKit.AlphaType.Premul; final ByteData? data = _encodeImage( skImage: skImage, format: format, @@ -358,7 +431,8 @@ class CkImage implements ui.Image, StackTraceDebugger { }) { Uint8List? bytes; - if (format == ui.ImageByteFormat.rawRgba || format == ui.ImageByteFormat.rawStraightRgba) { + if (format == ui.ImageByteFormat.rawRgba || + format == ui.ImageByteFormat.rawStraightRgba) { final SkImageInfo imageInfo = SkImageInfo( alphaType: alphaType, colorType: colorType, From 6b248d104d20a7bea8e8bde2c9ffb77030eab3ea Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Wed, 29 May 2024 13:16:21 -0700 Subject: [PATCH 02/15] start touching up tests --- lib/web_ui/lib/src/engine/canvaskit/image.dart | 3 ++- lib/web_ui/test/canvaskit/image_golden_test.dart | 16 ++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart index f21047ff5a8f5..388cd8c6500e2 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -13,14 +13,15 @@ import 'package:ui/ui_web/src/ui_web.dart' as ui_web; /// Instantiates a [ui.Codec] backed by an `SkAnimatedImage` from Skia. Future skiaInstantiateImageCodec(Uint8List list, [int? targetWidth, int? targetHeight]) async { - // If we have either a target width or target height, use canvaskit to decode. ui.Codec codec; + print('browser supports image decoder? $browserSupportsImageDecoder'); if (browserSupportsImageDecoder) { codec = await CkBrowserImageDecoder.create( data: list, debugSource: 'encoded image bytes', ); } else { + // TODO(harryterkelsen): If the image is animated, then use Skia to decode. final DomBlob blob = createDomBlob([list.buffer]); codec = CkImageBlobCodec(blob); } diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index 6d3ee7e298859..4e5959ce980ed 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -294,7 +294,8 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { } }); - test('instantiateImageCodec with multi-frame image does not support targetWidth/targetHeight', + test( + 'instantiateImageCodec with multi-frame image supports targetWidth/targetHeight', () async { final ui.Codec codec = await ui.instantiateImageCodec( kAnimatedGif, @@ -303,18 +304,9 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { ); final ui.Image image = (await codec.getNextFrame()).image; - expect( - warnings, - containsAllInOrder( - [ - 'targetWidth and targetHeight for multi-frame images not supported', - ], - ), - ); - // expect the re-size did not happen, kAnimatedGif is [1x1] - expect(image.width, 1); - expect(image.height, 1); + expect(image.width, 2); + expect(image.height, 3); image.dispose(); codec.dispose(); }); From 4e43312eccd84c8d7033d5c4b5fdcc97cd2ed822 Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Fri, 31 May 2024 13:37:09 -0700 Subject: [PATCH 03/15] eagerly decode --- .../lib/src/engine/canvaskit/image.dart | 42 ++- .../engine/canvaskit/image_web_codecs.dart | 20 +- .../src/engine/html_image_element_codec.dart | 104 +++-- .../test/canvaskit/embedded_views_test.dart | 1 + .../test/canvaskit/image_golden_test.dart | 355 +++++++++++------- 5 files changed, 330 insertions(+), 192 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart index 388cd8c6500e2..b83a9d278713c 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -4,6 +4,7 @@ import 'dart:async'; import 'dart:js_interop'; +import 'dart:math' as math; import 'dart:typed_data'; import 'package:ui/src/engine.dart'; @@ -14,16 +15,20 @@ import 'package:ui/ui_web/src/ui_web.dart' as ui_web; Future skiaInstantiateImageCodec(Uint8List list, [int? targetWidth, int? targetHeight]) async { ui.Codec codec; - print('browser supports image decoder? $browserSupportsImageDecoder'); + // ImageDecoder does not detect image type automatically. It requires us to + // tell it what the image type is. + final String contentType = tryDetectContentType(list, 'encoded image bytes'); + if (browserSupportsImageDecoder) { codec = await CkBrowserImageDecoder.create( data: list, + contentType: contentType, debugSource: 'encoded image bytes', ); } else { // TODO(harryterkelsen): If the image is animated, then use Skia to decode. final DomBlob blob = createDomBlob([list.buffer]); - codec = CkImageBlobCodec(blob); + codec = await decodeBlobToCkImage(blob); } return ResizingCodec( codec, @@ -90,6 +95,13 @@ class CkImageBlobCodec extends HtmlBlobCodec { } } +/// Creates and decodes an image using HtmlImageElement. +Future decodeBlobToCkImage(DomBlob blob) async { + final CkImageBlobCodec codec = CkImageBlobCodec(blob); + await codec.decode(); + return codec; +} + void skiaDecodeImageFromPixels( Uint8List pixels, int width, @@ -227,8 +239,10 @@ const String _kNetworkImageMessage = 'Failed to load network image.'; Future skiaInstantiateWebImageCodec( String url, ui_web.ImageCodecChunkCallback? chunkCallback) async { final Uint8List list = await fetchImage(url, chunkCallback); + final String contentType = tryDetectContentType(list, url); if (browserSupportsImageDecoder) { - return CkBrowserImageDecoder.create(data: list, debugSource: url); + return CkBrowserImageDecoder.create( + data: list, contentType: contentType, debugSource: url); } else { return CkAnimatedImage.decodeFromBytes(list, url); } @@ -455,3 +469,25 @@ class CkImage implements ui.Image, StackTraceDebugger { return '[$width\u00D7$height]'; } } + +/// Detect the content type or throw an error if content type can't be detected. +String tryDetectContentType(Uint8List data, String debugSource) { + // ImageDecoder does not detect image type automatically. It requires us to + // tell it what the image type is. + final String? contentType = detectContentType(data); + + if (contentType == null) { + final String fileHeader; + if (data.isNotEmpty) { + fileHeader = + '[${bytesToHexString(data.sublist(0, math.min(10, data.length)))}]'; + } else { + fileHeader = 'empty'; + } + throw ImageCodecException( + 'Failed to detect image file format using the file header.\n' + 'File header was $fileHeader.\n' + 'Image source: $debugSource'); + } + return contentType; +} diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart index f2b213c731cd2..4f78f2cfb5bee 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart @@ -11,7 +11,6 @@ import 'dart:async'; import 'dart:convert' show base64; import 'dart:js_interop'; -import 'dart:math' as math; import 'dart:typed_data'; import 'package:ui/src/engine.dart'; @@ -27,26 +26,9 @@ class CkBrowserImageDecoder extends BrowserImageDecoder { static Future create({ required Uint8List data, + required String contentType, required String debugSource, }) async { - // ImageDecoder does not detect image type automatically. It requires us to - // tell it what the image type is. - final String? contentType = detectContentType(data); - - if (contentType == null) { - final String fileHeader; - if (data.isNotEmpty) { - fileHeader = '[${bytesToHexString(data.sublist(0, math.min(10, data.length)))}]'; - } else { - fileHeader = 'empty'; - } - throw ImageCodecException( - 'Failed to detect image file format using the file header.\n' - 'File header was $fileHeader.\n' - 'Image source: $debugSource' - ); - } - final CkBrowserImageDecoder decoder = CkBrowserImageDecoder._( contentType: contentType, dataSource: data.toJS, diff --git a/lib/web_ui/lib/src/engine/html_image_element_codec.dart b/lib/web_ui/lib/src/engine/html_image_element_codec.dart index b40fec7dabcf0..e21fd6c5917af 100644 --- a/lib/web_ui/lib/src/engine/html_image_element_codec.dart +++ b/lib/web_ui/lib/src/engine/html_image_element_codec.dart @@ -4,6 +4,7 @@ import 'dart:async'; +import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; import 'package:ui/ui_web/src/ui_web.dart' as ui_web; @@ -24,10 +25,11 @@ final bool _supportsDecode = _jsImageDecodeFunction != null; typedef WebOnlyImageCodecChunkCallback = ui_web.ImageCodecChunkCallback; abstract class HtmlImageElementCodec implements ui.Codec { - HtmlImageElementCodec(this.src, {this.chunkCallback}); + HtmlImageElementCodec(this.src, {this.chunkCallback, this.debugSource}); final String src; final ui_web.ImageCodecChunkCallback? chunkCallback; + final String? debugSource; @override int get frameCount => 1; @@ -35,39 +37,34 @@ abstract class HtmlImageElementCodec implements ui.Codec { @override int get repetitionCount => 0; - @override - Future getNextFrame() async { - final Completer completer = Completer(); + /// The Image() element backing this codec. + DomHTMLImageElement? imgElement; + + /// A Future which completes when the Image element backing this codec has + /// been loaded and decoded. + Future? decodeFuture; + + Future decode() async { + if (decodeFuture != null) { + return decodeFuture; + } + final Completer completer = Completer(); + decodeFuture = completer.future; // Currently there is no way to watch decode progress, so // we add 0/100 , 100/100 progress callbacks to enable loading progress // builders to create UI. chunkCallback?.call(0, 100); if (_supportsDecode) { - final DomHTMLImageElement imgElement = createDomHTMLImageElement(); - imgElement.src = src; - setJsProperty(imgElement, 'decoding', 'async'); + imgElement = createDomHTMLImageElement(); + imgElement!.src = src; + setJsProperty(imgElement!, 'decoding', 'async'); // Ignoring the returned future on purpose because we're communicating // through the `completer`. // ignore: unawaited_futures - imgElement.decode().then((dynamic _) { + imgElement!.decode().then((dynamic _) { chunkCallback?.call(100, 100); - int naturalWidth = imgElement.naturalWidth.toInt(); - int naturalHeight = imgElement.naturalHeight.toInt(); - // Workaround for https://bugzilla.mozilla.org/show_bug.cgi?id=700533. - if (naturalWidth == 0 && - naturalHeight == 0 && - ui_web.browser.browserEngine == ui_web.BrowserEngine.firefox) { - const int kDefaultImageSizeFallback = 300; - naturalWidth = kDefaultImageSizeFallback; - naturalHeight = kDefaultImageSizeFallback; - } - final ui.Image image = createImageFromHTMLImageElement( - imgElement, - naturalWidth, - naturalHeight, - ); - completer.complete(SingleFrameInfo(image)); + completer.complete(); }).catchError((dynamic e) { // This code path is hit on Chrome 80.0.3987.16 when too many // images are on the page (~1000). @@ -80,8 +77,31 @@ abstract class HtmlImageElementCodec implements ui.Codec { return completer.future; } - void _decodeUsingOnLoad(Completer completer) { - final DomHTMLImageElement imgElement = createDomHTMLImageElement(); + @override + Future getNextFrame() async { + await decode(); + int naturalWidth = imgElement!.naturalWidth.toInt(); + int naturalHeight = imgElement!.naturalHeight.toInt(); + // Workaround for https://bugzilla.mozilla.org/show_bug.cgi?id=700533. + if (naturalWidth == 0 && + naturalHeight == 0 && + ui_web.browser.browserEngine == ui_web.BrowserEngine.firefox) { + const int kDefaultImageSizeFallback = 300; + naturalWidth = kDefaultImageSizeFallback; + naturalHeight = kDefaultImageSizeFallback; + } + final ui.Image image = createImageFromHTMLImageElement( + imgElement!, + naturalWidth, + naturalHeight, + ); + return SingleFrameInfo(image); + } + + // TODO(harryterkelsen): All browsers support Image.decode now. Should we + // remove this code path? + void _decodeUsingOnLoad(Completer completer) { + imgElement = createDomHTMLImageElement(); // If the browser doesn't support asynchronous decoding of an image, // then use the `onload` event to decide when it's ready to paint to the // DOM. Unfortunately, this will cause the image to be decoded synchronously @@ -90,27 +110,25 @@ abstract class HtmlImageElementCodec implements ui.Codec { DomEventListener? loadListener; errorListener = createDomEventListener((DomEvent event) { if (loadListener != null) { - imgElement.removeEventListener('load', loadListener); + imgElement!.removeEventListener('load', loadListener); } - imgElement.removeEventListener('error', errorListener); - completer.completeError(event); + imgElement!.removeEventListener('error', errorListener); + completer.completeError(ImageCodecException( + 'Failed to decode image data.\n' + 'Image source: $debugSource', + )); }); - imgElement.addEventListener('error', errorListener); + imgElement!.addEventListener('error', errorListener); loadListener = createDomEventListener((DomEvent event) { if (chunkCallback != null) { chunkCallback!(100, 100); } - imgElement.removeEventListener('load', loadListener); - imgElement.removeEventListener('error', errorListener); - final ui.Image image = createImageFromHTMLImageElement( - imgElement, - imgElement.naturalWidth.toInt(), - imgElement.naturalHeight.toInt(), - ); - completer.complete(SingleFrameInfo(image)); + imgElement!.removeEventListener('load', loadListener); + imgElement!.removeEventListener('error', errorListener); + completer.complete(); }); - imgElement.addEventListener('load', loadListener); - imgElement.src = src; + imgElement!.addEventListener('load', loadListener); + imgElement!.src = src; } /// Creates a [ui.Image] from an [HTMLImageElement] that has been loaded. @@ -125,7 +143,11 @@ abstract class HtmlImageElementCodec implements ui.Codec { } abstract class HtmlBlobCodec extends HtmlImageElementCodec { - HtmlBlobCodec(this.blob) : super(domWindow.URL.createObjectURL(blob)); + HtmlBlobCodec(this.blob) + : super( + domWindow.URL.createObjectURL(blob), + debugSource: 'encoded image bytes', + ); final DomBlob blob; diff --git a/lib/web_ui/test/canvaskit/embedded_views_test.dart b/lib/web_ui/test/canvaskit/embedded_views_test.dart index a69579bb1d78e..2033352504c90 100644 --- a/lib/web_ui/test/canvaskit/embedded_views_test.dart +++ b/lib/web_ui/test/canvaskit/embedded_views_test.dart @@ -597,6 +597,7 @@ void testMain() { await createPlatformView(0, 'test-platform-view'); final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( + contentType: 'image/gif', data: kAnimatedGif, debugSource: 'test', ); diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index 4e5959ce980ed..273e03bbc2421 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -40,15 +40,43 @@ void testMain() { expect(isAvif(Uint8List.fromList([1, 2, 3])), isFalse); expect( isAvif(Uint8List.fromList([ - 0x00, 0x00, 0x00, 0x1c, 0x66, 0x74, 0x79, 0x70, - 0x61, 0x76, 0x69, 0x66, 0x00, 0x00, 0x00, 0x00, + 0x00, + 0x00, + 0x00, + 0x1c, + 0x66, + 0x74, + 0x79, + 0x70, + 0x61, + 0x76, + 0x69, + 0x66, + 0x00, + 0x00, + 0x00, + 0x00, ])), isTrue, ); expect( isAvif(Uint8List.fromList([ - 0x00, 0x00, 0x00, 0x20, 0x66, 0x74, 0x79, 0x70, - 0x61, 0x76, 0x69, 0x66, 0x00, 0x00, 0x00, 0x00, + 0x00, + 0x00, + 0x00, + 0x20, + 0x66, + 0x74, + 0x79, + 0x70, + 0x61, + 0x76, + 0x69, + 0x66, + 0x00, + 0x00, + 0x00, + 0x00, ])), isTrue, ); @@ -83,7 +111,8 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { }); test('CkAnimatedImage can be explicitly disposed of', () { - final CkAnimatedImage image = CkAnimatedImage.decodeFromBytes(kTransparentImage, 'test'); + final CkAnimatedImage image = + CkAnimatedImage.decodeFromBytes(kTransparentImage, 'test'); expect(image.debugDisposed, isFalse); image.dispose(); expect(image.debugDisposed, isTrue); @@ -98,7 +127,8 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { }); test('CkAnimatedImage iterates frames correctly', () async { - final CkAnimatedImage image = CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); + final CkAnimatedImage image = + CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); expect(image.frameCount, 3); expect(image.repetitionCount, -1); @@ -170,17 +200,24 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { .makeImageAtCurrentFrame(); final CkImage image = CkImage(skImage); expect((await image.toByteData()).lengthInBytes, greaterThan(0)); - expect((await image.toByteData(format: ui.ImageByteFormat.png)).lengthInBytes, greaterThan(0)); + expect( + (await image.toByteData(format: ui.ImageByteFormat.png)) + .lengthInBytes, + greaterThan(0)); }); - test('toByteData with decodeImageFromPixels on videoFrame formats', () async { + test('toByteData with decodeImageFromPixels on videoFrame formats', + () async { // This test ensures that toByteData() returns pixels that can be used by decodeImageFromPixels // for the following videoFrame formats: // [BGRX, I422, I420, I444, BGRA] - final HttpFetchResponse listingResponse = await httpFetch('/test_images/'); - final List testFiles = (await listingResponse.json() as List).cast(); + final HttpFetchResponse listingResponse = + await httpFetch('/test_images/'); + final List testFiles = + (await listingResponse.json() as List).cast(); - Future testDecodeFromPixels(Uint8List pixels, int width, int height) async { + Future testDecodeFromPixels( + Uint8List pixels, int width, int height) async { final Completer completer = Completer(); ui.decodeImageFromPixels( pixels, @@ -205,7 +242,8 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { expect(testFiles, contains(matches(RegExp(r'.*\.bmp')))); for (final String testFile in testFiles) { - final HttpFetchResponse imageResponse = await httpFetch('/test_images/$testFile'); + final HttpFetchResponse imageResponse = + await httpFetch('/test_images/$testFile'); final Uint8List imageData = await imageResponse.asUint8List(); final ui.Codec codec = await skiaInstantiateImageCodec(imageData); expect(codec.frameCount, greaterThan(0)); @@ -213,11 +251,13 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { final ui.FrameInfo frame = await codec.getNextFrame(); final CkImage ckImage = frame.image as CkImage; + print('testing $testFile'); final ByteData imageBytes = await ckImage.toByteData(); expect(imageBytes.lengthInBytes, greaterThan(0)); final Uint8List pixels = imageBytes.buffer.asUint8List(); - final ui.Image testImage = await testDecodeFromPixels(pixels, ckImage.width, ckImage.height); + final ui.Image testImage = + await testDecodeFromPixels(pixels, ckImage.width, ckImage.height); expect(testImage, isNotNull); codec.dispose(); } @@ -228,6 +268,7 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { test('CkImage.clone also clones the VideoFrame', () async { final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( + contentType: 'image/gif', data: kAnimatedGif, debugSource: 'test', ); @@ -238,13 +279,14 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { final CkImage imageClone = ckImage.clone(); expect(imageClone.videoFrame, isNotNull); - final ByteData png = await imageClone.toByteData(format: ui.ImageByteFormat.png); + final ByteData png = + await imageClone.toByteData(format: ui.ImageByteFormat.png); expect(png, isNotNull); // The precise PNG encoding is browser-specific, but we can check the file // signature. expect(detectContentType(png.buffer.asUint8List()), 'image/png'); - // TODO(hterkelsen): Firefox and Safari do not currently support ImageDecoder. + // TODO(hterkelsen): Firefox and Safari do not currently support ImageDecoder. }, skip: isFirefox || isSafari); test('skiaInstantiateWebImageCodec loads an image from the network', @@ -265,8 +307,7 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { expect(image.width, 1); }); - test('instantiateImageCodec respects target image size', - () async { + test('instantiateImageCodec respects target image size', () async { const List> targetSizes = >[ [1, 1], [1, 2], @@ -297,24 +338,25 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { test( 'instantiateImageCodec with multi-frame image supports targetWidth/targetHeight', () async { - final ui.Codec codec = await ui.instantiateImageCodec( - kAnimatedGif, - targetWidth: 2, - targetHeight: 3, - ); - final ui.Image image = (await codec.getNextFrame()).image; + final ui.Codec codec = await ui.instantiateImageCodec( + kAnimatedGif, + targetWidth: 2, + targetHeight: 3, + ); + final ui.Image image = (await codec.getNextFrame()).image; - // expect the re-size did not happen, kAnimatedGif is [1x1] + // expect the re-size did not happen, kAnimatedGif is [1x1] expect(image.width, 2); expect(image.height, 3); - image.dispose(); - codec.dispose(); + image.dispose(); + codec.dispose(); }); test('skiaInstantiateWebImageCodec throws exception on request error', () async { mockHttpFetchResponseFactory = (String url) async { - throw HttpFetchError(url, requestError: 'This is a test request error.'); + throw HttpFetchError(url, + requestError: 'This is a test request error.'); }; try { @@ -346,7 +388,8 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { } }); - test('skiaInstantiateWebImageCodec includes URL in the error for malformed image', + test( + 'skiaInstantiateWebImageCodec includes URL in the error for malformed image', () async { mockHttpFetchResponseFactory = (String url) async { return MockHttpFetchResponse( @@ -357,23 +400,16 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { }; try { - await skiaInstantiateWebImageCodec('http://image-server.com/picture.jpg', null); + await skiaInstantiateWebImageCodec( + 'http://image-server.com/picture.jpg', null); fail('Expected to throw'); } on ImageCodecException catch (exception) { - if (!browserSupportsImageDecoder) { - expect( - exception.toString(), - 'ImageCodecException: Failed to decode image data.\n' - 'Image source: http://image-server.com/picture.jpg', - ); - } else { - expect( - exception.toString(), - 'ImageCodecException: Failed to detect image file format using the file header.\n' - 'File header was empty.\n' - 'Image source: http://image-server.com/picture.jpg', - ); - } + expect( + exception.toString(), + 'ImageCodecException: Failed to detect image file format using the file header.\n' + 'File header was empty.\n' + 'Image source: http://image-server.com/picture.jpg', + ); } }); @@ -382,76 +418,74 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { await ui.instantiateImageCodec(Uint8List(0)); fail('Expected to throw'); } on ImageCodecException catch (exception) { - if (!browserSupportsImageDecoder) { - expect( - exception.toString(), - 'ImageCodecException: Failed to decode image data.\n' - 'Image source: encoded image bytes', - ); - } else { - expect( - exception.toString(), - 'ImageCodecException: Failed to detect image file format using the file header.\n' - 'File header was empty.\n' - 'Image source: encoded image bytes', - ); - } + expect( + exception.toString(), + 'ImageCodecException: Failed to detect image file format using the file header.\n' + 'File header was empty.\n' + 'Image source: encoded image bytes', + ); } }); test('Reports error when failing to decode malformed image data', () async { try { await ui.instantiateImageCodec(Uint8List.fromList([ - 0xFF, 0xD8, 0xFF, 0xDB, 0x00, 0x00, 0x00, + 0xFF, + 0xD8, + 0xFF, + 0xDB, + 0x00, + 0x00, + 0x00, ])); fail('Expected to throw'); } on ImageCodecException catch (exception) { if (!browserSupportsImageDecoder) { expect( - exception.toString(), - 'ImageCodecException: Failed to decode image data.\n' - 'Image source: encoded image bytes' - ); + exception.toString(), + 'ImageCodecException: Failed to decode image data.\n' + 'Image source: encoded image bytes'); } else { expect( - exception.toString(), - // Browser error message is not checked as it can depend on the - // browser engine and version. - matches(RegExp( - r"ImageCodecException: Failed to decode image using the browser's ImageDecoder API.\n" - r'Image source: encoded image bytes\n' - r'Original browser error: .+' - )) - ); + exception.toString(), + // Browser error message is not checked as it can depend on the + // browser engine and version. + matches(RegExp( + r"ImageCodecException: Failed to decode image using the browser's ImageDecoder API.\n" + r'Image source: encoded image bytes\n' + r'Original browser error: .+'))); } } }); - test('Includes file header in the error message when fails to detect file type', () async { + test( + 'Includes file header in the error message when fails to detect file type', + () async { try { await ui.instantiateImageCodec(Uint8List.fromList([ - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x00, ])); fail('Expected to throw'); } on ImageCodecException catch (exception) { - if (!browserSupportsImageDecoder) { - expect( - exception.toString(), - 'ImageCodecException: Failed to decode image data.\n' - 'Image source: encoded image bytes' - ); - } else { - expect( + expect( exception.toString(), 'ImageCodecException: Failed to detect image file format using the file header.\n' 'File header was [0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x00].\n' - 'Image source: encoded image bytes' - ); - } + 'Image source: encoded image bytes'); } }); - test('Provides readable error message when image type is unsupported', () async { + test('Provides readable error message when image type is unsupported', + () async { addTearDown(() { debugContentTypeDetector = null; }); @@ -460,22 +494,29 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { }; try { await ui.instantiateImageCodec(Uint8List.fromList([ - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x00, ])); fail('Expected to throw'); } on ImageCodecException catch (exception) { if (!browserSupportsImageDecoder) { expect( - exception.toString(), - 'ImageCodecException: Failed to decode image data.\n' - 'Image source: encoded image bytes' - ); + exception.toString(), + 'ImageCodecException: Failed to decode image data.\n' + 'Image source: encoded image bytes'); } else { expect( - exception.toString(), - "ImageCodecException: Image file format (unsupported/image-type) is not supported by this browser's ImageDecoder API.\n" - 'Image source: encoded image bytes' - ); + exception.toString(), + "ImageCodecException: Image file format (unsupported/image-type) is not supported by this browser's ImageDecoder API.\n" + 'Image source: encoded image bytes'); } } }); @@ -507,7 +548,8 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { }); test('decodeImageFromPixels respects target image size', () async { - Future testDecodeFromPixels(int width, int height, int targetWidth, int targetHeight) async { + Future testDecodeFromPixels( + int width, int height, int targetWidth, int targetHeight) async { final Completer completer = Completer(); ui.decodeImageFromPixels( Uint8List.fromList(List.filled(width * height * 4, 0)), @@ -536,7 +578,8 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { final int targetWidth = targetSize[0]; final int targetHeight = targetSize[1]; - final ui.Image image = await testDecodeFromPixels(10, 20, targetWidth, targetHeight); + final ui.Image image = + await testDecodeFromPixels(10, 20, targetWidth, targetHeight); expect(image.width, targetWidth); expect(image.height, targetHeight); @@ -544,29 +587,28 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { } }); - test('decodeImageFromPixels upscale when allowUpscaling is false', () async { + test('decodeImageFromPixels upscale when allowUpscaling is false', + () async { Future testDecodeFromPixels(int width, int height) async { final Completer completer = Completer(); ui.decodeImageFromPixels( - Uint8List.fromList(List.filled(width * height * 4, 0)), - width, - height, - ui.PixelFormat.rgba8888, - (ui.Image image) { - completer.complete(image); - }, - targetWidth: 20, - targetHeight: 30, - allowUpscaling: false - ); + Uint8List.fromList(List.filled(width * height * 4, 0)), + width, + height, + ui.PixelFormat.rgba8888, (ui.Image image) { + completer.complete(image); + }, targetWidth: 20, targetHeight: 30, allowUpscaling: false); return completer.future; } + expect(() async => testDecodeFromPixels(10, 20), throwsAssertionError); }); test('Decode test images', () async { - final HttpFetchResponse listingResponse = await httpFetch('/test_images/'); - final List testFiles = (await listingResponse.json() as List).cast(); + final HttpFetchResponse listingResponse = + await httpFetch('/test_images/'); + final List testFiles = + (await listingResponse.json() as List).cast(); // Sanity-check the test file list. If suddenly test files are moved or // deleted, and the test server returns an empty list, or is missing some @@ -579,7 +621,8 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { expect(testFiles, contains(matches(RegExp(r'.*\.bmp')))); for (final String testFile in testFiles) { - final HttpFetchResponse imageResponse = await httpFetch('/test_images/$testFile'); + final HttpFetchResponse imageResponse = + await httpFetch('/test_images/$testFile'); final Uint8List imageData = await imageResponse.asUint8List(); final ui.Codec codec = await skiaInstantiateImageCodec(imageData); expect(codec.frameCount, greaterThan(0)); @@ -595,7 +638,8 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { // Reproduces https://skbug.com/12721 test('decoded image can be read back from picture', () async { - final HttpFetchResponse imageResponse = await httpFetch('/test_images/mandrill_128.png'); + final HttpFetchResponse imageResponse = + await httpFetch('/test_images/mandrill_128.png'); final Uint8List imageData = await imageResponse.asUint8List(); final ui.Codec codec = await skiaInstantiateImageCodec(imageData); final ui.FrameInfo frame = await codec.getNextFrame(); @@ -685,7 +729,8 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { }); test('toImageSync with texture-backed image', () async { - final HttpFetchResponse imageResponse = await httpFetch('/test_images/mandrill_128.png'); + final HttpFetchResponse imageResponse = + await httpFetch('/test_images/mandrill_128.png'); final Uint8List imageData = await imageResponse.asUint8List(); final ui.Codec codec = await skiaInstantiateImageCodec(imageData); final ui.FrameInfo frame = await codec.getNextFrame(); @@ -728,7 +773,8 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { }); test('decoded image can be read back from picture', () async { - final HttpFetchResponse imageResponse = await httpFetch('/test_images/mandrill_128.png'); + final HttpFetchResponse imageResponse = + await httpFetch('/test_images/mandrill_128.png'); final Uint8List imageData = await imageResponse.asUint8List(); final ui.Codec codec = await skiaInstantiateImageCodec(imageData); final ui.FrameInfo frame = await codec.getNextFrame(); @@ -772,19 +818,33 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { test('can detect JPEG from just magic number', () async { expect( - detectContentType( - Uint8List.fromList([0xff, 0xd8, 0xff, 0xe2, 0x0c, 0x58, 0x49, 0x43, 0x43, 0x5f])), - 'image/jpeg'); + detectContentType(Uint8List.fromList([ + 0xff, + 0xd8, + 0xff, + 0xe2, + 0x0c, + 0x58, + 0x49, + 0x43, + 0x43, + 0x5f + ])), + 'image/jpeg'); }); - }, timeout: const Timeout.factor(10)); // These tests can take a while. Allow for a longer timeout. + }, + timeout: const Timeout.factor( + 10)); // These tests can take a while. Allow for a longer timeout. } /// Tests specific to WASM codecs bundled with CanvasKit. void _testCkAnimatedImage() { test('ImageDecoder toByteData(PNG)', () async { - final CkAnimatedImage image = CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); + final CkAnimatedImage image = + CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); final ui.FrameInfo frame = await image.getNextFrame(); - final ByteData? png = await frame.image.toByteData(format: ui.ImageByteFormat.png); + final ByteData? png = + await frame.image.toByteData(format: ui.ImageByteFormat.png); expect(png, isNotNull); // The precise PNG encoding is browser-specific, but we can check the file @@ -793,7 +853,8 @@ void _testCkAnimatedImage() { }); test('CkAnimatedImage toByteData(RGBA)', () async { - final CkAnimatedImage image = CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); + final CkAnimatedImage image = + CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); const List> expectedColors = >[ [255, 0, 0, 255], [0, 255, 0, 255], @@ -814,11 +875,13 @@ void _testCkBrowserImageDecoder() { test('ImageDecoder toByteData(PNG)', () async { final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( + contentType: 'image/gif', data: kAnimatedGif, debugSource: 'test', ); final ui.FrameInfo frame = await image.getNextFrame(); - final ByteData? png = await frame.image.toByteData(format: ui.ImageByteFormat.png); + final ByteData? png = + await frame.image.toByteData(format: ui.ImageByteFormat.png); expect(png, isNotNull); // The precise PNG encoding is browser-specific, but we can check the file @@ -828,6 +891,7 @@ void _testCkBrowserImageDecoder() { test('ImageDecoder toByteData(RGBA)', () async { final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( + contentType: 'image/gif', data: kAnimatedGif, debugSource: 'test', ); @@ -849,6 +913,7 @@ void _testCkBrowserImageDecoder() { debugOverrideWebDecoderExpireDuration(testExpireDuration); final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( + contentType: 'image/gif', data: kAnimatedGif, debugSource: 'test', ); @@ -890,20 +955,52 @@ void _testCkBrowserImageDecoder() { test('ImageDecoder toByteData(translucent PNG)', () async { final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( + contentType: 'image/png', data: kTranslucentPng, debugSource: 'test', ); final ui.FrameInfo frame = await image.getNextFrame(); - ByteData? data = await frame.image.toByteData(format: ui.ImageByteFormat.rawStraightRgba); - expect(data!.buffer.asUint8List(), - [0x22, 0x44, 0x66, 0x80, 0x22, 0x44, 0x66, 0x80, - 0x22, 0x44, 0x66, 0x80, 0x22, 0x44, 0x66, 0x80]); + ByteData? data = await frame.image + .toByteData(format: ui.ImageByteFormat.rawStraightRgba); + expect(data!.buffer.asUint8List(), [ + 0x22, + 0x44, + 0x66, + 0x80, + 0x22, + 0x44, + 0x66, + 0x80, + 0x22, + 0x44, + 0x66, + 0x80, + 0x22, + 0x44, + 0x66, + 0x80 + ]); data = await frame.image.toByteData(); - expect(data!.buffer.asUint8List(), - [0x11, 0x22, 0x33, 0x80, 0x11, 0x22, 0x33, 0x80, - 0x11, 0x22, 0x33, 0x80, 0x11, 0x22, 0x33, 0x80]); + expect(data!.buffer.asUint8List(), [ + 0x11, + 0x22, + 0x33, + 0x80, + 0x11, + 0x22, + 0x33, + 0x80, + 0x11, + 0x22, + 0x33, + 0x80, + 0x11, + 0x22, + 0x33, + 0x80 + ]); }); } From e6d4d44c9febd0f6589308db3ebe3a7fa1a1df93 Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Tue, 4 Jun 2024 11:26:45 -0700 Subject: [PATCH 04/15] wip --- .../src/engine/canvaskit/canvaskit_api.dart | 6 ++-- .../lib/src/engine/canvaskit/image.dart | 36 +++++++++++++++---- .../lib/src/engine/canvaskit/picture.dart | 9 +++-- .../test/canvaskit/image_golden_test.dart | 1 - 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart b/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart index c0284632069d1..ab7ea271e3419 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart @@ -1183,10 +1183,10 @@ extension SkImageExtension on SkImage { matrix?.toJS); @JS('readPixels') - external JSUint8Array _readPixels( + external JSUint8Array? _readPixels( JSNumber srcX, JSNumber srcY, SkImageInfo imageInfo); - Uint8List readPixels(double srcX, double srcY, SkImageInfo imageInfo) => - _readPixels(srcX.toJS, srcY.toJS, imageInfo).toDart; + Uint8List? readPixels(double srcX, double srcY, SkImageInfo imageInfo) => + _readPixels(srcX.toJS, srcY.toJS, imageInfo)?.toDart; @JS('encodeToBytes') external JSUint8Array? _encodeToBytes(); diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart index b83a9d278713c..3ff792c58dcd0 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -412,14 +412,20 @@ class CkImage implements ui.Image, StackTraceDebugger { videoFrame!.format != 'I422') { return readPixelsFromVideoFrame(videoFrame!, format); } else { - return _readPixelsFromSkImage(format); + ByteData? data = _readPixelsFromSkImage(format); + data ??= _readPixelsFromImageViaSurface(format); + if (data == null) { + return Future.error('Failed to encode the image into bytes.'); + } else { + return Future.value(data); + } } } @override ui.ColorSpace get colorSpace => ui.ColorSpace.sRGB; - Future _readPixelsFromSkImage(ui.ImageByteFormat format) { + ByteData? _readPixelsFromSkImage(ui.ImageByteFormat format) { final SkAlphaType alphaType = format == ui.ImageByteFormat.rawStraightRgba ? canvasKit.AlphaType.Unpremul : canvasKit.AlphaType.Premul; @@ -430,11 +436,29 @@ class CkImage implements ui.Image, StackTraceDebugger { colorType: canvasKit.ColorType.RGBA_8888, colorSpace: SkColorSpaceSRGB, ); - if (data == null) { - return Future.error('Failed to encode the image into bytes.'); - } else { - return Future.value(data); + return data; + } + + ByteData? _readPixelsFromImageViaSurface(ui.ImageByteFormat format) { + final Surface surface = CanvasKitRenderer.instance.pictureToImageSurface; + final CkSurface ckSurface = + surface.createOrUpdateSurface(BitmapSize(width, height)); + final CkCanvas ckCanvas = ckSurface.getCanvas(); + ckCanvas.clear(const ui.Color(0x00000000)); + ckCanvas.drawImage(this, ui.Offset.zero, CkPaint()); + final SkImage skImage = ckSurface.surface.makeImageSnapshot(); + final SkImageInfo imageInfo = SkImageInfo( + alphaType: canvasKit.AlphaType.Premul, + colorType: canvasKit.ColorType.RGBA_8888, + colorSpace: SkColorSpaceSRGB, + width: width.toDouble(), + height: height.toDouble(), + ); + final Uint8List? pixels = skImage.readPixels(0, 0, imageInfo); + if (pixels == null) { + throw StateError('Unable to convert read pixels from SkImage.'); } + return pixels.buffer.asByteData(); } static ByteData? _encodeImage({ diff --git a/lib/web_ui/lib/src/engine/canvaskit/picture.dart b/lib/web_ui/lib/src/engine/canvaskit/picture.dart index ba30b55d2fc70..97d4978c4fa64 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/picture.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/picture.dart @@ -101,8 +101,8 @@ class CkPicture implements ScenePicture { assert(debugCheckNotDisposed('Cannot convert picture to image.')); final Surface surface = CanvasKitRenderer.instance.pictureToImageSurface; - final CkSurface ckSurface = surface - .createOrUpdateSurface(BitmapSize(width, height)); + final CkSurface ckSurface = + surface.createOrUpdateSurface(BitmapSize(width, height)); final CkCanvas ckCanvas = ckSurface.getCanvas(); ckCanvas.clear(const ui.Color(0x00000000)); ckCanvas.drawPicture(this); @@ -114,7 +114,10 @@ class CkPicture implements ScenePicture { width: width.toDouble(), height: height.toDouble(), ); - final Uint8List pixels = skImage.readPixels(0, 0, imageInfo); + final Uint8List? pixels = skImage.readPixels(0, 0, imageInfo); + if (pixels == null) { + throw StateError('Unable to convert read pixels from SkImage.'); + } final SkImage? rasterImage = canvasKit.MakeImage(imageInfo, pixels, (4 * width).toDouble()); if (rasterImage == null) { diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index 273e03bbc2421..5738c8af5550a 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -251,7 +251,6 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { final ui.FrameInfo frame = await codec.getNextFrame(); final CkImage ckImage = frame.image as CkImage; - print('testing $testFile'); final ByteData imageBytes = await ckImage.toByteData(); expect(imageBytes.lengthInBytes, greaterThan(0)); From 88d685dd93afb2491ba1c1bfe05cbc90a7a65eb5 Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Tue, 2 Jul 2024 10:16:14 -0700 Subject: [PATCH 05/15] wip --- .../lib/src/engine/canvaskit/image.dart | 140 +++++---- .../engine/canvaskit/image_web_codecs.dart | 53 +++- .../lib/src/engine/canvaskit/renderer.dart | 3 +- .../src/engine/html_image_element_codec.dart | 3 - .../test/canvaskit/image_golden_test.dart | 267 ++++++++++++++++-- 5 files changed, 381 insertions(+), 85 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart index 3ff792c58dcd0..bf1a8d58356a5 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -13,7 +13,7 @@ import 'package:ui/ui_web/src/ui_web.dart' as ui_web; /// Instantiates a [ui.Codec] backed by an `SkAnimatedImage` from Skia. Future skiaInstantiateImageCodec(Uint8List list, - [int? targetWidth, int? targetHeight]) async { + [int? targetWidth, int? targetHeight, bool allowUpscaling = true]) async { ui.Codec codec; // ImageDecoder does not detect image type automatically. It requires us to // tell it what the image type is. @@ -27,43 +27,48 @@ Future skiaInstantiateImageCodec(Uint8List list, ); } else { // TODO(harryterkelsen): If the image is animated, then use Skia to decode. - final DomBlob blob = createDomBlob([list.buffer]); + final DomBlob blob = createDomBlob([list.buffer]); codec = await decodeBlobToCkImage(blob); } return ResizingCodec( codec, targetWidth: targetWidth, targetHeight: targetHeight, + allowUpscaling: allowUpscaling, ); } -class CkHtmlImageElementCodec extends HtmlImageElementCodec { - CkHtmlImageElementCodec(super.src, {super.chunkCallback}); +ui.Image createCkImageFromImageElement( + DomHTMLImageElement image, + int naturalWidth, + int naturalHeight, +) { + final SkImage? skImage = canvasKit.MakeLazyImageFromTextureSourceWithInfo( + image, + SkPartialImageInfo( + alphaType: canvasKit.AlphaType.Premul, + colorType: canvasKit.ColorType.RGBA_8888, + colorSpace: SkColorSpaceSRGB, + width: naturalWidth.toDouble(), + height: naturalHeight.toDouble(), + ), + ); + if (skImage == null) { + throw ImageCodecException( + 'Failed to create image from Image.decode', + ); + } + + return CkImage(skImage, imageElement: image); +} + +class CkImageElementCodec extends HtmlImageElementCodec { + CkImageElementCodec(super.src); @override ui.Image createImageFromHTMLImageElement( - DomHTMLImageElement image, - int naturalWidth, - int naturalHeight, - ) { - final SkImage? skImage = canvasKit.MakeLazyImageFromTextureSourceWithInfo( - image, - SkPartialImageInfo( - alphaType: canvasKit.AlphaType.Premul, - colorType: canvasKit.ColorType.RGBA_8888, - colorSpace: SkColorSpaceSRGB, - width: naturalWidth.toDouble(), - height: naturalHeight.toDouble(), - ), - ); - if (skImage == null) { - throw ImageCodecException( - 'Failed to create image from Image.decode', - ); - } - - return CkImage(skImage); - } + DomHTMLImageElement image, int naturalWidth, int naturalHeight) => + createCkImageFromImageElement(image, naturalWidth, naturalHeight); } class CkImageBlobCodec extends HtmlBlobCodec { @@ -71,28 +76,8 @@ class CkImageBlobCodec extends HtmlBlobCodec { @override ui.Image createImageFromHTMLImageElement( - DomHTMLImageElement image, - int naturalWidth, - int naturalHeight, - ) { - final SkImage? skImage = canvasKit.MakeLazyImageFromTextureSourceWithInfo( - image, - SkPartialImageInfo( - alphaType: canvasKit.AlphaType.Premul, - colorType: canvasKit.ColorType.RGBA_8888, - colorSpace: SkColorSpaceSRGB, - width: naturalWidth.toDouble(), - height: naturalHeight.toDouble(), - ), - ); - if (skImage == null) { - throw ImageCodecException( - 'Failed to create image from Image.decode', - ); - } - - return CkImage(skImage); - } + DomHTMLImageElement image, int naturalWidth, int naturalHeight) => + createCkImageFromImageElement(image, naturalWidth, naturalHeight); } /// Creates and decodes an image using HtmlImageElement. @@ -102,6 +87,12 @@ Future decodeBlobToCkImage(DomBlob blob) async { return codec; } +Future decodeUrlToCkImage(String src) async { + final CkImageElementCodec codec = CkImageElementCodec(src); + await codec.decode(); + return codec; +} + void skiaDecodeImageFromPixels( Uint8List pixels, int width, @@ -221,6 +212,9 @@ CkImage scaleImage(SkImage image, int? targetWidth, int? targetHeight) { return ckImage; } +CkImage scaleImageWithCanvas( + SkImage image, int? targetWidth, int? targetHeight) {} + /// Thrown when the web engine fails to decode an image, either due to a /// network issue, corrupted image contents, or missing codec. class ImageCodecException implements Exception { @@ -238,13 +232,29 @@ const String _kNetworkImageMessage = 'Failed to load network image.'; /// requesting from URI. Future skiaInstantiateWebImageCodec( String url, ui_web.ImageCodecChunkCallback? chunkCallback) async { - final Uint8List list = await fetchImage(url, chunkCallback); - final String contentType = tryDetectContentType(list, url); - if (browserSupportsImageDecoder) { - return CkBrowserImageDecoder.create( - data: list, contentType: contentType, debugSource: url); - } else { - return CkAnimatedImage.decodeFromBytes(list, url); + final CkImageElementCodec imageElementCodec = CkImageElementCodec(url); + try { + await imageElementCodec.decode(); + return imageElementCodec; + } on ImageCodecException { + imageElementCodec.dispose(); + final Uint8List list = await fetchImage(url, chunkCallback); + final String contentType = tryDetectContentType(list, url); + if (browserSupportsImageDecoder) { + return CkBrowserImageDecoder.create( + data: list, contentType: contentType, debugSource: url); + } else { + final DomBlob blob = createDomBlob([list.buffer]); + final CkImageBlobCodec codec = CkImageBlobCodec(blob); + + try { + await codec.decode(); + return codec; + } on ImageCodecException { + codec.dispose(); + return CkAnimatedImage.decodeFromBytes(list, url); + } + } } } @@ -297,12 +307,12 @@ Future readChunked(HttpFetchPayload payload, int contentLength, /// A [ui.Image] backed by an `SkImage` from Skia. class CkImage implements ui.Image, StackTraceDebugger { - CkImage(SkImage skImage, {this.videoFrame}) { + CkImage(SkImage skImage, {this.videoFrame, this.imageElement}) { box = CountedRef(skImage, this, 'SkImage'); _init(); } - CkImage.cloneOf(this.box, {this.videoFrame}) { + CkImage.cloneOf(this.box, {this.videoFrame, this.imageElement}) { _init(); box.ref(this); } @@ -331,6 +341,14 @@ class CkImage implements ui.Image, StackTraceDebugger { /// the video frame until the image is [dispose]d of. VideoFrame? videoFrame; + /// For images which are decoded via an HTML Image element, this field holds + /// the image element from which this image was created. + /// + /// Skia owns the image element and will close it when it's no longer used. + /// However, Flutter co-owns the [SkImage] and therefore it's safe to access + /// the image element until the image is [dispose]d of. + DomHTMLImageElement? imageElement; + /// The underlying Skia image object. /// /// Do not store the returned value. It is memory-managed by [CountedRef]. @@ -374,7 +392,11 @@ class CkImage implements ui.Image, StackTraceDebugger { @override CkImage clone() { assert(_debugCheckIsNotDisposed()); - return CkImage.cloneOf(box, videoFrame: videoFrame?.clone()); + return CkImage.cloneOf( + box, + videoFrame: videoFrame?.clone(), + imageElement: imageElement, + ); } @override @@ -411,6 +433,8 @@ class CkImage implements ui.Image, StackTraceDebugger { videoFrame!.format != 'I444' && videoFrame!.format != 'I422') { return readPixelsFromVideoFrame(videoFrame!, format); + } else if (imageElement != null) { + return readPixelsFromImageElement(imageElement!, format); } else { ByteData? data = _readPixelsFromSkImage(format); data ??= _readPixelsFromImageViaSurface(format); diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart index 4f78f2cfb5bee..de93dea19e99a 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart @@ -62,7 +62,8 @@ class CkBrowserImageDecoder extends BrowserImageDecoder { } } -Future readPixelsFromVideoFrame(VideoFrame videoFrame, ui.ImageByteFormat format) async { +Future readPixelsFromVideoFrame( + VideoFrame videoFrame, ui.ImageByteFormat format) async { if (format == ui.ImageByteFormat.png) { final Uint8List png = await encodeVideoFrameAsPng(videoFrame); return png.buffer.asByteData(); @@ -94,6 +95,17 @@ Future readPixelsFromVideoFrame(VideoFrame videoFrame, ui.ImageByteFor return pixels.asByteData(); } +Future readPixelsFromImageElement( + DomHTMLImageElement imageElement, ui.ImageByteFormat format) async { + if (format == ui.ImageByteFormat.png) { + final Uint8List png = await encodeImageElementAsPng(imageElement); + return png.buffer.asByteData(); + } + + final ByteBuffer pixels = readImageElementPixelsUnmodified(imageElement); + return pixels.asByteData(); +} + /// Mutates the [pixels], converting them from BGRX/BGRA to RGBA. void _bgrToStraightRgba(ByteBuffer pixels, bool isBgrx) { final Uint8List pixelBytes = pixels.asUint8List(); @@ -141,14 +153,16 @@ void _bgrToRawRgba(ByteBuffer pixels) { } } -bool _shouldReadPixelsUnmodified(VideoFrame videoFrame, ui.ImageByteFormat format) { +bool _shouldReadPixelsUnmodified( + VideoFrame videoFrame, ui.ImageByteFormat format) { if (format == ui.ImageByteFormat.rawUnmodified) { return true; } // Do not convert if the requested format is RGBA and the video frame is // encoded as either RGBA or RGBX. - final bool isRgbFrame = videoFrame.format == 'RGBA' || videoFrame.format == 'RGBX'; + final bool isRgbFrame = + videoFrame.format == 'RGBA' || videoFrame.format == 'RGBX'; return format == ui.ImageByteFormat.rawStraightRgba && isRgbFrame; } @@ -166,13 +180,40 @@ Future readVideoFramePixelsUnmodified(VideoFrame videoFrame) async { return destination.toDart.buffer; } +ByteBuffer readImageElementPixelsUnmodified(DomHTMLImageElement imageElement) { + final int width = imageElement.naturalWidth.toInt(); + final int height = imageElement.naturalHeight.toInt(); + + final DomCanvasElement htmlCanvas = + createDomCanvasElement(width: width, height: height); + final DomCanvasRenderingContext2D ctx = + htmlCanvas.getContext('2d')! as DomCanvasRenderingContext2D; + ctx.drawImage(imageElement, 0, 0); + final DomImageData imageData = ctx.getImageData(0, 0, width, height); + return imageData.data.buffer; +} + Future encodeVideoFrameAsPng(VideoFrame videoFrame) async { final int width = videoFrame.displayWidth.toInt(); final int height = videoFrame.displayHeight.toInt(); - final DomCanvasElement canvas = createDomCanvasElement(width: width, height: - height); + final DomCanvasElement canvas = + createDomCanvasElement(width: width, height: height); final DomCanvasRenderingContext2D ctx = canvas.context2D; ctx.drawImage(videoFrame, 0, 0); - final String pngBase64 = canvas.toDataURL().substring('data:image/png;base64,'.length); + final String pngBase64 = + canvas.toDataURL().substring('data:image/png;base64,'.length); + return base64.decode(pngBase64); +} + +Future encodeImageElementAsPng( + DomHTMLImageElement imageElement) async { + final int width = imageElement.naturalWidth.toInt(); + final int height = imageElement.naturalHeight.toInt(); + final DomCanvasElement canvas = + createDomCanvasElement(width: width, height: height); + final DomCanvasRenderingContext2D ctx = canvas.context2D; + ctx.drawImage(imageElement, 0, 0); + final String pngBase64 = + canvas.toDataURL().substring('data:image/png;base64,'.length); return base64.decode(pngBase64); } diff --git a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart index 7f2e9ba31e684..ae411dd939ab5 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart @@ -221,7 +221,8 @@ class CanvasKitRenderer implements Renderer { {int? targetWidth, int? targetHeight, bool allowUpscaling = true}) async => - skiaInstantiateImageCodec(list, targetWidth, targetHeight); + skiaInstantiateImageCodec( + list, targetWidth, targetHeight, allowUpscaling); @override Future instantiateImageCodecFromUrl(Uri uri, diff --git a/lib/web_ui/lib/src/engine/html_image_element_codec.dart b/lib/web_ui/lib/src/engine/html_image_element_codec.dart index e21fd6c5917af..40ceb6418a562 100644 --- a/lib/web_ui/lib/src/engine/html_image_element_codec.dart +++ b/lib/web_ui/lib/src/engine/html_image_element_codec.dart @@ -8,9 +8,6 @@ import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; import 'package:ui/ui_web/src/ui_web.dart' as ui_web; -import 'dom.dart'; -import 'safe_browser_api.dart'; - Object? get _jsImageDecodeFunction => getJsProperty( getJsProperty( getJsProperty(domWindow, 'Image'), diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index 5738c8af5550a..80b2fa08c57b4 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -3,6 +3,7 @@ // found in the LICENSE file. import 'dart:async'; +import 'dart:js_interop'; import 'dart:typed_data'; import 'package:test/bootstrap/browser.dart'; @@ -19,6 +20,122 @@ void main() { internalBootstrapBrowserTest(() => testMain); } +abstract class TestCodec { + TestCodec({required this.description}); + final String description; + + ui.Codec? _cachedCodec; + + Future getCodec() async => _cachedCodec ??= await createCodec(); + + Future createCodec(); +} + +abstract class TestFileCodec extends TestCodec { + TestFileCodec.fromTestFile(this.testFile, {required super.description}); + + final String testFile; + + Future createCodecFromTestFile(String testFile); + + @override + Future createCodec() { + return createCodecFromTestFile(testFile); + } +} + +class UrlTestCodec extends TestFileCodec { + UrlTestCodec(super.testFile, this.codecFactory, String function) + : super.fromTestFile(description: 'created with $function("$testFile")'); + + final Future Function(String) codecFactory; + + @override + Future createCodecFromTestFile(String testFile) { + return codecFactory(testFile); + } +} + +class FetchTestCodec extends TestFileCodec { + FetchTestCodec( + super.testFile, + this.codecFactory, + String function, + ) : super.fromTestFile( + description: 'created with $function from bytes ' + 'fetch()\'ed from "$testFile"'); + + final Future Function(Uint8List) codecFactory; + + @override + Future createCodecFromTestFile(String testFile) async { + final HttpFetchResponse response = await httpFetch(testFile); + + if (!response.hasPayload) { + throw Exception('Unable to fetch() image test file "$testFile"'); + } + + final Uint8List responseBytes = await response.asUint8List(); + return codecFactory(responseBytes); + } +} + +class BitmapTestCodec extends TestFileCodec { + BitmapTestCodec( + super.testFile, + this.codecFactory, + String function, + ) : super.fromTestFile( + description: 'created with $function from ImageBitmap' + ' created from "$testFile"'); + + final Future Function(DomImageBitmap) codecFactory; + + @override + Future createCodecFromTestFile(String testFile) async { + final DomHTMLImageElement imageElement = createDomHTMLImageElement(); + imageElement.src = testFile; + setJsProperty(imageElement, 'decoding', 'async'); + + await imageElement.decode(); + + final DomImageBitmap bitmap = + await createImageBitmap(imageElement as JSObject, ( + x: 0, + y: 0, + width: imageElement.naturalWidth.toInt(), + height: imageElement.naturalHeight.toInt(), + )); + + final ui.Image image = await codecFactory(bitmap); + return BitmapSingleFrameCodec(bitmap, image); + } +} + +class BitmapSingleFrameCodec implements ui.Codec { + BitmapSingleFrameCodec(this.bitmap, this.image); + + final DomImageBitmap bitmap; + final ui.Image image; + + @override + void dispose() { + image.dispose(); + bitmap.close(); + } + + @override + int get frameCount => 1; + + @override + Future getNextFrame() async { + return SingleFrameInfo(image); + } + + @override + int get repetitionCount => 0; +} + void testMain() { group('CanvasKit Images', () { setUpCanvasKitTest(withImplicitView: true); @@ -27,13 +144,133 @@ void testMain() { mockHttpFetchResponseFactory = null; }); + group('Codecs', () { + List? testCodecs; + + setUpAll(() async { + Future> createTestCodecs( + {int testTargetWidth = 300, int testTargetHeight = 300}) async { + final HttpFetchResponse listingResponse = + await httpFetch('/test_images/'); + final List testFiles = + (await listingResponse.json() as List).cast(); + + // Sanity-check the test file list. If suddenly test files are moved or + // deleted, and the test server returns an empty list, or is missing some + // important test files, we want to know. + expect(testFiles, isNotEmpty); + expect(testFiles, contains(matches(RegExp(r'.*\.jpg')))); + expect(testFiles, contains(matches(RegExp(r'.*\.png')))); + expect(testFiles, contains(matches(RegExp(r'.*\.gif')))); + expect(testFiles, contains(matches(RegExp(r'.*\.webp')))); + expect(testFiles, contains(matches(RegExp(r'.*\.bmp')))); + + final List testCodecs = []; + for (final String testFile in testFiles) { + testCodecs.add(UrlTestCodec( + testFile, + (String file) => renderer.instantiateImageCodecFromUrl( + Uri.tryParse('/test_images/$file')!, + ), + 'renderer.instantiateImageFromUrl', + )); + } + for (final String testFile in testFiles) { + testCodecs.add( + FetchTestCodec( + '/test_images/$testFile', + (Uint8List bytes) => renderer.instantiateImageCodec(bytes), + 'renderer.instantiateImageCodec', + ), + ); + testCodecs.add( + FetchTestCodec( + '/test_images/$testFile', + (Uint8List bytes) => renderer.instantiateImageCodec( + bytes, + targetWidth: testTargetWidth, + targetHeight: testTargetHeight, + ), + 'renderer.instantiateImageCodec (target size ' + '$testTargetWidth x $testTargetHeight)', + ), + ); + } + for (final String testFile in testFiles) { + testCodecs.add( + BitmapTestCodec( + 'test_images/$testFile', + (DomImageBitmap bitmap) async => + renderer.createImageFromImageBitmap(bitmap), + 'renderer.createImageFromImageBitmap', + ), + ); + } + + return testCodecs; + } + + testCodecs = await createTestCodecs(); + }); + + test('can create images', () async { + for (final TestCodec testCodec in testCodecs!) { + try { + final ui.Codec codec = await testCodec.getCodec(); + final ui.FrameInfo frameInfo = await codec.getNextFrame(); + final ui.Image image = frameInfo.image; + expect(image, isNotNull); + expect(image.width, isNonZero); + expect(image.height, isNonZero); + expect(image.colorSpace, isNotNull); + } catch (e) { + throw TestFailure( + 'Failed to get image for ${testCodec.description}: $e'); + } + } + }); + + test('images can be decoded with toByteData', () async { + for (final TestCodec testCodec in testCodecs!) { + ui.Image image; + try { + final ui.Codec codec = await testCodec.getCodec(); + final ui.FrameInfo frameInfo = await codec.getNextFrame(); + image = frameInfo.image; + } catch (e) { + throw TestFailure( + 'Failed to get image for ${testCodec.description}: $e'); + } + + final ByteData? byteData = await image.toByteData(); + expect( + byteData, + isNotNull, + reason: '${testCodec.description} toByteData() should not be null', + ); + expect( + byteData!.lengthInBytes, + isNonZero, + reason: '${testCodec.description} toByteData() should not be empty', + ); + expect( + byteData.buffer.asUint8List().any((int byte) => byte > 0), + isTrue, + reason: '${testCodec.description} toByteData() should ' + 'contain nonzero value', + ); + } + }); + + //testCodecs?.forEach(_testForImageCodecs); + }); + _testCkAnimatedImage(); - _testForImageCodecs(useBrowserImageDecoder: false); - if (browserSupportsImageDecoder) { - _testForImageCodecs(useBrowserImageDecoder: true); - _testCkBrowserImageDecoder(); - } + // if (browserSupportsImageDecoder) { + // _testForImageCodecs(useBrowserImageDecoder: true); + // _testCkBrowserImageDecoder(); + // } test('isAvif', () { expect(isAvif(Uint8List.fromList([])), isFalse); @@ -84,14 +321,13 @@ void testMain() { }, skip: isSafari); } -void _testForImageCodecs({required bool useBrowserImageDecoder}) { - final String mode = useBrowserImageDecoder ? 'webcodecs' : 'wasm'; +void _testForImageCodecs(TestCodec codec) { + final String mode = codec.description; final List warnings = []; late void Function(String) oldPrintWarning; group('($mode)', () { setUp(() { - browserSupportsImageDecoder = useBrowserImageDecoder; warnings.clear(); }); @@ -206,11 +442,7 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { greaterThan(0)); }); - test('toByteData with decodeImageFromPixels on videoFrame formats', - () async { - // This test ensures that toByteData() returns pixels that can be used by decodeImageFromPixels - // for the following videoFrame formats: - // [BGRX, I422, I420, I444, BGRA] + test('toByteData with decodeImageFromPixels', () async { final HttpFetchResponse listingResponse = await httpFetch('/test_images/'); final List testFiles = @@ -253,6 +485,9 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { final CkImage ckImage = frame.image as CkImage; final ByteData imageBytes = await ckImage.toByteData(); expect(imageBytes.lengthInBytes, greaterThan(0)); + // Sanity check the image isn't all transparent black. + expect(imageBytes.buffer.asUint32List().any((int byte) => byte != 0), + isTrue); final Uint8List pixels = imageBytes.buffer.asUint8List(); final ui.Image testImage = @@ -260,10 +495,9 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { expect(testImage, isNotNull); codec.dispose(); } - // TODO(hterkelsen): Firefox and Safari do not currently support ImageDecoder. // TODO(jacksongardner): enable on wasm // see https://github.com/flutter/flutter/issues/118334 - }, skip: isFirefox || isSafari || isWasm); + }, skip: isWasm); test('CkImage.clone also clones the VideoFrame', () async { final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( @@ -285,8 +519,7 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { // The precise PNG encoding is browser-specific, but we can check the file // signature. expect(detectContentType(png.buffer.asUint8List()), 'image/png'); - // TODO(hterkelsen): Firefox and Safari do not currently support ImageDecoder. - }, skip: isFirefox || isSafari); + }, skip: !browserSupportsImageDecoder); test('skiaInstantiateWebImageCodec loads an image from the network', () async { From 8a1a7ce5a8cb842dfeac770ca9629d2a0adf86e5 Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Wed, 10 Jul 2024 13:16:29 -0700 Subject: [PATCH 06/15] WIP --- lib/web_ui/lib/semantics.dart | 8 +- .../lib/src/engine/canvaskit/image.dart | 104 ++++++++- lib/web_ui/lib/src/engine/image_decoder.dart | 19 +- .../test/canvaskit/image_golden_test.dart | 204 +++++++++++------- 4 files changed, 242 insertions(+), 93 deletions(-) diff --git a/lib/web_ui/lib/semantics.dart b/lib/web_ui/lib/semantics.dart index 34242b9830508..9635a89a2638c 100644 --- a/lib/web_ui/lib/semantics.dart +++ b/lib/web_ui/lib/semantics.dart @@ -280,14 +280,14 @@ class SemanticsUpdateBuilder { required List decreasedValueAttributes, required String hint, required List hintAttributes, - String? tooltip, - TextDirection? textDirection, + required String? tooltip, + required TextDirection? textDirection, required Float64List transform, required Int32List childrenInTraversalOrder, required Int32List childrenInHitTestOrder, required Int32List additionalActions, - int headingLevel = 0, - String? linkUrl, + required int headingLevel, + required String? linkUrl, }) { if (transform.length != 16) { throw ArgumentError('transform argument must have 16 entries.'); diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart index bf1a8d58356a5..84786011272e6 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -30,7 +30,7 @@ Future skiaInstantiateImageCodec(Uint8List list, final DomBlob blob = createDomBlob([list.buffer]); codec = await decodeBlobToCkImage(blob); } - return ResizingCodec( + return CkResizingCodec( codec, targetWidth: targetWidth, targetHeight: targetHeight, @@ -38,6 +38,105 @@ Future skiaInstantiateImageCodec(Uint8List list, ); } +/// A resizing codec which uses an HTML element to scale the image if +/// it is backed by an HTML Image element. +class CkResizingCodec extends ResizingCodec { + CkResizingCodec( + super.delegate, { + super.targetWidth, + super.targetHeight, + super.allowUpscaling, + }); + + @override + ui.Image scaleImage( + ui.Image image, { + int? targetWidth, + int? targetHeight, + bool allowUpscaling = true, + }) { + final CkImage ckImage = image as CkImage; + if (ckImage.imageElement == null) { + return scaleImageIfNeeded( + image, + targetWidth: targetWidth, + targetHeight: targetHeight, + allowUpscaling: allowUpscaling, + ); + } else { + return _scaleImageUsingDomCanvas( + ckImage, + targetWidth: targetWidth, + targetHeight: targetHeight, + allowUpscaling: allowUpscaling, + ); + } + } + + CkImage _scaleImageUsingDomCanvas( + CkImage image, { + int? targetWidth, + int? targetHeight, + bool allowUpscaling = true, + }) { + assert(image.imageElement != null); + final int width = image.width; + final int height = image.height; + final ui.Size? scaledSize = + scaledImageSize(width, height, targetWidth, targetHeight); + if (scaledSize == null) { + return image; + } + if (!allowUpscaling && + (scaledSize.width > width || scaledSize.height > height)) { + return image; + } + + final int scaledWidth = scaledSize.width.toInt(); + final int scaledHeight = scaledSize.height.toInt(); + + final DomCanvasElement htmlCanvas = createDomCanvasElement( + width: scaledWidth, + height: scaledHeight, + ); + final DomCanvasRenderingContext2D ctx = + htmlCanvas.getContext('2d')! as DomCanvasRenderingContext2D; + ctx.drawImage( + image.imageElement!, + 0, + 0, + width, + height, + 0, + 0, + scaledWidth, + scaledHeight, + ); + final DomImageData imageData = + ctx.getImageData(0, 0, scaledWidth, scaledHeight); + final Uint8List pixels = imageData.data.buffer.asUint8List(); + + final SkImage? skImage = canvasKit.MakeImage( + SkImageInfo( + width: scaledWidth.toDouble(), + height: scaledHeight.toDouble(), + colorType: canvasKit.ColorType.RGBA_8888, + alphaType: canvasKit.AlphaType.Premul, + colorSpace: SkColorSpaceSRGB, + ), + pixels, + (4 * scaledWidth).toDouble(), + ); + + if (skImage == null) { + domWindow.console.warn('Failed to scale image.'); + return image; + } + + return CkImage(skImage); + } +} + ui.Image createCkImageFromImageElement( DomHTMLImageElement image, int naturalWidth, @@ -212,9 +311,6 @@ CkImage scaleImage(SkImage image, int? targetWidth, int? targetHeight) { return ckImage; } -CkImage scaleImageWithCanvas( - SkImage image, int? targetWidth, int? targetHeight) {} - /// Thrown when the web engine fails to decode an image, either due to a /// network issue, corrupted image contents, or missing codec. class ImageCodecException implements Exception { diff --git a/lib/web_ui/lib/src/engine/image_decoder.dart b/lib/web_ui/lib/src/engine/image_decoder.dart index 94d564952dd1a..1804f5c4a2af3 100644 --- a/lib/web_ui/lib/src/engine/image_decoder.dart +++ b/lib/web_ui/lib/src/engine/image_decoder.dart @@ -381,18 +381,31 @@ class ResizingCodec implements ui.Codec { final ui.FrameInfo frameInfo = await delegate.getNextFrame(); return AnimatedImageFrameInfo( frameInfo.duration, - scaleImageIfNeeded(frameInfo.image, + scaleImage(frameInfo.image, targetWidth: targetWidth, targetHeight: targetHeight, allowUpscaling: allowUpscaling), ); } + ui.Image scaleImage( + ui.Image image, { + int? targetWidth, + int? targetHeight, + bool allowUpscaling = true, + }) => + scaleImageIfNeeded( + image, + targetWidth: targetWidth, + targetHeight: targetHeight, + allowUpscaling: allowUpscaling, + ); + @override int get repetitionCount => delegate.frameCount; } -ui.Size? _scaledSize( +ui.Size? scaledImageSize( int width, int height, int? targetWidth, @@ -427,7 +440,7 @@ ui.Image scaleImageIfNeeded( final int width = image.width; final int height = image.height; final ui.Size? scaledSize = - _scaledSize(width, height, targetWidth, targetHeight); + scaledImageSize(width, height, targetWidth, targetHeight); if (scaledSize == null) { return image; } diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index 80b2fa08c57b4..e5dd4166bbae5 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -16,13 +16,24 @@ import '../common/matchers.dart'; import 'common.dart'; import 'test_data.dart'; +List? testCodecs; + void main() { internalBootstrapBrowserTest(() => testMain); } +String _getBaseName(String fileName) { + final int lastDotIndex = fileName.lastIndexOf('.'); + if (lastDotIndex == -1) { + return fileName; + } + return fileName.substring(0, lastDotIndex).replaceAll('/', '_'); +} + abstract class TestCodec { TestCodec({required this.description}); final String description; + String get goldenFileName; ui.Codec? _cachedCodec; @@ -45,9 +56,16 @@ abstract class TestFileCodec extends TestCodec { } class UrlTestCodec extends TestFileCodec { - UrlTestCodec(super.testFile, this.codecFactory, String function) + UrlTestCodec(super.testFile, this.codecFactory, this.function) : super.fromTestFile(description: 'created with $function("$testFile")'); + final String function; + + @override + String get goldenFileName { + return '${_getBaseName(testFile)}_${function.replaceAll(' ', '_')}_url.png'; + } + final Future Function(String) codecFactory; @override @@ -60,11 +78,18 @@ class FetchTestCodec extends TestFileCodec { FetchTestCodec( super.testFile, this.codecFactory, - String function, + this.function, ) : super.fromTestFile( description: 'created with $function from bytes ' 'fetch()\'ed from "$testFile"'); + final String function; + + @override + String get goldenFileName { + return '${_getBaseName(testFile)}_${function.replaceAll(' ', '_')}_fetched_bytes.png'; + } + final Future Function(Uint8List) codecFactory; @override @@ -84,10 +109,16 @@ class BitmapTestCodec extends TestFileCodec { BitmapTestCodec( super.testFile, this.codecFactory, - String function, + this.function, ) : super.fromTestFile( description: 'created with $function from ImageBitmap' ' created from "$testFile"'); + final String function; + + @override + String get goldenFileName { + return '${_getBaseName(testFile)}_${function.replaceAll(' ', '_')}_imagebitmap.png'; + } final Future Function(DomImageBitmap) codecFactory; @@ -136,7 +167,66 @@ class BitmapSingleFrameCodec implements ui.Codec { int get repetitionCount => 0; } -void testMain() { +void testMain() async { + Future> createTestCodecs( + {int testTargetWidth = 300, int testTargetHeight = 300}) async { + final HttpFetchResponse listingResponse = await httpFetch('/test_images/'); + final List testFiles = + (await listingResponse.json() as List).cast(); + + // Sanity-check the test file list. If suddenly test files are moved or + // deleted, and the test server returns an empty list, or is missing some + // important test files, we want to know. + assert(testFiles.isNotEmpty); + assert(testFiles.any((String testFile) => testFile.endsWith('.jpg'))); + assert(testFiles.any((String testFile) => testFile.endsWith('.png'))); + assert(testFiles.any((String testFile) => testFile.endsWith('.gif'))); + assert(testFiles.any((String testFile) => testFile.endsWith('.webp'))); + assert(testFiles.any((String testFile) => testFile.endsWith('.bmp'))); + + final List testCodecs = []; + for (final String testFile in testFiles) { + testCodecs.add(UrlTestCodec( + testFile, + (String file) => renderer.instantiateImageCodecFromUrl( + Uri.tryParse('/test_images/$file')!, + ), + 'renderer.instantiateImageFromUrl', + )); + testCodecs.add( + FetchTestCodec( + '/test_images/$testFile', + (Uint8List bytes) => renderer.instantiateImageCodec(bytes), + 'renderer.instantiateImageCodec', + ), + ); + testCodecs.add( + FetchTestCodec( + '/test_images/$testFile', + (Uint8List bytes) => renderer.instantiateImageCodec( + bytes, + targetWidth: testTargetWidth, + targetHeight: testTargetHeight, + ), + 'renderer.instantiateImageCodec ' + '($testTargetWidth x $testTargetHeight)', + ), + ); + testCodecs.add( + BitmapTestCodec( + 'test_images/$testFile', + (DomImageBitmap bitmap) async => + renderer.createImageFromImageBitmap(bitmap), + 'renderer.createImageFromImageBitmap', + ), + ); + } + + return testCodecs; + } + + testCodecs = await createTestCodecs(); + group('CanvasKit Images', () { setUpCanvasKitTest(withImplicitView: true); @@ -145,76 +235,8 @@ void testMain() { }); group('Codecs', () { - List? testCodecs; - - setUpAll(() async { - Future> createTestCodecs( - {int testTargetWidth = 300, int testTargetHeight = 300}) async { - final HttpFetchResponse listingResponse = - await httpFetch('/test_images/'); - final List testFiles = - (await listingResponse.json() as List).cast(); - - // Sanity-check the test file list. If suddenly test files are moved or - // deleted, and the test server returns an empty list, or is missing some - // important test files, we want to know. - expect(testFiles, isNotEmpty); - expect(testFiles, contains(matches(RegExp(r'.*\.jpg')))); - expect(testFiles, contains(matches(RegExp(r'.*\.png')))); - expect(testFiles, contains(matches(RegExp(r'.*\.gif')))); - expect(testFiles, contains(matches(RegExp(r'.*\.webp')))); - expect(testFiles, contains(matches(RegExp(r'.*\.bmp')))); - - final List testCodecs = []; - for (final String testFile in testFiles) { - testCodecs.add(UrlTestCodec( - testFile, - (String file) => renderer.instantiateImageCodecFromUrl( - Uri.tryParse('/test_images/$file')!, - ), - 'renderer.instantiateImageFromUrl', - )); - } - for (final String testFile in testFiles) { - testCodecs.add( - FetchTestCodec( - '/test_images/$testFile', - (Uint8List bytes) => renderer.instantiateImageCodec(bytes), - 'renderer.instantiateImageCodec', - ), - ); - testCodecs.add( - FetchTestCodec( - '/test_images/$testFile', - (Uint8List bytes) => renderer.instantiateImageCodec( - bytes, - targetWidth: testTargetWidth, - targetHeight: testTargetHeight, - ), - 'renderer.instantiateImageCodec (target size ' - '$testTargetWidth x $testTargetHeight)', - ), - ); - } - for (final String testFile in testFiles) { - testCodecs.add( - BitmapTestCodec( - 'test_images/$testFile', - (DomImageBitmap bitmap) async => - renderer.createImageFromImageBitmap(bitmap), - 'renderer.createImageFromImageBitmap', - ), - ); - } - - return testCodecs; - } - - testCodecs = await createTestCodecs(); - }); - - test('can create images', () async { - for (final TestCodec testCodec in testCodecs!) { + for (final TestCodec testCodec in testCodecs!) { + test('${testCodec.description} can create an image', () async { try { final ui.Codec codec = await testCodec.getCodec(); final ui.FrameInfo frameInfo = await codec.getNextFrame(); @@ -227,11 +249,31 @@ void testMain() { throw TestFailure( 'Failed to get image for ${testCodec.description}: $e'); } - } - }); + }); - test('images can be decoded with toByteData', () async { - for (final TestCodec testCodec in testCodecs!) { + test('${testCodec.description} can draw an image', () async { + ui.Image image; + try { + final ui.Codec codec = await testCodec.getCodec(); + final ui.FrameInfo frameInfo = await codec.getNextFrame(); + image = frameInfo.image; + } catch (e) { + throw TestFailure( + 'Failed to get image for ${testCodec.description}: $e'); + } + final LayerSceneBuilder sb = LayerSceneBuilder(); + final CkPictureRecorder recorder = CkPictureRecorder(); + final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); + canvas.drawImage(image as CkImage, ui.Offset.zero, CkPaint()); + sb.addPicture(ui.Offset.zero, recorder.endRecording()); + + await matchSceneGolden(testCodec.goldenFileName, sb.build(), + region: ui.Rect.fromLTRB( + 0, 0, image.width.toDouble(), image.height.toDouble())); + }); + + test('${testCodec.description} can be decoded with toByteData', + () async { ui.Image image; try { final ui.Codec codec = await testCodec.getCodec(); @@ -259,10 +301,8 @@ void testMain() { reason: '${testCodec.description} toByteData() should ' 'contain nonzero value', ); - } - }); - - //testCodecs?.forEach(_testForImageCodecs); + }); + } }); _testCkAnimatedImage(); From d900deda8f45a433c6a5a759cc2c501b3da848af Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Thu, 11 Jul 2024 12:57:13 -0700 Subject: [PATCH 07/15] Fix ImageBitmap toByteData case --- .../lib/src/engine/canvaskit/image.dart | 25 +++++++++- .../engine/canvaskit/image_web_codecs.dart | 48 +++++++++++-------- .../lib/src/engine/canvaskit/renderer.dart | 2 +- lib/web_ui/lib/src/engine/dom.dart | 2 +- .../test/canvaskit/image_golden_test.dart | 21 -------- 5 files changed, 54 insertions(+), 44 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart index 84786011272e6..92054134e6abe 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -403,7 +403,8 @@ Future readChunked(HttpFetchPayload payload, int contentLength, /// A [ui.Image] backed by an `SkImage` from Skia. class CkImage implements ui.Image, StackTraceDebugger { - CkImage(SkImage skImage, {this.videoFrame, this.imageElement}) { + CkImage(SkImage skImage, + {this.videoFrame, this.imageElement, this.imageBitmap}) { box = CountedRef(skImage, this, 'SkImage'); _init(); } @@ -445,6 +446,14 @@ class CkImage implements ui.Image, StackTraceDebugger { /// the image element until the image is [dispose]d of. DomHTMLImageElement? imageElement; + /// For images which are decoded via an HTML ImageBitmap, this field holds + /// the image element from which this image was created. + /// + /// Skia owns the image bitmap and will close it when it's no longer used. + /// However, Flutter co-owns the [SkImage] and therefore it's safe to access + /// the image element until the image is [dispose]d of. + DomImageBitmap? imageBitmap; + /// The underlying Skia image object. /// /// Do not store the returned value. It is memory-managed by [CountedRef]. @@ -530,7 +539,19 @@ class CkImage implements ui.Image, StackTraceDebugger { videoFrame!.format != 'I422') { return readPixelsFromVideoFrame(videoFrame!, format); } else if (imageElement != null) { - return readPixelsFromImageElement(imageElement!, format); + return readPixelsFromDomImageSource( + imageElement!, + format, + imageElement!.naturalWidth.toInt(), + imageElement!.naturalHeight.toInt(), + ); + } else if (imageBitmap != null) { + return readPixelsFromDomImageSource( + imageBitmap!, + format, + imageBitmap!.width.toDartInt, + imageBitmap!.height.toDartInt, + ); } else { ByteData? data = _readPixelsFromSkImage(format); data ??= _readPixelsFromImageViaSurface(format); diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart index de93dea19e99a..c62393e58fc48 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart @@ -65,7 +65,11 @@ class CkBrowserImageDecoder extends BrowserImageDecoder { Future readPixelsFromVideoFrame( VideoFrame videoFrame, ui.ImageByteFormat format) async { if (format == ui.ImageByteFormat.png) { - final Uint8List png = await encodeVideoFrameAsPng(videoFrame); + final Uint8List png = await encodeDomImageSourceAsPng( + videoFrame, + videoFrame.displayWidth.toInt(), + videoFrame.displayHeight.toInt(), + ); return png.buffer.asByteData(); } @@ -95,14 +99,23 @@ Future readPixelsFromVideoFrame( return pixels.asByteData(); } -Future readPixelsFromImageElement( - DomHTMLImageElement imageElement, ui.ImageByteFormat format) async { +Future readPixelsFromDomImageSource( + DomCanvasImageSource imageSource, + ui.ImageByteFormat format, + int width, + int height, +) async { if (format == ui.ImageByteFormat.png) { - final Uint8List png = await encodeImageElementAsPng(imageElement); + final Uint8List png = await encodeDomImageSourceAsPng( + imageSource, + width, + height, + ); return png.buffer.asByteData(); } - final ByteBuffer pixels = readImageElementPixelsUnmodified(imageElement); + final ByteBuffer pixels = + readDomImageSourcePixelsUnmodified(imageSource, width, height); return pixels.asByteData(); } @@ -193,26 +206,23 @@ ByteBuffer readImageElementPixelsUnmodified(DomHTMLImageElement imageElement) { return imageData.data.buffer; } -Future encodeVideoFrameAsPng(VideoFrame videoFrame) async { - final int width = videoFrame.displayWidth.toInt(); - final int height = videoFrame.displayHeight.toInt(); - final DomCanvasElement canvas = +ByteBuffer readDomImageSourcePixelsUnmodified( + DomCanvasImageSource imageSource, int width, int height) { + final DomCanvasElement htmlCanvas = createDomCanvasElement(width: width, height: height); - final DomCanvasRenderingContext2D ctx = canvas.context2D; - ctx.drawImage(videoFrame, 0, 0); - final String pngBase64 = - canvas.toDataURL().substring('data:image/png;base64,'.length); - return base64.decode(pngBase64); + final DomCanvasRenderingContext2D ctx = + htmlCanvas.getContext('2d')! as DomCanvasRenderingContext2D; + ctx.drawImage(imageSource, 0, 0); + final DomImageData imageData = ctx.getImageData(0, 0, width, height); + return imageData.data.buffer; } -Future encodeImageElementAsPng( - DomHTMLImageElement imageElement) async { - final int width = imageElement.naturalWidth.toInt(); - final int height = imageElement.naturalHeight.toInt(); +Future encodeDomImageSourceAsPng( + DomCanvasImageSource imageSource, int width, int height) async { final DomCanvasElement canvas = createDomCanvasElement(width: width, height: height); final DomCanvasRenderingContext2D ctx = canvas.context2D; - ctx.drawImage(imageElement, 0, 0); + ctx.drawImage(imageSource, 0, 0); final String pngBase64 = canvas.toDataURL().substring('data:image/png;base64,'.length); return base64.decode(pngBase64); diff --git a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart index d04df038b00a0..cbe1fcffe5d7e 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart @@ -237,7 +237,7 @@ class CanvasKitRenderer implements Renderer { if (skImage == null) { throw Exception('Failed to convert image bitmap to an SkImage.'); } - return CkImage(skImage); + return CkImage(skImage, imageBitmap: imageBitmap); } @override diff --git a/lib/web_ui/lib/src/engine/dom.dart b/lib/web_ui/lib/src/engine/dom.dart index 3239c62173a94..45f7732006059 100644 --- a/lib/web_ui/lib/src/engine/dom.dart +++ b/lib/web_ui/lib/src/engine/dom.dart @@ -1540,7 +1540,7 @@ extension DomImageDataExtension on DomImageData { @JS('ImageBitmap') @staticInterop -class DomImageBitmap {} +class DomImageBitmap implements DomCanvasImageSource {} extension DomImageBitmapExtension on DomImageBitmap { external JSNumber get width; diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index e5dd4166bbae5..43615bd259265 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -251,27 +251,6 @@ void testMain() async { } }); - test('${testCodec.description} can draw an image', () async { - ui.Image image; - try { - final ui.Codec codec = await testCodec.getCodec(); - final ui.FrameInfo frameInfo = await codec.getNextFrame(); - image = frameInfo.image; - } catch (e) { - throw TestFailure( - 'Failed to get image for ${testCodec.description}: $e'); - } - final LayerSceneBuilder sb = LayerSceneBuilder(); - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.drawImage(image as CkImage, ui.Offset.zero, CkPaint()); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - - await matchSceneGolden(testCodec.goldenFileName, sb.build(), - region: ui.Rect.fromLTRB( - 0, 0, image.width.toDouble(), image.height.toDouble())); - }); - test('${testCodec.description} can be decoded with toByteData', () async { ui.Image image; From 355891b38974d3ec17db335698c861e84e52745d Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Thu, 11 Jul 2024 13:25:50 -0700 Subject: [PATCH 08/15] Fall back to Skia decoding for GIF and WEBP images --- lib/web_ui/lib/src/engine/canvaskit/image.dart | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart index 92054134e6abe..3bb6ab048ff79 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -27,8 +27,16 @@ Future skiaInstantiateImageCodec(Uint8List list, ); } else { // TODO(harryterkelsen): If the image is animated, then use Skia to decode. - final DomBlob blob = createDomBlob([list.buffer]); - codec = await decodeBlobToCkImage(blob); + // This is currently too conservative, assuming all GIF and WEBP images are + // animated. We should detect if they are actually animated by reading the + // image headers. + if (contentType == 'image/gif' || contentType == 'image/webp') { + codec = CkAnimatedImage.decodeFromBytes(list, 'decoded bytes', + targetWidth: targetWidth, targetHeight: targetHeight); + } else { + final DomBlob blob = createDomBlob([list.buffer]); + codec = await decodeBlobToCkImage(blob); + } } return CkResizingCodec( codec, From d317302fdd5bf1fd6f94694c643e6f56df5a2319 Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Thu, 11 Jul 2024 13:40:22 -0700 Subject: [PATCH 09/15] Undo semantics change --- lib/web_ui/lib/semantics.dart | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/web_ui/lib/semantics.dart b/lib/web_ui/lib/semantics.dart index 9635a89a2638c..34242b9830508 100644 --- a/lib/web_ui/lib/semantics.dart +++ b/lib/web_ui/lib/semantics.dart @@ -280,14 +280,14 @@ class SemanticsUpdateBuilder { required List decreasedValueAttributes, required String hint, required List hintAttributes, - required String? tooltip, - required TextDirection? textDirection, + String? tooltip, + TextDirection? textDirection, required Float64List transform, required Int32List childrenInTraversalOrder, required Int32List childrenInHitTestOrder, required Int32List additionalActions, - required int headingLevel, - required String? linkUrl, + int headingLevel = 0, + String? linkUrl, }) { if (transform.length != 16) { throw ArgumentError('transform argument must have 16 entries.'); From c579c8ef9c8d4c4b2216348b2049b421cca3369a Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Thu, 11 Jul 2024 13:47:49 -0700 Subject: [PATCH 10/15] Remove unused readImageElementPixelsUnmodified method --- .../lib/src/engine/canvaskit/image_web_codecs.dart | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart index c62393e58fc48..4f81bb0a1c90d 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart @@ -193,19 +193,6 @@ Future readVideoFramePixelsUnmodified(VideoFrame videoFrame) async { return destination.toDart.buffer; } -ByteBuffer readImageElementPixelsUnmodified(DomHTMLImageElement imageElement) { - final int width = imageElement.naturalWidth.toInt(); - final int height = imageElement.naturalHeight.toInt(); - - final DomCanvasElement htmlCanvas = - createDomCanvasElement(width: width, height: height); - final DomCanvasRenderingContext2D ctx = - htmlCanvas.getContext('2d')! as DomCanvasRenderingContext2D; - ctx.drawImage(imageElement, 0, 0); - final DomImageData imageData = ctx.getImageData(0, 0, width, height); - return imageData.data.buffer; -} - ByteBuffer readDomImageSourcePixelsUnmodified( DomCanvasImageSource imageSource, int width, int height) { final DomCanvasElement htmlCanvas = From dd5c8d62ac7f2acd8d1ad08a0b7b5fe0b2f897a7 Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Thu, 11 Jul 2024 13:50:30 -0700 Subject: [PATCH 11/15] remove outdated comment in test --- lib/web_ui/test/canvaskit/image_golden_test.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index 43615bd259265..d78ca2429a941 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -596,7 +596,6 @@ void _testForImageCodecs(TestCodec codec) { ); final ui.Image image = (await codec.getNextFrame()).image; - // expect the re-size did not happen, kAnimatedGif is [1x1] expect(image.width, 2); expect(image.height, 3); image.dispose(); From f3997c160b8763340da65e111deaed01b6bb18bd Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Thu, 11 Jul 2024 14:48:31 -0700 Subject: [PATCH 12/15] delete old unused tests --- .../test/canvaskit/image_golden_test.dart | 921 +----------------- 1 file changed, 4 insertions(+), 917 deletions(-) diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index d78ca2429a941..7d4c1296be281 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -33,7 +33,6 @@ String _getBaseName(String fileName) { abstract class TestCodec { TestCodec({required this.description}); final String description; - String get goldenFileName; ui.Codec? _cachedCodec; @@ -56,16 +55,9 @@ abstract class TestFileCodec extends TestCodec { } class UrlTestCodec extends TestFileCodec { - UrlTestCodec(super.testFile, this.codecFactory, this.function) + UrlTestCodec(super.testFile, this.codecFactory, String function) : super.fromTestFile(description: 'created with $function("$testFile")'); - final String function; - - @override - String get goldenFileName { - return '${_getBaseName(testFile)}_${function.replaceAll(' ', '_')}_url.png'; - } - final Future Function(String) codecFactory; @override @@ -78,18 +70,11 @@ class FetchTestCodec extends TestFileCodec { FetchTestCodec( super.testFile, this.codecFactory, - this.function, + String function, ) : super.fromTestFile( description: 'created with $function from bytes ' 'fetch()\'ed from "$testFile"'); - final String function; - - @override - String get goldenFileName { - return '${_getBaseName(testFile)}_${function.replaceAll(' ', '_')}_fetched_bytes.png'; - } - final Future Function(Uint8List) codecFactory; @override @@ -109,16 +94,10 @@ class BitmapTestCodec extends TestFileCodec { BitmapTestCodec( super.testFile, this.codecFactory, - this.function, + String function, ) : super.fromTestFile( description: 'created with $function from ImageBitmap' ' created from "$testFile"'); - final String function; - - @override - String get goldenFileName { - return '${_getBaseName(testFile)}_${function.replaceAll(' ', '_')}_imagebitmap.png'; - } final Future Function(DomImageBitmap) codecFactory; @@ -167,7 +146,7 @@ class BitmapSingleFrameCodec implements ui.Codec { int get repetitionCount => 0; } -void testMain() async { +Future testMain() async { Future> createTestCodecs( {int testTargetWidth = 300, int testTargetHeight = 300}) async { final HttpFetchResponse listingResponse = await httpFetch('/test_images/'); @@ -286,11 +265,6 @@ void testMain() async { _testCkAnimatedImage(); - // if (browserSupportsImageDecoder) { - // _testForImageCodecs(useBrowserImageDecoder: true); - // _testCkBrowserImageDecoder(); - // } - test('isAvif', () { expect(isAvif(Uint8List.fromList([])), isFalse); expect(isAvif(Uint8List.fromList([1, 2, 3])), isFalse); @@ -340,753 +314,6 @@ void testMain() async { }, skip: isSafari); } -void _testForImageCodecs(TestCodec codec) { - final String mode = codec.description; - final List warnings = []; - late void Function(String) oldPrintWarning; - - group('($mode)', () { - setUp(() { - warnings.clear(); - }); - - setUpAll(() { - oldPrintWarning = printWarning; - printWarning = (String warning) { - warnings.add(warning); - }; - }); - - tearDown(() { - debugResetBrowserSupportsImageDecoder(); - }); - - tearDownAll(() { - printWarning = oldPrintWarning; - }); - - test('CkAnimatedImage can be explicitly disposed of', () { - final CkAnimatedImage image = - CkAnimatedImage.decodeFromBytes(kTransparentImage, 'test'); - expect(image.debugDisposed, isFalse); - image.dispose(); - expect(image.debugDisposed, isTrue); - - // Disallow usage after disposal - expect(() => image.frameCount, throwsAssertionError); - expect(() => image.repetitionCount, throwsAssertionError); - expect(() => image.getNextFrame(), throwsAssertionError); - - // Disallow double-dispose. - expect(() => image.dispose(), throwsAssertionError); - }); - - test('CkAnimatedImage iterates frames correctly', () async { - final CkAnimatedImage image = - CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); - expect(image.frameCount, 3); - expect(image.repetitionCount, -1); - - final ui.FrameInfo frame1 = await image.getNextFrame(); - await expectFrameData(frame1, [255, 0, 0, 255]); - final ui.FrameInfo frame2 = await image.getNextFrame(); - await expectFrameData(frame2, [0, 255, 0, 255]); - final ui.FrameInfo frame3 = await image.getNextFrame(); - await expectFrameData(frame3, [0, 0, 255, 255]); - }); - - test('CkImage toString', () { - final SkImage skImage = - canvasKit.MakeAnimatedImageFromEncoded(kTransparentImage)! - .makeImageAtCurrentFrame(); - final CkImage image = CkImage(skImage); - expect(image.toString(), '[1×1]'); - image.dispose(); - }); - - test('CkImage can be explicitly disposed of', () { - final SkImage skImage = - canvasKit.MakeAnimatedImageFromEncoded(kTransparentImage)! - .makeImageAtCurrentFrame(); - final CkImage image = CkImage(skImage); - expect(image.debugDisposed, isFalse); - expect(image.box.isDisposed, isFalse); - image.dispose(); - expect(image.debugDisposed, isTrue); - expect(image.box.isDisposed, isTrue); - - // Disallow double-dispose. - expect(() => image.dispose(), throwsAssertionError); - }); - - test('CkImage can be explicitly disposed of when cloned', () async { - final SkImage skImage = - canvasKit.MakeAnimatedImageFromEncoded(kTransparentImage)! - .makeImageAtCurrentFrame(); - final CkImage image = CkImage(skImage); - final CountedRef box = image.box; - expect(box.refCount, 1); - expect(box.debugGetStackTraces().length, 1); - - final CkImage clone = image.clone(); - expect(box.refCount, 2); - expect(box.debugGetStackTraces().length, 2); - - expect(image.isCloneOf(clone), isTrue); - expect(box.isDisposed, isFalse); - - expect(skImage.isDeleted(), isFalse); - image.dispose(); - expect(box.refCount, 1); - expect(box.isDisposed, isFalse); - - expect(skImage.isDeleted(), isFalse); - clone.dispose(); - expect(box.refCount, 0); - expect(box.isDisposed, isTrue); - - expect(skImage.isDeleted(), isTrue); - expect(box.debugGetStackTraces().length, 0); - }); - - test('CkImage toByteData', () async { - final SkImage skImage = - canvasKit.MakeAnimatedImageFromEncoded(kTransparentImage)! - .makeImageAtCurrentFrame(); - final CkImage image = CkImage(skImage); - expect((await image.toByteData()).lengthInBytes, greaterThan(0)); - expect( - (await image.toByteData(format: ui.ImageByteFormat.png)) - .lengthInBytes, - greaterThan(0)); - }); - - test('toByteData with decodeImageFromPixels', () async { - final HttpFetchResponse listingResponse = - await httpFetch('/test_images/'); - final List testFiles = - (await listingResponse.json() as List).cast(); - - Future testDecodeFromPixels( - Uint8List pixels, int width, int height) async { - final Completer completer = Completer(); - ui.decodeImageFromPixels( - pixels, - width, - height, - ui.PixelFormat.rgba8888, - (ui.Image image) { - completer.complete(image); - }, - ); - return completer.future; - } - - // Sanity-check the test file list. If suddenly test files are moved or - // deleted, and the test server returns an empty list, or is missing some - // important test files, we want to know. - expect(testFiles, isNotEmpty); - expect(testFiles, contains(matches(RegExp(r'.*\.jpg')))); - expect(testFiles, contains(matches(RegExp(r'.*\.png')))); - expect(testFiles, contains(matches(RegExp(r'.*\.gif')))); - expect(testFiles, contains(matches(RegExp(r'.*\.webp')))); - expect(testFiles, contains(matches(RegExp(r'.*\.bmp')))); - - for (final String testFile in testFiles) { - final HttpFetchResponse imageResponse = - await httpFetch('/test_images/$testFile'); - final Uint8List imageData = await imageResponse.asUint8List(); - final ui.Codec codec = await skiaInstantiateImageCodec(imageData); - expect(codec.frameCount, greaterThan(0)); - expect(codec.repetitionCount, isNotNull); - - final ui.FrameInfo frame = await codec.getNextFrame(); - final CkImage ckImage = frame.image as CkImage; - final ByteData imageBytes = await ckImage.toByteData(); - expect(imageBytes.lengthInBytes, greaterThan(0)); - // Sanity check the image isn't all transparent black. - expect(imageBytes.buffer.asUint32List().any((int byte) => byte != 0), - isTrue); - - final Uint8List pixels = imageBytes.buffer.asUint8List(); - final ui.Image testImage = - await testDecodeFromPixels(pixels, ckImage.width, ckImage.height); - expect(testImage, isNotNull); - codec.dispose(); - } - // TODO(jacksongardner): enable on wasm - // see https://github.com/flutter/flutter/issues/118334 - }, skip: isWasm); - - test('CkImage.clone also clones the VideoFrame', () async { - final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( - contentType: 'image/gif', - data: kAnimatedGif, - debugSource: 'test', - ); - final ui.FrameInfo frame = await image.getNextFrame(); - final CkImage ckImage = frame.image as CkImage; - expect(ckImage.videoFrame, isNotNull); - - final CkImage imageClone = ckImage.clone(); - expect(imageClone.videoFrame, isNotNull); - - final ByteData png = - await imageClone.toByteData(format: ui.ImageByteFormat.png); - expect(png, isNotNull); - - // The precise PNG encoding is browser-specific, but we can check the file - // signature. - expect(detectContentType(png.buffer.asUint8List()), 'image/png'); - }, skip: !browserSupportsImageDecoder); - - test('skiaInstantiateWebImageCodec loads an image from the network', - () async { - mockHttpFetchResponseFactory = (String url) async { - return MockHttpFetchResponse( - url: url, - status: 200, - payload: MockHttpFetchPayload(byteBuffer: kTransparentImage.buffer), - ); - }; - - final ui.Codec codec = await skiaInstantiateWebImageCodec( - 'http://image-server.com/picture.jpg', null); - expect(codec.frameCount, 1); - final ui.Image image = (await codec.getNextFrame()).image; - expect(image.height, 1); - expect(image.width, 1); - }); - - test('instantiateImageCodec respects target image size', () async { - const List> targetSizes = >[ - [1, 1], - [1, 2], - [2, 3], - [3, 4], - [4, 4], - [10, 20], - ]; - - for (final List targetSize in targetSizes) { - final int targetWidth = targetSize[0]; - final int targetHeight = targetSize[1]; - - final ui.Codec codec = await ui.instantiateImageCodec( - k4x4PngImage, - targetWidth: targetWidth, - targetHeight: targetHeight, - ); - - final ui.Image image = (await codec.getNextFrame()).image; - expect(image.width, targetWidth); - expect(image.height, targetHeight); - image.dispose(); - codec.dispose(); - } - }); - - test( - 'instantiateImageCodec with multi-frame image supports targetWidth/targetHeight', - () async { - final ui.Codec codec = await ui.instantiateImageCodec( - kAnimatedGif, - targetWidth: 2, - targetHeight: 3, - ); - final ui.Image image = (await codec.getNextFrame()).image; - - expect(image.width, 2); - expect(image.height, 3); - image.dispose(); - codec.dispose(); - }); - - test('skiaInstantiateWebImageCodec throws exception on request error', - () async { - mockHttpFetchResponseFactory = (String url) async { - throw HttpFetchError(url, - requestError: 'This is a test request error.'); - }; - - try { - await skiaInstantiateWebImageCodec('url-does-not-matter', null); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - expect( - exception.toString(), - 'ImageCodecException: Failed to load network image.\n' - 'Image URL: url-does-not-matter\n' - 'Trying to load an image from another domain? Find answers at:\n' - 'https://flutter.dev/docs/development/platform-integration/web-images', - ); - } - }); - - test('skiaInstantiateWebImageCodec throws exception on HTTP error', - () async { - try { - await skiaInstantiateWebImageCodec('/does-not-exist.jpg', null); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - expect( - exception.toString(), - 'ImageCodecException: Failed to load network image.\n' - 'Image URL: /does-not-exist.jpg\n' - 'Server response code: 404', - ); - } - }); - - test( - 'skiaInstantiateWebImageCodec includes URL in the error for malformed image', - () async { - mockHttpFetchResponseFactory = (String url) async { - return MockHttpFetchResponse( - url: url, - status: 200, - payload: MockHttpFetchPayload(byteBuffer: Uint8List(0).buffer), - ); - }; - - try { - await skiaInstantiateWebImageCodec( - 'http://image-server.com/picture.jpg', null); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - expect( - exception.toString(), - 'ImageCodecException: Failed to detect image file format using the file header.\n' - 'File header was empty.\n' - 'Image source: http://image-server.com/picture.jpg', - ); - } - }); - - test('Reports error when failing to decode empty image data', () async { - try { - await ui.instantiateImageCodec(Uint8List(0)); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - expect( - exception.toString(), - 'ImageCodecException: Failed to detect image file format using the file header.\n' - 'File header was empty.\n' - 'Image source: encoded image bytes', - ); - } - }); - - test('Reports error when failing to decode malformed image data', () async { - try { - await ui.instantiateImageCodec(Uint8List.fromList([ - 0xFF, - 0xD8, - 0xFF, - 0xDB, - 0x00, - 0x00, - 0x00, - ])); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - if (!browserSupportsImageDecoder) { - expect( - exception.toString(), - 'ImageCodecException: Failed to decode image data.\n' - 'Image source: encoded image bytes'); - } else { - expect( - exception.toString(), - // Browser error message is not checked as it can depend on the - // browser engine and version. - matches(RegExp( - r"ImageCodecException: Failed to decode image using the browser's ImageDecoder API.\n" - r'Image source: encoded image bytes\n' - r'Original browser error: .+'))); - } - } - }); - - test( - 'Includes file header in the error message when fails to detect file type', - () async { - try { - await ui.instantiateImageCodec(Uint8List.fromList([ - 0x01, - 0x02, - 0x03, - 0x04, - 0x05, - 0x06, - 0x07, - 0x08, - 0x09, - 0x00, - ])); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - expect( - exception.toString(), - 'ImageCodecException: Failed to detect image file format using the file header.\n' - 'File header was [0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x00].\n' - 'Image source: encoded image bytes'); - } - }); - - test('Provides readable error message when image type is unsupported', - () async { - addTearDown(() { - debugContentTypeDetector = null; - }); - debugContentTypeDetector = (_) { - return 'unsupported/image-type'; - }; - try { - await ui.instantiateImageCodec(Uint8List.fromList([ - 0x01, - 0x02, - 0x03, - 0x04, - 0x05, - 0x06, - 0x07, - 0x08, - 0x09, - 0x00, - ])); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - if (!browserSupportsImageDecoder) { - expect( - exception.toString(), - 'ImageCodecException: Failed to decode image data.\n' - 'Image source: encoded image bytes'); - } else { - expect( - exception.toString(), - "ImageCodecException: Image file format (unsupported/image-type) is not supported by this browser's ImageDecoder API.\n" - 'Image source: encoded image bytes'); - } - } - }); - - test('decodeImageFromPixels', () async { - Future testDecodeFromPixels(int width, int height) async { - final Completer completer = Completer(); - ui.decodeImageFromPixels( - Uint8List.fromList(List.filled(width * height * 4, 0)), - width, - height, - ui.PixelFormat.rgba8888, - (ui.Image image) { - completer.complete(image); - }, - ); - return completer.future; - } - - final ui.Image image1 = await testDecodeFromPixels(10, 20); - expect(image1, isNotNull); - expect(image1.width, 10); - expect(image1.height, 20); - - final ui.Image image2 = await testDecodeFromPixels(40, 100); - expect(image2, isNotNull); - expect(image2.width, 40); - expect(image2.height, 100); - }); - - test('decodeImageFromPixels respects target image size', () async { - Future testDecodeFromPixels( - int width, int height, int targetWidth, int targetHeight) async { - final Completer completer = Completer(); - ui.decodeImageFromPixels( - Uint8List.fromList(List.filled(width * height * 4, 0)), - width, - height, - ui.PixelFormat.rgba8888, - (ui.Image image) { - completer.complete(image); - }, - targetWidth: targetWidth, - targetHeight: targetHeight, - ); - return completer.future; - } - - const List> targetSizes = >[ - [1, 1], - [1, 2], - [2, 3], - [3, 4], - [4, 4], - [10, 20], - ]; - - for (final List targetSize in targetSizes) { - final int targetWidth = targetSize[0]; - final int targetHeight = targetSize[1]; - - final ui.Image image = - await testDecodeFromPixels(10, 20, targetWidth, targetHeight); - - expect(image.width, targetWidth); - expect(image.height, targetHeight); - image.dispose(); - } - }); - - test('decodeImageFromPixels upscale when allowUpscaling is false', - () async { - Future testDecodeFromPixels(int width, int height) async { - final Completer completer = Completer(); - ui.decodeImageFromPixels( - Uint8List.fromList(List.filled(width * height * 4, 0)), - width, - height, - ui.PixelFormat.rgba8888, (ui.Image image) { - completer.complete(image); - }, targetWidth: 20, targetHeight: 30, allowUpscaling: false); - return completer.future; - } - - expect(() async => testDecodeFromPixels(10, 20), throwsAssertionError); - }); - - test('Decode test images', () async { - final HttpFetchResponse listingResponse = - await httpFetch('/test_images/'); - final List testFiles = - (await listingResponse.json() as List).cast(); - - // Sanity-check the test file list. If suddenly test files are moved or - // deleted, and the test server returns an empty list, or is missing some - // important test files, we want to know. - expect(testFiles, isNotEmpty); - expect(testFiles, contains(matches(RegExp(r'.*\.jpg')))); - expect(testFiles, contains(matches(RegExp(r'.*\.png')))); - expect(testFiles, contains(matches(RegExp(r'.*\.gif')))); - expect(testFiles, contains(matches(RegExp(r'.*\.webp')))); - expect(testFiles, contains(matches(RegExp(r'.*\.bmp')))); - - for (final String testFile in testFiles) { - final HttpFetchResponse imageResponse = - await httpFetch('/test_images/$testFile'); - final Uint8List imageData = await imageResponse.asUint8List(); - final ui.Codec codec = await skiaInstantiateImageCodec(imageData); - expect(codec.frameCount, greaterThan(0)); - expect(codec.repetitionCount, isNotNull); - for (int i = 0; i < codec.frameCount; i++) { - final ui.FrameInfo frame = await codec.getNextFrame(); - expect(frame.duration, isNotNull); - expect(frame.image, isNotNull); - } - codec.dispose(); - } - }); - - // Reproduces https://skbug.com/12721 - test('decoded image can be read back from picture', () async { - final HttpFetchResponse imageResponse = - await httpFetch('/test_images/mandrill_128.png'); - final Uint8List imageData = await imageResponse.asUint8List(); - final ui.Codec codec = await skiaInstantiateImageCodec(imageData); - final ui.FrameInfo frame = await codec.getNextFrame(); - final CkImage image = frame.image as CkImage; - - final CkImage snapshot; - { - final LayerSceneBuilder sb = LayerSceneBuilder(); - sb.pushOffset(10, 10); - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.drawRect( - const ui.Rect.fromLTRB(5, 5, 20, 20), - CkPaint(), - ); - canvas.drawImage(image, ui.Offset.zero, CkPaint()); - canvas.drawRect( - const ui.Rect.fromLTRB(90, 90, 105, 105), - CkPaint(), - ); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - sb.pop(); - snapshot = await sb.build().toImage(150, 150) as CkImage; - } - - { - final LayerSceneBuilder sb = LayerSceneBuilder(); - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.drawImage(snapshot, ui.Offset.zero, CkPaint()); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - - await matchSceneGolden( - 'canvaskit_read_back_decoded_image_$mode.png', sb.build(), - region: const ui.Rect.fromLTRB(0, 0, 150, 150)); - } - - image.dispose(); - codec.dispose(); - }); - - // This is a regression test for the issues with transferring textures from - // one GL context to another, such as: - // - // * https://github.com/flutter/flutter/issues/86809 - // * https://github.com/flutter/flutter/issues/91881 - test('the same image can be rendered on difference surfaces', () async { - ui_web.platformViewRegistry.registerViewFactory( - 'test-platform-view', - (int viewId) => createDomHTMLDivElement()..id = 'view-0', - ); - await createPlatformView(0, 'test-platform-view'); - - final ui.Codec codec = await ui.instantiateImageCodec(k4x4PngImage); - final CkImage image = (await codec.getNextFrame()).image as CkImage; - - final LayerSceneBuilder sb = LayerSceneBuilder(); - sb.pushOffset(4, 4); - { - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.save(); - canvas.scale(16, 16); - canvas.drawImage(image, ui.Offset.zero, CkPaint()); - canvas.restore(); - canvas.drawParagraph(makeSimpleText('1'), const ui.Offset(4, 4)); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - } - sb.addPlatformView(0, width: 100, height: 100); - sb.pushOffset(20, 20); - { - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.save(); - canvas.scale(16, 16); - canvas.drawImage(image, ui.Offset.zero, CkPaint()); - canvas.restore(); - canvas.drawParagraph(makeSimpleText('2'), const ui.Offset(2, 2)); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - } - - await matchSceneGolden( - 'canvaskit_cross_gl_context_image_$mode.png', sb.build(), - region: const ui.Rect.fromLTRB(0, 0, 100, 100)); - - await disposePlatformView(0); - }); - - test('toImageSync with texture-backed image', () async { - final HttpFetchResponse imageResponse = - await httpFetch('/test_images/mandrill_128.png'); - final Uint8List imageData = await imageResponse.asUint8List(); - final ui.Codec codec = await skiaInstantiateImageCodec(imageData); - final ui.FrameInfo frame = await codec.getNextFrame(); - final CkImage mandrill = frame.image as CkImage; - final ui.PictureRecorder recorder = ui.PictureRecorder(); - final ui.Canvas canvas = ui.Canvas(recorder); - canvas.drawImageRect( - mandrill, - const ui.Rect.fromLTWH(0, 0, 128, 128), - const ui.Rect.fromLTWH(0, 0, 128, 128), - ui.Paint(), - ); - final ui.Picture picture = recorder.endRecording(); - final ui.Image image = picture.toImageSync(50, 50); - - expect(image.width, 50); - expect(image.height, 50); - - final ByteData? data = await image.toByteData(); - expect(data, isNotNull); - expect(data!.lengthInBytes, 50 * 50 * 4); - expect(data.buffer.asUint32List().any((int byte) => byte != 0), isTrue); - - final LayerSceneBuilder sb = LayerSceneBuilder(); - sb.pushOffset(0, 0); - { - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.save(); - canvas.drawImage(image as CkImage, ui.Offset.zero, CkPaint()); - canvas.restore(); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - } - - await matchSceneGolden( - 'canvaskit_picture_texture_toimage.png', sb.build(), - region: const ui.Rect.fromLTRB(0, 0, 128, 128)); - mandrill.dispose(); - codec.dispose(); - }); - - test('decoded image can be read back from picture', () async { - final HttpFetchResponse imageResponse = - await httpFetch('/test_images/mandrill_128.png'); - final Uint8List imageData = await imageResponse.asUint8List(); - final ui.Codec codec = await skiaInstantiateImageCodec(imageData); - final ui.FrameInfo frame = await codec.getNextFrame(); - final CkImage image = frame.image as CkImage; - - final CkImage snapshot; - { - final LayerSceneBuilder sb = LayerSceneBuilder(); - sb.pushOffset(10, 10); - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.drawRect( - const ui.Rect.fromLTRB(5, 5, 20, 20), - CkPaint(), - ); - canvas.drawImage(image, ui.Offset.zero, CkPaint()); - canvas.drawRect( - const ui.Rect.fromLTRB(90, 90, 105, 105), - CkPaint(), - ); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - sb.pop(); - snapshot = await sb.build().toImage(150, 150) as CkImage; - } - - { - final LayerSceneBuilder sb = LayerSceneBuilder(); - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.drawImage(snapshot, ui.Offset.zero, CkPaint()); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - - await matchSceneGolden( - 'canvaskit_read_back_decoded_image_$mode.png', sb.build(), - region: const ui.Rect.fromLTRB(0, 0, 150, 150)); - } - - image.dispose(); - codec.dispose(); - }); - - test('can detect JPEG from just magic number', () async { - expect( - detectContentType(Uint8List.fromList([ - 0xff, - 0xd8, - 0xff, - 0xe2, - 0x0c, - 0x58, - 0x49, - 0x43, - 0x43, - 0x5f - ])), - 'image/jpeg'); - }); - }, - timeout: const Timeout.factor( - 10)); // These tests can take a while. Allow for a longer timeout. -} - /// Tests specific to WASM codecs bundled with CanvasKit. void _testCkAnimatedImage() { test('ImageDecoder toByteData(PNG)', () async { @@ -1118,143 +345,3 @@ void _testCkAnimatedImage() { } }); } - -/// Tests specific to browser image codecs based functionality. -void _testCkBrowserImageDecoder() { - assert(browserSupportsImageDecoder); - - test('ImageDecoder toByteData(PNG)', () async { - final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( - contentType: 'image/gif', - data: kAnimatedGif, - debugSource: 'test', - ); - final ui.FrameInfo frame = await image.getNextFrame(); - final ByteData? png = - await frame.image.toByteData(format: ui.ImageByteFormat.png); - expect(png, isNotNull); - - // The precise PNG encoding is browser-specific, but we can check the file - // signature. - expect(detectContentType(png!.buffer.asUint8List()), 'image/png'); - }); - - test('ImageDecoder toByteData(RGBA)', () async { - final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( - contentType: 'image/gif', - data: kAnimatedGif, - debugSource: 'test', - ); - const List> expectedColors = >[ - [255, 0, 0, 255], - [0, 255, 0, 255], - [0, 0, 255, 255], - ]; - for (int i = 0; i < image.frameCount; i++) { - final ui.FrameInfo frame = await image.getNextFrame(); - final ByteData? rgba = await frame.image.toByteData(); - expect(rgba, isNotNull); - expect(rgba!.buffer.asUint8List(), expectedColors[i]); - } - }); - - test('ImageDecoder expires after inactivity', () async { - const Duration testExpireDuration = Duration(milliseconds: 100); - debugOverrideWebDecoderExpireDuration(testExpireDuration); - - final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( - contentType: 'image/gif', - data: kAnimatedGif, - debugSource: 'test', - ); - - // ImageDecoder is initialized eagerly to populate `frameCount` and - // `repetitionCount`. - final ImageDecoder? decoder1 = image.debugCachedWebDecoder; - expect(decoder1, isNotNull); - expect(image.frameCount, 3); - expect(image.repetitionCount, -1); - - // A frame can be decoded right away. - final ui.FrameInfo frame1 = await image.getNextFrame(); - await expectFrameData(frame1, [255, 0, 0, 255]); - expect(frame1, isNotNull); - - // The cached decoder should not yet expire. - await Future.delayed(testExpireDuration ~/ 2); - expect(image.debugCachedWebDecoder, same(decoder1)); - - // Now it expires. - await Future.delayed(testExpireDuration); - expect(image.debugCachedWebDecoder, isNull); - - // A new decoder should be created upon the next frame request. - final ui.FrameInfo frame2 = await image.getNextFrame(); - - // Check that the cached decoder is indeed new. - final ImageDecoder? decoder2 = image.debugCachedWebDecoder; - expect(decoder2, isNot(same(decoder1))); - await expectFrameData(frame2, [0, 255, 0, 255]); - - // Check that the new decoder remembers the last frame index. - final ui.FrameInfo frame3 = await image.getNextFrame(); - await expectFrameData(frame3, [0, 0, 255, 255]); - - debugRestoreWebDecoderExpireDuration(); - }); - - test('ImageDecoder toByteData(translucent PNG)', () async { - final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( - contentType: 'image/png', - data: kTranslucentPng, - debugSource: 'test', - ); - final ui.FrameInfo frame = await image.getNextFrame(); - - ByteData? data = await frame.image - .toByteData(format: ui.ImageByteFormat.rawStraightRgba); - expect(data!.buffer.asUint8List(), [ - 0x22, - 0x44, - 0x66, - 0x80, - 0x22, - 0x44, - 0x66, - 0x80, - 0x22, - 0x44, - 0x66, - 0x80, - 0x22, - 0x44, - 0x66, - 0x80 - ]); - - data = await frame.image.toByteData(); - expect(data!.buffer.asUint8List(), [ - 0x11, - 0x22, - 0x33, - 0x80, - 0x11, - 0x22, - 0x33, - 0x80, - 0x11, - 0x22, - 0x33, - 0x80, - 0x11, - 0x22, - 0x33, - 0x80 - ]); - }); -} - -Future expectFrameData(ui.FrameInfo frame, List data) async { - final ByteData frameData = (await frame.image.toByteData())!; - expect(frameData.buffer.asUint8List(), Uint8List.fromList(data)); -} From 52b83308836be26ca57a308f05b2de605f1e388c Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Thu, 11 Jul 2024 15:26:20 -0700 Subject: [PATCH 13/15] fix analysis errors --- lib/web_ui/test/canvaskit/image_golden_test.dart | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index 7d4c1296be281..94d169ae7afcc 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -10,9 +10,7 @@ import 'package:test/bootstrap/browser.dart'; import 'package:test/test.dart'; import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; -import 'package:ui/ui_web/src/ui_web.dart' as ui_web; -import '../common/matchers.dart'; import 'common.dart'; import 'test_data.dart'; @@ -22,14 +20,6 @@ void main() { internalBootstrapBrowserTest(() => testMain); } -String _getBaseName(String fileName) { - final int lastDotIndex = fileName.lastIndexOf('.'); - if (lastDotIndex == -1) { - return fileName; - } - return fileName.substring(0, lastDotIndex).replaceAll('/', '_'); -} - abstract class TestCodec { TestCodec({required this.description}); final String description; From 7c6ab13dfe40a1e17dc813700b27ccc256b44af3 Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Wed, 17 Jul 2024 16:10:17 -0700 Subject: [PATCH 14/15] Respond to review comments --- .../lib/src/engine/canvaskit/image.dart | 216 +++++++++++------- .../engine/canvaskit/image_web_codecs.dart | 10 +- .../lib/src/engine/canvaskit/picture.dart | 2 +- .../lib/src/engine/canvaskit/renderer.dart | 2 +- lib/web_ui/lib/src/engine/html/image.dart | 4 + .../src/engine/html_image_element_codec.dart | 75 ++---- lib/web_ui/lib/src/engine/image_decoder.dart | 22 +- 7 files changed, 172 insertions(+), 159 deletions(-) diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart index 3bb6ab048ff79..c55dff726455d 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -29,9 +29,9 @@ Future skiaInstantiateImageCodec(Uint8List list, // TODO(harryterkelsen): If the image is animated, then use Skia to decode. // This is currently too conservative, assuming all GIF and WEBP images are // animated. We should detect if they are actually animated by reading the - // image headers. + // image headers, https://github.com/flutter/flutter/issues/151911. if (contentType == 'image/gif' || contentType == 'image/webp') { - codec = CkAnimatedImage.decodeFromBytes(list, 'decoded bytes', + codec = CkAnimatedImage.decodeFromBytes(list, 'encoded image bytes', targetWidth: targetWidth, targetHeight: targetHeight); } else { final DomBlob blob = createDomBlob([list.buffer]); @@ -64,7 +64,7 @@ class CkResizingCodec extends ResizingCodec { bool allowUpscaling = true, }) { final CkImage ckImage = image as CkImage; - if (ckImage.imageElement == null) { + if (ckImage.imageSource == null) { return scaleImageIfNeeded( image, targetWidth: targetWidth, @@ -87,10 +87,10 @@ class CkResizingCodec extends ResizingCodec { int? targetHeight, bool allowUpscaling = true, }) { - assert(image.imageElement != null); + assert(image.imageSource != null); final int width = image.width; final int height = image.height; - final ui.Size? scaledSize = + final BitmapSize? scaledSize = scaledImageSize(width, height, targetWidth, targetHeight); if (scaledSize == null) { return image; @@ -100,17 +100,17 @@ class CkResizingCodec extends ResizingCodec { return image; } - final int scaledWidth = scaledSize.width.toInt(); - final int scaledHeight = scaledSize.height.toInt(); + final int scaledWidth = scaledSize.width; + final int scaledHeight = scaledSize.height; - final DomCanvasElement htmlCanvas = createDomCanvasElement( - width: scaledWidth, - height: scaledHeight, + final DomOffscreenCanvas offscreenCanvas = createDomOffscreenCanvas( + scaledWidth, + scaledHeight, ); final DomCanvasRenderingContext2D ctx = - htmlCanvas.getContext('2d')! as DomCanvasRenderingContext2D; + offscreenCanvas.getContext('2d')! as DomCanvasRenderingContext2D; ctx.drawImage( - image.imageElement!, + image.imageSource!.canvasImageSource, 0, 0, width, @@ -120,28 +120,21 @@ class CkResizingCodec extends ResizingCodec { scaledWidth, scaledHeight, ); - final DomImageData imageData = - ctx.getImageData(0, 0, scaledWidth, scaledHeight); - final Uint8List pixels = imageData.data.buffer.asUint8List(); + final DomImageBitmap bitmap = offscreenCanvas.transferToImageBitmap(); + final SkImage? skImage = + canvasKit.MakeLazyImageFromImageBitmap(bitmap, true); - final SkImage? skImage = canvasKit.MakeImage( - SkImageInfo( - width: scaledWidth.toDouble(), - height: scaledHeight.toDouble(), - colorType: canvasKit.ColorType.RGBA_8888, - alphaType: canvasKit.AlphaType.Premul, - colorSpace: SkColorSpaceSRGB, - ), - pixels, - (4 * scaledWidth).toDouble(), - ); + // Resize the canvas to 0x0 to cause the browser to eagerly reclaim its + // memory. + offscreenCanvas.width = 0; + offscreenCanvas.height = 0; if (skImage == null) { domWindow.console.warn('Failed to scale image.'); return image; } - return CkImage(skImage); + return CkImage(skImage, imageSource: ImageBitmapImageSource(bitmap)); } } @@ -166,7 +159,7 @@ ui.Image createCkImageFromImageElement( ); } - return CkImage(skImage, imageElement: image); + return CkImage(skImage, imageSource: ImageElementImageSource(image)); } class CkImageElementCodec extends HtmlImageElementCodec { @@ -411,13 +404,12 @@ Future readChunked(HttpFetchPayload payload, int contentLength, /// A [ui.Image] backed by an `SkImage` from Skia. class CkImage implements ui.Image, StackTraceDebugger { - CkImage(SkImage skImage, - {this.videoFrame, this.imageElement, this.imageBitmap}) { + CkImage(SkImage skImage, {this.imageSource}) { box = CountedRef(skImage, this, 'SkImage'); _init(); } - CkImage.cloneOf(this.box, {this.videoFrame, this.imageElement}) { + CkImage.cloneOf(this.box, {this.imageSource}) { _init(); box.ref(this); } @@ -438,29 +430,11 @@ class CkImage implements ui.Image, StackTraceDebugger { // being garbage-collected, or by an explicit call to [delete]. late final CountedRef box; - /// For browsers that support `ImageDecoder` this field holds the video frame - /// from which this image was created. - /// - /// Skia owns the video frame and will close it when it's no longer used. - /// However, Flutter co-owns the [SkImage] and therefore it's safe to access - /// the video frame until the image is [dispose]d of. - VideoFrame? videoFrame; - - /// For images which are decoded via an HTML Image element, this field holds - /// the image element from which this image was created. - /// - /// Skia owns the image element and will close it when it's no longer used. - /// However, Flutter co-owns the [SkImage] and therefore it's safe to access - /// the image element until the image is [dispose]d of. - DomHTMLImageElement? imageElement; - - /// For images which are decoded via an HTML ImageBitmap, this field holds - /// the image element from which this image was created. - /// - /// Skia owns the image bitmap and will close it when it's no longer used. - /// However, Flutter co-owns the [SkImage] and therefore it's safe to access - /// the image element until the image is [dispose]d of. - DomImageBitmap? imageBitmap; + /// If this [CkImage] is backed by an image source (either VideoFrame, + /// element, or ImageBitmap), this is the backing image source. We read pixels + /// and byte data from the backing image source rather than from the [SkImage] + /// because of this bug: https://issues.skia.org/issues/40043810. + ImageSource? imageSource; /// The underlying Skia image object. /// @@ -484,6 +458,7 @@ class CkImage implements ui.Image, StackTraceDebugger { ui.Image.onDispose?.call(this); _disposed = true; box.unref(this); + imageSource?.close(); } @override @@ -507,8 +482,7 @@ class CkImage implements ui.Image, StackTraceDebugger { assert(_debugCheckIsNotDisposed()); return CkImage.cloneOf( box, - videoFrame: videoFrame?.clone(), - imageElement: imageElement, + imageSource: imageSource, ); } @@ -539,35 +513,41 @@ class CkImage implements ui.Image, StackTraceDebugger { ui.ImageByteFormat format = ui.ImageByteFormat.rawRgba, }) { assert(_debugCheckIsNotDisposed()); - // readPixelsFromVideoFrame currently does not convert I420, I444, I422 - // videoFrame formats to RGBA - if (videoFrame != null && - videoFrame!.format != 'I420' && - videoFrame!.format != 'I444' && - videoFrame!.format != 'I422') { - return readPixelsFromVideoFrame(videoFrame!, format); - } else if (imageElement != null) { - return readPixelsFromDomImageSource( - imageElement!, - format, - imageElement!.naturalWidth.toInt(), - imageElement!.naturalHeight.toInt(), - ); - } else if (imageBitmap != null) { - return readPixelsFromDomImageSource( - imageBitmap!, - format, - imageBitmap!.width.toDartInt, - imageBitmap!.height.toDartInt, - ); + switch (imageSource) { + case ImageElementImageSource(): + final DomHTMLImageElement imageElement = + (imageSource! as ImageElementImageSource).imageElement; + return readPixelsFromDomImageSource( + imageElement, + format, + imageElement.naturalWidth.toInt(), + imageElement.naturalHeight.toInt(), + ); + case ImageBitmapImageSource(): + final DomImageBitmap imageBitmap = + (imageSource! as ImageBitmapImageSource).imageBitmap; + return readPixelsFromDomImageSource( + imageBitmap, + format, + imageBitmap.width.toDartInt, + imageBitmap.height.toDartInt, + ); + case VideoFrameImageSource(): + final VideoFrame videoFrame = + (imageSource! as VideoFrameImageSource).videoFrame; + if (videoFrame.format != 'I420' && + videoFrame.format != 'I444' && + videoFrame.format != 'I422') { + return readPixelsFromVideoFrame(videoFrame, format); + } + case null: + } + ByteData? data = _readPixelsFromSkImage(format); + data ??= _readPixelsFromImageViaSurface(format); + if (data == null) { + return Future.error('Failed to encode the image into bytes.'); } else { - ByteData? data = _readPixelsFromSkImage(format); - data ??= _readPixelsFromImageViaSurface(format); - if (data == null) { - return Future.error('Failed to encode the image into bytes.'); - } else { - return Future.value(data); - } + return Future.value(data); } } @@ -664,3 +644,71 @@ String tryDetectContentType(Uint8List data, String debugSource) { } return contentType; } + +sealed class ImageSource { + DomCanvasImageSource get canvasImageSource; + int get width; + int get height; + void close(); +} + +class VideoFrameImageSource extends ImageSource { + VideoFrameImageSource(this.videoFrame); + + final VideoFrame videoFrame; + + @override + void close() { + // Do nothing. Skia will close the VideoFrame when the SkImage is disposed. + } + + @override + int get height => videoFrame.displayHeight.toInt(); + + @override + int get width => videoFrame.displayWidth.toInt(); + + @override + DomCanvasImageSource get canvasImageSource => videoFrame; +} + +class ImageElementImageSource extends ImageSource { + ImageElementImageSource(this.imageElement); + + final DomHTMLImageElement imageElement; + + @override + void close() { + // There's no way to immediately close the element. Just let the + // browser garbage collect it. + } + + @override + int get height => imageElement.naturalHeight.toInt(); + + @override + int get width => imageElement.naturalWidth.toInt(); + + @override + DomCanvasImageSource get canvasImageSource => imageElement; +} + +class ImageBitmapImageSource extends ImageSource { + ImageBitmapImageSource(this.imageBitmap); + + final DomImageBitmap imageBitmap; + + @override + void close() { + imageBitmap.close(); + } + + @override + int get height => imageBitmap.height.toDartInt; + + @override + int get width => imageBitmap.width.toDartInt; + + @override + DomCanvasImageSource get canvasImageSource => imageBitmap; +} diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart index 4f81bb0a1c90d..f9c583aa9b3b3 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart @@ -58,7 +58,7 @@ class CkBrowserImageDecoder extends BrowserImageDecoder { ); } - return CkImage(skImage, videoFrame: frame); + return CkImage(skImage, imageSource: VideoFrameImageSource(frame)); } } @@ -201,6 +201,10 @@ ByteBuffer readDomImageSourcePixelsUnmodified( htmlCanvas.getContext('2d')! as DomCanvasRenderingContext2D; ctx.drawImage(imageSource, 0, 0); final DomImageData imageData = ctx.getImageData(0, 0, width, height); + // Resize the canvas to 0x0 to cause the browser to reclaim its memory + // eagerly. + htmlCanvas.width = 0; + htmlCanvas.height = 0; return imageData.data.buffer; } @@ -212,5 +216,9 @@ Future encodeDomImageSourceAsPng( ctx.drawImage(imageSource, 0, 0); final String pngBase64 = canvas.toDataURL().substring('data:image/png;base64,'.length); + // Resize the canvas to 0x0 to cause the browser to reclaim its memory + // eagerly. + canvas.width = 0; + canvas.height = 0; return base64.decode(pngBase64); } diff --git a/lib/web_ui/lib/src/engine/canvaskit/picture.dart b/lib/web_ui/lib/src/engine/canvaskit/picture.dart index 97d4978c4fa64..addb864c441ee 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/picture.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/picture.dart @@ -116,7 +116,7 @@ class CkPicture implements ScenePicture { ); final Uint8List? pixels = skImage.readPixels(0, 0, imageInfo); if (pixels == null) { - throw StateError('Unable to convert read pixels from SkImage.'); + throw StateError('Unable to read pixels from SkImage.'); } final SkImage? rasterImage = canvasKit.MakeImage(imageInfo, pixels, (4 * width).toDouble()); diff --git a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart index cbe1fcffe5d7e..ff5c7006f28b3 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart @@ -237,7 +237,7 @@ class CanvasKitRenderer implements Renderer { if (skImage == null) { throw Exception('Failed to convert image bitmap to an SkImage.'); } - return CkImage(skImage, imageBitmap: imageBitmap); + return CkImage(skImage, imageSource: ImageBitmapImageSource(imageBitmap)); } @override diff --git a/lib/web_ui/lib/src/engine/html/image.dart b/lib/web_ui/lib/src/engine/html/image.dart index 72c9f9ddbbda9..b1a5b6d3cfb65 100644 --- a/lib/web_ui/lib/src/engine/html/image.dart +++ b/lib/web_ui/lib/src/engine/html/image.dart @@ -98,6 +98,10 @@ class HtmlImage implements ui.Image { final DomCanvasRenderingContext2D ctx = canvas.context2D; ctx.drawImage(imgElement, 0, 0); final DomImageData imageData = ctx.getImageData(0, 0, width, height); + // Resize the canvas to 0x0 to cause the browser to reclaim its memory + // eagerly. + canvas.width = 0; + canvas.height = 0; return Future.value(imageData.data.buffer.asByteData()); default: if (imgElement.src?.startsWith('data:') ?? false) { diff --git a/lib/web_ui/lib/src/engine/html_image_element_codec.dart b/lib/web_ui/lib/src/engine/html_image_element_codec.dart index 40ceb6418a562..2bd6ccabc54f0 100644 --- a/lib/web_ui/lib/src/engine/html_image_element_codec.dart +++ b/lib/web_ui/lib/src/engine/html_image_element_codec.dart @@ -8,15 +8,6 @@ import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; import 'package:ui/ui_web/src/ui_web.dart' as ui_web; -Object? get _jsImageDecodeFunction => getJsProperty( - getJsProperty( - getJsProperty(domWindow, 'Image'), - 'prototype', - ), - 'decode', - ); -final bool _supportsDecode = _jsImageDecodeFunction != null; - // TODO(mdebbar): Deprecate this and remove it. // https://github.com/flutter/flutter/issues/127395 typedef WebOnlyImageCodecChunkCallback = ui_web.ImageCodecChunkCallback; @@ -51,26 +42,19 @@ abstract class HtmlImageElementCodec implements ui.Codec { // we add 0/100 , 100/100 progress callbacks to enable loading progress // builders to create UI. chunkCallback?.call(0, 100); - if (_supportsDecode) { - imgElement = createDomHTMLImageElement(); - imgElement!.src = src; - setJsProperty(imgElement!, 'decoding', 'async'); - - // Ignoring the returned future on purpose because we're communicating - // through the `completer`. - // ignore: unawaited_futures - imgElement!.decode().then((dynamic _) { - chunkCallback?.call(100, 100); - completer.complete(); - }).catchError((dynamic e) { - // This code path is hit on Chrome 80.0.3987.16 when too many - // images are on the page (~1000). - // Fallback here is to load using onLoad instead. - _decodeUsingOnLoad(completer); - }); - } else { - _decodeUsingOnLoad(completer); - } + imgElement = createDomHTMLImageElement(); + imgElement!.src = src; + setJsProperty(imgElement!, 'decoding', 'async'); + + // Ignoring the returned future on purpose because we're communicating + // through the `completer`. + // ignore: unawaited_futures + imgElement!.decode().then((dynamic _) { + chunkCallback?.call(100, 100); + completer.complete(); + }).catchError((dynamic e) { + completer.completeError(e.toString()); + }); return completer.future; } @@ -95,39 +79,6 @@ abstract class HtmlImageElementCodec implements ui.Codec { return SingleFrameInfo(image); } - // TODO(harryterkelsen): All browsers support Image.decode now. Should we - // remove this code path? - void _decodeUsingOnLoad(Completer completer) { - imgElement = createDomHTMLImageElement(); - // If the browser doesn't support asynchronous decoding of an image, - // then use the `onload` event to decide when it's ready to paint to the - // DOM. Unfortunately, this will cause the image to be decoded synchronously - // on the main thread, and may cause dropped framed. - late DomEventListener errorListener; - DomEventListener? loadListener; - errorListener = createDomEventListener((DomEvent event) { - if (loadListener != null) { - imgElement!.removeEventListener('load', loadListener); - } - imgElement!.removeEventListener('error', errorListener); - completer.completeError(ImageCodecException( - 'Failed to decode image data.\n' - 'Image source: $debugSource', - )); - }); - imgElement!.addEventListener('error', errorListener); - loadListener = createDomEventListener((DomEvent event) { - if (chunkCallback != null) { - chunkCallback!(100, 100); - } - imgElement!.removeEventListener('load', loadListener); - imgElement!.removeEventListener('error', errorListener); - completer.complete(); - }); - imgElement!.addEventListener('load', loadListener); - imgElement!.src = src; - } - /// Creates a [ui.Image] from an [HTMLImageElement] that has been loaded. ui.Image createImageFromHTMLImageElement( DomHTMLImageElement image, diff --git a/lib/web_ui/lib/src/engine/image_decoder.dart b/lib/web_ui/lib/src/engine/image_decoder.dart index 1804f5c4a2af3..fa2667cbf9a29 100644 --- a/lib/web_ui/lib/src/engine/image_decoder.dart +++ b/lib/web_ui/lib/src/engine/image_decoder.dart @@ -381,10 +381,12 @@ class ResizingCodec implements ui.Codec { final ui.FrameInfo frameInfo = await delegate.getNextFrame(); return AnimatedImageFrameInfo( frameInfo.duration, - scaleImage(frameInfo.image, - targetWidth: targetWidth, - targetHeight: targetHeight, - allowUpscaling: allowUpscaling), + await scaleImage( + frameInfo.image, + targetWidth: targetWidth, + targetHeight: targetHeight, + allowUpscaling: allowUpscaling, + ), ); } @@ -405,7 +407,7 @@ class ResizingCodec implements ui.Codec { int get repetitionCount => delegate.frameCount; } -ui.Size? scaledImageSize( +BitmapSize? scaledImageSize( int width, int height, int? targetWidth, @@ -428,7 +430,7 @@ ui.Size? scaledImageSize( } targetHeight = (height * targetWidth / width).round(); } - return ui.Size(targetWidth.toDouble(), targetHeight.toDouble()); + return BitmapSize(targetWidth, targetHeight); } ui.Image scaleImageIfNeeded( @@ -439,7 +441,7 @@ ui.Image scaleImageIfNeeded( }) { final int width = image.width; final int height = image.height; - final ui.Size? scaledSize = + final BitmapSize? scaledSize = scaledImageSize(width, height, targetWidth, targetHeight); if (scaledSize == null) { return image; @@ -449,8 +451,8 @@ ui.Image scaleImageIfNeeded( return image; } - final ui.Rect outputRect = - ui.Rect.fromLTWH(0, 0, scaledSize.width, scaledSize.height); + final ui.Rect outputRect = ui.Rect.fromLTWH( + 0, 0, scaledSize.width.toDouble(), scaledSize.height.toDouble()); final ui.PictureRecorder recorder = ui.PictureRecorder(); final ui.Canvas canvas = ui.Canvas(recorder, outputRect); @@ -462,7 +464,7 @@ ui.Image scaleImageIfNeeded( ); final ui.Picture picture = recorder.endRecording(); final ui.Image finalImage = - picture.toImageSync(scaledSize.width.round(), scaledSize.height.round()); + picture.toImageSync(scaledSize.width, scaledSize.height); picture.dispose(); image.dispose(); return finalImage; From e7aff4633b39ad7943a5a2d4a845adb0359b8968 Mon Sep 17 00:00:00 2001 From: Harry Terkelsen Date: Thu, 18 Jul 2024 14:28:15 -0700 Subject: [PATCH 15/15] Remove unnecessary `await` --- lib/web_ui/lib/src/engine/image_decoder.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/web_ui/lib/src/engine/image_decoder.dart b/lib/web_ui/lib/src/engine/image_decoder.dart index fa2667cbf9a29..ca321456731bb 100644 --- a/lib/web_ui/lib/src/engine/image_decoder.dart +++ b/lib/web_ui/lib/src/engine/image_decoder.dart @@ -381,7 +381,7 @@ class ResizingCodec implements ui.Codec { final ui.FrameInfo frameInfo = await delegate.getNextFrame(); return AnimatedImageFrameInfo( frameInfo.duration, - await scaleImage( + scaleImage( frameInfo.image, targetWidth: targetWidth, targetHeight: targetHeight,