diff --git a/lib/web_ui/lib/src/engine/alarm_clock.dart b/lib/web_ui/lib/src/engine/alarm_clock.dart index 900286e77dcc7..8a03bccec16bc 100644 --- a/lib/web_ui/lib/src/engine/alarm_clock.dart +++ b/lib/web_ui/lib/src/engine/alarm_clock.dart @@ -33,7 +33,11 @@ class AlarmClock { DateTime? _datetime; /// The callback called when the alarm goes off. - late ui.VoidCallback callback; + /// + /// If this is null, the alarm goes off without calling the callback. Set the + /// callback to null if the callback is a closure holding onto expensive + /// resources. + ui.VoidCallback? callback; /// The time when the alarm clock will go off. /// @@ -103,7 +107,7 @@ class AlarmClock { // zero difference between now and _datetime. if (!now.isBefore(_datetime!)) { _timer = null; - callback(); + callback?.call(); } else { // The timer fired before the target date. We need to reschedule. _timer = Timer(_datetime!.difference(now), _timerDidFire); diff --git a/lib/web_ui/lib/src/engine/canvaskit/image.dart b/lib/web_ui/lib/src/engine/canvaskit/image.dart index e51d71c9f1d93..3e980842eda24 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -148,7 +148,7 @@ Future fetchImage( /// A [ui.Image] backed by an `SkImage` from Skia. class CkImage implements ui.Image, StackTraceDebugger { - CkImage(SkImage skImage) { + CkImage(SkImage skImage, { this.videoFrame }) { if (assertionsEnabled) { _debugStackTrace = StackTrace.current; } @@ -214,6 +214,14 @@ class CkImage implements ui.Image, StackTraceDebugger { // being garbage-collected, or by an explicit call to [delete]. late final SkiaObjectBox box; + /// For browsers that support `ImageDecoder` this field holds the video frame + /// from which this image was created. + /// + /// Skia owns the video frame and will close it when it's no longer used. + /// However, Flutter co-owns the [SkImage] and therefore it's safe to access + /// the video frame until the image is [dispose]d of. + VideoFrame? videoFrame; + /// The underlying Skia image object. /// /// Do not store the returned value. It is memory-managed by [SkiaObjectBox]. @@ -279,6 +287,14 @@ class CkImage implements ui.Image, StackTraceDebugger { ui.ImageByteFormat format = ui.ImageByteFormat.rawRgba, }) { assert(_debugCheckIsNotDisposed()); + if (videoFrame != null) { + return readPixelsFromVideoFrame(videoFrame!, format); + } else { + return _readPixelsFromSkImage(format); + } + } + + Future _readPixelsFromSkImage(ui.ImageByteFormat format) { final SkAlphaType alphaType = format == ui.ImageByteFormat.rawStraightRgba ? canvasKit.AlphaType.Unpremul : canvasKit.AlphaType.Premul; final ByteData? data = _encodeImage( skImage: skImage, @@ -313,7 +329,7 @@ class CkImage implements ui.Image, StackTraceDebugger { ); bytes = skImage.readPixels(0, 0, imageInfo); } else { - bytes = skImage.encodeToBytes(); //defaults to PNG 100% + bytes = skImage.encodeToBytes(); // defaults to PNG 100% } return bytes?.buffer.asByteData(0, bytes.length); diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart index 48af52b6cbd1e..19e5884c985fb 100644 --- a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart +++ b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart @@ -9,17 +9,37 @@ /// * `image_wasm_codecs.dart`, which uses codecs supplied by the CanvasKit WASM bundle. import 'dart:async'; +import 'dart:convert' show base64; import 'dart:html' as html; import 'dart:math' as math; import 'dart:typed_data'; +import 'package:meta/meta.dart'; import 'package:ui/ui.dart' as ui; +import '../alarm_clock.dart'; import '../safe_browser_api.dart'; import '../util.dart'; import 'canvaskit_api.dart'; import 'image.dart'; +Duration _kDefaultWebDecoderExpireDuration = const Duration(seconds: 3); +Duration _kWebDecoderExpireDuration = _kDefaultWebDecoderExpireDuration; + +/// Overrides the inactivity duration after which the web decoder is closed. +/// +/// This should only be used in tests. +void debugOverrideWebDecoderExpireDuration(Duration override) { + _kWebDecoderExpireDuration = override; +} + +/// Restores the web decoder inactivity expiry duration to its original value. +/// +/// This should only be used in tests. +void debugRestoreWebDecoderExpireDuration() { + _kWebDecoderExpireDuration = _kDefaultWebDecoderExpireDuration; +} + /// Image decoder backed by the browser's `ImageDecoder`. class CkBrowserImageDecoder implements ui.Codec { static Future create({ @@ -46,12 +66,99 @@ class CkBrowserImageDecoder implements ui.Codec { ); } + final CkBrowserImageDecoder decoder = CkBrowserImageDecoder._( + contentType: contentType, + targetWidth: targetWidth, + targetHeight: targetHeight, + data: data, + debugSource: debugSource, + ); + + // Call once to initialize the decoder and populate late fields. + await decoder._getOrCreateWebDecoder(); + return decoder; + } + + CkBrowserImageDecoder._({ + required this.contentType, + required this.targetWidth, + required this.targetHeight, + required this.data, + required this.debugSource, + }); + + final String contentType; + final int? targetWidth; + final int? targetHeight; + final Uint8List data; + final String debugSource; + + @override + late int frameCount; + + @override + late int repetitionCount; + + /// Whether this decoder has been disposed of. + /// + /// Once this turns true it stays true forever, and this decoder becomes + /// unusable. + bool _isDisposed = false; + + @override + void dispose() { + _isDisposed = true; + + // This releases all resources, including any currently running decoding work. + _cachedWebDecoder?.close(); + _cachedWebDecoder = null; + } + + void _debugCheckNotDisposed() { + assert( + !_isDisposed, + 'Cannot use this image decoder. It has been disposed of.' + ); + } + + /// The index of the frame that will be decoded on the next call of [getNextFrame]; + int _nextFrameIndex = 0; + + /// Creating a new decoder is expensive, so we cache the decoder for reuse. + /// + /// This decoder is closed and the field is nulled out after some time of + /// inactivity. + ImageDecoder? _cachedWebDecoder; + + /// The underlying image decoder used to decode images. + /// + /// This value is volatile. It may be closed or become null any time. + /// + /// + /// This is only meant to be used in tests. + @visibleForTesting + ImageDecoder? get debugCachedWebDecoder => _cachedWebDecoder; + + final AlarmClock _cacheExpirationClock = AlarmClock(() => DateTime.now()); + + Future _getOrCreateWebDecoder() async { + if (_cachedWebDecoder != null) { + // Give the cached value some time for reuse, e.g. if the image is + // currently animating. + _cacheExpirationClock.datetime = DateTime.now().add(_kWebDecoderExpireDuration); + return _cachedWebDecoder!; + } + + // Null out the callback so the clock doesn't try to expire the decoder + // while it's initializing. There's no way to tell how long the + // initialization will take place. We just let it proceed at its own pace. + _cacheExpirationClock.callback = null; try { final ImageDecoder webDecoder = ImageDecoder(ImageDecoderOptions( type: contentType, data: data, - // Flutter always uses premultiplied alpha. + // Flutter always uses premultiplied alpha when decoding. premultiplyAlpha: 'premultiply', desiredWidth: targetWidth, desiredHeight: targetHeight, @@ -72,7 +179,28 @@ class CkBrowserImageDecoder implements ui.Codec { // package:js bindings don't work with getters that return a Promise, which // is why js_util is used instead. await promiseToFuture(getJsProperty(webDecoder, 'completed')); - return CkBrowserImageDecoder._(webDecoder, debugSource); + frameCount = webDecoder.tracks.selectedTrack!.frameCount; + repetitionCount = webDecoder.tracks.selectedTrack!.repetitionCount; + + _cachedWebDecoder = webDecoder; + + // Expire the decoder if it's not used for several seconds. If the image is + // not animated, it could mean that the framework has cached the frame and + // therefore doesn't need the decoder any more, or it could mean that the + // widget is gone and it's time to collect resources associated with it. + // If it's an animated image it means the animation has stopped, otherwise + // we'd see calls to [getNextFrame] which would update the expiry date on + // the decoder. If the animation is stopped for long enough, it's better + // to collect resources. If and when the animation resumes, a new decoder + // will be instantiated. + _cacheExpirationClock.callback = () { + _cachedWebDecoder?.close(); + _cachedWebDecoder = null; + _cacheExpirationClock.callback = null; + }; + _cacheExpirationClock.datetime = DateTime.now().add(_kWebDecoderExpireDuration); + + return webDecoder; } catch (error) { if (error is html.DomException) { if (error.name == html.DomException.NOT_SUPPORTED) { @@ -90,44 +218,10 @@ class CkBrowserImageDecoder implements ui.Codec { } } - CkBrowserImageDecoder._(this.webDecoder, this.debugSource); - - final ImageDecoder webDecoder; - final String debugSource; - - /// Whether this decoded has been disposed of. - /// - /// Once this turns true it stays true forever, and this decoder becomes - /// unusable. - bool _isDisposed = false; - - @override - void dispose() { - _isDisposed = true; - - // This releases all resources, including any currently running decoding work. - webDecoder.close(); - } - - void _debugCheckNotDisposed() { - assert( - !_isDisposed, - 'Cannot use this image decoder. It has been disposed of.' - ); - } - - @override - int get frameCount { - _debugCheckNotDisposed(); - return webDecoder.tracks.selectedTrack!.frameCount; - } - - /// The index of the frame that will be decoded on the next call of [getNextFrame]; - int _nextFrameIndex = 0; - @override Future getNextFrame() async { _debugCheckNotDisposed(); + final ImageDecoder webDecoder = await _getOrCreateWebDecoder(); final DecodeResult result = await promiseToFuture( webDecoder.decode(DecodeOptions(frameIndex: _nextFrameIndex)), ); @@ -156,15 +250,9 @@ class CkBrowserImageDecoder implements ui.Codec { ); } - final CkImage image = CkImage(skImage); + final CkImage image = CkImage(skImage, videoFrame: frame); return Future.value(AnimatedImageFrameInfo(duration, image)); } - - @override - int get repetitionCount { - _debugCheckNotDisposed(); - return webDecoder.tracks.selectedTrack!.repetitionCount; - } } /// Represents an image file format, such as PNG or JPEG. @@ -302,3 +390,79 @@ bool isAvif(Uint8List data) { } return false; } + +Future readPixelsFromVideoFrame(VideoFrame videoFrame, ui.ImageByteFormat format) async { + if (format == ui.ImageByteFormat.png) { + final Uint8List png = await encodeVideoFrameAsPng(videoFrame); + return png.buffer.asByteData(); + } + + final ByteBuffer pixels = await readVideoFramePixelsUnmodified(videoFrame); + + // Check if the pixels are already in the right format and if so, return the + // original pixels without modification. + if (_shouldReadPixelsUnmodified(videoFrame, format)) { + return pixels.asByteData(); + } + + // At this point we know we want to read unencoded pixels, and that the video + // frame is _not_ using the same format as the requested one. + final bool isBgrFrame = videoFrame.format == 'BGRA' || videoFrame.format == 'BGRX'; + if (format == ui.ImageByteFormat.rawRgba && isBgrFrame) { + _bgrToRgba(pixels); + return pixels.asByteData(); + } + + // Last resort, just return the original pixels. + return pixels.asByteData(); +} + +/// Mutates the [pixels], converting them from BGRX/BGRA to RGBA. +void _bgrToRgba(ByteBuffer pixels) { + final int pixelCount = pixels.lengthInBytes ~/ 4; + final Uint8List pixelBytes = pixels.asUint8List(); + for (int i = 0; i < pixelCount; i += 4) { + // It seems even in little-endian machines the BGR_ pixels are encoded as + // big-endian, i.e. the blue byte is written into the lowest byte in the + // memory address space. + final int b = pixelBytes[i]; + final int r = pixelBytes[i + 2]; + + // So far the codec has reported 255 for the X component, so there's no + // special treatment for alpha. This may need to change if we ever face + // codecs that do something different. + pixelBytes[i] = r; + pixelBytes[i + 2] = b; + } +} + +bool _shouldReadPixelsUnmodified(VideoFrame videoFrame, ui.ImageByteFormat format) { + if (format == ui.ImageByteFormat.rawUnmodified) { + return true; + } + + // Do not convert if the requested format is RGBA and the video frame is + // encoded as either RGBA or RGBX. + final bool isRgbFrame = videoFrame.format == 'RGBA' || videoFrame.format == 'RGBX'; + return format == ui.ImageByteFormat.rawRgba && isRgbFrame; +} + +Future readVideoFramePixelsUnmodified(VideoFrame videoFrame) async { + final int size = videoFrame.allocationSize(); + final Uint8List destination = Uint8List(size); + final JsPromise copyPromise = videoFrame.copyTo(destination); + await promiseToFuture(copyPromise); + return destination.buffer; +} + +Future encodeVideoFrameAsPng(VideoFrame videoFrame) async { + final int width = videoFrame.displayWidth; + final int height = videoFrame.displayHeight; + final html.CanvasElement canvas = html.CanvasElement() + ..width = width + ..height = height; + final html.CanvasRenderingContext2D ctx = canvas.context2D; + ctx.drawImage(videoFrame, 0, 0); + final String pngBase64 = canvas.toDataUrl().substring('data:image/png;base64,'.length); + return base64.decode(pngBase64); +} diff --git a/lib/web_ui/lib/src/engine/safe_browser_api.dart b/lib/web_ui/lib/src/engine/safe_browser_api.dart index 4e5b17ff077a8..b196f745e371e 100644 --- a/lib/web_ui/lib/src/engine/safe_browser_api.dart +++ b/lib/web_ui/lib/src/engine/safe_browser_api.dart @@ -304,7 +304,7 @@ class DecodeOptions { /// * https://www.w3.org/TR/webcodecs/#videoframe-interface @JS() @anonymous -class VideoFrame { +class VideoFrame implements html.CanvasImageSource { external int allocationSize(); external JsPromise copyTo(Uint8List destination); external String? get format; diff --git a/lib/web_ui/test/canvaskit/image_golden_test.dart b/lib/web_ui/test/canvaskit/image_golden_test.dart index 479e8262a9d8d..abfacd4bb5919 100644 --- a/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -28,10 +28,12 @@ void testMain() { debugRestoreHttpRequestFactory(); }); + _testCkAnimatedImage(); _testForImageCodecs(useBrowserImageDecoder: false); if (browserSupportsImageDecoder) { _testForImageCodecs(useBrowserImageDecoder: true); + _testCkBrowserImageDecoder(); } test('isAvif', () { @@ -87,19 +89,14 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { test('CkAnimatedImage remembers last animation position after resurrection', () async { browserSupportsFinalizationRegistry = false; - Future expectFrameData(ui.FrameInfo frame, List data) async { - final ByteData frameData = (await frame.image.toByteData())!; - expect(frameData.buffer.asUint8List(), Uint8List.fromList(data)); - } - final CkAnimatedImage image = CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); expect(image.frameCount, 3); expect(image.repetitionCount, -1); final ui.FrameInfo frame1 = await image.getNextFrame(); - expectFrameData(frame1, [0, 255, 0, 255]); + await expectFrameData(frame1, [0, 255, 0, 255]); final ui.FrameInfo frame2 = await image.getNextFrame(); - expectFrameData(frame2, [0, 0, 255, 255]); + await expectFrameData(frame2, [0, 0, 255, 255]); // Pretend that the image is temporarily deleted. image.delete(); @@ -107,7 +104,7 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { // Check that we got the 3rd frame after resurrection. final ui.FrameInfo frame3 = await image.getNextFrame(); - expectFrameData(frame3, [255, 0, 0, 255]); + await expectFrameData(frame3, [255, 0, 0, 255]); testCollector.collectNow(); }); @@ -535,6 +532,122 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) { }); } +/// Tests specific to WASM codecs bundled with CanvasKit. +void _testCkAnimatedImage() { + test('ImageDecoder toByteData(PNG)', () async { + final CkAnimatedImage image = CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); + final ui.FrameInfo frame = await image.getNextFrame(); + final ByteData? png = await frame.image.toByteData(format: ui.ImageByteFormat.png); + expect(png, isNotNull); + + // The precise PNG encoding is browser-specific, but we can check the file + // signature. + expect(detectContentType(png!.buffer.asUint8List()), 'image/png'); + testCollector.collectNow(); + }); + + test('CkAnimatedImage toByteData(RGBA)', () async { + final CkAnimatedImage image = CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); + // TODO(yjbanov): frame sequence is wrong (https://github.com/flutter/flutter/issues/95281) + const List> expectedColors = >[ + [0, 255, 0, 255], + [0, 0, 255, 255], + [255, 0, 0, 255], + ]; + for (int i = 0; i < image.frameCount; i++) { + final ui.FrameInfo frame = await image.getNextFrame(); + final ByteData? rgba = await frame.image.toByteData(format: ui.ImageByteFormat.rawRgba); + expect(rgba, isNotNull); + expect(rgba!.buffer.asUint8List(), expectedColors[i]); + } + testCollector.collectNow(); + }); +} + +/// Tests specific to browser image codecs based functionality. +void _testCkBrowserImageDecoder() { + assert(browserSupportsImageDecoder); + + test('ImageDecoder toByteData(PNG)', () async { + final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( + data: kAnimatedGif, + debugSource: 'test', + ); + final ui.FrameInfo frame = await image.getNextFrame(); + final ByteData? png = await frame.image.toByteData(format: ui.ImageByteFormat.png); + expect(png, isNotNull); + + // The precise PNG encoding is browser-specific, but we can check the file + // signature. + expect(detectContentType(png!.buffer.asUint8List()), 'image/png'); + testCollector.collectNow(); + }); + + test('ImageDecoder toByteData(RGBA)', () async { + final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( + data: kAnimatedGif, + debugSource: 'test', + ); + const List> expectedColors = >[ + [255, 0, 0, 255], + [0, 255, 0, 255], + [0, 0, 255, 255], + ]; + for (int i = 0; i < image.frameCount; i++) { + final ui.FrameInfo frame = await image.getNextFrame(); + final ByteData? rgba = await frame.image.toByteData(format: ui.ImageByteFormat.rawRgba); + expect(rgba, isNotNull); + expect(rgba!.buffer.asUint8List(), expectedColors[i]); + } + testCollector.collectNow(); + }); + + test('ImageDecoder expires after inactivity', () async { + const Duration testExpireDuration = Duration(milliseconds: 100); + debugOverrideWebDecoderExpireDuration(testExpireDuration); + + final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( + data: kAnimatedGif, + debugSource: 'test', + ); + + // ImageDecoder is initialized eagerly to populate `frameCount` and + // `repetitionCount`. + final ImageDecoder? decoder1 = image.debugCachedWebDecoder; + expect(decoder1, isNotNull); + expect(image.frameCount, 3); + expect(image.repetitionCount, double.infinity); + + // A frame can be decoded right away. + final ui.FrameInfo frame1 = await image.getNextFrame(); + await expectFrameData(frame1, [255, 0, 0, 255]); + expect(frame1, isNotNull); + + // The cached decoder should not yet expire. + await Future.delayed(testExpireDuration ~/ 2); + expect(image.debugCachedWebDecoder, same(decoder1)); + + // Now it expires. + await Future.delayed(testExpireDuration); + expect(image.debugCachedWebDecoder, isNull); + + // A new decoder should be created upon the next frame request. + final ui.FrameInfo frame2 = await image.getNextFrame(); + + // Check that the cached decoder is indeed new. + final ImageDecoder? decoder2 = image.debugCachedWebDecoder; + expect(decoder2, isNot(same(decoder1))); + await expectFrameData(frame2, [0, 255, 0, 255]); + + // Check that the new decoder remembers the last frame index. + final ui.FrameInfo frame3 = await image.getNextFrame(); + await expectFrameData(frame3, [0, 0, 255, 255]); + + testCollector.collectNow(); + debugRestoreWebDecoderExpireDuration(); + }); +} + class TestHttpRequest implements html.HttpRequest { @override String responseType = 'invalid'; @@ -646,3 +759,8 @@ class TestHttpRequest implements html.HttpRequest { @override html.HttpRequestUpload get upload => throw UnimplementedError(); } + +Future expectFrameData(ui.FrameInfo frame, List data) async { + final ByteData frameData = (await frame.image.toByteData())!; + expect(frameData.buffer.asUint8List(), Uint8List.fromList(data)); +}