From ce7a2065cba41c92b5e31ae74d9c8d52a1dc5a21 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 30 May 2023 10:42:34 -0700 Subject: [PATCH 01/24] Some things. --- lib/web_ui/skwasm/BUILD.gn | 1 + lib/web_ui/skwasm/surface.cpp | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/web_ui/skwasm/BUILD.gn b/lib/web_ui/skwasm/BUILD.gn index 0cba9b2a49956..784361420e9ed 100644 --- a/lib/web_ui/skwasm/BUILD.gn +++ b/lib/web_ui/skwasm/BUILD.gn @@ -47,6 +47,7 @@ wasm_lib("skwasm") { "-Wno-pthreads-mem-growth", "--js-library", rebase_path("library_skwasm_support.js"), + "--profiling", ] inputs = [ rebase_path("library_skwasm_support.js") ] diff --git a/lib/web_ui/skwasm/surface.cpp b/lib/web_ui/skwasm/surface.cpp index 10537ae963d52..4944964530ed7 100644 --- a/lib/web_ui/skwasm/surface.cpp +++ b/lib/web_ui/skwasm/surface.cpp @@ -113,6 +113,7 @@ void Surface::_init() { } makeCurrent(_glContext); + emscripten_webgl_enable_extension(_glContext, "WEBGL_debug_renderer_info"); _grContext = GrDirectContext::MakeGL(GrGLMakeNativeInterface()); @@ -196,7 +197,7 @@ void Surface::_rasterizeImage(SkImage* image, data = nullptr; } } - emscripten_sync_run_in_main_runtime_thread( + emscripten_async_run_in_main_runtime_thread( EM_FUNC_SIG_VIII, fOnRasterizeComplete, this, data.release(), callbackId); } @@ -210,7 +211,7 @@ void Surface::_onRasterizeComplete(SkData* data, uint32_t callbackId) { // Worker thread only void Surface::_notifyRenderComplete(uint32_t callbackId) { - emscripten_sync_run_in_main_runtime_thread(EM_FUNC_SIG_VII, fOnRenderComplete, + emscripten_async_run_in_main_runtime_thread(EM_FUNC_SIG_VII, fOnRenderComplete, this, callbackId); } From 2cfc26c2b5676194fe69721240ad35b676680560 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 6 Jun 2023 09:32:35 -0700 Subject: [PATCH 02/24] Rendering is mostly working, although there is some weird aliasing with drawAtlas. --- lib/web_ui/dev/steps/run_suite_step.dart | 6 +- lib/web_ui/lib/src/engine/dom.dart | 16 +++ .../skwasm/skwasm_impl/raw/raw_surface.dart | 17 +-- .../skwasm/skwasm_impl/raw/skwasm_module.dart | 3 + .../engine/skwasm/skwasm_impl/renderer.dart | 32 ++--- .../engine/skwasm/skwasm_impl/surface.dart | 17 ++- lib/web_ui/skwasm/BUILD.gn | 2 +- lib/web_ui/skwasm/library_skwasm_support.js | 89 +++++++++++--- lib/web_ui/skwasm/skwasm_support.h | 13 ++ lib/web_ui/skwasm/surface.cpp | 113 +++++++----------- lib/web_ui/skwasm/surface.h | 18 +-- .../test/ui/draw_atlas_golden_test.dart | 10 ++ 12 files changed, 194 insertions(+), 142 deletions(-) diff --git a/lib/web_ui/dev/steps/run_suite_step.dart b/lib/web_ui/dev/steps/run_suite_step.dart index b78e6ac9d71be..adb9a12d8bb4b 100644 --- a/lib/web_ui/dev/steps/run_suite_step.dart +++ b/lib/web_ui/dev/steps/run_suite_step.dart @@ -179,8 +179,12 @@ class RunSuiteStep implements PipelineStep { Future _createSkiaClient() async { final Renderer renderer = suite.testBundle.compileConfig.renderer; final CanvasKitVariant? variant = suite.runConfig.variant; + final io.Directory workDirectory = getSkiaGoldDirectoryForSuite(suite); + if (workDirectory.existsSync()) { + workDirectory.deleteSync(recursive: true); + } final SkiaGoldClient skiaClient = SkiaGoldClient( - getSkiaGoldDirectoryForSuite(suite), + workDirectory, dimensions: { 'Browser': suite.runConfig.browser.name, if (isWasm) 'Wasm': 'true', diff --git a/lib/web_ui/lib/src/engine/dom.dart b/lib/web_ui/lib/src/engine/dom.dart index 4a5ad17889a90..18eab800f44c2 100644 --- a/lib/web_ui/lib/src/engine/dom.dart +++ b/lib/web_ui/lib/src/engine/dom.dart @@ -1071,6 +1071,9 @@ extension DomCanvasElementExtension on DomCanvasElement { } return getContext('webgl2')! as WebGLContext; } + + DomCanvasRenderingContextBitmapRenderer get contextBitmapRenderer => + getContext('bitmaprenderer')! as DomCanvasRenderingContextBitmapRenderer; } @JS() @@ -1360,6 +1363,15 @@ extension DomCanvasRenderingContextWebGlExtension bool isContextLost() => _isContextLost().toDart; } +@JS() +@staticInterop +class DomCanvasRenderingContextBitmapRenderer {} + +extension DomCanvasRenderingContextBitmapRendererExtension + on DomCanvasRenderingContextBitmapRenderer { + external void transferFromImageBitmap(DomImageBitmap bitmap); +} + @JS('ImageData') @staticInterop class DomImageData { @@ -1375,6 +1387,10 @@ extension DomImageDataExtension on DomImageData { Uint8ClampedList get data => _data.toDart; } +@JS('ImageBitmap') +@staticInterop +class DomImageBitmap {} + @JS() @staticInterop class DomCanvasPattern {} diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart index 9aa48f69e01f0..f43196284af51 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart @@ -14,12 +14,8 @@ typedef SurfaceHandle = Pointer; final class RawRenderCallback extends Opaque {} typedef OnRenderCallbackHandle = Pointer; -@Native)>( - symbol: 'surface_createFromCanvas', - isLeaf: true) -external SurfaceHandle surfaceCreateFromCanvas( - Pointer querySelector -); +@Native(symbol: 'surface_create', isLeaf: true) +external SurfaceHandle surfaceCreate(); @Native(symbol: 'surface_getThreadId', isLeaf: true) external int surfaceGetThreadId(SurfaceHandle handle); @@ -37,15 +33,6 @@ external void surfaceSetCallbackHandler( isLeaf: true) external void surfaceDestroy(SurfaceHandle surface); -@Native( - symbol: 'surface_setCanvasSize', - isLeaf: true) -external void surfaceSetCanvasSize( - SurfaceHandle surface, - int width, - int height -); - @Native( symbol: 'surface_renderPicture', isLeaf: true) diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart index 76cec92901e79..0d1aac6159401 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart @@ -20,6 +20,9 @@ extension SkwasmInstanceExtension on SkwasmInstance { external JSNumber addFunction(JSFunction function, JSString signature); external void removeFunction(JSNumber functionPointer); + @JS('skwasm_generateUniqueId') + external JSNumber skwasmGenerateUniqueId(); + @JS('skwasm_registerObject') external void skwasmRegisterObject(JSNumber objectId, JSAny object); diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart index 46ec0fe541477..23bf9361f89e6 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart @@ -14,7 +14,6 @@ import 'package:ui/ui.dart' as ui; class SkwasmRenderer implements Renderer { late DomCanvasElement sceneElement; late SkwasmSurface surface; - ui.Size? surfaceSize; @override final SkwasmFontCollection fontCollection = SkwasmFontCollection(); @@ -352,9 +351,7 @@ class SkwasmRenderer implements Renderer { // to deal with those cases. sceneElement = createDomCanvasElement(); sceneElement.id = 'flt-scene'; - domDocument.body!.appendChild(sceneElement); - surface = SkwasmSurface('#flt-scene'); - domDocument.body!.removeChild(sceneElement); + surface = SkwasmSurface(); } @override @@ -403,19 +400,22 @@ class SkwasmRenderer implements Renderer { @override Future renderScene(ui.Scene scene) async { - final ui.Size frameSize = ui.window.physicalSize; - if (frameSize != surfaceSize) { - final double logicalWidth = frameSize.width.ceil() / window.devicePixelRatio; - final double logicalHeight = frameSize.height.ceil() / window.devicePixelRatio; - final DomCSSStyleDeclaration style = sceneElement.style; - style.width = '${logicalWidth}px'; - style.height = '${logicalHeight}px'; - - surface.setSize(frameSize.width.ceil(), frameSize.height.ceil()); - surfaceSize = frameSize; - } final SkwasmPicture picture = (scene as SkwasmScene).picture as SkwasmPicture; - await surface.renderPicture(picture); + final DomImageBitmap bitmap = await surface.renderPicture(picture); + + final DomCSSStyleDeclaration style = sceneElement.style; + final ui.Rect pictureRect = picture.cullRect; + final double logicalWidth = pictureRect.width / window.devicePixelRatio; + final double logicalHeight = pictureRect.height / window.devicePixelRatio; + style.width = '${logicalWidth}px'; + style.height = '${logicalHeight}px'; + style.position = 'absolute'; + style.left = '${pictureRect.left}px'; + style.top = '${pictureRect.top}px'; + sceneElement.width = pictureRect.width; + sceneElement.height = pictureRect.height; + final DomCanvasRenderingContextBitmapRenderer context = sceneElement.contextBitmapRenderer; + context.transferFromImageBitmap(bitmap); } @override diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart index 049e770203b9f..ce096f37ff3cf 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart @@ -7,14 +7,14 @@ import 'dart:ffi'; import 'dart:js_interop'; import 'dart:typed_data'; +import 'package:ui/src/engine.dart'; import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; import 'package:ui/ui.dart' as ui; class SkwasmSurface { - factory SkwasmSurface(String canvasQuerySelector) { + factory SkwasmSurface() { final SurfaceHandle surfaceHandle = withStackScope((StackScope scope) { - final Pointer pointer = scope.convertStringToNative(canvasQuerySelector); - return surfaceCreateFromCanvas(pointer); + return surfaceCreate(); }); final SkwasmSurface surface = SkwasmSurface._fromHandle(surfaceHandle); surface._initialize(); @@ -28,8 +28,7 @@ class SkwasmSurface { final int threadId; - int _currentObjectId = 0; - int acquireObjectId() => ++_currentObjectId; + int acquireObjectId() => skwasmInstance.skwasmGenerateUniqueId().toDart.toInt(); void _initialize() { _callbackHandle = @@ -42,12 +41,10 @@ class SkwasmSurface { surfaceSetCallbackHandler(handle, _callbackHandle); } - void setSize(int width, int height) => - surfaceSetCanvasSize(handle, width, height); - - Future renderPicture(SkwasmPicture picture) { + Future renderPicture(SkwasmPicture picture) async { final int callbackId = surfaceRenderPicture(handle, picture.handle); - return _registerCallback(callbackId); + await _registerCallback(callbackId); + return skwasmInstance.skwasmGetObject(callbackId.toJS) as DomImageBitmap; } Future rasterizeImage(SkwasmImage image, ui.ImageByteFormat format) async { diff --git a/lib/web_ui/skwasm/BUILD.gn b/lib/web_ui/skwasm/BUILD.gn index 784361420e9ed..37dc9ec2d6467 100644 --- a/lib/web_ui/skwasm/BUILD.gn +++ b/lib/web_ui/skwasm/BUILD.gn @@ -43,7 +43,7 @@ wasm_lib("skwasm") { "-sUSE_PTHREADS=1", "-lexports.js", "-sEXPORTED_FUNCTIONS=[stackAlloc]", - "-sEXPORTED_RUNTIME_METHODS=[addFunction,removeFunction,skwasm_registerObject,skwasm_unregisterObject,skwasm_getObject,skwasm_transferObjectToThread]", + "-sEXPORTED_RUNTIME_METHODS=[addFunction,removeFunction,skwasm_registerObject,skwasm_unregisterObject,skwasm_getObject,skwasm_transferObjectToThread,skwasm_generateUniqueId]", "-Wno-pthreads-mem-growth", "--js-library", rebase_path("library_skwasm_support.js"), diff --git a/lib/web_ui/skwasm/library_skwasm_support.js b/lib/web_ui/skwasm/library_skwasm_support.js index 431b4e7da3581..68ce0881983db 100644 --- a/lib/web_ui/skwasm/library_skwasm_support.js +++ b/lib/web_ui/skwasm/library_skwasm_support.js @@ -9,6 +9,12 @@ mergeInto(LibraryManager.library, { $skwasm_support_setup__postset: 'skwasm_support_setup();', $skwasm_support_setup: function() { const objectMap = new Map(); + const handleToCanvasMap = new Map(); + let currentUniqueId = 0; + _skwasm_generateUniqueId = function() { + return ++currentUniqueId; + } + skwasm_generateUniqueId = _skwasm_generateUniqueId; skwasm_registerObject = function(id, object) { objectMap.set(id, object); }; @@ -18,32 +24,67 @@ mergeInto(LibraryManager.library, { skwasm_getObject = function(id) { return objectMap.get(id); } - - addEventListener('message', function (event) { - const transfers = event.data.skwasmObjectTransfers; - if (!transfers) { - return; - } - transfers.forEach(function(object, objectId) { - objectMap.set(objectId, object); - }); - }); - skwasm_transferObjectToMain = function(objectId) { - postMessage({ + skwasm_transferObjectToThread = function(objectId, threadId) { + PThread.pthreads[threadId].postMessage({ skwasmObjectTransfers: new Map([ - [objectId, objectMap[objectId]] + [objectId, objectMap.get(objectId)] ]) }); objectMap.delete(objectId); } - skwasm_transferObjectToThread = function(objectId, threadId) { - PThread.pthreads[threadId].postMessage({ + _skwasm_transferObjectToMain = function(objectId) { + postMessage({ skwasmObjectTransfers: new Map([ [objectId, objectMap.get(objectId)] ]) }); objectMap.delete(objectId); } + _skwasm_registerMessageListener = function(threadId) { + eventListener = function(event) { + const transfers = event.data.skwasmObjectTransfers; + if (!transfers) { + return; + } + transfers.forEach(function(object, objectId) { + objectMap.set(objectId, object); + }); + }; + if (!threadId) { + addEventListener("message", eventListener); + } else { + PThread.pthreads[threadId].addEventListener("message", eventListener); + } + }; + _skwasm_createOffscreenCanvas = function(width, height) { + const canvas = new OffscreenCanvas(width, height); + var contextAttributes = { + 'majorVersion': 2, + 'alpha': true, + 'depth': true, + 'stencil': true, + 'antialias': false, + 'premultipliedAlpha': true, + 'preserveDrawingBuffer': false, + 'powerPreference': 'default', + 'failIfMajorPerformanceCaveat': false, + 'enableExtensionsByDefault': true, + }; + const contextHandle = GL.createContext(canvas, contextAttributes); + handleToCanvasMap.set(contextHandle, canvas); + return contextHandle; + } + _skwasm_resizeCanvas = function(contextHandle, width, height) { + const canvas = handleToCanvasMap.get(contextHandle); + canvas.width = width; + canvas.height = height; + } + _skwasm_captureImageBitmap = async function(surfaceHandle, contextHandle, bitmapId, width, height) { + const canvas = handleToCanvasMap.get(contextHandle); + const imageBitmap = await createImageBitmap(canvas, 0, 0, width, height); + objectMap.set(bitmapId, imageBitmap); + _surface_onCaptureComplete(surfaceHandle, bitmapId); + } _skwasm_createGlTextureFromVideoFrame = function(videoFrameId, width, height) { const videoFrame = skwasm_getObject(videoFrameId); const glCtx = GL.currentContext.GLctx; @@ -59,23 +100,35 @@ mergeInto(LibraryManager.library, { const textureId = GL.getNewId(GL.textures); GL.textures[textureId] = newTexture; return textureId; - }, + } _skwasm_disposeVideoFrame = function(videoFrameId) { const videoFrame = skwasm_getObject(videoFrameId); videoFrame.close(); skwasm_unregisterObject(videoFrameId); } }, + skwasm_generateUniqueId: function() {}, + skwasm_generateUniqueId__deps: ['$skwasm_support_setup'], + $skwasm_generateUniqueId: function() {}, + $skwasm_generateUniqueId__deps: ['$skwasm_support_setup'], $skwasm_registerObject: function() {}, $skwasm_registerObject__deps: ['$skwasm_support_setup'], $skwasm_unregisterObject: function() {}, $skwasm_unregisterObject__deps: ['$skwasm_support_setup'], $skwasm_getObject: function() {}, $skwasm_getObject__deps: ['$skwasm_support_setup'], - $skwasm_transferObjectToMain: function() {}, - $skwasm_transferObjectToMain__deps: ['$skwasm_support_setup'], $skwasm_transferObjectToThread: function() {}, $skwasm_transferObjectToThread__deps: ['$skwasm_support_setup'], + skwasm_transferObjectToMain: function() {}, + skwasm_transferObjectToMain__deps: ['$skwasm_support_setup'], + skwasm_registerMessageListener: function() {}, + skwasm_registerMessageListener__deps: ['$skwasm_support_setup'], + skwasm_createOffscreenCanvas: function () {}, + skwasm_createOffscreenCanvas__deps: ['$skwasm_support_setup'], + skwasm_resizeCanvas: function () {}, + skwasm_resizeCanvas__deps: ['$skwasm_support_setup'], + skwasm_captureImageBitmap: function () {}, + skwasm_captureImageBitmap__deps: ['$skwasm_support_setup'], skwasm_createGlTextureFromVideoFrame: function () {}, skwasm_createGlTextureFromVideoFrame__deps: ['$skwasm_support_setup', '$skwasm_getObject'], skwasm_disposeVideoFrame: function () {}, diff --git a/lib/web_ui/skwasm/skwasm_support.h b/lib/web_ui/skwasm/skwasm_support.h index 990f0de0963a6..46410b6f56c7e 100644 --- a/lib/web_ui/skwasm/skwasm_support.h +++ b/lib/web_ui/skwasm/skwasm_support.h @@ -5,12 +5,25 @@ #include #include +namespace Skwasm { +class Surface; +} + using SkwasmObjectId = uint32_t; extern "C" { +extern SkwasmObjectId skwasm_generateUniqueId(); extern void skwasm_transferObjectToMain(SkwasmObjectId objectId); extern void skwasm_transferObjectToThread(SkwasmObjectId objectId, pthread_t threadId); +extern void skwasm_registerMessageListener(pthread_t threadId); +extern uint32_t skwasm_createOffscreenCanvas(int width, int height); +extern void skwasm_resizeCanvas(uint32_t contextHandle, int width, int height); +extern void skwasm_captureImageBitmap(Skwasm::Surface* surfaceHandle, + uint32_t contextHandle, + uint32_t bitmapId, + int width, + int height); extern unsigned int skwasm_createGlTextureFromVideoFrame( SkwasmObjectId videoFrameId, int width, diff --git a/lib/web_ui/skwasm/surface.cpp b/lib/web_ui/skwasm/surface.cpp index 4944964530ed7..7e2208411adce 100644 --- a/lib/web_ui/skwasm/surface.cpp +++ b/lib/web_ui/skwasm/surface.cpp @@ -3,16 +3,16 @@ // found in the LICENSE file. #include "surface.h" +#include using namespace Skwasm; -Surface::Surface(const char* canvasID) : _canvasID(canvasID) { +Surface::Surface() { assert(emscripten_is_main_browser_thread()); pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - emscripten_pthread_attr_settransferredcanvases(&attr, _canvasID.c_str()); pthread_create( &_thread, &attr, @@ -21,6 +21,8 @@ Surface::Surface(const char* canvasID) : _canvasID(canvasID) { return nullptr; }, this); + // Listen to messages from the worker + skwasm_registerMessageListener(_thread); } // Main thread only @@ -31,37 +33,21 @@ void Surface::dispose() { this); } -// Main thread only -void Surface::setCanvasSize(int width, int height) { - assert(emscripten_is_main_browser_thread()); - emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VIII, - reinterpret_cast(fSetCanvasSize), - nullptr, this, width, height); -} - // Main thread only uint32_t Surface::renderPicture(SkPicture* picture) { assert(emscripten_is_main_browser_thread()); - uint32_t callbackId = ++_currentCallbackId; + uint32_t callbackId = skwasm_generateUniqueId(); picture->ref(); - emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VII, + emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VIII, reinterpret_cast(fRenderPicture), - nullptr, this, picture); - - // After drawing to the surface, the browser implicitly flushes the drawing - // commands at the end of the event loop. As a result, in order to make - // sure we call back after the rendering has actually occurred, we issue - // the callback in a subsequent event, after the flushing has happened. - emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VII, - reinterpret_cast(fNotifyRenderComplete), - nullptr, this, callbackId); + nullptr, this, picture, callbackId); return callbackId; } // Main thread only uint32_t Surface::rasterizeImage(SkImage* image, ImageByteFormat format) { assert(emscripten_is_main_browser_thread()); - uint32_t callbackId = ++_currentCallbackId; + uint32_t callbackId = skwasm_generateUniqueId(); image->ref(); emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VIIII, @@ -90,23 +76,9 @@ void Surface::_runWorker() { // Worker thread only void Surface::_init() { - EmscriptenWebGLContextAttributes attributes; - emscripten_webgl_init_context_attributes(&attributes); - - attributes.alpha = true; - attributes.depth = true; - attributes.stencil = true; - attributes.antialias = false; - attributes.premultipliedAlpha = true; - attributes.preserveDrawingBuffer = 0; - attributes.powerPreference = EM_WEBGL_POWER_PREFERENCE_DEFAULT; - attributes.failIfMajorPerformanceCaveat = false; - attributes.enableExtensionsByDefault = true; - attributes.explicitSwapControl = false; - attributes.renderViaOffscreenBackBuffer = true; - attributes.majorVersion = 2; - - _glContext = emscripten_webgl_create_context(_canvasID.c_str(), &attributes); + // Listen to messages from the main thread + skwasm_registerMessageListener(0); + _glContext = skwasm_createOffscreenCanvas(256, 256); if (!_glContext) { printf("Failed to create context!\n"); return; @@ -141,11 +113,10 @@ void Surface::_dispose() { } // Worker thread only -void Surface::_setCanvasSize(int width, int height) { - if (_canvasWidth != width || _canvasHeight != height) { - emscripten_set_canvas_element_size(_canvasID.c_str(), width, height); - _canvasWidth = width; - _canvasHeight = height; +void Surface::_resizeCanvasToFit(int width, int height) { + if (!_surface || width > _canvasWidth || height > _canvasHeight) { + _canvasWidth = std::max(width, _canvasWidth); + _canvasHeight = std::max(height, _canvasHeight); _recreateSurface(); } } @@ -153,6 +124,7 @@ void Surface::_setCanvasSize(int width, int height) { // Worker thread only void Surface::_recreateSurface() { makeCurrent(_glContext); + skwasm_resizeCanvas(_glContext, _canvasWidth, _canvasHeight); GrBackendRenderTarget target(_canvasWidth, _canvasHeight, _sampleCount, _stencil, _fbInfo); _surface = SkSurfaces::WrapBackendRenderTarget( @@ -161,16 +133,17 @@ void Surface::_recreateSurface() { } // Worker thread only -void Surface::_renderPicture(const SkPicture* picture) { - if (!_surface) { - printf("Can't render picture with no surface.\n"); - return; - } - +void Surface::_renderPicture(const SkPicture* picture, uint32_t callbackId) { + SkRect pictureRect = picture->cullRect(); + _resizeCanvasToFit(pictureRect.width(), pictureRect.height()); + SkMatrix matrix = SkMatrix::Translate(-pictureRect.fLeft, -pictureRect.fTop); makeCurrent(_glContext); auto canvas = _surface->getCanvas(); - canvas->drawPicture(picture); + canvas->drawColor(SK_ColorTRANSPARENT, SkBlendMode::kSrc); + canvas->drawPicture(sk_ref_sp(picture), &matrix, nullptr); _surface->flush(); + skwasm_captureImageBitmap(this, _glContext, callbackId, pictureRect.width(), + pictureRect.height()); } void Surface::_rasterizeImage(SkImage* image, @@ -210,9 +183,10 @@ void Surface::_onRasterizeComplete(SkData* data, uint32_t callbackId) { } // Worker thread only -void Surface::_notifyRenderComplete(uint32_t callbackId) { - emscripten_async_run_in_main_runtime_thread(EM_FUNC_SIG_VII, fOnRenderComplete, - this, callbackId); +void Surface::notifyRenderComplete(uint32_t callbackId) { + skwasm_transferObjectToMain(callbackId); + emscripten_async_run_in_main_runtime_thread( + EM_FUNC_SIG_VII, fOnRenderComplete, this, callbackId); } // Main thread only @@ -225,19 +199,13 @@ void Surface::fDispose(Surface* surface) { surface->_dispose(); } -void Surface::fSetCanvasSize(Surface* surface, int width, int height) { - surface->_setCanvasSize(width, height); -} - -void Surface::fRenderPicture(Surface* surface, SkPicture* picture) { - surface->_renderPicture(picture); +void Surface::fRenderPicture(Surface* surface, + SkPicture* picture, + uint32_t callbackId) { + surface->_renderPicture(picture, callbackId); picture->unref(); } -void Surface::fNotifyRenderComplete(Surface* surface, uint32_t callbackId) { - surface->_notifyRenderComplete(callbackId); -} - void Surface::fOnRenderComplete(Surface* surface, uint32_t callbackId) { surface->_onRenderComplete(callbackId); } @@ -261,8 +229,8 @@ void Surface::fDisposeVideoFrame(Surface* surface, surface->_disposeVideoFrame(videoFrameId); } -SKWASM_EXPORT Surface* surface_createFromCanvas(const char* canvasID) { - return new Surface(canvasID); +SKWASM_EXPORT Surface* surface_create() { + return new Surface(); } SKWASM_EXPORT unsigned long surface_getThreadId(Surface* surface) { @@ -279,12 +247,6 @@ SKWASM_EXPORT void surface_destroy(Surface* surface) { surface->dispose(); } -SKWASM_EXPORT void surface_setCanvasSize(Surface* surface, - int width, - int height) { - surface->setCanvasSize(width, height); -} - SKWASM_EXPORT uint32_t surface_renderPicture(Surface* surface, SkPicture* picture) { return surface->renderPicture(picture); @@ -295,3 +257,10 @@ SKWASM_EXPORT uint32_t surface_rasterizeImage(Surface* surface, ImageByteFormat format) { return surface->rasterizeImage(image, format); } + +// This is used by the skwasm JS support code to call back into C++ when the +// we finish creating the image bitmap, which is an asynchronous operation. +SKWASM_EXPORT void surface_onCaptureComplete(Surface* surface, + uint32_t bitmapId) { + return surface->notifyRenderComplete(bitmapId); +} diff --git a/lib/web_ui/skwasm/surface.h b/lib/web_ui/skwasm/surface.h index 3fc6d8edcf3c8..eb50567fbd5cb 100644 --- a/lib/web_ui/skwasm/surface.h +++ b/lib/web_ui/skwasm/surface.h @@ -39,13 +39,12 @@ class Surface { using CallbackHandler = void(uint32_t, void*); // Main thread only - Surface(const char* canvasID); + Surface(); unsigned long getThreadId() { return _thread; } // Main thread only void dispose(); - void setCanvasSize(int width, int height); uint32_t renderPicture(SkPicture* picture); uint32_t rasterizeImage(SkImage* image, ImageByteFormat format); void setCallbackHandler(CallbackHandler* callbackHandler); @@ -53,24 +52,25 @@ class Surface { // Any thread void disposeVideoFrame(SkwasmObjectId videoFrameId); + // Worker thread only. + void notifyRenderComplete(uint32_t callbackId); + private: void _runWorker(); void _init(); void _dispose(); - void _setCanvasSize(int width, int height); + void _resizeCanvasToFit(int width, int height); void _recreateSurface(); - void _renderPicture(const SkPicture* picture); + void _renderPicture(const SkPicture* picture, uint32_t callbackId); void _rasterizeImage(SkImage* image, ImageByteFormat format, uint32_t callbackId); void _disposeVideoFrame(SkwasmObjectId objectId); void _onRasterizeComplete(SkData* data, uint32_t callbackId); - void _notifyRenderComplete(uint32_t callbackId); void _onRenderComplete(uint32_t callbackId); std::string _canvasID; CallbackHandler* _callbackHandler = nullptr; - uint32_t _currentCallbackId = 0; int _canvasWidth = 0; int _canvasHeight = 0; @@ -85,9 +85,9 @@ class Surface { pthread_t _thread; static void fDispose(Surface* surface); - static void fSetCanvasSize(Surface* surface, int width, int height); - static void fRenderPicture(Surface* surface, SkPicture* picture); - static void fNotifyRenderComplete(Surface* surface, uint32_t callbackId); + static void fRenderPicture(Surface* surface, + SkPicture* picture, + uint32_t callbackId); static void fOnRenderComplete(Surface* surface, uint32_t callbackId); static void fRasterizeImage(Surface* surface, SkImage* image, diff --git a/lib/web_ui/test/ui/draw_atlas_golden_test.dart b/lib/web_ui/test/ui/draw_atlas_golden_test.dart index 30ade95b7c33c..a5d60b82432f6 100644 --- a/lib/web_ui/test/ui/draw_atlas_golden_test.dart +++ b/lib/web_ui/test/ui/draw_atlas_golden_test.dart @@ -104,6 +104,16 @@ Future testMain() async { ); const ui.Rect region = ui.Rect.fromLTWH(0, 0, 300, 300); + + test('render atlas', () async { + final ui.PictureRecorder recorder = ui.PictureRecorder(); + final ui.Canvas canvas = ui.Canvas(recorder, region); + final ui.Image atlas = generateAtlas(); + canvas.drawImage(atlas, ui.Offset.zero, ui.Paint()); + await drawPictureUsingCurrentRenderer(recorder.endRecording()); + await matchGoldenFile('ui_atlas.png', region: region); + }, skip: isHtml); // HTML renderer doesn't support drawAtlas + test('drawAtlas', () async { final ui.PictureRecorder recorder = ui.PictureRecorder(); final ui.Canvas canvas = ui.Canvas(recorder, region); From 3917961536207c6e76815852c6d9f85f36efd11a Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 18 Jul 2023 08:52:58 -0700 Subject: [PATCH 03/24] Fix tests. --- lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart index 284a3fe0da540..335a91e33ec87 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart @@ -28,7 +28,7 @@ class SkwasmSurface { final int threadId; - int acquireObjectId() => skwasmInstance.skwasmGenerateUniqueId().toDart.toInt(); + int acquireObjectId() => skwasmInstance.skwasmGenerateUniqueId().toDartDouble.toInt(); void _initialize() { _callbackHandle = From f6a002e47086951d5485dd48d2fccd8034d9cdde Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Mon, 24 Jul 2023 17:59:16 -0700 Subject: [PATCH 04/24] Occlusion stuff --- .../platform_views/content_manager.dart | 2 +- .../src/engine/skwasm/skwasm_impl/layers.dart | 192 +++++++++++++++--- .../skwasm/skwasm_impl/scene_builder.dart | 42 ++-- .../engine/skwasm/skwasm_impl/scene_view.dart | 31 +++ 4 files changed, 217 insertions(+), 50 deletions(-) create mode 100644 lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart diff --git a/lib/web_ui/lib/src/engine/platform_views/content_manager.dart b/lib/web_ui/lib/src/engine/platform_views/content_manager.dart index 288ab4374a32e..0fcaf0fbbb8fb 100644 --- a/lib/web_ui/lib/src/engine/platform_views/content_manager.dart +++ b/lib/web_ui/lib/src/engine/platform_views/content_manager.dart @@ -120,7 +120,7 @@ class PlatformViewManager { return _contents.putIfAbsent(viewId, () { final DomElement wrapper = domDocument - .createElement('flt-platform-view') + .createElement('flt-platform-view') ..setAttribute('slot', slotName); final Function factoryFunction = _factories[viewType]!; diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart index 12142f04332fa..f4985e59c412b 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart @@ -8,6 +8,8 @@ import 'package:ui/src/engine/scene_painting.dart'; import 'package:ui/src/engine/vector_math.dart'; import 'package:ui/ui.dart' as ui; +class RootLayer with PictureLayer {} + class BackdropFilterLayer with PictureLayer implements ui.BackdropFilterEngineLayer {} @@ -20,6 +22,9 @@ class BackdropFilterOperation implements LayerOperation { @override ui.Rect cullRect(ui.Rect contentRect) => contentRect; + @override + ui.Rect inverseMapRect(ui.Rect rect) => rect; + @override void pre(SceneCanvas canvas, ui.Rect contentRect) { canvas.saveLayerWithFilter(contentRect, ui.Paint()..blendMode = mode, filter); @@ -43,6 +48,9 @@ class ClipPathOperation implements LayerOperation { @override ui.Rect cullRect(ui.Rect contentRect) => contentRect.intersect(path.getBounds()); + @override + ui.Rect inverseMapRect(ui.Rect rect) => rect; + @override void pre(SceneCanvas canvas, ui.Rect contentRect) { canvas.save(); @@ -73,6 +81,9 @@ class ClipRectOperation implements LayerOperation { @override ui.Rect cullRect(ui.Rect contentRect) => contentRect.intersect(rect); + @override + ui.Rect inverseMapRect(ui.Rect rect) => rect; + @override void pre(SceneCanvas canvas, ui.Rect contentRect) { canvas.save(); @@ -103,6 +114,9 @@ class ClipRRectOperation implements LayerOperation { @override ui.Rect cullRect(ui.Rect contentRect) => contentRect.intersect(rrect.outerRect); + @override + ui.Rect inverseMapRect(ui.Rect rect) => rect; + @override void pre(SceneCanvas canvas, ui.Rect contentRect) { canvas.save(); @@ -132,6 +146,9 @@ class ColorFilterOperation implements LayerOperation { @override ui.Rect cullRect(ui.Rect contentRect) => contentRect; + @override + ui.Rect inverseMapRect(ui.Rect rect) => rect; + @override void pre(SceneCanvas canvas, ui.Rect contentRect) { canvas.saveLayer(contentRect, ui.Paint()..colorFilter = filter); @@ -155,6 +172,9 @@ class ImageFilterOperation implements LayerOperation { @override ui.Rect cullRect(ui.Rect contentRect) => contentRect; + @override + ui.Rect inverseMapRect(ui.Rect rect) => rect; + @override void pre(SceneCanvas canvas, ui.Rect contentRect) { if (offset != ui.Offset.zero) { @@ -187,6 +207,9 @@ class OffsetOperation implements LayerOperation { @override ui.Rect cullRect(ui.Rect contentRect) => contentRect.shift(ui.Offset(dx, dy)); + @override + ui.Rect inverseMapRect(ui.Rect rect) => rect.shift(ui.Offset(-dx, -dy)); + @override void pre(SceneCanvas canvas, ui.Rect cullRect) { canvas.save(); @@ -211,6 +234,9 @@ class OpacityOperation implements LayerOperation { @override ui.Rect cullRect(ui.Rect contentRect) => contentRect.shift(offset); + @override + ui.Rect inverseMapRect(ui.Rect rect) => rect; + @override void pre(SceneCanvas canvas, ui.Rect cullRect) { if (offset != ui.Offset.zero) { @@ -240,9 +266,16 @@ class TransformOperation implements LayerOperation { final Float64List transform; + Matrix4 getMatrix() => Matrix4.fromFloat32List(toMatrix32(transform)); + @override - ui.Rect cullRect(ui.Rect contentRect) => - Matrix4.fromFloat32List(toMatrix32(transform)).transformRect(contentRect); + ui.Rect cullRect(ui.Rect contentRect) => getMatrix().transformRect(contentRect); + + @override + ui.Rect inverseMapRect(ui.Rect rect) { + final Matrix4 matrix = getMatrix()..invert(); + return matrix.transformRect(rect); + } @override void pre(SceneCanvas canvas, ui.Rect cullRect) { @@ -269,6 +302,9 @@ class ShaderMaskOperation implements LayerOperation { @override ui.Rect cullRect(ui.Rect contentRect) => contentRect; + @override + ui.Rect inverseMapRect(ui.Rect rect) => rect; + @override void pre(SceneCanvas canvas, ui.Rect contentRect) { canvas.saveLayer( @@ -292,13 +328,51 @@ class ShaderMaskOperation implements LayerOperation { } } +class PlatformView { + PlatformView(this.viewId, this.rect); + + int viewId; + + // The bounds of this platform view, in the layer's local coordinate space. + ui.Rect rect; +} + +abstract class LayerSlice { + void dispose(); +} + +class PlatformViewSlice implements LayerSlice { + PlatformViewSlice(this.views, this.occlusionRect); + + List views; + + // A conservative estimate of what area platform views in this slice may cover. + // This is expressed in the coordinate space of the parent. + ui.Rect? occlusionRect; + + @override + void dispose() {} + + // TODO: Probably add some styling stuff in here +} + +class PictureSlice implements LayerSlice { + PictureSlice(this.picture); + + ui.Picture picture; + + @override + void dispose() => picture.dispose(); +} mixin PictureLayer implements ui.EngineLayer { - ui.Picture? picture; + List slices = []; @override void dispose() { - picture?.dispose(); + for (final LayerSlice slice in slices) { + slice.dispose(); + } } } @@ -306,6 +380,10 @@ abstract class LayerOperation { const LayerOperation(); ui.Rect cullRect(ui.Rect contentRect); + + // Takes a rectangle in the layer's coordinate space and maps it to the parent + // coordinate space. + ui.Rect inverseMapRect(ui.Rect rect); void pre(SceneCanvas canvas, ui.Rect contentRect); void post(SceneCanvas canvas, ui.Rect contentRect); } @@ -319,7 +397,7 @@ class PictureDrawCommand { class LayerBuilder { factory LayerBuilder.rootLayer() { - return LayerBuilder._(null, null, null); + return LayerBuilder._(null, RootLayer(), null); } factory LayerBuilder.childLayer({ @@ -333,36 +411,52 @@ class LayerBuilder { LayerBuilder._( this.parent, this.layer, - this.operation - ); + this.operation); final LayerBuilder? parent; - final PictureLayer? layer; + final PictureLayer layer; final LayerOperation? operation; - final List drawCommands = []; - ui.Rect? contentRect; - - ui.Picture build() { - final ui.Rect drawnRect = contentRect ?? ui.Rect.zero; - final ui.Rect rect = operation?.cullRect(drawnRect) ?? drawnRect; - final ui.PictureRecorder recorder = ui.PictureRecorder(); - final SceneCanvas canvas = ui.Canvas(recorder, rect) as SceneCanvas; - - operation?.pre(canvas, rect); - for (final PictureDrawCommand command in drawCommands) { - if (command.offset != ui.Offset.zero) { - canvas.save(); - canvas.translate(command.offset.dx, command.offset.dy); - canvas.drawPicture(command.picture); - canvas.restore(); - } else { - canvas.drawPicture(command.picture); + final List pendingPictures = []; + final List pendingPlatformViews = []; + ui.Rect? picturesRect; + ui.Rect? platformViewRect; + + void flushSlices() { + if (pendingPictures.isNotEmpty) { + final ui.Rect drawnRect = picturesRect ?? ui.Rect.zero; + final ui.Rect rect = operation?.cullRect(drawnRect) ?? drawnRect; + final ui.PictureRecorder recorder = ui.PictureRecorder(); + final SceneCanvas canvas = ui.Canvas(recorder, rect) as SceneCanvas; + + operation?.pre(canvas, rect); + for (final PictureDrawCommand command in pendingPictures) { + if (command.offset != ui.Offset.zero) { + canvas.save(); + canvas.translate(command.offset.dx, command.offset.dy); + canvas.drawPicture(command.picture); + canvas.restore(); + } else { + canvas.drawPicture(command.picture); + } } + operation?.post(canvas, rect); + final ui.Picture picture = recorder.endRecording(); + layer.slices.add(PictureSlice(picture)); } - operation?.post(canvas, rect); - final ui.Picture picture = recorder.endRecording(); - layer?.picture = picture; - return picture; + + if (pendingPlatformViews.isNotEmpty) { + ui.Rect? occlusionRect = platformViewRect; + if (occlusionRect != null && operation != null) { + occlusionRect = operation!.inverseMapRect(occlusionRect); + } + layer.slices.add(PlatformViewSlice(pendingPlatformViews, occlusionRect)); + // TODO: attach styling information + } + + pendingPictures.clear(); + pendingPlatformViews.clear(); + picturesRect = null; + platformViewRect = null; } void addPicture( @@ -371,9 +465,43 @@ class LayerBuilder { bool isComplexHint = false, bool willChangeHint = false }) { - drawCommands.add(PictureDrawCommand(offset, picture)); final ui.Rect cullRect = (picture as ScenePicture).cullRect; final ui.Rect shiftedRect = cullRect.shift(offset); - contentRect = contentRect?.expandToInclude(shiftedRect) ?? shiftedRect; + if (platformViewRect?.overlaps(shiftedRect) ?? false) { + flushSlices(); + } + pendingPictures.add(PictureDrawCommand(offset, picture)); + picturesRect = picturesRect?.expandToInclude(shiftedRect) ?? shiftedRect; + } + + void addPlatformView( + int viewId, { + ui.Offset offset = ui.Offset.zero, + double width = 0.0, + double height = 0.0 + }) { + final ui.Rect bounds = ui.Rect.fromLTWH(offset.dx, offset.dy, width, height); + platformViewRect = platformViewRect?.expandToInclude(bounds) ?? bounds; + pendingPlatformViews.add(PlatformView(viewId, bounds)); + } + + void mergeLayer(PictureLayer layer) { + for (final LayerSlice slice in layer.slices) { + switch (slice) { + case PictureSlice(): + addPicture(ui.Offset.zero, slice.picture); + case PlatformViewSlice(): + final ui.Rect? occlusionRect = slice.occlusionRect; + if (occlusionRect != null) { + platformViewRect = platformViewRect?.expandToInclude(occlusionRect) ?? occlusionRect; + } + slice.views.addAll(pendingPlatformViews); + } + } + } + + PictureLayer build() { + flushSlices(); + return layer; } } diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart index 4cee3683a3178..2154fc489c8e6 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart @@ -8,25 +8,32 @@ import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; import 'package:ui/ui.dart' as ui; class SkwasmScene implements ui.Scene { - SkwasmScene(this.picture); + SkwasmScene(this.rootLayer); - final ui.Picture picture; + final RootLayer rootLayer; @override void dispose() { - picture.dispose(); + rootLayer.dispose(); } @override - Future toImage(int width, int height) { - return picture.toImage(width, height); + Future toImage(int width, int height) async { + return toImageSync(width, height); } @override ui.Image toImageSync(int width, int height) { - return picture.toImageSync(width, height); - } + final ui.PictureRecorder recorder = ui.PictureRecorder(); + final ui.Rect canvasRect = ui.Rect.fromLTWH(0, 0, width.toDouble(), height.toDouble()); + final ui.Canvas canvas = ui.Canvas(recorder, canvasRect); + // Only rasterizes the picture slices. + for (final PictureSlice slice in rootLayer.slices.whereType()) { + canvas.drawPicture(slice.picture); + } + return recorder.endRecording().toImageSync(width, height); + } } class SkwasmSceneBuilder implements ui.SceneBuilder { @@ -61,16 +68,17 @@ class SkwasmSceneBuilder implements ui.SceneBuilder { double width = 0.0, double height = 0.0 }) { - throw UnimplementedError('Platform view not yet implemented with skwasm renderer.'); + currentBuilder.addPlatformView( + viewId, + offset: offset, + width: width, + height: height + ); } @override void addRetained(ui.EngineLayer retainedLayer) { - final ui.Picture? picture = (retainedLayer as PictureLayer).picture; - if (picture == null) { - throw StateError('Adding incomplete retained layer.'); - } - currentBuilder.addPicture(ui.Offset.zero, picture); + currentBuilder.mergeLayer(retainedLayer as PictureLayer); } @override @@ -214,19 +222,19 @@ class SkwasmSceneBuilder implements ui.SceneBuilder { while (currentBuilder.parent != null) { pop(); } - final ui.Picture finalPicture = currentBuilder.build(); - return SkwasmScene(finalPicture); + final PictureLayer rootLayer = currentBuilder.build(); + return SkwasmScene(rootLayer as RootLayer); } @override void pop() { - final ui.Picture picture = currentBuilder.build(); + final PictureLayer layer = currentBuilder.build(); final LayerBuilder? parentBuilder = currentBuilder.parent; if (parentBuilder == null) { throw StateError('Popped too many times.'); } currentBuilder = parentBuilder; - currentBuilder.addPicture(ui.Offset.zero, picture); + currentBuilder.mergeLayer(layer); } T pushLayer(T layer, LayerOperation operation) { diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart new file mode 100644 index 0000000000000..ec5325513cf57 --- /dev/null +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart @@ -0,0 +1,31 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:ui/src/engine.dart'; +import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; + +class SkwasmSceneView { + factory SkwasmSceneView(SkwasmSurface surface) { + final DomElement sceneElement = createDomElement('flt-scene'); + return SkwasmSceneView._(surface, sceneElement); + } + + SkwasmSceneView._(this.surface, this.sceneElement); + + final SkwasmSurface surface; + final DomElement sceneElement; + int queuedRenders = 0; + static const int kMaxQueuedRenders = 3; + + Future renderScene(SkwasmScene scene) async { + if (queuedRenders >= kMaxQueuedRenders) { + return; + } + queuedRenders += 1; + + + + queuedRenders -= 1; + } +} From e25bad4eed346bda77935c01cc183cd53dd15309 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Thu, 27 Jul 2023 08:10:24 -0700 Subject: [PATCH 05/24] asdf --- .../src/engine/skwasm/skwasm_impl/layers.dart | 4 +- .../engine/skwasm/skwasm_impl/scene_view.dart | 61 ++++++++++++++++++- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart index f4985e59c412b..94dbf2e769622 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart @@ -337,7 +337,7 @@ class PlatformView { ui.Rect rect; } -abstract class LayerSlice { +sealed class LayerSlice { void dispose(); } @@ -359,7 +359,7 @@ class PlatformViewSlice implements LayerSlice { class PictureSlice implements LayerSlice { PictureSlice(this.picture); - ui.Picture picture; + ScenePicture picture; @override void dispose() => picture.dispose(); diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart index ec5325513cf57..93d62142ed9cf 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart @@ -2,9 +2,18 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:html'; + +import 'package:ui/ui.dart' as ui; import 'package:ui/src/engine.dart'; import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; +const String kCanvasContainerTag = 'flt-canvas-container'; +const String kPlatformViewContainerTag = 'flt-platform-view'; + +class SliceContainer { +} + class SkwasmSceneView { factory SkwasmSceneView(SkwasmSurface surface) { final DomElement sceneElement = createDomElement('flt-scene'); @@ -15,6 +24,7 @@ class SkwasmSceneView { final SkwasmSurface surface; final DomElement sceneElement; + int queuedRenders = 0; static const int kMaxQueuedRenders = 3; @@ -24,8 +34,57 @@ class SkwasmSceneView { } queuedRenders += 1; - + final List slices = scene.rootLayer.slices; + final Iterable> renderFutures = slices.map( + (LayerSlice slice) async => switch (slice) { + PlatformViewSlice() => null, + PictureSlice() => surface.renderPicture(slice.picture as SkwasmPicture), + } + ); + final List renderedBitmaps = await Future.wait(renderFutures); + + // TODO: optimize this. + DomElement? lastAdded; + for (int i = 0; i < slices.length; i++) { + final DomNode? nextNode = lastAdded == null + ? sceneElement.firstChild + : lastAdded.nextSibling; + final LayerSlice slice = slices[i]; + switch (slice) { + case PictureSlice(): + final DomNode? candidate = findBestExistingPictureNode(slice); + if (candidate != null) { + if (nextNode != candidate) { + sceneElement.removeChild(candidate); + if (lastAdded == null) { + sceneElement.prepend(candidate); + } else { + lastAdded.after() + } + sceneElement.prepend( + } + } + case PlatformViewSlice(): + final + } + final DomNode? candidate = lastAdded?.nextSibling ?? sceneElement.firstChild; + + } queuedRenders -= 1; } + + DomElement? findBestExistingPictureNode(PictureSlice slice) { + final ui.Rect bounds = slice.picture.cullRect; + for (DomElement? current = sceneElement.firstElementChild; current != null; current = current.nextSibling) { + if (current.tagName != kCanvasContainerTag) { + continue; + } + final DomCanvasElement canvas = current.firstChild! as DomCanvasElement; + if (canvas.width == bounds.width && canvas.height == bounds.height) { + return current; + } + } + return null; + } } From 765a39c68947e27152b986ca89110832f109e055 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Wed, 2 Aug 2023 17:03:48 -0700 Subject: [PATCH 06/24] The single picture case at least is working. --- lib/web_ui/lib/src/engine/dom.dart | 2 + .../platform_views/content_manager.dart | 2 +- .../lib/src/engine/skwasm/skwasm_impl.dart | 1 + .../src/engine/skwasm/skwasm_impl/layers.dart | 4 +- .../engine/skwasm/skwasm_impl/renderer.dart | 29 +-- .../skwasm/skwasm_impl/scene_builder.dart | 4 +- .../engine/skwasm/skwasm_impl/scene_view.dart | 166 ++++++++++++++---- 7 files changed, 143 insertions(+), 65 deletions(-) diff --git a/lib/web_ui/lib/src/engine/dom.dart b/lib/web_ui/lib/src/engine/dom.dart index 5736455df65f4..50be162f56dc6 100644 --- a/lib/web_ui/lib/src/engine/dom.dart +++ b/lib/web_ui/lib/src/engine/dom.dart @@ -530,6 +530,8 @@ extension DomElementExtension on DomElement { external DomElement? get firstElementChild; + external DomElement? get nextElementSibling; + @JS('clientHeight') external JSNumber get _clientHeight; double get clientHeight => _clientHeight.toDartDouble; diff --git a/lib/web_ui/lib/src/engine/platform_views/content_manager.dart b/lib/web_ui/lib/src/engine/platform_views/content_manager.dart index 13906b5433a51..b5579bc4def25 100644 --- a/lib/web_ui/lib/src/engine/platform_views/content_manager.dart +++ b/lib/web_ui/lib/src/engine/platform_views/content_manager.dart @@ -133,7 +133,7 @@ class PlatformViewManager { return _contents.putIfAbsent(viewId, () { final DomElement wrapper = domDocument - .createElement('flt-platform-view') + .createElement('flt-platform-view') ..setAttribute('slot', slotName); final Function factoryFunction = _factories[viewType]!; diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart index ab0b58d728387..aab16e6735b3d 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart @@ -45,6 +45,7 @@ export 'skwasm_impl/raw/text/raw_strut_style.dart'; export 'skwasm_impl/raw/text/raw_text_style.dart'; export 'skwasm_impl/renderer.dart'; export 'skwasm_impl/scene_builder.dart'; +export 'skwasm_impl/scene_view.dart'; export 'skwasm_impl/shaders.dart'; export 'skwasm_impl/surface.dart'; export 'skwasm_impl/vertices.dart'; diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart index 94dbf2e769622..932da13134c6c 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart @@ -270,7 +270,7 @@ class TransformOperation implements LayerOperation { @override ui.Rect cullRect(ui.Rect contentRect) => getMatrix().transformRect(contentRect); - + @override ui.Rect inverseMapRect(ui.Rect rect) { final Matrix4 matrix = getMatrix()..invert(); @@ -441,7 +441,7 @@ class LayerBuilder { } operation?.post(canvas, rect); final ui.Picture picture = recorder.endRecording(); - layer.slices.add(PictureSlice(picture)); + layer.slices.add(PictureSlice(picture as ScenePicture)); } if (pendingPlatformViews.isNotEmpty) { diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart index c1aab552a14ad..029f2011b3baf 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart @@ -13,8 +13,8 @@ import 'package:ui/ui.dart' as ui; import 'package:ui/ui_web/src/ui_web.dart' as ui_web; class SkwasmRenderer implements Renderer { - late DomCanvasElement sceneElement; late SkwasmSurface surface; + late SkwasmSceneView sceneView; @override final SkwasmFontCollection fontCollection = SkwasmFontCollection(); @@ -347,12 +347,8 @@ class SkwasmRenderer implements Renderer { @override FutureOr initialize() { - // TODO(jacksongardner): This is very basic and doesn't work for element - // embedding or with platform views. We need to update this at some point - // to deal with those cases. - sceneElement = createDomCanvasElement(); - sceneElement.id = 'flt-scene'; surface = SkwasmSurface(); + sceneView = SkwasmSceneView(surface); } @override @@ -403,31 +399,14 @@ class SkwasmRenderer implements Renderer { } @override - Future renderScene(ui.Scene scene) async { - final SkwasmPicture picture = (scene as SkwasmScene).picture as SkwasmPicture; - final DomImageBitmap bitmap = await surface.renderPicture(picture); - - final DomCSSStyleDeclaration style = sceneElement.style; - final ui.Rect pictureRect = picture.cullRect; - final double logicalWidth = pictureRect.width / window.devicePixelRatio; - final double logicalHeight = pictureRect.height / window.devicePixelRatio; - style.width = '${logicalWidth}px'; - style.height = '${logicalHeight}px'; - style.position = 'absolute'; - style.left = '${pictureRect.left}px'; - style.top = '${pictureRect.top}px'; - sceneElement.width = pictureRect.width; - sceneElement.height = pictureRect.height; - final DomCanvasRenderingContextBitmapRenderer context = sceneElement.contextBitmapRenderer; - context.transferFromImageBitmap(bitmap); - } + Future renderScene(ui.Scene scene) => sceneView.renderScene(scene as SkwasmScene); @override String get rendererTag => 'skwasm'; @override void reset(FlutterViewEmbedder embedder) { - embedder.addSceneToSceneHost(sceneElement); + embedder.addSceneToSceneHost(sceneView.sceneElement); } static final Map> _programs = >{}; diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart index 2154fc489c8e6..6c14d8202400f 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart @@ -30,7 +30,7 @@ class SkwasmScene implements ui.Scene { // Only rasterizes the picture slices. for (final PictureSlice slice in rootLayer.slices.whereType()) { - canvas.drawPicture(slice.picture); + canvas.drawPicture(slice.picture); } return recorder.endRecording().toImageSync(width, height); } @@ -69,7 +69,7 @@ class SkwasmSceneBuilder implements ui.SceneBuilder { double height = 0.0 }) { currentBuilder.addPlatformView( - viewId, + viewId, offset: offset, width: width, height: height diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart index 93d62142ed9cf..6e2b270c48594 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart @@ -2,16 +2,88 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:html'; - -import 'package:ui/ui.dart' as ui; import 'package:ui/src/engine.dart'; import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; +import 'package:ui/ui.dart' as ui; const String kCanvasContainerTag = 'flt-canvas-container'; const String kPlatformViewContainerTag = 'flt-platform-view'; -class SliceContainer { +sealed class SliceContainer { + DomElement get container; + + void updateContents(); +} + +final class PictureSliceContainer extends SliceContainer { + factory PictureSliceContainer(ui.Rect bounds) { + final DomElement container = domDocument.createElement(kCanvasContainerTag); + final DomCanvasElement canvas = createDomCanvasElement( + width: bounds.width.toInt(), + height: bounds.height.toInt() + ); + container.appendChild(canvas); + return PictureSliceContainer._(bounds, container, canvas); + } + + PictureSliceContainer._(this._bounds, this.container, this.canvas); + + ui.Rect _bounds; + bool _dirty = true; + + ui.Rect get bounds => _bounds; + set bounds(ui.Rect bounds) { + if (_bounds != bounds) { + _bounds = bounds; + _dirty = true; + } + } + + @override + void updateContents() { + if (_dirty) { + _dirty = false; + final DomCSSStyleDeclaration style = canvas.style; + final double logicalWidth = bounds.width / window.devicePixelRatio; + final double logicalHeight = bounds.height / window.devicePixelRatio; + style.width = '${logicalWidth}px'; + style.height = '${logicalHeight}px'; + style.position = 'absolute'; + style.left = '${bounds.left}px'; + style.top = '${bounds.top}px'; + canvas.width = bounds.width; + canvas.height = bounds.height; + } + } + + void renderBitmap(DomImageBitmap bitmap) { + final DomCanvasRenderingContextBitmapRenderer ctx = canvas.contextBitmapRenderer; + ctx.transferFromImageBitmap(bitmap); + } + + @override + final DomElement container; + final DomCanvasElement canvas; +} + +final class PlatformViewSliceContainer extends SliceContainer { + PlatformViewSliceContainer(this._views); + + List _views; + + @override + final DomElement container = domDocument.createElement(kPlatformViewContainerTag); + + set views(List views) { + if (_views != views) { + _views = views; + } + } + + @override + void updateContents() { + // TODO: Actually add the views needed + } } class SkwasmSceneView { @@ -25,6 +97,8 @@ class SkwasmSceneView { final SkwasmSurface surface; final DomElement sceneElement; + List containers = []; + int queuedRenders = 0; static const int kMaxQueuedRenders = 3; @@ -42,49 +116,71 @@ class SkwasmSceneView { } ); final List renderedBitmaps = await Future.wait(renderFutures); - - // TODO: optimize this. - DomElement? lastAdded; + final List reusableContainers = List.from(containers); + final List newContainers = []; for (int i = 0; i < slices.length; i++) { - final DomNode? nextNode = lastAdded == null - ? sceneElement.firstChild - : lastAdded.nextSibling; final LayerSlice slice = slices[i]; switch (slice) { case PictureSlice(): - final DomNode? candidate = findBestExistingPictureNode(slice); - if (candidate != null) { - if (nextNode != candidate) { - sceneElement.removeChild(candidate); - if (lastAdded == null) { - sceneElement.prepend(candidate); - } else { - lastAdded.after() - } - sceneElement.prepend( + PictureSliceContainer? container; + for (int j = 0; j < reusableContainers.length; j++) { + final SliceContainer? candidate = reusableContainers[j]; + if (candidate is PictureSliceContainer) { + container = candidate; + reusableContainers[j] = null; + break; } } + + if (container != null) { + container.bounds = slice.picture.cullRect; + } else { + container = PictureSliceContainer(slice.picture.cullRect); + } + container.updateContents(); + container.renderBitmap(renderedBitmaps[i]!); + newContainers.add(container); + case PlatformViewSlice(): - final + PlatformViewSliceContainer? container; + for (int j = 0; j < reusableContainers.length; j++) { + final SliceContainer? candidate = reusableContainers[j]; + if (candidate is PlatformViewSliceContainer) { + container = candidate; + reusableContainers[j] = null; + break; + } + } + if (container != null) { + container.views = slice.views; + } else { + container = PlatformViewSliceContainer(slice.views); + } + container.updateContents(); + newContainers.add(container); } - final DomNode? candidate = lastAdded?.nextSibling ?? sceneElement.firstChild; - } - queuedRenders -= 1; - } + containers = newContainers; - DomElement? findBestExistingPictureNode(PictureSlice slice) { - final ui.Rect bounds = slice.picture.cullRect; - for (DomElement? current = sceneElement.firstElementChild; current != null; current = current.nextSibling) { - if (current.tagName != kCanvasContainerTag) { - continue; - } - final DomCanvasElement canvas = current.firstChild! as DomCanvasElement; - if (canvas.width == bounds.width && canvas.height == bounds.height) { - return current; + DomElement? currentElement = sceneElement.firstElementChild; + for (final SliceContainer container in containers) { + if (currentElement == null) { + sceneElement.appendChild(container.container); + } else if (currentElement == container.container) { + currentElement = currentElement.nextElementSibling; + } else { + sceneElement.insertBefore(container.container, currentElement); } } - return null; + + // Remove any other + while (currentElement != null) { + final DomElement? sibling = currentElement.nextElementSibling; + sceneElement.removeChild(currentElement); + currentElement = sibling; + } + + queuedRenders -= 1; } } From b515342148af39bfba2373a4126d6ca444193fe3 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 8 Aug 2023 13:06:01 -0700 Subject: [PATCH 07/24] Some things working, some things broken. --- DEPS | 2 +- .../src/engine/skwasm/skwasm_impl/layers.dart | 48 ++++++- .../engine/skwasm/skwasm_impl/scene_view.dart | 45 ++++++- lib/web_ui/skwasm/BUILD.gn | 4 + lib/web_ui/test/ui/platform_view_test.dart | 117 ++++++++++++++++++ tools/activate_emsdk.py | 2 +- 6 files changed, 209 insertions(+), 9 deletions(-) create mode 100644 lib/web_ui/test/ui/platform_view_test.dart diff --git a/DEPS b/DEPS index 30d51587b3bac..27a08b4d1c426 100644 --- a/DEPS +++ b/DEPS @@ -783,7 +783,7 @@ deps = { }, 'src/buildtools/emsdk': { - 'url': Var('skia_git') + '/external/github.com/emscripten-core/emsdk.git' + '@' + 'da9699832b5df4e123403490e499c87000c22654', + 'url': Var('skia_git') + '/external/github.com/emscripten-core/emsdk.git' + '@' + 'a896e3d066448b3530dbcaa48869fafefd738f57', 'condition': 'download_emsdk', }, diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart index 932da13134c6c..c139c6c834119 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart @@ -335,6 +335,11 @@ class PlatformView { // The bounds of this platform view, in the layer's local coordinate space. ui.Rect rect; + + @override + String toString() { + return 'PlatformView(id: $viewId, rect: $rect)'; + } } sealed class LayerSlice { @@ -354,6 +359,11 @@ class PlatformViewSlice implements LayerSlice { void dispose() {} // TODO: Probably add some styling stuff in here + + @override + String toString() { + return 'PlatformViewSlice($views)'; + } } class PictureSlice implements LayerSlice { @@ -363,6 +373,11 @@ class PictureSlice implements LayerSlice { @override void dispose() => picture.dispose(); + + @override + String toString() { + return 'PictureSlice(${picture.cullRect})'; + } } mixin PictureLayer implements ui.EngineLayer { @@ -395,6 +410,26 @@ class PictureDrawCommand { ui.Picture picture; } +class PlatformViewPosition { + ui.Offset? offset; + Matrix4? transform; + + static PlatformViewPosition combine(PlatformViewPosition outer, PlatformViewPosition inner) { + // Implement this + return outer; + } +} + +class PlatformViewClip { + +} + +class PlatformViewStyling { + PlatformViewPosition? position; + PlatformViewClip? clip; + double? opacity; +} + class LayerBuilder { factory LayerBuilder.rootLayer() { return LayerBuilder._(null, RootLayer(), null); @@ -417,11 +452,12 @@ class LayerBuilder { final PictureLayer layer; final LayerOperation? operation; final List pendingPictures = []; - final List pendingPlatformViews = []; + List pendingPlatformViews = []; ui.Rect? picturesRect; ui.Rect? platformViewRect; void flushSlices() { + print('Flushing slices'); if (pendingPictures.isNotEmpty) { final ui.Rect drawnRect = picturesRect ?? ui.Rect.zero; final ui.Rect rect = operation?.cullRect(drawnRect) ?? drawnRect; @@ -442,6 +478,7 @@ class LayerBuilder { operation?.post(canvas, rect); final ui.Picture picture = recorder.endRecording(); layer.slices.add(PictureSlice(picture as ScenePicture)); + print('adding picture slice'); } if (pendingPlatformViews.isNotEmpty) { @@ -450,11 +487,12 @@ class LayerBuilder { occlusionRect = operation!.inverseMapRect(occlusionRect); } layer.slices.add(PlatformViewSlice(pendingPlatformViews, occlusionRect)); + print('adding platform view slice with views: $pendingPlatformViews'); // TODO: attach styling information } pendingPictures.clear(); - pendingPlatformViews.clear(); + pendingPlatformViews = []; picturesRect = null; platformViewRect = null; } @@ -486,22 +524,26 @@ class LayerBuilder { } void mergeLayer(PictureLayer layer) { + print('merging layer'); for (final LayerSlice slice in layer.slices) { switch (slice) { case PictureSlice(): + print('adding picture from merged layer slice with bounds: ${slice.picture.cullRect}'); addPicture(ui.Offset.zero, slice.picture); case PlatformViewSlice(): final ui.Rect? occlusionRect = slice.occlusionRect; if (occlusionRect != null) { platformViewRect = platformViewRect?.expandToInclude(occlusionRect) ?? occlusionRect; } - slice.views.addAll(pendingPlatformViews); + print('adding views from merged platform view slice: ${slice.views}'); + pendingPlatformViews.addAll(slice.views); } } } PictureLayer build() { flushSlices(); + print('built layer with slices: ${layer.slices}'); return layer; } } diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart index 6e2b270c48594..d3241523c26d1 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart @@ -7,7 +7,8 @@ import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; import 'package:ui/ui.dart' as ui; const String kCanvasContainerTag = 'flt-canvas-container'; -const String kPlatformViewContainerTag = 'flt-platform-view'; +const String kPlatformViewSliceContainerTag = 'flt-platform-view-slice'; +const String kPlatformViewContainerTag = 'flt-platform-view-container'; sealed class SliceContainer { DomElement get container; @@ -72,7 +73,7 @@ final class PlatformViewSliceContainer extends SliceContainer { List _views; @override - final DomElement container = domDocument.createElement(kPlatformViewContainerTag); + final DomElement container = domDocument.createElement(kPlatformViewSliceContainerTag); set views(List views) { if (_views != views) { @@ -82,7 +83,32 @@ final class PlatformViewSliceContainer extends SliceContainer { @override void updateContents() { - // TODO: Actually add the views needed + DomElement? currentContainer = container.firstElementChild; + for (final PlatformView view in _views) { + final DomElement platformView = platformViewManager.getViewById(view.viewId); + final DomElement viewContainer; + if (currentContainer != null && currentContainer.firstElementChild == platformView) { + viewContainer = currentContainer; + currentContainer = currentContainer.nextElementSibling; + } else { + viewContainer = domDocument.createElement(kPlatformViewContainerTag); + if (currentContainer != null) { + container.insertBefore(viewContainer, currentContainer); + } else { + container.appendChild(viewContainer); + } + viewContainer.appendChild(platformView); + } + // TODO: set all the styling here instead of just the position + final DomCSSStyleDeclaration style = viewContainer.style; + final double logicalWidth = view.rect.width / window.devicePixelRatio; + final double logicalHeight = view.rect.height / window.devicePixelRatio; + style.width = '${logicalWidth}px'; + style.height = '${logicalHeight}px'; + style.position = 'absolute'; + style.left = '${view.rect.left}px'; + style.top = '${view.rect.top}px'; + } } } @@ -109,6 +135,17 @@ class SkwasmSceneView { queuedRenders += 1; final List slices = scene.rootLayer.slices; + print('Rendering scene with slices: ${slices.map((LayerSlice slice) { + if (slice is PictureSlice) { + return 'PictureSlice(${slice.picture.cullRect})'; + } else if (slice is PlatformViewSlice) { + return 'PlatformViewSlice(${slice.views.map((PlatformView view) { + return 'PlatformView(${view.viewId}, ${view.rect})'; + })})'; + } else { + return 'Unknown'; + } + })}'); final Iterable> renderFutures = slices.map( (LayerSlice slice) async => switch (slice) { PlatformViewSlice() => null, @@ -174,7 +211,7 @@ class SkwasmSceneView { } } - // Remove any other + // Remove any other unused containers while (currentElement != null) { final DomElement? sibling = currentElement.nextElementSibling; sceneElement.removeChild(currentElement); diff --git a/lib/web_ui/skwasm/BUILD.gn b/lib/web_ui/skwasm/BUILD.gn index 37dc9ec2d6467..cc508a5fa6bb9 100644 --- a/lib/web_ui/skwasm/BUILD.gn +++ b/lib/web_ui/skwasm/BUILD.gn @@ -31,6 +31,10 @@ wasm_lib("skwasm") { "wrappers.h", ] + cflags = [ + "-mreference-types", + ] + ldflags = [ "-std=c++20", "-lGL", diff --git a/lib/web_ui/test/ui/platform_view_test.dart b/lib/web_ui/test/ui/platform_view_test.dart new file mode 100644 index 0000000000000..434fea330f199 --- /dev/null +++ b/lib/web_ui/test/ui/platform_view_test.dart @@ -0,0 +1,117 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:test/bootstrap/browser.dart'; +import 'package:test/test.dart'; +import 'package:ui/src/engine.dart'; +import 'package:ui/ui.dart' as ui; +import 'package:ui/ui_web/src/ui_web.dart' as ui_web; +import 'package:web_engine_tester/golden_tester.dart'; + +import '../common/test_initialization.dart'; + +void main() { + internalBootstrapBrowserTest(() => testMain); +} + +Future testMain() async { + setUpUnitTests( + emulateTesterEnvironment: false, + setUpTestViewDimensions: false, + ); + + const ui.Rect region = ui.Rect.fromLTWH(0, 0, 300, 300); + const String platformViewType = 'test-platform-view'; + + setUpAll(() { + ui_web.platformViewRegistry.registerViewFactory( + platformViewType, + (int viewId) { + final DomElement element = createDomHTMLDivElement(); + element.style.backgroundColor = 'blue'; + return element; + } + ); + }); + + test('Picture + Overlapping PlatformView', () async { + await _createPlatformView(1, platformViewType); + + final ui.PictureRecorder recorder = ui.PictureRecorder(); + final ui.Canvas canvas = ui.Canvas(recorder, region); + canvas.drawCircle( + ui.Offset.zero, + 50, + ui.Paint() + ..style = ui.PaintingStyle.fill + ..color = const ui.Color(0xFFFF0000) + ); + + final ui.SceneBuilder sb = ui.SceneBuilder(); + sb.pushOffset(0, 0); + sb.addPicture(const ui.Offset(150, 150), recorder.endRecording()); + + sb.addPlatformView( + 1, + offset: const ui.Offset(125, 125), + width: 50, + height: 50, + ); + await renderer.renderScene(sb.build()); + + await matchGoldenFile('picture_platformview_overlap.png', region: region); + }); + + test('PlatformView sandwich', () async { + await _createPlatformView(1, platformViewType); + + final ui.PictureRecorder recorder = ui.PictureRecorder(); + final ui.Canvas canvas = ui.Canvas(recorder, region); + canvas.drawCircle( + ui.Offset.zero, + 50, + ui.Paint() + ..style = ui.PaintingStyle.fill + ..color = const ui.Color(0xFF00FF00) + ); + + final ui.Picture picture = recorder.endRecording(); + + final ui.SceneBuilder sb = ui.SceneBuilder(); + sb.pushOffset(0, 0); + sb.addPicture(const ui.Offset(125, 125), picture); + + sb.addPlatformView( + 1, + offset: const ui.Offset(150, 150), + width: 50, + height: 50, + ); + + sb.addPicture(const ui.Offset(175, 175), picture); + await renderer.renderScene(sb.build()); + + await matchGoldenFile('picture_platformview_overlap.png', region: region); + }); +} + +// Sends a platform message to create a Platform View with the given id and viewType. +Future _createPlatformView(int id, String viewType) { + final Completer completer = Completer(); + const MethodCodec codec = StandardMethodCodec(); + window.sendPlatformMessage( + 'flutter/platform_views', + codec.encodeMethodCall(MethodCall( + 'create', + { + 'id': id, + 'viewType': viewType, + }, + )), + (dynamic _) => completer.complete(), + ); + return completer.future; +} diff --git a/tools/activate_emsdk.py b/tools/activate_emsdk.py index 9b0d11dbf740d..3653bc3eb9700 100644 --- a/tools/activate_emsdk.py +++ b/tools/activate_emsdk.py @@ -16,7 +16,7 @@ EMSDK_PATH = os.path.join(EMSDK_ROOT, 'emsdk.py') # See lib/web_ui/README.md for instructions on updating the EMSDK version. -EMSDK_VERSION = '3.1.32' +EMSDK_VERSION = '3.1.44' def main(): From 353d5f4c153fdefa8f7e1e31e08d67cf5b507eff Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Thu, 10 Aug 2023 17:32:32 -0700 Subject: [PATCH 08/24] Partway through __externref_t refactor --- .../src/engine/skwasm/skwasm_impl/codecs.dart | 7 +- .../skwasm/skwasm_impl/raw/raw_image.dart | 48 ++++++++++--- .../skwasm/skwasm_impl/raw/raw_surface.dart | 6 +- .../skwasm/skwasm_impl/raw/skwasm_module.dart | 15 ---- .../engine/skwasm/skwasm_impl/surface.dart | 21 +++--- lib/web_ui/skwasm/image.cpp | 14 ++-- lib/web_ui/skwasm/library_skwasm_support.js | 69 +++++-------------- lib/web_ui/skwasm/skwasm_support.h | 10 +-- lib/web_ui/skwasm/surface.cpp | 38 +++++----- lib/web_ui/skwasm/surface.h | 17 +++-- 10 files changed, 113 insertions(+), 132 deletions(-) diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart index 91eedca9c8000..7c1a91cbd29c7 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:js_interop'; - import 'package:ui/src/engine.dart'; import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; import 'package:ui/ui.dart' as ui; @@ -20,11 +18,8 @@ class SkwasmImageDecoder extends BrowserImageDecoder { final int width = frame.codedWidth.toInt(); final int height = frame.codedHeight.toInt(); final SkwasmSurface surface = (renderer as SkwasmRenderer).surface; - final int videoFrameId = surface.acquireObjectId(); - skwasmInstance.skwasmRegisterObject(videoFrameId.toJS, frame as JSAny); - skwasmInstance.skwasmTransferObjectToThread(videoFrameId.toJS, surface.threadId.toJS); return SkwasmImage(imageCreateFromVideoFrame( - videoFrameId, + frame, width, height, surface.handle, diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart index 6f0eb810948cf..3da653df65800 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart @@ -6,7 +6,9 @@ library skwasm_impl; import 'dart:ffi'; +import 'dart:js_interop'; +import 'package:ui/src/engine.dart'; import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; final class RawImage extends Opaque {} @@ -38,17 +40,45 @@ external ImageHandle imageCreateFromPixels( int rowByteCount, ); -@Native(symbol: 'image_createFromVideoFrame', isLeaf: true) -external ImageHandle imageCreateFromVideoFrame( - int videoFrameId, +// We actually want this function to look something like this: +// +// @Native(symbol: 'image_createFromVideoFrame', isLeaf: true) +// external ImageHandle imageCreateFromVideoFrame( +// JSAny videoFrame, +// int width, +// int height, +// SurfaceHandle handle, +// ); +// +// However, this doesn't work because we cannot use extern refs as part of @Native +// annotations currently. For now, we can use JS interop to expose this function +// instead. +extension SkwasmImageExtension on SkwasmInstance { + @JS('image_createFromVideoFrame') + external JSNumber imageCreateFromVideoFrame( + VideoFrame frame, + JSNumber width, + JSNumber height, + JSNumber surfaceHandle, + ); +} +ImageHandle imageCreateFromVideoFrame( + VideoFrame frame, int width, int height, - SurfaceHandle handle, + SurfaceHandle handle +) => ImageHandle.fromAddress( + skwasmInstance.imageCreateFromVideoFrame( + frame, + width.toJS, + height.toJS, + handle.address.toJS, + ).toDartInt ); @Native(symbol:'image_ref', isLeaf: true) diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart index f43196284af51..3831188c4df63 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart @@ -14,6 +14,8 @@ typedef SurfaceHandle = Pointer; final class RawRenderCallback extends Opaque {} typedef OnRenderCallbackHandle = Pointer; +typedef CallbackId = int; + @Native(symbol: 'surface_create', isLeaf: true) external SurfaceHandle surfaceCreate(); @@ -36,14 +38,14 @@ external void surfaceDestroy(SurfaceHandle surface); @Native( symbol: 'surface_renderPicture', isLeaf: true) -external int surfaceRenderPicture(SurfaceHandle surface, PictureHandle picture); +external CallbackId surfaceRenderPicture(SurfaceHandle surface, PictureHandle picture); @Native(symbol: 'surface_rasterizeImage', isLeaf: true) -external int surfaceRasterizeImage( +external CallbackId surfaceRasterizeImage( SurfaceHandle handle, ImageHandle image, int format, diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart index 0d1aac6159401..a8b994bcb1e89 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart @@ -20,21 +20,6 @@ extension SkwasmInstanceExtension on SkwasmInstance { external JSNumber addFunction(JSFunction function, JSString signature); external void removeFunction(JSNumber functionPointer); - @JS('skwasm_generateUniqueId') - external JSNumber skwasmGenerateUniqueId(); - - @JS('skwasm_registerObject') - external void skwasmRegisterObject(JSNumber objectId, JSAny object); - - @JS('skwasm_unregisterObject') - external void skwasmUnregisterObject(JSNumber objectId); - - @JS('skwasm_getObject') - external JSAny skwasmGetObject(JSNumber objectId); - - @JS('skwasm_transferObjectToThread') - external void skwasmTransferObjectToThread(JSNumber objectId, JSNumber threadId); - external WebAssemblyMemory get wasmMemory; } diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart index 335a91e33ec87..6df842dcdc62e 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart @@ -24,12 +24,10 @@ class SkwasmSurface { SkwasmSurface._fromHandle(this.handle) : threadId = surfaceGetThreadId(handle); final SurfaceHandle handle; OnRenderCallbackHandle _callbackHandle = nullptr; - final Map> _pendingCallbacks = >{}; + final Map> _pendingCallbacks = >{}; final int threadId; - int acquireObjectId() => skwasmInstance.skwasmGenerateUniqueId().toDartDouble.toInt(); - void _initialize() { _callbackHandle = OnRenderCallbackHandle.fromAddress( @@ -43,8 +41,7 @@ class SkwasmSurface { Future renderPicture(SkwasmPicture picture) async { final int callbackId = surfaceRenderPicture(handle, picture.handle); - await _registerCallback(callbackId); - return skwasmInstance.skwasmGetObject(callbackId.toJS) as DomImageBitmap; + return (await _registerCallback(callbackId)) as DomImageBitmap; } Future rasterizeImage(SkwasmImage image, ui.ImageByteFormat format) async { @@ -53,7 +50,7 @@ class SkwasmSurface { image.handle, format.index, ); - final int context = await _registerCallback(callbackId); + final int context = (await _registerCallback(callbackId) as JSNumber).toDartInt; final SkDataHandle dataHandle = SkDataHandle.fromAddress(context); final int byteCount = skDataGetSize(dataHandle); final Pointer dataPointer = skDataGetConstPointer(dataHandle).cast(); @@ -65,16 +62,16 @@ class SkwasmSurface { return ByteData.sublistView(output); } - Future _registerCallback(int callbackId) { - final Completer completer = Completer(); + Future _registerCallback(int callbackId) { + final Completer completer = Completer(); _pendingCallbacks[callbackId] = completer; return completer.future; } - void _callbackHandler(JSNumber jsCallbackId, JSNumber jsPointer) { - final int callbackId = jsCallbackId.toDartDouble.toInt(); - final Completer completer = _pendingCallbacks.remove(callbackId)!; - completer.complete(jsPointer.toDartDouble.toInt()); + void _callbackHandler(JSNumber jsCallbackId, JSAny value) { + final CallbackId callbackId = jsCallbackId.toDartInt; + final Completer completer = _pendingCallbacks.remove(callbackId)!; + completer.complete(value); } void dispose() { diff --git a/lib/web_ui/skwasm/image.cpp b/lib/web_ui/skwasm/image.cpp index 8216f4132090a..c5fdd4c51fca5 100644 --- a/lib/web_ui/skwasm/image.cpp +++ b/lib/web_ui/skwasm/image.cpp @@ -81,14 +81,14 @@ class ExternalWebGLTexture : public GrExternalTexture { class VideoFrameImageGenerator : public GrExternalTextureGenerator { public: VideoFrameImageGenerator(SkImageInfo ii, - SkwasmObjectId videoFrameId, + SkwasmObject videoFrame, Skwasm::Surface* surface) : GrExternalTextureGenerator(ii), - _videoFrameId(videoFrameId), + _videoFrame(videoFrame), _surface(surface) {} ~VideoFrameImageGenerator() override { - _surface->disposeVideoFrame(_videoFrameId); + _surface->disposeVideoFrame(_videoFrame); } std::unique_ptr generateExternalTexture( @@ -96,7 +96,7 @@ class VideoFrameImageGenerator : public GrExternalTextureGenerator { GrMipMapped mipmapped) override { GrGLTextureInfo glInfo; glInfo.fID = skwasm_createGlTextureFromVideoFrame( - _videoFrameId, fInfo.width(), fInfo.height()); + _videoFrame, fInfo.width(), fInfo.height()); glInfo.fFormat = GL_RGBA8_OES; glInfo.fTarget = GL_TEXTURE_2D; @@ -107,7 +107,7 @@ class VideoFrameImageGenerator : public GrExternalTextureGenerator { } private: - SkwasmObjectId _videoFrameId; + SkwasmObject _videoFrame; Skwasm::Surface* _surface; }; @@ -134,7 +134,7 @@ SKWASM_EXPORT SkImage* image_createFromPixels(SkData* data, .release(); } -SKWASM_EXPORT SkImage* image_createFromVideoFrame(SkwasmObjectId videoFrameId, +SKWASM_EXPORT SkImage* image_createFromVideoFrame(SkwasmObject videoFrame, int width, int height, Skwasm::Surface* surface) { @@ -143,7 +143,7 @@ SKWASM_EXPORT SkImage* image_createFromVideoFrame(SkwasmObjectId videoFrameId, SkImageInfo::Make(width, height, SkColorType::kRGBA_8888_SkColorType, SkAlphaType::kPremul_SkAlphaType), - videoFrameId, surface)) + videoFrame, surface)) .release(); } diff --git a/lib/web_ui/skwasm/library_skwasm_support.js b/lib/web_ui/skwasm/library_skwasm_support.js index 68ce0881983db..12fd673b86019 100644 --- a/lib/web_ui/skwasm/library_skwasm_support.js +++ b/lib/web_ui/skwasm/library_skwasm_support.js @@ -8,54 +8,34 @@ mergeInto(LibraryManager.library, { $skwasm_support_setup__postset: 'skwasm_support_setup();', $skwasm_support_setup: function() { - const objectMap = new Map(); const handleToCanvasMap = new Map(); - let currentUniqueId = 0; - _skwasm_generateUniqueId = function() { - return ++currentUniqueId; - } - skwasm_generateUniqueId = _skwasm_generateUniqueId; - skwasm_registerObject = function(id, object) { - objectMap.set(id, object); - }; - skwasm_unregisterObject = function(id) { - objectMap.delete(id); - } - skwasm_getObject = function(id) { - return objectMap.get(id); - } - skwasm_transferObjectToThread = function(objectId, threadId) { + _skwasm_invokeFunctionOnThread = function(functionIndex, threadId, ...arguments) { PThread.pthreads[threadId].postMessage({ - skwasmObjectTransfers: new Map([ - [objectId, objectMap.get(objectId)] - ]) + functionIndex, + arguments, }); - objectMap.delete(objectId); } - _skwasm_transferObjectToMain = function(objectId) { + _skwasm_invokeFunctionOnMain = function(functionIndex, ...args) { postMessage({ - skwasmObjectTransfers: new Map([ - [objectId, objectMap.get(objectId)] - ]) + functionIndex, + args, }); - objectMap.delete(objectId); } _skwasm_registerMessageListener = function(threadId) { eventListener = function(event) { - const transfers = event.data.skwasmObjectTransfers; - if (!transfers) { - return; + const { functionIndex, args } = event.data; + if (!functionIndex || !args) { + return; } - transfers.forEach(function(object, objectId) { - objectMap.set(objectId, object); - }); + const invokeFunction = getWasmTableEntry(functionIndex); + invokeFunction(...args); }; if (!threadId) { addEventListener("message", eventListener); } else { PThread.pthreads[threadId].addEventListener("message", eventListener); } - }; + }; _skwasm_createOffscreenCanvas = function(width, height) { const canvas = new OffscreenCanvas(width, height); var contextAttributes = { @@ -85,8 +65,7 @@ mergeInto(LibraryManager.library, { objectMap.set(bitmapId, imageBitmap); _surface_onCaptureComplete(surfaceHandle, bitmapId); } - _skwasm_createGlTextureFromVideoFrame = function(videoFrameId, width, height) { - const videoFrame = skwasm_getObject(videoFrameId); + _skwasm_createGlTextureFromVideoFrame = function(videoFrame, width, height) { const glCtx = GL.currentContext.GLctx; const newTexture = glCtx.createTexture(); glCtx.bindTexture(glCtx.TEXTURE_2D, newTexture); @@ -101,26 +80,14 @@ mergeInto(LibraryManager.library, { GL.textures[textureId] = newTexture; return textureId; } - _skwasm_disposeVideoFrame = function(videoFrameId) { - const videoFrame = skwasm_getObject(videoFrameId); + _skwasm_disposeVideoFrame = function(videoFrame) { videoFrame.close(); - skwasm_unregisterObject(videoFrameId); } }, - skwasm_generateUniqueId: function() {}, - skwasm_generateUniqueId__deps: ['$skwasm_support_setup'], - $skwasm_generateUniqueId: function() {}, - $skwasm_generateUniqueId__deps: ['$skwasm_support_setup'], - $skwasm_registerObject: function() {}, - $skwasm_registerObject__deps: ['$skwasm_support_setup'], - $skwasm_unregisterObject: function() {}, - $skwasm_unregisterObject__deps: ['$skwasm_support_setup'], - $skwasm_getObject: function() {}, - $skwasm_getObject__deps: ['$skwasm_support_setup'], - $skwasm_transferObjectToThread: function() {}, - $skwasm_transferObjectToThread__deps: ['$skwasm_support_setup'], - skwasm_transferObjectToMain: function() {}, - skwasm_transferObjectToMain__deps: ['$skwasm_support_setup'], + $skwasm_invokeFunctionOnThread: function() {}, + $skwasm_invokeFunctionOnThread__deps: ['$skwasm_support_setup'], + skwasm_invokeFunctionOnMain: function() {}, + skwasm_invokeFunctionOnMain__deps: ['$skwasm_support_setup'], skwasm_registerMessageListener: function() {}, skwasm_registerMessageListener__deps: ['$skwasm_support_setup'], skwasm_createOffscreenCanvas: function () {}, diff --git a/lib/web_ui/skwasm/skwasm_support.h b/lib/web_ui/skwasm/skwasm_support.h index 46410b6f56c7e..645e54e4f799e 100644 --- a/lib/web_ui/skwasm/skwasm_support.h +++ b/lib/web_ui/skwasm/skwasm_support.h @@ -9,13 +9,9 @@ namespace Skwasm { class Surface; } -using SkwasmObjectId = uint32_t; +using SkwasmObject = __externref_t; extern "C" { -extern SkwasmObjectId skwasm_generateUniqueId(); -extern void skwasm_transferObjectToMain(SkwasmObjectId objectId); -extern void skwasm_transferObjectToThread(SkwasmObjectId objectId, - pthread_t threadId); extern void skwasm_registerMessageListener(pthread_t threadId); extern uint32_t skwasm_createOffscreenCanvas(int width, int height); extern void skwasm_resizeCanvas(uint32_t contextHandle, int width, int height); @@ -25,8 +21,8 @@ extern void skwasm_captureImageBitmap(Skwasm::Surface* surfaceHandle, int width, int height); extern unsigned int skwasm_createGlTextureFromVideoFrame( - SkwasmObjectId videoFrameId, + SkwasmObject videoFrame, int width, int height); -extern void skwasm_disposeVideoFrame(SkwasmObjectId videoFrameId); +extern void skwasm_disposeVideoFrame(SkwasmObject videoFrameId); } diff --git a/lib/web_ui/skwasm/surface.cpp b/lib/web_ui/skwasm/surface.cpp index 114b3ca6337a6..f600bab5e3400 100644 --- a/lib/web_ui/skwasm/surface.cpp +++ b/lib/web_ui/skwasm/surface.cpp @@ -39,7 +39,7 @@ void Surface::dispose() { // Main thread only uint32_t Surface::renderPicture(SkPicture* picture) { assert(emscripten_is_main_browser_thread()); - uint32_t callbackId = skwasm_generateUniqueId(); + uint32_t callbackId = ++_currentCallbackId; picture->ref(); emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VIII, reinterpret_cast(fRenderPicture), @@ -50,7 +50,7 @@ uint32_t Surface::renderPicture(SkPicture* picture) { // Main thread only uint32_t Surface::rasterizeImage(SkImage* image, ImageByteFormat format) { assert(emscripten_is_main_browser_thread()); - uint32_t callbackId = skwasm_generateUniqueId(); + uint32_t callbackId = ++_currentCallbackId; image->ref(); emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VIIII, @@ -59,10 +59,10 @@ uint32_t Surface::rasterizeImage(SkImage* image, ImageByteFormat format) { return callbackId; } -void Surface::disposeVideoFrame(SkwasmObjectId videoFrameId) { +void Surface::disposeVideoFrame(SkwasmObject videoFrame) { emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VII, reinterpret_cast(fDisposeVideoFrame), - nullptr, this, videoFrameId); + nullptr, this, videoFrame); } // Main thread only @@ -177,8 +177,8 @@ void Surface::_rasterizeImage(SkImage* image, EM_FUNC_SIG_VIII, fOnRasterizeComplete, this, data.release(), callbackId); } -void Surface::_disposeVideoFrame(SkwasmObjectId objectId) { - skwasm_disposeVideoFrame(objectId); +void Surface::_disposeVideoFrame(SkwasmObject videoFrame) { + skwasm_disposeVideoFrame(videoFrame); } void Surface::_onRasterizeComplete(SkData* data, uint32_t callbackId) { @@ -186,16 +186,15 @@ void Surface::_onRasterizeComplete(SkData* data, uint32_t callbackId) { } // Worker thread only -void Surface::notifyRenderComplete(uint32_t callbackId) { - skwasm_transferObjectToMain(callbackId); +void Surface::notifyRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap) { emscripten_async_run_in_main_runtime_thread( - EM_FUNC_SIG_VII, fOnRenderComplete, this, callbackId); + EM_FUNC_SIG_VII, fOnRenderComplete, this, imageBitmap); } // Main thread only -void Surface::_onRenderComplete(uint32_t callbackId) { +void Surface::_onRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap) { assert(emscripten_is_main_browser_thread()); - _callbackHandler(callbackId, nullptr); + _callbackHandler(callbackId, imageBitmap); } void Surface::fDispose(Surface* surface) { @@ -209,8 +208,12 @@ void Surface::fRenderPicture(Surface* surface, picture->unref(); } -void Surface::fOnRenderComplete(Surface* surface, uint32_t callbackId) { - surface->_onRenderComplete(callbackId); +void Surface::fOnRenderComplete( + Surface* surface, + uint32_t callbackId, + SkwasmObject imageBitmap +) { + surface->_onRenderComplete(callbackId, imageBitmap); } void Surface::fOnRasterizeComplete(Surface* surface, @@ -228,8 +231,8 @@ void Surface::fRasterizeImage(Surface* surface, } void Surface::fDisposeVideoFrame(Surface* surface, - SkwasmObjectId videoFrameId) { - surface->_disposeVideoFrame(videoFrameId); + SkwasmObject videoFrame) { + surface->_disposeVideoFrame(videoFrame); } SKWASM_EXPORT Surface* surface_create() { @@ -264,6 +267,7 @@ SKWASM_EXPORT uint32_t surface_rasterizeImage(Surface* surface, // This is used by the skwasm JS support code to call back into C++ when the // we finish creating the image bitmap, which is an asynchronous operation. SKWASM_EXPORT void surface_onCaptureComplete(Surface* surface, - uint32_t bitmapId) { - return surface->notifyRenderComplete(bitmapId); + uint32_t callbackId, + SkwasmObject imageBitmap) { + return surface->notifyRenderComplete(callbackId, imageBitmap); } diff --git a/lib/web_ui/skwasm/surface.h b/lib/web_ui/skwasm/surface.h index eb50567fbd5cb..855a5138d6d60 100644 --- a/lib/web_ui/skwasm/surface.h +++ b/lib/web_ui/skwasm/surface.h @@ -50,10 +50,10 @@ class Surface { void setCallbackHandler(CallbackHandler* callbackHandler); // Any thread - void disposeVideoFrame(SkwasmObjectId videoFrameId); + void disposeVideoFrame(SkwasmObject videoFrame); // Worker thread only. - void notifyRenderComplete(uint32_t callbackId); + void notifyRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap); private: void _runWorker(); @@ -65,12 +65,13 @@ class Surface { void _rasterizeImage(SkImage* image, ImageByteFormat format, uint32_t callbackId); - void _disposeVideoFrame(SkwasmObjectId objectId); + void _disposeVideoFrame(SkwasmObject videoFrame); void _onRasterizeComplete(SkData* data, uint32_t callbackId); - void _onRenderComplete(uint32_t callbackId); + void _onRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap); std::string _canvasID; CallbackHandler* _callbackHandler = nullptr; + uint32_t _currentCallbackId = 0; int _canvasWidth = 0; int _canvasHeight = 0; @@ -88,7 +89,11 @@ class Surface { static void fRenderPicture(Surface* surface, SkPicture* picture, uint32_t callbackId); - static void fOnRenderComplete(Surface* surface, uint32_t callbackId); + static void fOnRenderComplete( + Surface* surface, + uint32_t callbackId, + SkwasmObject imageBitmap + ); static void fRasterizeImage(Surface* surface, SkImage* image, ImageByteFormat format, @@ -96,6 +101,6 @@ class Surface { static void fOnRasterizeComplete(Surface* surface, SkData* imageData, uint32_t callbackId); - static void fDisposeVideoFrame(Surface* surface, SkwasmObjectId videoFrameId); + static void fDisposeVideoFrame(Surface* surface, SkwasmObject videoFrame); }; } // namespace Skwasm From d91ae9a911b0d242a13a642862c154fbe7a8951c Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 15 Aug 2023 15:14:57 -0700 Subject: [PATCH 09/24] Fiiiiinally got the sandwich rendering properly in skwasm. --- lib/web_ui/dev/test_dart2wasm.js | 2 +- lib/web_ui/lib/src/engine/dom.dart | 5 ++ .../platform_views/content_manager.dart | 1 + .../src/engine/skwasm/skwasm_impl/layers.dart | 10 ++-- .../skwasm/skwasm_impl/raw/raw_image.dart | 2 +- .../skwasm/skwasm_impl/raw/skwasm_module.dart | 28 ++++++++- .../engine/skwasm/skwasm_impl/surface.dart | 39 +++++++++---- lib/web_ui/skwasm/BUILD.gn | 4 +- lib/web_ui/skwasm/image.cpp | 16 ++---- lib/web_ui/skwasm/library_skwasm_support.js | 57 ++++++++++++------- lib/web_ui/skwasm/skwasm_support.h | 4 +- lib/web_ui/skwasm/surface.cpp | 32 +++-------- lib/web_ui/skwasm/surface.h | 27 ++++++--- lib/web_ui/test/ui/platform_view_test.dart | 12 ++-- 14 files changed, 148 insertions(+), 91 deletions(-) diff --git a/lib/web_ui/dev/test_dart2wasm.js b/lib/web_ui/dev/test_dart2wasm.js index ea30e6e16758c..b48e2403a3535 100644 --- a/lib/web_ui/dev/test_dart2wasm.js +++ b/lib/web_ui/dev/test_dart2wasm.js @@ -64,7 +64,7 @@ window.onload = async function () { const skwasmInstance = await skwasm(); window._flutter_skwasmInstance = skwasmInstance; resolve({ - "skwasm": skwasmInstance.asm, + "skwasm": skwasmInstance.asm ?? skwasmInstance.wasmExports, "ffi": { "memory": skwasmInstance.wasmMemory, } diff --git a/lib/web_ui/lib/src/engine/dom.dart b/lib/web_ui/lib/src/engine/dom.dart index 50be162f56dc6..7f8394bc8ffe2 100644 --- a/lib/web_ui/lib/src/engine/dom.dart +++ b/lib/web_ui/lib/src/engine/dom.dart @@ -1427,6 +1427,11 @@ extension DomImageDataExtension on DomImageData { @staticInterop class DomImageBitmap {} +extension DomImageBitmapExtension on DomImageBitmap { + external JSNumber get width; + external JSNumber get height; +} + @JS() @staticInterop class DomCanvasPattern {} diff --git a/lib/web_ui/lib/src/engine/platform_views/content_manager.dart b/lib/web_ui/lib/src/engine/platform_views/content_manager.dart index 73d951596a65d..8c84abd1bd3f9 100644 --- a/lib/web_ui/lib/src/engine/platform_views/content_manager.dart +++ b/lib/web_ui/lib/src/engine/platform_views/content_manager.dart @@ -125,6 +125,7 @@ class PlatformViewManager { int viewId, Object? params, ) { + print('render content $viewId'); assert(knowsViewType(viewType), 'Attempted to render contents of unregistered viewType: $viewType'); diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart index c139c6c834119..4948652733158 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart @@ -457,7 +457,7 @@ class LayerBuilder { ui.Rect? platformViewRect; void flushSlices() { - print('Flushing slices'); + //print('Flushing slices'); if (pendingPictures.isNotEmpty) { final ui.Rect drawnRect = picturesRect ?? ui.Rect.zero; final ui.Rect rect = operation?.cullRect(drawnRect) ?? drawnRect; @@ -524,18 +524,18 @@ class LayerBuilder { } void mergeLayer(PictureLayer layer) { - print('merging layer'); + //print('merging layer'); for (final LayerSlice slice in layer.slices) { switch (slice) { case PictureSlice(): - print('adding picture from merged layer slice with bounds: ${slice.picture.cullRect}'); + //print('adding picture from merged layer slice with bounds: ${slice.picture.cullRect}'); addPicture(ui.Offset.zero, slice.picture); case PlatformViewSlice(): final ui.Rect? occlusionRect = slice.occlusionRect; if (occlusionRect != null) { platformViewRect = platformViewRect?.expandToInclude(occlusionRect) ?? occlusionRect; } - print('adding views from merged platform view slice: ${slice.views}'); + //print('adding views from merged platform view slice: ${slice.views}'); pendingPlatformViews.addAll(slice.views); } } @@ -543,7 +543,7 @@ class LayerBuilder { PictureLayer build() { flushSlices(); - print('built layer with slices: ${layer.slices}'); + //print('built layer with slices: ${layer.slices}'); return layer; } } diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart index 3da653df65800..fb16ad086f9f1 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart @@ -59,7 +59,7 @@ external ImageHandle imageCreateFromPixels( // annotations currently. For now, we can use JS interop to expose this function // instead. extension SkwasmImageExtension on SkwasmInstance { - @JS('image_createFromVideoFrame') + @JS('wasmExports.image_createFromVideoFrame') external JSNumber imageCreateFromVideoFrame( VideoFrame frame, JSNumber width, diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart index a8b994bcb1e89..c117c5dc12735 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart @@ -17,7 +17,14 @@ extension WebAssemblyMemoryExtension on WebAssemblyMemory { class SkwasmInstance {} extension SkwasmInstanceExtension on SkwasmInstance { - external JSNumber addFunction(JSFunction function, JSString signature); + external JSNumber getEmptyTableSlot(); + + // The function here *must* be a directly exported wasm function, not a + // JavaScript function. If you actually need to add a JavaScript function, + // use `addFunction` instead. + external void setWasmTableEntry(JSNumber index, JSAny function); + + external JSNumber addFunction(WebAssemblyFunction function); external void removeFunction(JSNumber functionPointer); external WebAssemblyMemory get wasmMemory; @@ -25,3 +32,22 @@ extension SkwasmInstanceExtension on SkwasmInstance { @JS('window._flutter_skwasmInstance') external SkwasmInstance get skwasmInstance; + +@JS() +@staticInterop +@anonymous +class WebAssemblyFunctionType { + external factory WebAssemblyFunctionType({ + required JSArray parameters, + required JSArray results, + }); +} + +@JS('WebAssembly.Function') +@staticInterop +class WebAssemblyFunction { + external factory WebAssemblyFunction( + WebAssemblyFunctionType functionType, + JSFunction function + ); +} diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart index 6df842dcdc62e..b828eab94375f 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart @@ -11,6 +11,8 @@ import 'package:ui/src/engine.dart'; import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; import 'package:ui/ui.dart' as ui; + + class SkwasmSurface { factory SkwasmSurface() { final SurfaceHandle surfaceHandle = withStackScope((StackScope scope) { @@ -29,19 +31,29 @@ class SkwasmSurface { final int threadId; void _initialize() { - _callbackHandle = - OnRenderCallbackHandle.fromAddress( - skwasmInstance.addFunction( - _callbackHandler.toJS, - 'vii'.toJS - ).toDartDouble.toInt() - ); + final WebAssemblyFunction wasmFunction = WebAssemblyFunction( + WebAssemblyFunctionType( + parameters: [ + 'i32'.toJS, + 'i32'.toJS, + 'externref'.toJS + ].toJS, + results: [].toJS + ), + _callbackHandler.toJS, + ); + _callbackHandle = OnRenderCallbackHandle.fromAddress( + skwasmInstance.addFunction(wasmFunction).toDartInt, + ); surfaceSetCallbackHandler(handle, _callbackHandle); } Future renderPicture(SkwasmPicture picture) async { final int callbackId = surfaceRenderPicture(handle, picture.handle); - return (await _registerCallback(callbackId)) as DomImageBitmap; + print('render picture with callbackID: $callbackId, rect: ${picture.cullRect}'); + final DomImageBitmap bitmap = (await _registerCallback(callbackId)) as DomImageBitmap; + print('picture rendered with callbackID: $callbackId, rect: ${bitmap.width.toDartDouble} x ${bitmap.height.toDartDouble}'); + return bitmap; } Future rasterizeImage(SkwasmImage image, ui.ImageByteFormat format) async { @@ -68,10 +80,13 @@ class SkwasmSurface { return completer.future; } - void _callbackHandler(JSNumber jsCallbackId, JSAny value) { - final CallbackId callbackId = jsCallbackId.toDartInt; - final Completer completer = _pendingCallbacks.remove(callbackId)!; - completer.complete(value); + void _callbackHandler(JSNumber callbackId, JSNumber context, JSAny jsContext) { + final Completer completer = _pendingCallbacks.remove(callbackId.toDartInt)!; + if (jsContext.isUndefinedOrNull) { + completer.complete(context); + } else { + completer.complete(jsContext); + } } void dispose() { diff --git a/lib/web_ui/skwasm/BUILD.gn b/lib/web_ui/skwasm/BUILD.gn index cc508a5fa6bb9..5742d0dd1d728 100644 --- a/lib/web_ui/skwasm/BUILD.gn +++ b/lib/web_ui/skwasm/BUILD.gn @@ -33,6 +33,7 @@ wasm_lib("skwasm") { cflags = [ "-mreference-types", + "-pthread", ] ldflags = [ @@ -44,10 +45,9 @@ wasm_lib("skwasm") { "-sPTHREAD_POOL_SIZE=1", "-sALLOW_MEMORY_GROWTH", "-sALLOW_TABLE_GROWTH", - "-sUSE_PTHREADS=1", "-lexports.js", "-sEXPORTED_FUNCTIONS=[stackAlloc]", - "-sEXPORTED_RUNTIME_METHODS=[addFunction,removeFunction,skwasm_registerObject,skwasm_unregisterObject,skwasm_getObject,skwasm_transferObjectToThread,skwasm_generateUniqueId]", + "-sEXPORTED_RUNTIME_METHODS=[addFunction,wasmExports]", "-Wno-pthreads-mem-growth", "--js-library", rebase_path("library_skwasm_support.js"), diff --git a/lib/web_ui/skwasm/image.cpp b/lib/web_ui/skwasm/image.cpp index c5fdd4c51fca5..9050ee2db4bf7 100644 --- a/lib/web_ui/skwasm/image.cpp +++ b/lib/web_ui/skwasm/image.cpp @@ -84,19 +84,14 @@ class VideoFrameImageGenerator : public GrExternalTextureGenerator { SkwasmObject videoFrame, Skwasm::Surface* surface) : GrExternalTextureGenerator(ii), - _videoFrame(videoFrame), - _surface(surface) {} - - ~VideoFrameImageGenerator() override { - _surface->disposeVideoFrame(_videoFrame); - } + _videoFrameWrapper(surface->createVideoFrameWrapper(videoFrame)) {} std::unique_ptr generateExternalTexture( GrRecordingContext* context, GrMipMapped mipmapped) override { GrGLTextureInfo glInfo; glInfo.fID = skwasm_createGlTextureFromVideoFrame( - _videoFrame, fInfo.width(), fInfo.height()); + _videoFrameWrapper->getVideoFrame(), fInfo.width(), fInfo.height()); glInfo.fFormat = GL_RGBA8_OES; glInfo.fTarget = GL_TEXTURE_2D; @@ -107,8 +102,7 @@ class VideoFrameImageGenerator : public GrExternalTextureGenerator { } private: - SkwasmObject _videoFrame; - Skwasm::Surface* _surface; + std::unique_ptr _videoFrameWrapper; }; SKWASM_EXPORT SkImage* image_createFromPicture(SkPicture* picture, @@ -139,11 +133,11 @@ SKWASM_EXPORT SkImage* image_createFromVideoFrame(SkwasmObject videoFrame, int height, Skwasm::Surface* surface) { return SkImages::DeferredFromTextureGenerator( - std::make_unique( + std::unique_ptr(new VideoFrameImageGenerator( SkImageInfo::Make(width, height, SkColorType::kRGBA_8888_SkColorType, SkAlphaType::kPremul_SkAlphaType), - videoFrame, surface)) + videoFrame, surface))) .release(); } diff --git a/lib/web_ui/skwasm/library_skwasm_support.js b/lib/web_ui/skwasm/library_skwasm_support.js index 12fd673b86019..3121199a861f2 100644 --- a/lib/web_ui/skwasm/library_skwasm_support.js +++ b/lib/web_ui/skwasm/library_skwasm_support.js @@ -9,26 +9,34 @@ mergeInto(LibraryManager.library, { $skwasm_support_setup__postset: 'skwasm_support_setup();', $skwasm_support_setup: function() { const handleToCanvasMap = new Map(); - _skwasm_invokeFunctionOnThread = function(functionIndex, threadId, ...arguments) { + const associatedObjectsMap = new Map(); + _skwasm_setAssociatedObjectOnThread = function(threadId, pointer, object) { PThread.pthreads[threadId].postMessage({ - functionIndex, - arguments, + skwasmMessage: 'setAssociatedObject', + pointer, + object, }); } - _skwasm_invokeFunctionOnMain = function(functionIndex, ...args) { - postMessage({ - functionIndex, - args, - }); + _skwasm_getAssociatedObject = function(pointer) { + return associatedObjectsMap.get(pointer); } _skwasm_registerMessageListener = function(threadId) { - eventListener = function(event) { - const { functionIndex, args } = event.data; - if (!functionIndex || !args) { + eventListener = function({data}) { + console.log(`message received in ${threadId}: ${JSON.stringify(data)}`); + const skwasmMessage = data.skwasmMessage; + if (!skwasmMessage) { return; } - const invokeFunction = getWasmTableEntry(functionIndex); - invokeFunction(...args); + switch (skwasmMessage) { + case 'onRenderComplete': + console.log(`onrenderComplete: ${data.callbackId}`); + _surface_onRenderComplete(data.surface, data.callbackId, data.imageBitmap); + return; + case 'setAssociatedObject': + associatedObjectsMap.set(data.pointer, data.object); + default: + console.warn('unrecognized skwasm message'); + } }; if (!threadId) { addEventListener("message", eventListener); @@ -59,11 +67,16 @@ mergeInto(LibraryManager.library, { canvas.width = width; canvas.height = height; } - _skwasm_captureImageBitmap = async function(surfaceHandle, contextHandle, bitmapId, width, height) { + _skwasm_captureImageBitmap = async function(surfaceHandle, contextHandle, callbackId, width, height) { + console.log(`captureImageBitmap ${callbackId}`); const canvas = handleToCanvasMap.get(contextHandle); const imageBitmap = await createImageBitmap(canvas, 0, 0, width, height); - objectMap.set(bitmapId, imageBitmap); - _surface_onCaptureComplete(surfaceHandle, bitmapId); + postMessage({ + skwasmMessage: 'onRenderComplete', + surface: surfaceHandle, + callbackId, + imageBitmap, + }); } _skwasm_createGlTextureFromVideoFrame = function(videoFrame, width, height) { const glCtx = GL.currentContext.GLctx; @@ -84,10 +97,10 @@ mergeInto(LibraryManager.library, { videoFrame.close(); } }, - $skwasm_invokeFunctionOnThread: function() {}, - $skwasm_invokeFunctionOnThread__deps: ['$skwasm_support_setup'], - skwasm_invokeFunctionOnMain: function() {}, - skwasm_invokeFunctionOnMain__deps: ['$skwasm_support_setup'], + skwasm_setAssociatedObjectOnThread: function () {}, + skwasm_setAssociatedObjectOnThread__deps: ['$skwasm_support_setup'], + skwasm_getAssociatedObject: function () {}, + skwasm_getAssociatedObject__deps: ['$skwasm_support_setup'], skwasm_registerMessageListener: function() {}, skwasm_registerMessageListener__deps: ['$skwasm_support_setup'], skwasm_createOffscreenCanvas: function () {}, @@ -97,8 +110,8 @@ mergeInto(LibraryManager.library, { skwasm_captureImageBitmap: function () {}, skwasm_captureImageBitmap__deps: ['$skwasm_support_setup'], skwasm_createGlTextureFromVideoFrame: function () {}, - skwasm_createGlTextureFromVideoFrame__deps: ['$skwasm_support_setup', '$skwasm_getObject'], + skwasm_createGlTextureFromVideoFrame__deps: ['$skwasm_support_setup'], skwasm_disposeVideoFrame: function () {}, - skwasm_disposeVideoFrame__deps: ['$skwasm_support_setup', '$skwasm_getObject', '$skwasm_unregisterObject'], + skwasm_disposeVideoFrame__deps: ['$skwasm_support_setup'], }); \ No newline at end of file diff --git a/lib/web_ui/skwasm/skwasm_support.h b/lib/web_ui/skwasm/skwasm_support.h index 645e54e4f799e..4c645974df650 100644 --- a/lib/web_ui/skwasm/skwasm_support.h +++ b/lib/web_ui/skwasm/skwasm_support.h @@ -12,6 +12,8 @@ class Surface; using SkwasmObject = __externref_t; extern "C" { +extern void skwasm_setAssociatedObjectOnThread(unsigned long threadId, void *pointer, SkwasmObject object); +extern SkwasmObject skwasm_getAssociatedObject(void *pointer); extern void skwasm_registerMessageListener(pthread_t threadId); extern uint32_t skwasm_createOffscreenCanvas(int width, int height); extern void skwasm_resizeCanvas(uint32_t contextHandle, int width, int height); @@ -24,5 +26,5 @@ extern unsigned int skwasm_createGlTextureFromVideoFrame( SkwasmObject videoFrame, int width, int height); -extern void skwasm_disposeVideoFrame(SkwasmObject videoFrameId); +extern void skwasm_disposeVideoFrame(SkwasmObject videoFrame); } diff --git a/lib/web_ui/skwasm/surface.cpp b/lib/web_ui/skwasm/surface.cpp index f600bab5e3400..6d2065b9ca91c 100644 --- a/lib/web_ui/skwasm/surface.cpp +++ b/lib/web_ui/skwasm/surface.cpp @@ -59,10 +59,8 @@ uint32_t Surface::rasterizeImage(SkImage* image, ImageByteFormat format) { return callbackId; } -void Surface::disposeVideoFrame(SkwasmObject videoFrame) { - emscripten_dispatch_to_thread(_thread, EM_FUNC_SIG_VII, - reinterpret_cast(fDisposeVideoFrame), - nullptr, this, videoFrame); +std::unique_ptr Surface::createVideoFrameWrapper(SkwasmObject videoFrame) { + return std::unique_ptr(new VideoFrameWrapper(_thread, videoFrame)); } // Main thread only @@ -74,7 +72,7 @@ void Surface::setCallbackHandler(CallbackHandler* callbackHandler) { // Worker thread only void Surface::_runWorker() { _init(); - emscripten_unwind_to_js_event_loop(); + emscripten_exit_with_live_runtime(); } // Worker thread only @@ -182,19 +180,13 @@ void Surface::_disposeVideoFrame(SkwasmObject videoFrame) { } void Surface::_onRasterizeComplete(SkData* data, uint32_t callbackId) { - _callbackHandler(callbackId, data); -} - -// Worker thread only -void Surface::notifyRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap) { - emscripten_async_run_in_main_runtime_thread( - EM_FUNC_SIG_VII, fOnRenderComplete, this, imageBitmap); + _callbackHandler(callbackId, data, __builtin_wasm_ref_null_extern()); } // Main thread only -void Surface::_onRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap) { +void Surface::onRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap) { assert(emscripten_is_main_browser_thread()); - _callbackHandler(callbackId, imageBitmap); + _callbackHandler(callbackId, nullptr, imageBitmap); } void Surface::fDispose(Surface* surface) { @@ -208,14 +200,6 @@ void Surface::fRenderPicture(Surface* surface, picture->unref(); } -void Surface::fOnRenderComplete( - Surface* surface, - uint32_t callbackId, - SkwasmObject imageBitmap -) { - surface->_onRenderComplete(callbackId, imageBitmap); -} - void Surface::fOnRasterizeComplete(Surface* surface, SkData* imageData, uint32_t callbackId) { @@ -266,8 +250,8 @@ SKWASM_EXPORT uint32_t surface_rasterizeImage(Surface* surface, // This is used by the skwasm JS support code to call back into C++ when the // we finish creating the image bitmap, which is an asynchronous operation. -SKWASM_EXPORT void surface_onCaptureComplete(Surface* surface, +SKWASM_EXPORT void surface_onRenderComplete(Surface* surface, uint32_t callbackId, SkwasmObject imageBitmap) { - return surface->notifyRenderComplete(callbackId, imageBitmap); + return surface->onRenderComplete(callbackId, imageBitmap); } diff --git a/lib/web_ui/skwasm/surface.h b/lib/web_ui/skwasm/surface.h index 855a5138d6d60..34433a70e09a6 100644 --- a/lib/web_ui/skwasm/surface.h +++ b/lib/web_ui/skwasm/surface.h @@ -33,10 +33,26 @@ enum class ImageByteFormat { png, }; +class VideoFrameWrapper { + public: + VideoFrameWrapper(unsigned long threadId, SkwasmObject videoFrame) : _rasterThreadId(threadId) { + skwasm_setAssociatedObjectOnThread(_rasterThreadId, this, videoFrame); + } + + ~VideoFrameWrapper() { + skwasm_setAssociatedObjectOnThread(_rasterThreadId, this, __builtin_wasm_ref_null_extern()); + } + + SkwasmObject getVideoFrame() { + return skwasm_getAssociatedObject(this); + } + private: + unsigned long _rasterThreadId; +}; + class Surface { public: - public: - using CallbackHandler = void(uint32_t, void*); + using CallbackHandler = void(uint32_t, void*, SkwasmObject); // Main thread only Surface(); @@ -48,12 +64,10 @@ class Surface { uint32_t renderPicture(SkPicture* picture); uint32_t rasterizeImage(SkImage* image, ImageByteFormat format); void setCallbackHandler(CallbackHandler* callbackHandler); + void onRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap); // Any thread - void disposeVideoFrame(SkwasmObject videoFrame); - - // Worker thread only. - void notifyRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap); + std::unique_ptr createVideoFrameWrapper(SkwasmObject videoFrame); private: void _runWorker(); @@ -67,7 +81,6 @@ class Surface { uint32_t callbackId); void _disposeVideoFrame(SkwasmObject videoFrame); void _onRasterizeComplete(SkData* data, uint32_t callbackId); - void _onRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap); std::string _canvasID; CallbackHandler* _callbackHandler = nullptr; diff --git a/lib/web_ui/test/ui/platform_view_test.dart b/lib/web_ui/test/ui/platform_view_test.dart index 434fea330f199..6e34c15a2929c 100644 --- a/lib/web_ui/test/ui/platform_view_test.dart +++ b/lib/web_ui/test/ui/platform_view_test.dart @@ -26,7 +26,7 @@ Future testMain() async { const ui.Rect region = ui.Rect.fromLTWH(0, 0, 300, 300); const String platformViewType = 'test-platform-view'; - setUpAll(() { + setUp(() { ui_web.platformViewRegistry.registerViewFactory( platformViewType, (int viewId) { @@ -37,11 +37,15 @@ Future testMain() async { ); }); + tearDown(() { + platformViewManager.debugClear(); + }); + test('Picture + Overlapping PlatformView', () async { await _createPlatformView(1, platformViewType); final ui.PictureRecorder recorder = ui.PictureRecorder(); - final ui.Canvas canvas = ui.Canvas(recorder, region); + final ui.Canvas canvas = ui.Canvas(recorder); canvas.drawCircle( ui.Offset.zero, 50, @@ -69,7 +73,7 @@ Future testMain() async { await _createPlatformView(1, platformViewType); final ui.PictureRecorder recorder = ui.PictureRecorder(); - final ui.Canvas canvas = ui.Canvas(recorder, region); + final ui.Canvas canvas = ui.Canvas(recorder); canvas.drawCircle( ui.Offset.zero, 50, @@ -94,7 +98,7 @@ Future testMain() async { sb.addPicture(const ui.Offset(175, 175), picture); await renderer.renderScene(sb.build()); - await matchGoldenFile('picture_platformview_overlap.png', region: region); + await matchGoldenFile('picture_platformview_sandwich.png', region: region); }); } From bfcd7593105e90f96aa181c60537cc50de1993bf Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Wed, 16 Aug 2023 15:27:11 -0700 Subject: [PATCH 10/24] Implement platform view transforms. --- .../platform_views/content_manager.dart | 1 - .../src/engine/skwasm/skwasm_impl/layers.dart | 179 +++++++++++++++--- .../skwasm/skwasm_impl/raw/skwasm_module.dart | 2 +- .../engine/skwasm/skwasm_impl/scene_view.dart | 32 ++-- .../engine/skwasm/skwasm_impl/surface.dart | 2 - lib/web_ui/skwasm/image.cpp | 13 +- lib/web_ui/skwasm/library_skwasm_support.js | 3 - lib/web_ui/skwasm/skwasm_support.h | 6 +- lib/web_ui/skwasm/surface.cpp | 13 +- lib/web_ui/skwasm/surface.h | 40 ++-- lib/web_ui/test/ui/platform_view_test.dart | 40 +++- 11 files changed, 240 insertions(+), 91 deletions(-) diff --git a/lib/web_ui/lib/src/engine/platform_views/content_manager.dart b/lib/web_ui/lib/src/engine/platform_views/content_manager.dart index 8c84abd1bd3f9..73d951596a65d 100644 --- a/lib/web_ui/lib/src/engine/platform_views/content_manager.dart +++ b/lib/web_ui/lib/src/engine/platform_views/content_manager.dart @@ -125,7 +125,6 @@ class PlatformViewManager { int viewId, Object? params, ) { - print('render content $viewId'); assert(knowsViewType(viewType), 'Attempted to render contents of unregistered viewType: $viewType'); diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart index 4948652733158..44db1cdbbddb4 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart @@ -34,6 +34,9 @@ class BackdropFilterOperation implements LayerOperation { void post(SceneCanvas canvas, ui.Rect contentRect) { canvas.restore(); } + + @override + PlatformViewStyling createPlatformViewStyling() => const PlatformViewStyling(); } class ClipPathLayer @@ -67,6 +70,12 @@ class ClipPathOperation implements LayerOperation { } canvas.restore(); } + + @override + PlatformViewStyling createPlatformViewStyling() { + // TODO: implement this + return const PlatformViewStyling(); + } } class ClipRectLayer @@ -100,6 +109,12 @@ class ClipRectOperation implements LayerOperation { } canvas.restore(); } + + @override + PlatformViewStyling createPlatformViewStyling() { + // TODO: implement createPlatformViewStyling + return const PlatformViewStyling(); + } } class ClipRRectLayer @@ -133,6 +148,12 @@ class ClipRRectOperation implements LayerOperation { } canvas.restore(); } + + @override + PlatformViewStyling createPlatformViewStyling() { + // TODO: implement createPlatformViewStyling + return const PlatformViewStyling(); + } } class ColorFilterLayer @@ -158,6 +179,9 @@ class ColorFilterOperation implements LayerOperation { void post(SceneCanvas canvas, ui.Rect contentRect) { canvas.restore(); } + + @override + PlatformViewStyling createPlatformViewStyling() => const PlatformViewStyling(); } class ImageFilterLayer @@ -193,6 +217,17 @@ class ImageFilterOperation implements LayerOperation { } canvas.restore(); } + + @override + PlatformViewStyling createPlatformViewStyling() { + if (offset != ui.Offset.zero) { + return PlatformViewStyling( + position: PlatformViewPosition(offset: offset) + ); + } else { + return const PlatformViewStyling(); + } + } } class OffsetLayer @@ -220,6 +255,11 @@ class OffsetOperation implements LayerOperation { void post(SceneCanvas canvas, ui.Rect contentRect) { canvas.restore(); } + + @override + PlatformViewStyling createPlatformViewStyling() => PlatformViewStyling( + position: PlatformViewPosition(offset: ui.Offset(dx, dy)) + ); } class OpacityLayer @@ -256,6 +296,12 @@ class OpacityOperation implements LayerOperation { canvas.restore(); } } + + @override + PlatformViewStyling createPlatformViewStyling() => PlatformViewStyling( + position: offset != ui.Offset.zero ? PlatformViewPosition(offset: offset) : const PlatformViewPosition(), + opacity: alpha.toDouble() / 255.0, + ); } class TransformLayer @@ -287,6 +333,11 @@ class TransformOperation implements LayerOperation { void post(SceneCanvas canvas, ui.Rect contentRect) { canvas.restore(); } + + @override + PlatformViewStyling createPlatformViewStyling() => PlatformViewStyling( + position: PlatformViewPosition(transform: getMatrix()), + ); } class ShaderMaskLayer @@ -326,20 +377,23 @@ class ShaderMaskOperation implements LayerOperation { canvas.restore(); canvas.restore(); } + + @override + PlatformViewStyling createPlatformViewStyling() { + // TODO: implement createPlatformViewStyling + return const PlatformViewStyling(); + } } class PlatformView { - PlatformView(this.viewId, this.rect); + PlatformView(this.viewId, this.size, this.styling); int viewId; // The bounds of this platform view, in the layer's local coordinate space. - ui.Rect rect; + ui.Size size; - @override - String toString() { - return 'PlatformView(id: $viewId, rect: $rect)'; - } + PlatformViewStyling styling; } sealed class LayerSlice { @@ -357,13 +411,6 @@ class PlatformViewSlice implements LayerSlice { @override void dispose() {} - - // TODO: Probably add some styling stuff in here - - @override - String toString() { - return 'PlatformViewSlice($views)'; - } } class PictureSlice implements LayerSlice { @@ -401,6 +448,8 @@ abstract class LayerOperation { ui.Rect inverseMapRect(ui.Rect rect); void pre(SceneCanvas canvas, ui.Rect contentRect); void post(SceneCanvas canvas, ui.Rect contentRect); + + PlatformViewStyling createPlatformViewStyling(); } class PictureDrawCommand { @@ -411,23 +460,74 @@ class PictureDrawCommand { } class PlatformViewPosition { - ui.Offset? offset; - Matrix4? transform; + const PlatformViewPosition({this.offset, this.transform}); + const PlatformViewPosition.zero() : offset = null, transform = null; + + final ui.Offset? offset; + final Matrix4? transform; static PlatformViewPosition combine(PlatformViewPosition outer, PlatformViewPosition inner) { - // Implement this - return outer; + final ui.Offset? outerOffset = outer.offset; + final Matrix4? outerTransform = outer.transform; + final ui.Offset? innerOffset = inner.offset; + final Matrix4? innerTransform = inner.transform; + if (innerTransform != null) { + if (innerOffset != null) { + if (outerTransform != null) { + final Matrix4 newTransform = innerTransform.clone(); + newTransform.translate(innerOffset.dx, innerOffset.dy); + newTransform.multiply(outerTransform); + return PlatformViewPosition(offset: outerOffset, transform: newTransform); + } else { + final ui.Offset finalOffset = outerOffset != null ? (innerOffset + outerOffset) : innerOffset; + return PlatformViewPosition(offset: finalOffset, transform: innerTransform); + } + } else { + if (outerTransform != null) { + final Matrix4 newTransform = innerTransform.clone(); + newTransform.multiply(outerTransform); + return PlatformViewPosition(offset: outerOffset, transform: newTransform); + } else { + return PlatformViewPosition(offset: outerOffset, transform: innerTransform); + } + } + } else { + if (innerOffset != null) { + if (outerTransform != null) { + final Matrix4 newTransform = Matrix4.translationValues(innerOffset.dx, innerOffset.dy, 0); + newTransform.multiply(outerTransform); + return PlatformViewPosition(offset: outerOffset, transform: newTransform); + } else { + final ui.Offset finalOffset = outerOffset != null ? (innerOffset + outerOffset) : innerOffset; + return PlatformViewPosition(offset: finalOffset); + } + } else { + return outer; + } + } } } class PlatformViewClip { - + const PlatformViewClip(); } class PlatformViewStyling { - PlatformViewPosition? position; - PlatformViewClip? clip; - double? opacity; + const PlatformViewStyling({ + this.position = const PlatformViewPosition.zero(), + this.clip = const PlatformViewClip(), + this.opacity = 1.0 + }); + + final PlatformViewPosition position; + final PlatformViewClip clip; + final double opacity; + + static PlatformViewStyling combine(PlatformViewStyling outer, PlatformViewStyling inner) { + return PlatformViewStyling( + position: PlatformViewPosition.combine(outer.position, inner.position), + ); + } } class LayerBuilder { @@ -456,8 +556,25 @@ class LayerBuilder { ui.Rect? picturesRect; ui.Rect? platformViewRect; + PlatformViewStyling? _memoizedPlatformViewStyling; + + PlatformViewStyling _createPlatformViewStyling() { + final PlatformViewStyling? innerStyling = operation?.createPlatformViewStyling(); + final PlatformViewStyling? outerStyling = parent?.platformViewStyling; + if (innerStyling == null) { + return outerStyling ?? const PlatformViewStyling(); + } + if (outerStyling == null) { + return innerStyling; + } + return PlatformViewStyling.combine(outerStyling, innerStyling); + } + + PlatformViewStyling get platformViewStyling { + return _memoizedPlatformViewStyling ??= _createPlatformViewStyling(); + } + void flushSlices() { - //print('Flushing slices'); if (pendingPictures.isNotEmpty) { final ui.Rect drawnRect = picturesRect ?? ui.Rect.zero; final ui.Rect rect = operation?.cullRect(drawnRect) ?? drawnRect; @@ -478,7 +595,6 @@ class LayerBuilder { operation?.post(canvas, rect); final ui.Picture picture = recorder.endRecording(); layer.slices.add(PictureSlice(picture as ScenePicture)); - print('adding picture slice'); } if (pendingPlatformViews.isNotEmpty) { @@ -487,8 +603,6 @@ class LayerBuilder { occlusionRect = operation!.inverseMapRect(occlusionRect); } layer.slices.add(PlatformViewSlice(pendingPlatformViews, occlusionRect)); - print('adding platform view slice with views: $pendingPlatformViews'); - // TODO: attach styling information } pendingPictures.clear(); @@ -520,22 +634,28 @@ class LayerBuilder { }) { final ui.Rect bounds = ui.Rect.fromLTWH(offset.dx, offset.dy, width, height); platformViewRect = platformViewRect?.expandToInclude(bounds) ?? bounds; - pendingPlatformViews.add(PlatformView(viewId, bounds)); + final PlatformViewStyling layerStyling = platformViewStyling; + final PlatformViewStyling viewStyling = offset == ui.Offset.zero + ? layerStyling + : PlatformViewStyling.combine( + PlatformViewStyling( + position: PlatformViewPosition(offset: offset), + ), + layerStyling, + ); + pendingPlatformViews.add(PlatformView(viewId, ui.Size(width, height), viewStyling)); } void mergeLayer(PictureLayer layer) { - //print('merging layer'); for (final LayerSlice slice in layer.slices) { switch (slice) { case PictureSlice(): - //print('adding picture from merged layer slice with bounds: ${slice.picture.cullRect}'); addPicture(ui.Offset.zero, slice.picture); case PlatformViewSlice(): final ui.Rect? occlusionRect = slice.occlusionRect; if (occlusionRect != null) { platformViewRect = platformViewRect?.expandToInclude(occlusionRect) ?? occlusionRect; } - //print('adding views from merged platform view slice: ${slice.views}'); pendingPlatformViews.addAll(slice.views); } } @@ -543,7 +663,6 @@ class LayerBuilder { PictureLayer build() { flushSlices(); - //print('built layer with slices: ${layer.slices}'); return layer; } } diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart index c117c5dc12735..3063a557aaed1 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/skwasm_module.dart @@ -34,7 +34,7 @@ extension SkwasmInstanceExtension on SkwasmInstance { external SkwasmInstance get skwasmInstance; @JS() -@staticInterop +@staticInterop @anonymous class WebAssemblyFunctionType { external factory WebAssemblyFunctionType({ diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart index d3241523c26d1..c842c8f7c02f1 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart @@ -101,13 +101,26 @@ final class PlatformViewSliceContainer extends SliceContainer { } // TODO: set all the styling here instead of just the position final DomCSSStyleDeclaration style = viewContainer.style; - final double logicalWidth = view.rect.width / window.devicePixelRatio; - final double logicalHeight = view.rect.height / window.devicePixelRatio; + final double logicalWidth = view.size.width / window.devicePixelRatio; + final double logicalHeight = view.size.height / window.devicePixelRatio; style.width = '${logicalWidth}px'; style.height = '${logicalHeight}px'; style.position = 'absolute'; - style.left = '${view.rect.left}px'; - style.top = '${view.rect.top}px'; + + final ui.Offset? offset = view.styling.position.offset; + style.left = '${offset?.dx ?? 0}px'; + style.top = '${offset?.dy ?? 0}px'; + + final Matrix4? transform = view.styling.position.transform; + if (transform != null) { + style.transform = float64ListToCssTransform3d(transform.storage); + } + } + + while (currentContainer != null) { + final DomElement? next = currentContainer.nextElementSibling; + currentContainer.remove(); + currentContainer = next; } } } @@ -135,17 +148,6 @@ class SkwasmSceneView { queuedRenders += 1; final List slices = scene.rootLayer.slices; - print('Rendering scene with slices: ${slices.map((LayerSlice slice) { - if (slice is PictureSlice) { - return 'PictureSlice(${slice.picture.cullRect})'; - } else if (slice is PlatformViewSlice) { - return 'PlatformViewSlice(${slice.views.map((PlatformView view) { - return 'PlatformView(${view.viewId}, ${view.rect})'; - })})'; - } else { - return 'Unknown'; - } - })}'); final Iterable> renderFutures = slices.map( (LayerSlice slice) async => switch (slice) { PlatformViewSlice() => null, diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart index b828eab94375f..bce7a8a62edf0 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart @@ -50,9 +50,7 @@ class SkwasmSurface { Future renderPicture(SkwasmPicture picture) async { final int callbackId = surfaceRenderPicture(handle, picture.handle); - print('render picture with callbackID: $callbackId, rect: ${picture.cullRect}'); final DomImageBitmap bitmap = (await _registerCallback(callbackId)) as DomImageBitmap; - print('picture rendered with callbackID: $callbackId, rect: ${bitmap.width.toDartDouble} x ${bitmap.height.toDartDouble}'); return bitmap; } diff --git a/lib/web_ui/skwasm/image.cpp b/lib/web_ui/skwasm/image.cpp index 3a80237411f11..70553cfe98a9a 100644 --- a/lib/web_ui/skwasm/image.cpp +++ b/lib/web_ui/skwasm/image.cpp @@ -92,7 +92,7 @@ class VideoFrameImageGenerator : public GrExternalTextureGenerator { GrMipMapped mipmapped) override { GrGLTextureInfo glInfo; glInfo.fID = skwasm_createGlTextureFromVideoFrame( - _videoFrameWrapper->getVideoFrame(), fInfo.width(), fInfo.height()); + _videoFrameWrapper->getVideoFrame(), fInfo.width(), fInfo.height()); glInfo.fFormat = GL_RGBA8_OES; glInfo.fTarget = GL_TEXTURE_2D; @@ -134,11 +134,12 @@ SKWASM_EXPORT SkImage* image_createFromVideoFrame(SkwasmObject videoFrame, int height, Skwasm::Surface* surface) { return SkImages::DeferredFromTextureGenerator( - std::unique_ptr(new VideoFrameImageGenerator( - SkImageInfo::Make(width, height, - SkColorType::kRGBA_8888_SkColorType, - SkAlphaType::kPremul_SkAlphaType), - videoFrame, surface))) + std::unique_ptr( + new VideoFrameImageGenerator( + SkImageInfo::Make(width, height, + SkColorType::kRGBA_8888_SkColorType, + SkAlphaType::kPremul_SkAlphaType), + videoFrame, surface))) .release(); } diff --git a/lib/web_ui/skwasm/library_skwasm_support.js b/lib/web_ui/skwasm/library_skwasm_support.js index 3121199a861f2..134bcb65345f4 100644 --- a/lib/web_ui/skwasm/library_skwasm_support.js +++ b/lib/web_ui/skwasm/library_skwasm_support.js @@ -22,14 +22,12 @@ mergeInto(LibraryManager.library, { } _skwasm_registerMessageListener = function(threadId) { eventListener = function({data}) { - console.log(`message received in ${threadId}: ${JSON.stringify(data)}`); const skwasmMessage = data.skwasmMessage; if (!skwasmMessage) { return; } switch (skwasmMessage) { case 'onRenderComplete': - console.log(`onrenderComplete: ${data.callbackId}`); _surface_onRenderComplete(data.surface, data.callbackId, data.imageBitmap); return; case 'setAssociatedObject': @@ -68,7 +66,6 @@ mergeInto(LibraryManager.library, { canvas.height = height; } _skwasm_captureImageBitmap = async function(surfaceHandle, contextHandle, callbackId, width, height) { - console.log(`captureImageBitmap ${callbackId}`); const canvas = handleToCanvasMap.get(contextHandle); const imageBitmap = await createImageBitmap(canvas, 0, 0, width, height); postMessage({ diff --git a/lib/web_ui/skwasm/skwasm_support.h b/lib/web_ui/skwasm/skwasm_support.h index 4c645974df650..5af38a6f2b64b 100644 --- a/lib/web_ui/skwasm/skwasm_support.h +++ b/lib/web_ui/skwasm/skwasm_support.h @@ -12,8 +12,10 @@ class Surface; using SkwasmObject = __externref_t; extern "C" { -extern void skwasm_setAssociatedObjectOnThread(unsigned long threadId, void *pointer, SkwasmObject object); -extern SkwasmObject skwasm_getAssociatedObject(void *pointer); +extern void skwasm_setAssociatedObjectOnThread(unsigned long threadId, + void* pointer, + SkwasmObject object); +extern SkwasmObject skwasm_getAssociatedObject(void* pointer); extern void skwasm_registerMessageListener(pthread_t threadId); extern uint32_t skwasm_createOffscreenCanvas(int width, int height); extern void skwasm_resizeCanvas(uint32_t contextHandle, int width, int height); diff --git a/lib/web_ui/skwasm/surface.cpp b/lib/web_ui/skwasm/surface.cpp index e3cf21fc488ae..c5098f348d70f 100644 --- a/lib/web_ui/skwasm/surface.cpp +++ b/lib/web_ui/skwasm/surface.cpp @@ -60,8 +60,10 @@ uint32_t Surface::rasterizeImage(SkImage* image, ImageByteFormat format) { return callbackId; } -std::unique_ptr Surface::createVideoFrameWrapper(SkwasmObject videoFrame) { - return std::unique_ptr(new VideoFrameWrapper(_thread, videoFrame)); +std::unique_ptr Surface::createVideoFrameWrapper( + SkwasmObject videoFrame) { + return std::unique_ptr( + new VideoFrameWrapper(_thread, videoFrame)); } // Main thread only @@ -215,8 +217,7 @@ void Surface::fRasterizeImage(Surface* surface, image->unref(); } -void Surface::fDisposeVideoFrame(Surface* surface, - SkwasmObject videoFrame) { +void Surface::fDisposeVideoFrame(Surface* surface, SkwasmObject videoFrame) { surface->_disposeVideoFrame(videoFrame); } @@ -252,7 +253,7 @@ SKWASM_EXPORT uint32_t surface_rasterizeImage(Surface* surface, // This is used by the skwasm JS support code to call back into C++ when the // we finish creating the image bitmap, which is an asynchronous operation. SKWASM_EXPORT void surface_onRenderComplete(Surface* surface, - uint32_t callbackId, - SkwasmObject imageBitmap) { + uint32_t callbackId, + SkwasmObject imageBitmap) { return surface->onRenderComplete(callbackId, imageBitmap); } diff --git a/lib/web_ui/skwasm/surface.h b/lib/web_ui/skwasm/surface.h index 34433a70e09a6..d93e4a81638b0 100644 --- a/lib/web_ui/skwasm/surface.h +++ b/lib/web_ui/skwasm/surface.h @@ -34,20 +34,21 @@ enum class ImageByteFormat { }; class VideoFrameWrapper { - public: - VideoFrameWrapper(unsigned long threadId, SkwasmObject videoFrame) : _rasterThreadId(threadId) { - skwasm_setAssociatedObjectOnThread(_rasterThreadId, this, videoFrame); - } - - ~VideoFrameWrapper() { - skwasm_setAssociatedObjectOnThread(_rasterThreadId, this, __builtin_wasm_ref_null_extern()); - } - - SkwasmObject getVideoFrame() { - return skwasm_getAssociatedObject(this); - } - private: - unsigned long _rasterThreadId; + public: + VideoFrameWrapper(unsigned long threadId, SkwasmObject videoFrame) + : _rasterThreadId(threadId) { + skwasm_setAssociatedObjectOnThread(_rasterThreadId, this, videoFrame); + } + + ~VideoFrameWrapper() { + skwasm_setAssociatedObjectOnThread(_rasterThreadId, this, + __builtin_wasm_ref_null_extern()); + } + + SkwasmObject getVideoFrame() { return skwasm_getAssociatedObject(this); } + + private: + unsigned long _rasterThreadId; }; class Surface { @@ -67,7 +68,8 @@ class Surface { void onRenderComplete(uint32_t callbackId, SkwasmObject imageBitmap); // Any thread - std::unique_ptr createVideoFrameWrapper(SkwasmObject videoFrame); + std::unique_ptr createVideoFrameWrapper( + SkwasmObject videoFrame); private: void _runWorker(); @@ -102,11 +104,9 @@ class Surface { static void fRenderPicture(Surface* surface, SkPicture* picture, uint32_t callbackId); - static void fOnRenderComplete( - Surface* surface, - uint32_t callbackId, - SkwasmObject imageBitmap - ); + static void fOnRenderComplete(Surface* surface, + uint32_t callbackId, + SkwasmObject imageBitmap); static void fRasterizeImage(Surface* surface, SkImage* image, ImageByteFormat format, diff --git a/lib/web_ui/test/ui/platform_view_test.dart b/lib/web_ui/test/ui/platform_view_test.dart index 6e34c15a2929c..982c48eaa9650 100644 --- a/lib/web_ui/test/ui/platform_view_test.dart +++ b/lib/web_ui/test/ui/platform_view_test.dart @@ -3,6 +3,7 @@ // found in the LICENSE file. import 'dart:async'; +import 'dart:math' as math; import 'package:test/bootstrap/browser.dart'; import 'package:test/test.dart'; @@ -29,7 +30,7 @@ Future testMain() async { setUp(() { ui_web.platformViewRegistry.registerViewFactory( platformViewType, - (int viewId) { + (int viewId) { final DomElement element = createDomHTMLDivElement(); element.style.backgroundColor = 'blue'; return element; @@ -62,7 +63,7 @@ Future testMain() async { 1, offset: const ui.Offset(125, 125), width: 50, - height: 50, + height: 50, ); await renderer.renderScene(sb.build()); @@ -90,9 +91,9 @@ Future testMain() async { sb.addPlatformView( 1, - offset: const ui.Offset(150, 150), - width: 50, - height: 50, + offset: const ui.Offset(100, 100), + width: 100, + height: 100, ); sb.addPicture(const ui.Offset(175, 175), picture); @@ -100,6 +101,35 @@ Future testMain() async { await matchGoldenFile('picture_platformview_sandwich.png', region: region); }); + + test('Transformed platformview', () async { + await _createPlatformView(1, platformViewType); + + final ui.PictureRecorder recorder = ui.PictureRecorder(); + final ui.Canvas canvas = ui.Canvas(recorder); + canvas.drawCircle( + ui.Offset.zero, + 50, + ui.Paint() + ..style = ui.PaintingStyle.fill + ..color = const ui.Color(0xFFFF0000) + ); + + final ui.SceneBuilder sb = ui.SceneBuilder(); + sb.pushOffset(0, 0); + sb.addPicture(const ui.Offset(150, 150), recorder.endRecording()); + + sb.pushTransform(Matrix4.rotationZ(math.pi / 3.0).toFloat64()); + sb.addPlatformView( + 1, + offset: const ui.Offset(125, 125), + width: 50, + height: 50, + ); + await renderer.renderScene(sb.build()); + + await matchGoldenFile('platformview_transformed.png', region: region); + }); } // Sends a platform message to create a Platform View with the given id and viewType. From f39ee43d7aac945e55bbaa6565f9a445f7e79782 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Wed, 16 Aug 2023 16:11:04 -0700 Subject: [PATCH 11/24] Implemented opacity. --- .../src/engine/skwasm/skwasm_impl/layers.dart | 6 +-- .../engine/skwasm/skwasm_impl/scene_view.dart | 4 ++ lib/web_ui/test/ui/platform_view_test.dart | 49 +++++++++++++++---- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart index 44db1cdbbddb4..d853352ef8883 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart @@ -420,11 +420,6 @@ class PictureSlice implements LayerSlice { @override void dispose() => picture.dispose(); - - @override - String toString() { - return 'PictureSlice(${picture.cullRect})'; - } } mixin PictureLayer implements ui.EngineLayer { @@ -526,6 +521,7 @@ class PlatformViewStyling { static PlatformViewStyling combine(PlatformViewStyling outer, PlatformViewStyling inner) { return PlatformViewStyling( position: PlatformViewPosition.combine(outer.position, inner.position), + opacity: outer.opacity * inner.opacity, ); } } diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart index c842c8f7c02f1..5e185188748f6 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart @@ -115,6 +115,10 @@ final class PlatformViewSliceContainer extends SliceContainer { if (transform != null) { style.transform = float64ListToCssTransform3d(transform.storage); } + + if (view.styling.opacity != 1.0) { + style.opacity = '${view.styling.opacity}'; + } } while (currentContainer != null) { diff --git a/lib/web_ui/test/ui/platform_view_test.dart b/lib/web_ui/test/ui/platform_view_test.dart index 982c48eaa9650..f85ef809f3c1d 100644 --- a/lib/web_ui/test/ui/platform_view_test.dart +++ b/lib/web_ui/test/ui/platform_view_test.dart @@ -42,13 +42,13 @@ Future testMain() async { platformViewManager.debugClear(); }); - test('Picture + Overlapping PlatformView', () async { + test('picture + overlapping platformView', () async { await _createPlatformView(1, platformViewType); final ui.PictureRecorder recorder = ui.PictureRecorder(); final ui.Canvas canvas = ui.Canvas(recorder); canvas.drawCircle( - ui.Offset.zero, + const ui.Offset(50, 50), 50, ui.Paint() ..style = ui.PaintingStyle.fill @@ -57,7 +57,7 @@ Future testMain() async { final ui.SceneBuilder sb = ui.SceneBuilder(); sb.pushOffset(0, 0); - sb.addPicture(const ui.Offset(150, 150), recorder.endRecording()); + sb.addPicture(const ui.Offset(100, 100), recorder.endRecording()); sb.addPlatformView( 1, @@ -70,13 +70,13 @@ Future testMain() async { await matchGoldenFile('picture_platformview_overlap.png', region: region); }); - test('PlatformView sandwich', () async { + test('platformView sandwich', () async { await _createPlatformView(1, platformViewType); final ui.PictureRecorder recorder = ui.PictureRecorder(); final ui.Canvas canvas = ui.Canvas(recorder); canvas.drawCircle( - ui.Offset.zero, + const ui.Offset(50, 50), 50, ui.Paint() ..style = ui.PaintingStyle.fill @@ -87,7 +87,7 @@ Future testMain() async { final ui.SceneBuilder sb = ui.SceneBuilder(); sb.pushOffset(0, 0); - sb.addPicture(const ui.Offset(125, 125), picture); + sb.addPicture(const ui.Offset(75, 75), picture); sb.addPlatformView( 1, @@ -96,19 +96,19 @@ Future testMain() async { height: 100, ); - sb.addPicture(const ui.Offset(175, 175), picture); + sb.addPicture(const ui.Offset(125, 125), picture); await renderer.renderScene(sb.build()); await matchGoldenFile('picture_platformview_sandwich.png', region: region); }); - test('Transformed platformview', () async { + test('transformed platformview', () async { await _createPlatformView(1, platformViewType); final ui.PictureRecorder recorder = ui.PictureRecorder(); final ui.Canvas canvas = ui.Canvas(recorder); canvas.drawCircle( - ui.Offset.zero, + const ui.Offset(50, 50), 50, ui.Paint() ..style = ui.PaintingStyle.fill @@ -117,7 +117,7 @@ Future testMain() async { final ui.SceneBuilder sb = ui.SceneBuilder(); sb.pushOffset(0, 0); - sb.addPicture(const ui.Offset(150, 150), recorder.endRecording()); + sb.addPicture(const ui.Offset(100, 100), recorder.endRecording()); sb.pushTransform(Matrix4.rotationZ(math.pi / 3.0).toFloat64()); sb.addPlatformView( @@ -130,6 +130,35 @@ Future testMain() async { await matchGoldenFile('platformview_transformed.png', region: region); }); + + test('platformview with opacity', () async { + await _createPlatformView(1, platformViewType); + + final ui.PictureRecorder recorder = ui.PictureRecorder(); + final ui.Canvas canvas = ui.Canvas(recorder); + canvas.drawCircle( + const ui.Offset(50, 50), + 50, + ui.Paint() + ..style = ui.PaintingStyle.fill + ..color = const ui.Color(0xFFFF0000) + ); + + final ui.SceneBuilder sb = ui.SceneBuilder(); + sb.pushOffset(0, 0); + sb.addPicture(const ui.Offset(100, 100), recorder.endRecording()); + + sb.pushOpacity(127); + sb.addPlatformView( + 1, + offset: const ui.Offset(125, 125), + width: 50, + height: 50, + ); + await renderer.renderScene(sb.build()); + + await matchGoldenFile('platformview_opacity.png', region: region); + }); } // Sends a platform message to create a Platform View with the given id and viewType. From a402bac88eee6668fc71c99772774caffab2f3a8 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Thu, 17 Aug 2023 11:48:04 -0700 Subject: [PATCH 12/24] Pull scene builder, layer builder and scene view out of skwasm into the core engine code. --- lib/web_ui/lib/src/engine.dart | 3 ++ .../{skwasm/skwasm_impl => }/layers.dart | 53 ++++++++----------- .../skwasm_impl => }/scene_builder.dart | 20 +++---- .../{skwasm/skwasm_impl => }/scene_view.dart | 24 +++++---- .../lib/src/engine/skwasm/skwasm_impl.dart | 3 -- .../engine/skwasm/skwasm_impl/renderer.dart | 18 +++++-- 6 files changed, 64 insertions(+), 57 deletions(-) rename lib/web_ui/lib/src/engine/{skwasm/skwasm_impl => }/layers.dart (94%) rename lib/web_ui/lib/src/engine/{skwasm/skwasm_impl => }/scene_builder.dart (91%) rename lib/web_ui/lib/src/engine/{skwasm/skwasm_impl => }/scene_view.dart (92%) diff --git a/lib/web_ui/lib/src/engine.dart b/lib/web_ui/lib/src/engine.dart index 70f3b31de825e..73ecc287611fc 100644 --- a/lib/web_ui/lib/src/engine.dart +++ b/lib/web_ui/lib/src/engine.dart @@ -110,6 +110,7 @@ export 'engine/js_interop/js_promise.dart'; export 'engine/js_interop/js_typed_data.dart'; export 'engine/key_map.g.dart'; export 'engine/keyboard_binding.dart'; +export 'engine/layers.dart'; export 'engine/mouse_cursor.dart'; export 'engine/navigation/history.dart'; export 'engine/noto_font.dart'; @@ -129,7 +130,9 @@ export 'engine/raw_keyboard.dart'; export 'engine/renderer.dart'; export 'engine/rrect_renderer.dart'; export 'engine/safe_browser_api.dart'; +export 'engine/scene_builder.dart'; export 'engine/scene_painting.dart'; +export 'engine/scene_view.dart'; export 'engine/semantics/accessibility.dart'; export 'engine/semantics/checkable.dart'; export 'engine/semantics/dialog.dart'; diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart b/lib/web_ui/lib/src/engine/layers.dart similarity index 94% rename from lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart rename to lib/web_ui/lib/src/engine/layers.dart index d853352ef8883..06a89fc3d3566 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart +++ b/lib/web_ui/lib/src/engine/layers.dart @@ -8,10 +8,10 @@ import 'package:ui/src/engine/scene_painting.dart'; import 'package:ui/src/engine/vector_math.dart'; import 'package:ui/ui.dart' as ui; -class RootLayer with PictureLayer {} +class EngineRootLayer with PictureEngineLayer {} class BackdropFilterLayer - with PictureLayer + with PictureEngineLayer implements ui.BackdropFilterEngineLayer {} class BackdropFilterOperation implements LayerOperation { BackdropFilterOperation(this.filter, this.mode); @@ -40,7 +40,7 @@ class BackdropFilterOperation implements LayerOperation { } class ClipPathLayer - with PictureLayer + with PictureEngineLayer implements ui.ClipPathEngineLayer {} class ClipPathOperation implements LayerOperation { ClipPathOperation(this.path, this.clip); @@ -73,13 +73,13 @@ class ClipPathOperation implements LayerOperation { @override PlatformViewStyling createPlatformViewStyling() { - // TODO: implement this + // TODO(jacksongardner): implement clip styling for platform views return const PlatformViewStyling(); } } class ClipRectLayer - with PictureLayer + with PictureEngineLayer implements ui.ClipRectEngineLayer {} class ClipRectOperation implements LayerOperation { const ClipRectOperation(this.rect, this.clip); @@ -112,13 +112,13 @@ class ClipRectOperation implements LayerOperation { @override PlatformViewStyling createPlatformViewStyling() { - // TODO: implement createPlatformViewStyling + // TODO(jacksongardner): implement clip styling for platform views return const PlatformViewStyling(); } } class ClipRRectLayer - with PictureLayer + with PictureEngineLayer implements ui.ClipRRectEngineLayer {} class ClipRRectOperation implements LayerOperation { const ClipRRectOperation(this.rrect, this.clip); @@ -151,13 +151,13 @@ class ClipRRectOperation implements LayerOperation { @override PlatformViewStyling createPlatformViewStyling() { - // TODO: implement createPlatformViewStyling + // TODO(jacksongardner): implement clip styling for platform views return const PlatformViewStyling(); } } class ColorFilterLayer - with PictureLayer + with PictureEngineLayer implements ui.ColorFilterEngineLayer {} class ColorFilterOperation implements LayerOperation { ColorFilterOperation(this.filter); @@ -185,7 +185,7 @@ class ColorFilterOperation implements LayerOperation { } class ImageFilterLayer - with PictureLayer + with PictureEngineLayer implements ui.ImageFilterEngineLayer {} class ImageFilterOperation implements LayerOperation { ImageFilterOperation(this.filter, this.offset); @@ -194,7 +194,7 @@ class ImageFilterOperation implements LayerOperation { final ui.Offset offset; @override - ui.Rect cullRect(ui.Rect contentRect) => contentRect; + ui.Rect cullRect(ui.Rect contentRect) => (filter as SceneImageFilter).filterBounds(contentRect); @override ui.Rect inverseMapRect(ui.Rect rect) => rect; @@ -231,7 +231,7 @@ class ImageFilterOperation implements LayerOperation { } class OffsetLayer - with PictureLayer + with PictureEngineLayer implements ui.OffsetEngineLayer {} class OffsetOperation implements LayerOperation { OffsetOperation(this.dx, this.dy); @@ -263,7 +263,7 @@ class OffsetOperation implements LayerOperation { } class OpacityLayer - with PictureLayer + with PictureEngineLayer implements ui.OpacityEngineLayer {} class OpacityOperation implements LayerOperation { OpacityOperation(this.alpha, this.offset); @@ -305,7 +305,7 @@ class OpacityOperation implements LayerOperation { } class TransformLayer - with PictureLayer + with PictureEngineLayer implements ui.TransformEngineLayer {} class TransformOperation implements LayerOperation { TransformOperation(this.transform); @@ -341,7 +341,7 @@ class TransformOperation implements LayerOperation { } class ShaderMaskLayer - with PictureLayer + with PictureEngineLayer implements ui.ShaderMaskEngineLayer {} class ShaderMaskOperation implements LayerOperation { ShaderMaskOperation(this.shader, this.maskRect, this.blendMode); @@ -379,10 +379,7 @@ class ShaderMaskOperation implements LayerOperation { } @override - PlatformViewStyling createPlatformViewStyling() { - // TODO: implement createPlatformViewStyling - return const PlatformViewStyling(); - } + PlatformViewStyling createPlatformViewStyling() => const PlatformViewStyling(); } class PlatformView { @@ -422,7 +419,7 @@ class PictureSlice implements LayerSlice { void dispose() => picture.dispose(); } -mixin PictureLayer implements ui.EngineLayer { +mixin PictureEngineLayer implements ui.EngineLayer { List slices = []; @override @@ -503,19 +500,13 @@ class PlatformViewPosition { } } -class PlatformViewClip { - const PlatformViewClip(); -} - class PlatformViewStyling { const PlatformViewStyling({ this.position = const PlatformViewPosition.zero(), - this.clip = const PlatformViewClip(), this.opacity = 1.0 }); final PlatformViewPosition position; - final PlatformViewClip clip; final double opacity; static PlatformViewStyling combine(PlatformViewStyling outer, PlatformViewStyling inner) { @@ -528,12 +519,12 @@ class PlatformViewStyling { class LayerBuilder { factory LayerBuilder.rootLayer() { - return LayerBuilder._(null, RootLayer(), null); + return LayerBuilder._(null, EngineRootLayer(), null); } factory LayerBuilder.childLayer({ required LayerBuilder parent, - required PictureLayer layer, + required PictureEngineLayer layer, required LayerOperation operation }) { return LayerBuilder._(parent, layer, operation); @@ -545,7 +536,7 @@ class LayerBuilder { this.operation); final LayerBuilder? parent; - final PictureLayer layer; + final PictureEngineLayer layer; final LayerOperation? operation; final List pendingPictures = []; List pendingPlatformViews = []; @@ -642,7 +633,7 @@ class LayerBuilder { pendingPlatformViews.add(PlatformView(viewId, ui.Size(width, height), viewStyling)); } - void mergeLayer(PictureLayer layer) { + void mergeLayer(PictureEngineLayer layer) { for (final LayerSlice slice in layer.slices) { switch (slice) { case PictureSlice(): @@ -657,7 +648,7 @@ class LayerBuilder { } } - PictureLayer build() { + PictureEngineLayer build() { flushSlices(); return layer; } diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart b/lib/web_ui/lib/src/engine/scene_builder.dart similarity index 91% rename from lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart rename to lib/web_ui/lib/src/engine/scene_builder.dart index 6c14d8202400f..20076e2b56c17 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart +++ b/lib/web_ui/lib/src/engine/scene_builder.dart @@ -4,13 +4,13 @@ import 'dart:typed_data'; -import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; +import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; -class SkwasmScene implements ui.Scene { - SkwasmScene(this.rootLayer); +class EngineScene implements ui.Scene { + EngineScene(this.rootLayer); - final RootLayer rootLayer; + final EngineRootLayer rootLayer; @override void dispose() { @@ -36,7 +36,7 @@ class SkwasmScene implements ui.Scene { } } -class SkwasmSceneBuilder implements ui.SceneBuilder { +class EngineSceneBuilder implements ui.SceneBuilder { LayerBuilder currentBuilder = LayerBuilder.rootLayer(); @override @@ -78,7 +78,7 @@ class SkwasmSceneBuilder implements ui.SceneBuilder { @override void addRetained(ui.EngineLayer retainedLayer) { - currentBuilder.mergeLayer(retainedLayer as PictureLayer); + currentBuilder.mergeLayer(retainedLayer as PictureEngineLayer); } @override @@ -222,13 +222,13 @@ class SkwasmSceneBuilder implements ui.SceneBuilder { while (currentBuilder.parent != null) { pop(); } - final PictureLayer rootLayer = currentBuilder.build(); - return SkwasmScene(rootLayer as RootLayer); + final PictureEngineLayer rootLayer = currentBuilder.build(); + return EngineScene(rootLayer as EngineRootLayer); } @override void pop() { - final PictureLayer layer = currentBuilder.build(); + final PictureEngineLayer layer = currentBuilder.build(); final LayerBuilder? parentBuilder = currentBuilder.parent; if (parentBuilder == null) { throw StateError('Popped too many times.'); @@ -237,7 +237,7 @@ class SkwasmSceneBuilder implements ui.SceneBuilder { currentBuilder.mergeLayer(layer); } - T pushLayer(T layer, LayerOperation operation) { + T pushLayer(T layer, LayerOperation operation) { currentBuilder = LayerBuilder.childLayer( parent: currentBuilder, layer: layer, diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart b/lib/web_ui/lib/src/engine/scene_view.dart similarity index 92% rename from lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart rename to lib/web_ui/lib/src/engine/scene_view.dart index 5e185188748f6..06b4bb6cd344c 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_view.dart +++ b/lib/web_ui/lib/src/engine/scene_view.dart @@ -2,8 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:async'; + import 'package:ui/src/engine.dart'; -import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; import 'package:ui/ui.dart' as ui; const String kCanvasContainerTag = 'flt-canvas-container'; @@ -99,7 +100,6 @@ final class PlatformViewSliceContainer extends SliceContainer { } viewContainer.appendChild(platformView); } - // TODO: set all the styling here instead of just the position final DomCSSStyleDeclaration style = viewContainer.style; final double logicalWidth = view.size.width / window.devicePixelRatio; final double logicalHeight = view.size.height / window.devicePixelRatio; @@ -119,6 +119,8 @@ final class PlatformViewSliceContainer extends SliceContainer { if (view.styling.opacity != 1.0) { style.opacity = '${view.styling.opacity}'; } + + // TODO(jacksongardner): Implement clip styling for platform views } while (currentContainer != null) { @@ -129,15 +131,19 @@ final class PlatformViewSliceContainer extends SliceContainer { } } -class SkwasmSceneView { - factory SkwasmSceneView(SkwasmSurface surface) { +abstract class PictureRenderer { + FutureOr renderPicture(ScenePicture picture); +} + +class EngineSceneView { + factory EngineSceneView(PictureRenderer pictureRenderer) { final DomElement sceneElement = createDomElement('flt-scene'); - return SkwasmSceneView._(surface, sceneElement); + return EngineSceneView._(pictureRenderer, sceneElement); } - SkwasmSceneView._(this.surface, this.sceneElement); + EngineSceneView._(this.pictureRenderer, this.sceneElement); - final SkwasmSurface surface; + final PictureRenderer pictureRenderer; final DomElement sceneElement; List containers = []; @@ -145,7 +151,7 @@ class SkwasmSceneView { int queuedRenders = 0; static const int kMaxQueuedRenders = 3; - Future renderScene(SkwasmScene scene) async { + Future renderScene(EngineScene scene) async { if (queuedRenders >= kMaxQueuedRenders) { return; } @@ -155,7 +161,7 @@ class SkwasmSceneView { final Iterable> renderFutures = slices.map( (LayerSlice slice) async => switch (slice) { PlatformViewSlice() => null, - PictureSlice() => surface.renderPicture(slice.picture as SkwasmPicture), + PictureSlice() => pictureRenderer.renderPicture(slice.picture), } ); final List renderedBitmaps = await Future.wait(renderFutures); diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart index aab16e6735b3d..fd370c7a0fb18 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart @@ -14,7 +14,6 @@ export 'skwasm_impl/codecs.dart'; export 'skwasm_impl/filters.dart'; export 'skwasm_impl/font_collection.dart'; export 'skwasm_impl/image.dart'; -export 'skwasm_impl/layers.dart'; export 'skwasm_impl/memory.dart'; export 'skwasm_impl/paint.dart'; export 'skwasm_impl/paragraph.dart'; @@ -44,8 +43,6 @@ export 'skwasm_impl/raw/text/raw_paragraph_style.dart'; export 'skwasm_impl/raw/text/raw_strut_style.dart'; export 'skwasm_impl/raw/text/raw_text_style.dart'; export 'skwasm_impl/renderer.dart'; -export 'skwasm_impl/scene_builder.dart'; -export 'skwasm_impl/scene_view.dart'; export 'skwasm_impl/shaders.dart'; export 'skwasm_impl/surface.dart'; export 'skwasm_impl/vertices.dart'; diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart index 029f2011b3baf..e2ea3f591cba7 100644 --- a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart +++ b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart @@ -14,7 +14,7 @@ import 'package:ui/ui_web/src/ui_web.dart' as ui_web; class SkwasmRenderer implements Renderer { late SkwasmSurface surface; - late SkwasmSceneView sceneView; + late EngineSceneView sceneView; @override final SkwasmFontCollection fontCollection = SkwasmFontCollection(); @@ -193,7 +193,7 @@ class SkwasmRenderer implements Renderer { ); @override - ui.SceneBuilder createSceneBuilder() => SkwasmSceneBuilder(); + ui.SceneBuilder createSceneBuilder() => EngineSceneBuilder(); @override ui.StrutStyle createStrutStyle({ @@ -348,7 +348,7 @@ class SkwasmRenderer implements Renderer { @override FutureOr initialize() { surface = SkwasmSurface(); - sceneView = SkwasmSceneView(surface); + sceneView = EngineSceneView(SkwasmPictureRenderer(surface)); } @override @@ -399,7 +399,7 @@ class SkwasmRenderer implements Renderer { } @override - Future renderScene(ui.Scene scene) => sceneView.renderScene(scene as SkwasmScene); + Future renderScene(ui.Scene scene) => sceneView.renderScene(scene as EngineScene); @override String get rendererTag => 'skwasm'; @@ -449,3 +449,13 @@ class SkwasmRenderer implements Renderer { lineNumber: lineNumber ); } + +class SkwasmPictureRenderer implements PictureRenderer { + SkwasmPictureRenderer(this.surface); + + SkwasmSurface surface; + + @override + FutureOr renderPicture(ScenePicture picture) => + surface.renderPicture(picture as SkwasmPicture); +} From ae37aa4b4a3f5cfeb5497eb2343eb87346436b3f Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Thu, 17 Aug 2023 12:02:11 -0700 Subject: [PATCH 13/24] Fix licenses golden. --- ci/licenses_golden/licenses_flutter | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 97ab61f6f88dd..052084d7ce908 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -2018,6 +2018,7 @@ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/js_interop/js_promise.dart + ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/js_interop/js_typed_data.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/key_map.g.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/keyboard_binding.dart + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/layers.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/mouse_cursor.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/navigation/history.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/noto_font.dart + ../../../flutter/LICENSE @@ -2037,7 +2038,9 @@ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/raw_keyboard.dart + ../../../ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/renderer.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/rrect_renderer.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/safe_browser_api.dart + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/scene_builder.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/scene_painting.dart + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/scene_view.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/semantics.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/semantics/accessibility.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/semantics/checkable.dart + ../../../flutter/LICENSE @@ -2066,7 +2069,6 @@ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dar ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/filters.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/font_collection.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart + ../../../flutter/LICENSE -ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/memory.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/paint.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/paragraph.dart + ../../../flutter/LICENSE @@ -2096,7 +2098,6 @@ ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/text/r ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/text/raw_strut_style.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/text/raw_text_style.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart + ../../../flutter/LICENSE -ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/shaders.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart + ../../../flutter/LICENSE ORIGIN: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/vertices.dart + ../../../flutter/LICENSE @@ -4746,6 +4747,7 @@ FILE: ../../../flutter/lib/web_ui/lib/src/engine/js_interop/js_promise.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/js_interop/js_typed_data.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/key_map.g.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/keyboard_binding.dart +FILE: ../../../flutter/lib/web_ui/lib/src/engine/layers.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/mouse_cursor.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/navigation/history.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/noto_font.dart @@ -4765,7 +4767,9 @@ FILE: ../../../flutter/lib/web_ui/lib/src/engine/raw_keyboard.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/renderer.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/rrect_renderer.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/safe_browser_api.dart +FILE: ../../../flutter/lib/web_ui/lib/src/engine/scene_builder.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/scene_painting.dart +FILE: ../../../flutter/lib/web_ui/lib/src/engine/scene_view.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/semantics.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/semantics/accessibility.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/semantics/checkable.dart @@ -4794,7 +4798,6 @@ FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/filters.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/font_collection.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart -FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/layers.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/memory.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/paint.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/paragraph.dart @@ -4824,7 +4827,6 @@ FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/text/raw FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/text/raw_strut_style.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/text/raw_text_style.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart -FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/scene_builder.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/shaders.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart FILE: ../../../flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/vertices.dart From 586f54fb22ce5bbe1e72fe50f6d2b7b057c2602a Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Thu, 17 Aug 2023 15:06:59 -0700 Subject: [PATCH 14/24] Don't dispose Scenes while they are rendering. --- lib/web_ui/lib/src/engine/scene_builder.dart | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/web_ui/lib/src/engine/scene_builder.dart b/lib/web_ui/lib/src/engine/scene_builder.dart index 20076e2b56c17..7fe5744c525a5 100644 --- a/lib/web_ui/lib/src/engine/scene_builder.dart +++ b/lib/web_ui/lib/src/engine/scene_builder.dart @@ -12,9 +12,29 @@ class EngineScene implements ui.Scene { final EngineRootLayer rootLayer; + bool _disposeCalled = false; + bool _isRendering = false; + bool _isDisposed = false; + + set isRendering(bool isRendering) { + if (_isRendering != isRendering) { + _isRendering = isRendering; + _updateDispose(); + } + } + @override void dispose() { + _disposeCalled = true; + _updateDispose(); + } + + void _updateDispose() { + if (_isDisposed || _isRendering || !_disposeCalled) { + return; + } rootLayer.dispose(); + _isDisposed = true; } @override From 0d54d75d09fc5c21958b007d855004abef26a727 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Thu, 17 Aug 2023 15:07:53 -0700 Subject: [PATCH 15/24] Actually tell the scene it's rendering. --- lib/web_ui/lib/src/engine/scene_view.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/web_ui/lib/src/engine/scene_view.dart b/lib/web_ui/lib/src/engine/scene_view.dart index 06b4bb6cd344c..9747e0b60d6a5 100644 --- a/lib/web_ui/lib/src/engine/scene_view.dart +++ b/lib/web_ui/lib/src/engine/scene_view.dart @@ -157,6 +157,7 @@ class EngineSceneView { } queuedRenders += 1; + scene.isRendering = true; final List slices = scene.rootLayer.slices; final Iterable> renderFutures = slices.map( (LayerSlice slice) async => switch (slice) { @@ -229,6 +230,7 @@ class EngineSceneView { sceneElement.removeChild(currentElement); currentElement = sibling; } + scene.isRendering = false; queuedRenders -= 1; } From 534b3d3f8414684050054dfdccf6da2516950c10 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Fri, 18 Aug 2023 10:14:13 -0700 Subject: [PATCH 16/24] Using slots, which is the correct way to add/remove platform views. --- lib/web_ui/lib/src/engine/scene_builder.dart | 35 +++++++++++--------- lib/web_ui/lib/src/engine/scene_view.dart | 21 ++++++------ lib/web_ui/skwasm/library_skwasm_support.js | 1 + 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/lib/web_ui/lib/src/engine/scene_builder.dart b/lib/web_ui/lib/src/engine/scene_builder.dart index 7fe5744c525a5..566e585c71c28 100644 --- a/lib/web_ui/lib/src/engine/scene_builder.dart +++ b/lib/web_ui/lib/src/engine/scene_builder.dart @@ -12,29 +12,32 @@ class EngineScene implements ui.Scene { final EngineRootLayer rootLayer; - bool _disposeCalled = false; - bool _isRendering = false; - bool _isDisposed = false; - - set isRendering(bool isRendering) { - if (_isRendering != isRendering) { - _isRendering = isRendering; - _updateDispose(); - } + // We keep a refcount here because this can be asynchronously rendered, so we + // don't necessarily want to dispose immediately when the user calls dispose. + // Instead, we need to stay alive until we're done rendering. + int _refCount = 1; + + void beginRender() { + assert(_refCount > 0); + _refCount++; + } + + void endRender() { + _refCount--; + _disposeIfNeeded(); } @override void dispose() { - _disposeCalled = true; - _updateDispose(); + _refCount--; + _disposeIfNeeded(); } - void _updateDispose() { - if (_isDisposed || _isRendering || !_disposeCalled) { - return; + void _disposeIfNeeded() { + assert(_refCount >= 0); + if (_refCount == 0) { + rootLayer.dispose(); } - rootLayer.dispose(); - _isDisposed = true; } @override diff --git a/lib/web_ui/lib/src/engine/scene_view.dart b/lib/web_ui/lib/src/engine/scene_view.dart index 9747e0b60d6a5..0185c58f1d788 100644 --- a/lib/web_ui/lib/src/engine/scene_view.dart +++ b/lib/web_ui/lib/src/engine/scene_view.dart @@ -72,6 +72,7 @@ final class PlatformViewSliceContainer extends SliceContainer { PlatformViewSliceContainer(this._views); List _views; + Map _slots = {}; @override final DomElement container = domDocument.createElement(kPlatformViewSliceContainerTag); @@ -85,8 +86,10 @@ final class PlatformViewSliceContainer extends SliceContainer { @override void updateContents() { DomElement? currentContainer = container.firstElementChild; + final Map newSlots = {}; for (final PlatformView view in _views) { - final DomElement platformView = platformViewManager.getViewById(view.viewId); + final DomElement platformView = _slots[view.viewId] ?? createPlatformViewSlot(view.viewId); + newSlots[view.viewId] = platformView; final DomElement viewContainer; if (currentContainer != null && currentContainer.firstElementChild == platformView) { viewContainer = currentContainer; @@ -112,14 +115,8 @@ final class PlatformViewSliceContainer extends SliceContainer { style.top = '${offset?.dy ?? 0}px'; final Matrix4? transform = view.styling.position.transform; - if (transform != null) { - style.transform = float64ListToCssTransform3d(transform.storage); - } - - if (view.styling.opacity != 1.0) { - style.opacity = '${view.styling.opacity}'; - } - + style.transform = transform != null ? float64ListToCssTransform3d(transform.storage) : ''; + style.opacity = view.styling.opacity != 1.0 ? '${view.styling.opacity}' : ''; // TODO(jacksongardner): Implement clip styling for platform views } @@ -128,6 +125,8 @@ final class PlatformViewSliceContainer extends SliceContainer { currentContainer.remove(); currentContainer = next; } + + _slots = newSlots; } } @@ -157,7 +156,7 @@ class EngineSceneView { } queuedRenders += 1; - scene.isRendering = true; + scene.beginRender(); final List slices = scene.rootLayer.slices; final Iterable> renderFutures = slices.map( (LayerSlice slice) async => switch (slice) { @@ -230,7 +229,7 @@ class EngineSceneView { sceneElement.removeChild(currentElement); currentElement = sibling; } - scene.isRendering = false; + scene.endRender(); queuedRenders -= 1; } diff --git a/lib/web_ui/skwasm/library_skwasm_support.js b/lib/web_ui/skwasm/library_skwasm_support.js index 134bcb65345f4..fe50a1d0a5918 100644 --- a/lib/web_ui/skwasm/library_skwasm_support.js +++ b/lib/web_ui/skwasm/library_skwasm_support.js @@ -32,6 +32,7 @@ mergeInto(LibraryManager.library, { return; case 'setAssociatedObject': associatedObjectsMap.set(data.pointer, data.object); + return; default: console.warn('unrecognized skwasm message'); } From 1ca0436304391d6e0d691b7c18cd29955170c2ee Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Mon, 21 Aug 2023 13:38:07 -0700 Subject: [PATCH 17/24] Fix position combining for platform views. --- lib/web_ui/lib/src/engine/layers.dart | 14 +++++++------- lib/web_ui/lib/src/engine/scene_builder.dart | 1 - lib/web_ui/test/ui/platform_view_test.dart | 3 +-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/web_ui/lib/src/engine/layers.dart b/lib/web_ui/lib/src/engine/layers.dart index 06a89fc3d3566..32bc944d4771d 100644 --- a/lib/web_ui/lib/src/engine/layers.dart +++ b/lib/web_ui/lib/src/engine/layers.dart @@ -466,9 +466,9 @@ class PlatformViewPosition { if (innerTransform != null) { if (innerOffset != null) { if (outerTransform != null) { - final Matrix4 newTransform = innerTransform.clone(); + final Matrix4 newTransform = outerTransform.clone(); newTransform.translate(innerOffset.dx, innerOffset.dy); - newTransform.multiply(outerTransform); + newTransform.multiply(innerTransform); return PlatformViewPosition(offset: outerOffset, transform: newTransform); } else { final ui.Offset finalOffset = outerOffset != null ? (innerOffset + outerOffset) : innerOffset; @@ -476,8 +476,8 @@ class PlatformViewPosition { } } else { if (outerTransform != null) { - final Matrix4 newTransform = innerTransform.clone(); - newTransform.multiply(outerTransform); + final Matrix4 newTransform = outerTransform.clone(); + newTransform.multiply(innerTransform); return PlatformViewPosition(offset: outerOffset, transform: newTransform); } else { return PlatformViewPosition(offset: outerOffset, transform: innerTransform); @@ -486,8 +486,8 @@ class PlatformViewPosition { } else { if (innerOffset != null) { if (outerTransform != null) { - final Matrix4 newTransform = Matrix4.translationValues(innerOffset.dx, innerOffset.dy, 0); - newTransform.multiply(outerTransform); + final Matrix4 newTransform = outerTransform.clone(); + newTransform.translate(innerOffset.dx, innerOffset.dy); return PlatformViewPosition(offset: outerOffset, transform: newTransform); } else { final ui.Offset finalOffset = outerOffset != null ? (innerOffset + outerOffset) : innerOffset; @@ -625,10 +625,10 @@ class LayerBuilder { final PlatformViewStyling viewStyling = offset == ui.Offset.zero ? layerStyling : PlatformViewStyling.combine( + layerStyling, PlatformViewStyling( position: PlatformViewPosition(offset: offset), ), - layerStyling, ); pendingPlatformViews.add(PlatformView(viewId, ui.Size(width, height), viewStyling)); } diff --git a/lib/web_ui/lib/src/engine/scene_builder.dart b/lib/web_ui/lib/src/engine/scene_builder.dart index 566e585c71c28..db4a6d93aa177 100644 --- a/lib/web_ui/lib/src/engine/scene_builder.dart +++ b/lib/web_ui/lib/src/engine/scene_builder.dart @@ -113,7 +113,6 @@ class EngineSceneBuilder implements ui.SceneBuilder { bool freeze = false, ui.FilterQuality filterQuality = ui.FilterQuality.low }) { - // TODO(jacksongardner): implement addTexture } @override diff --git a/lib/web_ui/test/ui/platform_view_test.dart b/lib/web_ui/test/ui/platform_view_test.dart index f85ef809f3c1d..7c934caf6087b 100644 --- a/lib/web_ui/test/ui/platform_view_test.dart +++ b/lib/web_ui/test/ui/platform_view_test.dart @@ -3,7 +3,6 @@ // found in the LICENSE file. import 'dart:async'; -import 'dart:math' as math; import 'package:test/bootstrap/browser.dart'; import 'package:test/test.dart'; @@ -119,7 +118,7 @@ Future testMain() async { sb.pushOffset(0, 0); sb.addPicture(const ui.Offset(100, 100), recorder.endRecording()); - sb.pushTransform(Matrix4.rotationZ(math.pi / 3.0).toFloat64()); + sb.pushTransform(Matrix4.rotationZ(0.1).toFloat64()); sb.addPlatformView( 1, offset: const ui.Offset(125, 125), From 7394dea31f72ca2fcb20af9aa8306e1bb2b9878e Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 22 Aug 2023 10:10:17 -0700 Subject: [PATCH 18/24] Add some comments. --- lib/web_ui/lib/src/engine/layers.dart | 140 +++++++++++++------ lib/web_ui/lib/src/engine/scene_builder.dart | 16 +++ 2 files changed, 114 insertions(+), 42 deletions(-) diff --git a/lib/web_ui/lib/src/engine/layers.dart b/lib/web_ui/lib/src/engine/layers.dart index 32bc944d4771d..e8f7d38e4a351 100644 --- a/lib/web_ui/lib/src/engine/layers.dart +++ b/lib/web_ui/lib/src/engine/layers.dart @@ -4,6 +4,7 @@ import 'dart:typed_data'; +import 'package:meta/meta.dart'; import 'package:ui/src/engine/scene_painting.dart'; import 'package:ui/src/engine/vector_math.dart'; import 'package:ui/ui.dart' as ui; @@ -222,7 +223,7 @@ class ImageFilterOperation implements LayerOperation { PlatformViewStyling createPlatformViewStyling() { if (offset != ui.Offset.zero) { return PlatformViewStyling( - position: PlatformViewPosition(offset: offset) + position: PlatformViewPosition.offset(offset) ); } else { return const PlatformViewStyling(); @@ -258,7 +259,7 @@ class OffsetOperation implements LayerOperation { @override PlatformViewStyling createPlatformViewStyling() => PlatformViewStyling( - position: PlatformViewPosition(offset: ui.Offset(dx, dy)) + position: PlatformViewPosition.offset(ui.Offset(dx, dy)) ); } @@ -299,7 +300,7 @@ class OpacityOperation implements LayerOperation { @override PlatformViewStyling createPlatformViewStyling() => PlatformViewStyling( - position: offset != ui.Offset.zero ? PlatformViewPosition(offset: offset) : const PlatformViewPosition(), + position: offset != ui.Offset.zero ? PlatformViewPosition.offset(offset) : const PlatformViewPosition.zero(), opacity: alpha.toDouble() / 255.0, ); } @@ -336,7 +337,7 @@ class TransformOperation implements LayerOperation { @override PlatformViewStyling createPlatformViewStyling() => PlatformViewStyling( - position: PlatformViewPosition(transform: getMatrix()), + position: PlatformViewPosition.transform(getMatrix()), ); } @@ -397,6 +398,7 @@ sealed class LayerSlice { void dispose(); } +// A slice that contains one or more platform views to be rendered. class PlatformViewSlice implements LayerSlice { PlatformViewSlice(this.views, this.occlusionRect); @@ -410,6 +412,8 @@ class PlatformViewSlice implements LayerSlice { void dispose() {} } +// A slice that contains flutter content to be rendered int he form of a single +// ScenePicture. class PictureSlice implements LayerSlice { PictureSlice(this.picture); @@ -420,6 +424,9 @@ class PictureSlice implements LayerSlice { } mixin PictureEngineLayer implements ui.EngineLayer { + // Each layer is represented as a series of "slices" which contain either + // flutter content or platform views. Slices in this list are ordered from + // bottom to top. List slices = []; @override @@ -433,6 +440,9 @@ mixin PictureEngineLayer implements ui.EngineLayer { abstract class LayerOperation { const LayerOperation(); + // Given an input content rectangle, this returns a conservative estimate of + // the covering rectangle of the content after it has been processed by the + // layer operation. ui.Rect cullRect(ui.Rect contentRect); // Takes a rectangle in the layer's coordinate space and maps it to the parent @@ -451,65 +461,80 @@ class PictureDrawCommand { ui.Picture picture; } +// Represents how a platform view should be positioned in the scene. +// This object is immutable, so it can be reused across different platform +// views that have the same positioning. class PlatformViewPosition { - const PlatformViewPosition({this.offset, this.transform}); + // No transformation at all. We leave both fields null. const PlatformViewPosition.zero() : offset = null, transform = null; + // A simple offset is the most common scenario. In those cases, we only + // store the offset and leave the transform as null + const PlatformViewPosition.offset(this.offset) : transform = null; + + // In more complex cases, we store the transform. In those cases, the offset + // is left as null. + const PlatformViewPosition.transform(this.transform) : offset = null; + + bool get isZero => (offset == null) && (transform == null); + final ui.Offset? offset; final Matrix4? transform; static PlatformViewPosition combine(PlatformViewPosition outer, PlatformViewPosition inner) { + // We try to reuse existing objects if possible, if they are immutable. + if (outer.isZero) { + return inner; + } + if (inner.isZero) { + return outer; + } final ui.Offset? outerOffset = outer.offset; - final Matrix4? outerTransform = outer.transform; final ui.Offset? innerOffset = inner.offset; - final Matrix4? innerTransform = inner.transform; - if (innerTransform != null) { - if (innerOffset != null) { - if (outerTransform != null) { - final Matrix4 newTransform = outerTransform.clone(); - newTransform.translate(innerOffset.dx, innerOffset.dy); - newTransform.multiply(innerTransform); - return PlatformViewPosition(offset: outerOffset, transform: newTransform); - } else { - final ui.Offset finalOffset = outerOffset != null ? (innerOffset + outerOffset) : innerOffset; - return PlatformViewPosition(offset: finalOffset, transform: innerTransform); - } - } else { - if (outerTransform != null) { - final Matrix4 newTransform = outerTransform.clone(); - newTransform.multiply(innerTransform); - return PlatformViewPosition(offset: outerOffset, transform: newTransform); - } else { - return PlatformViewPosition(offset: outerOffset, transform: innerTransform); - } - } + if (outerOffset != null && innerOffset != null) { + // Both positions are simple offsets, so they can be combined cheaply + // into another offset. + return PlatformViewPosition.offset(outerOffset + innerOffset); + } + + // Otherwise, at least one of the positions involves a matrix transform. + final Matrix4 newTransform; + if (outerOffset != null) { + newTransform = Matrix4.translationValues(outerOffset.dx, outerOffset.dy, 0); } else { - if (innerOffset != null) { - if (outerTransform != null) { - final Matrix4 newTransform = outerTransform.clone(); - newTransform.translate(innerOffset.dx, innerOffset.dy); - return PlatformViewPosition(offset: outerOffset, transform: newTransform); - } else { - final ui.Offset finalOffset = outerOffset != null ? (innerOffset + outerOffset) : innerOffset; - return PlatformViewPosition(offset: finalOffset); - } - } else { - return outer; - } + newTransform = outer.transform!.clone(); + } + if (innerOffset != null) { + newTransform.translate(innerOffset.dx, innerOffset.dy); + } else { + newTransform.multiply(inner.transform!); } + return PlatformViewPosition.transform(newTransform); } } +// Represents the styling to be performed on a platform view when it is +// composited. This object is immutable so that it can be reused with different +// platform views that have the same styling. class PlatformViewStyling { const PlatformViewStyling({ this.position = const PlatformViewPosition.zero(), this.opacity = 1.0 }); + bool get isDefault => position.isZero && (opacity == 1.0); + final PlatformViewPosition position; final double opacity; static PlatformViewStyling combine(PlatformViewStyling outer, PlatformViewStyling inner) { + // Attempt to reuse one of the existing immutable objects. + if (outer.isDefault) { + return inner; + } + if (inner.isDefault) { + return outer; + } return PlatformViewStyling( position: PlatformViewPosition.combine(outer.position, inner.position), opacity: outer.opacity * inner.opacity, @@ -535,6 +560,9 @@ class LayerBuilder { this.layer, this.operation); + @visibleForTesting + static (ui.PictureRecorder, SceneCanvas) Function(ui.Rect)? debugRecorderFactory; + final LayerBuilder? parent; final PictureEngineLayer layer; final LayerOperation? operation; @@ -561,12 +589,22 @@ class LayerBuilder { return _memoizedPlatformViewStyling ??= _createPlatformViewStyling(); } + (ui.PictureRecorder, SceneCanvas) _createRecorder(ui.Rect rect) { + if (debugRecorderFactory != null) { + return debugRecorderFactory!(rect); + } + final ui.PictureRecorder recorder = ui.PictureRecorder(); + final SceneCanvas canvas = ui.Canvas(recorder, rect) as SceneCanvas; + return (recorder, canvas); + } + void flushSlices() { if (pendingPictures.isNotEmpty) { + // Merge the existing draw commands into a single picture and add a slice + // with that picture to the slice list. final ui.Rect drawnRect = picturesRect ?? ui.Rect.zero; final ui.Rect rect = operation?.cullRect(drawnRect) ?? drawnRect; - final ui.PictureRecorder recorder = ui.PictureRecorder(); - final SceneCanvas canvas = ui.Canvas(recorder, rect) as SceneCanvas; + final (ui.PictureRecorder recorder, SceneCanvas canvas) = _createRecorder(rect); operation?.pre(canvas, rect); for (final PictureDrawCommand command in pendingPictures) { @@ -585,6 +623,8 @@ class LayerBuilder { } if (pendingPlatformViews.isNotEmpty) { + // Take any pending platform views and lower them into a platform view + // slice. ui.Rect? occlusionRect = platformViewRect; if (occlusionRect != null && operation != null) { occlusionRect = operation!.inverseMapRect(occlusionRect); @@ -594,6 +634,9 @@ class LayerBuilder { pendingPictures.clear(); pendingPlatformViews = []; + + // All the pictures and platform views have been lowered into slices. Clear + // our occlusion rectangles. picturesRect = null; platformViewRect = null; } @@ -606,7 +649,16 @@ class LayerBuilder { }) { final ui.Rect cullRect = (picture as ScenePicture).cullRect; final ui.Rect shiftedRect = cullRect.shift(offset); + + // Whenever we add a picture to our layer, we try to see if the picture + // will overlap with any platform views that are currently on top of our + // drawing surface. If they don't overlap with the platform views, they can + // be composited into the existing drawing surface. if (platformViewRect?.overlaps(shiftedRect) ?? false) { + // If they do overlap with the platform views, however, we need to create + // a new drawing surface to composite into. This lowers the current set + // of picture drawing commands into a picture slice and lowers the current + // set of platform views into a platform view slice. flushSlices(); } pendingPictures.add(PictureDrawCommand(offset, picture)); @@ -627,13 +679,16 @@ class LayerBuilder { : PlatformViewStyling.combine( layerStyling, PlatformViewStyling( - position: PlatformViewPosition(offset: offset), + position: PlatformViewPosition.offset(offset), ), ); pendingPlatformViews.add(PlatformView(viewId, ui.Size(width, height), viewStyling)); } void mergeLayer(PictureEngineLayer layer) { + // When we merge layers, we attempt to merge slices as much as possible as + // well, based on ordering of pictures and platform views and reusing the + // occlusion logic for determining where we can lower each picture. for (final LayerSlice slice in layer.slices) { switch (slice) { case PictureSlice(): @@ -649,6 +704,7 @@ class LayerBuilder { } PictureEngineLayer build() { + // Lower any pending pictures or platform views to their respective slices. flushSlices(); return layer; } diff --git a/lib/web_ui/lib/src/engine/scene_builder.dart b/lib/web_ui/lib/src/engine/scene_builder.dart index db4a6d93aa177..2ec9f6bc7523d 100644 --- a/lib/web_ui/lib/src/engine/scene_builder.dart +++ b/lib/web_ui/lib/src/engine/scene_builder.dart @@ -7,6 +7,17 @@ import 'dart:typed_data'; import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; +// This file implements a SceneBuilder and Scene that works with any renderer +// implementation that provides: +// * A `ui.Canvas` that conforms to `SceneCanvas` +// * A `ui.Picture` that conforms to `ScenePicture` +// * A `ui.ImageFilter` that conforms to `SceneImageFilter` +// +// These contain a few augmentations to the normal `dart:ui` API that provide +// additional sizing information that the scene builder uses to determine how +// these object might occlude one another. + + class EngineScene implements ui.Scene { EngineScene(this.rootLayer); @@ -113,6 +124,7 @@ class EngineSceneBuilder implements ui.SceneBuilder { bool freeze = false, ui.FilterQuality filterQuality = ui.FilterQuality.low }) { + // addTexture is not implemented on web. } @override @@ -217,10 +229,12 @@ class EngineSceneBuilder implements ui.SceneBuilder { @override void setCheckerboardOffscreenLayers(bool checkerboard) { + // Not implemented on web } @override void setCheckerboardRasterCacheImages(bool checkerboard) { + // Not implemented on web } @override @@ -233,10 +247,12 @@ class EngineSceneBuilder implements ui.SceneBuilder { double insetLeft, bool focusable ) { + // Not implemented on web } @override void setRasterizerTracingThreshold(int frameInterval) { + // Not implemented on web } @override From 4a36c48f487d44511917de7513453819d3427a1e Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 22 Aug 2023 10:44:36 -0700 Subject: [PATCH 19/24] Flatten the SceneView hierarchy a bit. --- lib/web_ui/lib/src/engine/scene_view.dart | 262 +++++++++++----------- 1 file changed, 127 insertions(+), 135 deletions(-) diff --git a/lib/web_ui/lib/src/engine/scene_view.dart b/lib/web_ui/lib/src/engine/scene_view.dart index 0185c58f1d788..c61520fcbaa6e 100644 --- a/lib/web_ui/lib/src/engine/scene_view.dart +++ b/lib/web_ui/lib/src/engine/scene_view.dart @@ -8,132 +8,15 @@ import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; const String kCanvasContainerTag = 'flt-canvas-container'; -const String kPlatformViewSliceContainerTag = 'flt-platform-view-slice'; -const String kPlatformViewContainerTag = 'flt-platform-view-container'; - -sealed class SliceContainer { - DomElement get container; - - void updateContents(); -} - -final class PictureSliceContainer extends SliceContainer { - factory PictureSliceContainer(ui.Rect bounds) { - final DomElement container = domDocument.createElement(kCanvasContainerTag); - final DomCanvasElement canvas = createDomCanvasElement( - width: bounds.width.toInt(), - height: bounds.height.toInt() - ); - container.appendChild(canvas); - return PictureSliceContainer._(bounds, container, canvas); - } - - PictureSliceContainer._(this._bounds, this.container, this.canvas); - - ui.Rect _bounds; - bool _dirty = true; - - ui.Rect get bounds => _bounds; - set bounds(ui.Rect bounds) { - if (_bounds != bounds) { - _bounds = bounds; - _dirty = true; - } - } - - @override - void updateContents() { - if (_dirty) { - _dirty = false; - final DomCSSStyleDeclaration style = canvas.style; - final double logicalWidth = bounds.width / window.devicePixelRatio; - final double logicalHeight = bounds.height / window.devicePixelRatio; - style.width = '${logicalWidth}px'; - style.height = '${logicalHeight}px'; - style.position = 'absolute'; - style.left = '${bounds.left}px'; - style.top = '${bounds.top}px'; - canvas.width = bounds.width; - canvas.height = bounds.height; - } - } - - void renderBitmap(DomImageBitmap bitmap) { - final DomCanvasRenderingContextBitmapRenderer ctx = canvas.contextBitmapRenderer; - ctx.transferFromImageBitmap(bitmap); - } - - @override - final DomElement container; - final DomCanvasElement canvas; -} - -final class PlatformViewSliceContainer extends SliceContainer { - PlatformViewSliceContainer(this._views); - - List _views; - Map _slots = {}; - - @override - final DomElement container = domDocument.createElement(kPlatformViewSliceContainerTag); - - set views(List views) { - if (_views != views) { - _views = views; - } - } - - @override - void updateContents() { - DomElement? currentContainer = container.firstElementChild; - final Map newSlots = {}; - for (final PlatformView view in _views) { - final DomElement platformView = _slots[view.viewId] ?? createPlatformViewSlot(view.viewId); - newSlots[view.viewId] = platformView; - final DomElement viewContainer; - if (currentContainer != null && currentContainer.firstElementChild == platformView) { - viewContainer = currentContainer; - currentContainer = currentContainer.nextElementSibling; - } else { - viewContainer = domDocument.createElement(kPlatformViewContainerTag); - if (currentContainer != null) { - container.insertBefore(viewContainer, currentContainer); - } else { - container.appendChild(viewContainer); - } - viewContainer.appendChild(platformView); - } - final DomCSSStyleDeclaration style = viewContainer.style; - final double logicalWidth = view.size.width / window.devicePixelRatio; - final double logicalHeight = view.size.height / window.devicePixelRatio; - style.width = '${logicalWidth}px'; - style.height = '${logicalHeight}px'; - style.position = 'absolute'; - - final ui.Offset? offset = view.styling.position.offset; - style.left = '${offset?.dx ?? 0}px'; - style.top = '${offset?.dy ?? 0}px'; - - final Matrix4? transform = view.styling.position.transform; - style.transform = transform != null ? float64ListToCssTransform3d(transform.storage) : ''; - style.opacity = view.styling.opacity != 1.0 ? '${view.styling.opacity}' : ''; - // TODO(jacksongardner): Implement clip styling for platform views - } - - while (currentContainer != null) { - final DomElement? next = currentContainer.nextElementSibling; - currentContainer.remove(); - currentContainer = next; - } - - _slots = newSlots; - } -} +// This is an interface that renders a `ScenePicture` as a `DomImageBitmap`. +// It is optionally asynchronous. It is required for the `EngineSceneView` to +// composite pictures into the canvases in the DOM tree it builds. abstract class PictureRenderer { FutureOr renderPicture(ScenePicture picture); } +// This class builds a DOM tree that composites an `EngineScene`. class EngineSceneView { factory EngineSceneView(PictureRenderer pictureRenderer) { final DomElement sceneElement = createDomElement('flt-scene'); @@ -191,22 +74,23 @@ class EngineSceneView { newContainers.add(container); case PlatformViewSlice(): - PlatformViewSliceContainer? container; - for (int j = 0; j < reusableContainers.length; j++) { - final SliceContainer? candidate = reusableContainers[j]; - if (candidate is PlatformViewSliceContainer) { - container = candidate; - reusableContainers[j] = null; - break; + for (final PlatformView view in slice.views) { + // Attempt to reuse a container for the existing view + PlatformViewContainer? container; + for (int j = 0; j < reusableContainers.length; j++) { + final SliceContainer? candidate = reusableContainers[j]; + if (candidate is PlatformViewContainer && candidate.viewId == view.viewId) { + container = candidate; + reusableContainers[j] = null; + break; + } } + container ??= PlatformViewContainer(view.viewId); + container.size = view.size; + container.styling = view.styling; + container.updateContents(); + newContainers.add(container); } - if (container != null) { - container.views = slice.views; - } else { - container = PlatformViewSliceContainer(slice.views); - } - container.updateContents(); - newContainers.add(container); } } @@ -234,3 +118,111 @@ class EngineSceneView { queuedRenders -= 1; } } + +sealed class SliceContainer { + DomElement get container; + + void updateContents(); +} + +final class PictureSliceContainer extends SliceContainer { + factory PictureSliceContainer(ui.Rect bounds) { + final DomElement container = domDocument.createElement(kCanvasContainerTag); + final DomCanvasElement canvas = createDomCanvasElement( + width: bounds.width.toInt(), + height: bounds.height.toInt() + ); + container.appendChild(canvas); + return PictureSliceContainer._(bounds, container, canvas); + } + + PictureSliceContainer._(this._bounds, this.container, this.canvas); + + ui.Rect _bounds; + bool _dirty = true; + + ui.Rect get bounds => _bounds; + set bounds(ui.Rect bounds) { + if (_bounds != bounds) { + _bounds = bounds; + _dirty = true; + } + } + + @override + void updateContents() { + if (_dirty) { + _dirty = false; + final DomCSSStyleDeclaration style = canvas.style; + final double logicalWidth = bounds.width / window.devicePixelRatio; + final double logicalHeight = bounds.height / window.devicePixelRatio; + style.width = '${logicalWidth}px'; + style.height = '${logicalHeight}px'; + style.position = 'absolute'; + style.left = '${bounds.left}px'; + style.top = '${bounds.top}px'; + canvas.width = bounds.width; + canvas.height = bounds.height; + } + } + + void renderBitmap(DomImageBitmap bitmap) { + final DomCanvasRenderingContextBitmapRenderer ctx = canvas.contextBitmapRenderer; + ctx.transferFromImageBitmap(bitmap); + } + + @override + final DomElement container; + final DomCanvasElement canvas; +} + +final class PlatformViewContainer extends SliceContainer { + PlatformViewContainer(this.viewId) : container = createPlatformViewSlot(viewId); + + final int viewId; + PlatformViewStyling? _styling; + ui.Size? _size; + bool _dirty = false; + + @override + final DomElement container; + + set styling(PlatformViewStyling styling) { + if (_styling != styling) { + _styling = styling; + _dirty = true; + } + } + + set size(ui.Size size) { + if (_size != size) { + _size = size; + _dirty = true; + } + } + + @override + void updateContents() { + assert(_styling != null); + assert(_size != null); + if (_dirty) { + final DomCSSStyleDeclaration style = container.style; + final double logicalWidth = _size!.width / window.devicePixelRatio; + final double logicalHeight = _size!.height / window.devicePixelRatio; + style.width = '${logicalWidth}px'; + style.height = '${logicalHeight}px'; + style.position = 'absolute'; + + final ui.Offset? offset = _styling!.position.offset; + style.left = '${offset?.dx ?? 0}px'; + style.top = '${offset?.dy ?? 0}px'; + + final Matrix4? transform = _styling!.position.transform; + style.transform = transform != null ? float64ListToCssTransform3d(transform.storage) : ''; + style.opacity = _styling!.opacity != 1.0 ? '${_styling!.opacity}' : ''; + // TODO(jacksongardner): Implement clip styling for platform views + + _dirty = false; + } + } +} From 94c26c1b4b175189945bcdf6396e6f7c85d1d363 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 22 Aug 2023 13:58:06 -0700 Subject: [PATCH 20/24] Add unit tests for generic scene builder. --- lib/web_ui/lib/src/engine/layers.dart | 61 ++- .../test/engine/scene_builder_test.dart | 409 ++++++++++++++++++ 2 files changed, 459 insertions(+), 11 deletions(-) create mode 100644 lib/web_ui/test/engine/scene_builder_test.dart diff --git a/lib/web_ui/lib/src/engine/layers.dart b/lib/web_ui/lib/src/engine/layers.dart index e8f7d38e4a351..e5b52f762779a 100644 --- a/lib/web_ui/lib/src/engine/layers.dart +++ b/lib/web_ui/lib/src/engine/layers.dart @@ -462,7 +462,7 @@ class PictureDrawCommand { } // Represents how a platform view should be positioned in the scene. -// This object is immutable, so it can be reused across different platform +// This object is immutable, so it can be reused across different platform // views that have the same positioning. class PlatformViewPosition { // No transformation at all. We leave both fields null. @@ -511,6 +511,22 @@ class PlatformViewPosition { } return PlatformViewPosition.transform(newTransform); } + + @override + bool operator ==(Object other) { + if (identical(this, other)) { + return true; + } + if (other is! PlatformViewPosition) { + return false; + } + return (offset == other.offset) && (transform == other.transform); + } + + @override + int get hashCode { + return Object.hash(offset, transform); + } } // Represents the styling to be performed on a platform view when it is @@ -540,6 +556,22 @@ class PlatformViewStyling { opacity: outer.opacity * inner.opacity, ); } + + @override + bool operator ==(Object other) { + if (identical(this, other)) { + return true; + } + if (other is! PlatformViewStyling) { + return false; + } + return (position == other.position) && (opacity == other.opacity); + } + + @override + int get hashCode { + return Object.hash(position, opacity); + } } class LayerBuilder { @@ -650,16 +682,23 @@ class LayerBuilder { final ui.Rect cullRect = (picture as ScenePicture).cullRect; final ui.Rect shiftedRect = cullRect.shift(offset); - // Whenever we add a picture to our layer, we try to see if the picture - // will overlap with any platform views that are currently on top of our - // drawing surface. If they don't overlap with the platform views, they can - // be composited into the existing drawing surface. - if (platformViewRect?.overlaps(shiftedRect) ?? false) { - // If they do overlap with the platform views, however, we need to create - // a new drawing surface to composite into. This lowers the current set - // of picture drawing commands into a picture slice and lowers the current - // set of platform views into a platform view slice. - flushSlices(); + final ui.Rect? currentPlatformViewRect = platformViewRect; + if (currentPlatformViewRect != null) { + // Whenever we add a picture to our layer, we try to see if the picture + // will overlap with any platform views that are currently on top of our + // drawing surface. If they don't overlap with the platform views, they + // can be grouped with the existing pending pictures. + if (pendingPictures.isEmpty || currentPlatformViewRect.overlaps(shiftedRect)) { + // If they do overlap with the platform views, however, we should flush + // all the current content into slices and start anew with a fresh + // group of pictures and platform views that will be rendered on top of + // the previous content. Note that we also flush if we have no pending + // pictures to group with. This is the case when platform views are + // the first thing in our stack of objects to composite, and it doesn't + // make sense to try to put a picture slice below the first platform + // view slice, even if the picture doesn't overlap. + flushSlices(); + } } pendingPictures.add(PictureDrawCommand(offset, picture)); picturesRect = picturesRect?.expandToInclude(shiftedRect) ?? shiftedRect; diff --git a/lib/web_ui/test/engine/scene_builder_test.dart b/lib/web_ui/test/engine/scene_builder_test.dart new file mode 100644 index 0000000000000..693f39203dad0 --- /dev/null +++ b/lib/web_ui/test/engine/scene_builder_test.dart @@ -0,0 +1,409 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:typed_data'; + +import 'package:test/bootstrap/browser.dart'; +import 'package:test/test.dart'; +import 'package:ui/src/engine.dart'; + +import 'package:ui/ui.dart' as ui; + +void main() { + internalBootstrapBrowserTest(() => testMain); +} + +void testMain() { + setUpAll(() { + LayerBuilder.debugRecorderFactory = (ui.Rect rect) { + final StubSceneCanvas canvas = StubSceneCanvas(); + final StubPictureRecorder recorder = StubPictureRecorder(canvas); + return (recorder, canvas); + }; + }); + + tearDownAll(() { + LayerBuilder.debugRecorderFactory = null; + }); + + group('EngineSceneBuilder', () { + test('single picture', () { + final EngineSceneBuilder sceneBuilder = EngineSceneBuilder(); + + const ui.Rect pictureRect = ui.Rect.fromLTRB(100, 100, 200, 200); + sceneBuilder.addPicture(ui.Offset.zero, StubPicture(pictureRect)); + + final EngineScene scene = sceneBuilder.build() as EngineScene; + final List slices = scene.rootLayer.slices; + expect(slices.length, 1); + expect(slices[0], pictureSliceWithRect(pictureRect)); + }); + + test('two pictures', () { + final EngineSceneBuilder sceneBuilder = EngineSceneBuilder(); + + const ui.Rect pictureRect1 = ui.Rect.fromLTRB(100, 100, 200, 200); + const ui.Rect pictureRect2 = ui.Rect.fromLTRB(300, 400, 400, 400); + sceneBuilder.addPicture(ui.Offset.zero, StubPicture(pictureRect1)); + sceneBuilder.addPicture(ui.Offset.zero, StubPicture(pictureRect2)); + + final EngineScene scene = sceneBuilder.build() as EngineScene; + final List slices = scene.rootLayer.slices; + expect(slices.length, 1); + expect(slices[0], pictureSliceWithRect(const ui.Rect.fromLTRB(100, 100, 400, 400))); + }); + + test('picture + platform view (overlapping)', () { + final EngineSceneBuilder sceneBuilder = EngineSceneBuilder(); + + const ui.Rect pictureRect = ui.Rect.fromLTRB(100, 100, 200, 200); + const ui.Rect platformViewRect = ui.Rect.fromLTRB(150, 150, 250, 250); + sceneBuilder.addPicture(ui.Offset.zero, StubPicture(pictureRect)); + sceneBuilder.addPlatformView( + 1, + offset: platformViewRect.topLeft, + width: platformViewRect.width, + height: platformViewRect.height + ); + + final EngineScene scene = sceneBuilder.build() as EngineScene; + final List slices = scene.rootLayer.slices; + expect(slices.length, 2); + expect(slices[0], pictureSliceWithRect(pictureRect)); + expect(slices[1], platformViewSliceWithViews([ + PlatformView(1, platformViewRect.size, PlatformViewStyling( + position: PlatformViewPosition.offset(platformViewRect.topLeft) + )) + ])); + }); + + test('platform view + picture (overlapping)', () { + final EngineSceneBuilder sceneBuilder = EngineSceneBuilder(); + + const ui.Rect pictureRect = ui.Rect.fromLTRB(100, 100, 200, 200); + const ui.Rect platformViewRect = ui.Rect.fromLTRB(150, 150, 250, 250); + sceneBuilder.addPlatformView( + 1, + offset: platformViewRect.topLeft, + width: platformViewRect.width, + height: platformViewRect.height + ); + sceneBuilder.addPicture(ui.Offset.zero, StubPicture(pictureRect)); + + final EngineScene scene = sceneBuilder.build() as EngineScene; + final List slices = scene.rootLayer.slices; + expect(slices.length, 2); + expect(slices[0], platformViewSliceWithViews([ + PlatformView(1, platformViewRect.size, PlatformViewStyling( + position: PlatformViewPosition.offset(platformViewRect.topLeft) + )) + ])); + expect(slices[1], pictureSliceWithRect(pictureRect)); + }); + + test('platform view sandwich (overlapping)', () { + final EngineSceneBuilder sceneBuilder = EngineSceneBuilder(); + + const ui.Rect pictureRect1 = ui.Rect.fromLTRB(100, 100, 200, 200); + const ui.Rect platformViewRect = ui.Rect.fromLTRB(150, 150, 250, 250); + const ui.Rect pictureRect2 = ui.Rect.fromLTRB(200, 200, 300, 300); + sceneBuilder.addPicture(ui.Offset.zero, StubPicture(pictureRect1)); + sceneBuilder.addPlatformView( + 1, + offset: platformViewRect.topLeft, + width: platformViewRect.width, + height: platformViewRect.height + ); + sceneBuilder.addPicture(ui.Offset.zero, StubPicture(pictureRect2)); + + final EngineScene scene = sceneBuilder.build() as EngineScene; + final List slices = scene.rootLayer.slices; + expect(slices.length, 3); + expect(slices[0], pictureSliceWithRect(pictureRect1)); + expect(slices[1], platformViewSliceWithViews([ + PlatformView(1, platformViewRect.size, PlatformViewStyling( + position: PlatformViewPosition.offset(platformViewRect.topLeft) + )) + ])); + expect(slices[2], pictureSliceWithRect(pictureRect2)); + }); + + test('platform view sandwich (non-overlapping)', () { + final EngineSceneBuilder sceneBuilder = EngineSceneBuilder(); + + const ui.Rect pictureRect1 = ui.Rect.fromLTRB(100, 100, 200, 200); + const ui.Rect platformViewRect = ui.Rect.fromLTRB(150, 150, 250, 250); + const ui.Rect pictureRect2 = ui.Rect.fromLTRB(50, 50, 100, 100); + sceneBuilder.addPicture(ui.Offset.zero, StubPicture(pictureRect1)); + sceneBuilder.addPlatformView( + 1, + offset: platformViewRect.topLeft, + width: platformViewRect.width, + height: platformViewRect.height + ); + sceneBuilder.addPicture(ui.Offset.zero, StubPicture(pictureRect2)); + + final EngineScene scene = sceneBuilder.build() as EngineScene; + final List slices = scene.rootLayer.slices; + + // The top picture does not overlap with the platform view, so it should + // be grouped into the slice below it to reduce the number of canvases we + // need. + expect(slices.length, 2); + expect(slices[0], pictureSliceWithRect(const ui.Rect.fromLTRB(50, 50, 200, 200))); + expect(slices[1], platformViewSliceWithViews([ + PlatformView(1, platformViewRect.size, PlatformViewStyling( + position: PlatformViewPosition.offset(platformViewRect.topLeft) + )) + ])); + }); + }); +} + +PictureSliceMatcher pictureSliceWithRect(ui.Rect rect) => PictureSliceMatcher(rect); +PlatformViewSliceMatcher platformViewSliceWithViews(List views) + => PlatformViewSliceMatcher(views); + +class PictureSliceMatcher extends Matcher { + PictureSliceMatcher(this.expectedRect); + + final ui.Rect expectedRect; + + @override + Description describe(Description description) { + return description.add(''); + } + + @override + bool matches(dynamic item, Map matchState) { + if (item is! PictureSlice) { + return false; + } + final ScenePicture picture = item.picture; + if (picture is! StubPicture) { + return false; + } + + if (picture.cullRect != expectedRect) { + return false; + } + + return true; + } +} + +class PlatformViewSliceMatcher extends Matcher { + PlatformViewSliceMatcher(this.expectedPlatformViews); + + final List expectedPlatformViews; + + @override + Description describe(Description description) { + return description.add(''); + } + + @override + bool matches(dynamic item, Map matchState) { + if (item is! PlatformViewSlice) { + return false; + } + + if (item.views.length != expectedPlatformViews.length) { + return false; + } + + for (int i = 0; i < item.views.length; i++) { + final PlatformView expectedView = expectedPlatformViews[i]; + final PlatformView actualView = item.views[i]; + if (expectedView.viewId != actualView.viewId) { + return false; + } + if (expectedView.size != actualView.size) { + return false; + } + if (expectedView.styling != actualView.styling) { + return false; + } + } + return true; + } +} + +class StubPicture implements ScenePicture { + StubPicture(this.cullRect); + + @override + final ui.Rect cullRect; + + @override + int get approximateBytesUsed => throw UnimplementedError(); + + @override + bool get debugDisposed => throw UnimplementedError(); + + @override + void dispose() {} + + @override + Future toImage(int width, int height) { + throw UnimplementedError(); + } + + @override + ui.Image toImageSync(int width, int height) { + throw UnimplementedError(); + } +} + +class StubCompositePicture extends StubPicture { + StubCompositePicture(this.children) : super( + children.fold(null, (ui.Rect? previousValue, StubPicture child) { + return previousValue?.expandToInclude(child.cullRect) ?? child.cullRect; + })! + ); + + final List children; +} + +class StubPictureRecorder implements ui.PictureRecorder { + StubPictureRecorder(this.canvas); + + final StubSceneCanvas canvas; + + @override + ui.Picture endRecording() { + return StubCompositePicture(canvas.pictures); + } + + @override + bool get isRecording => throw UnimplementedError(); +} + +class StubSceneCanvas implements SceneCanvas { + List pictures = []; + + @override + void drawPicture(ui.Picture picture) { + pictures.add(picture as StubPicture); + } + + @override + void clipPath(ui.Path path, {bool doAntiAlias = true}) {} + + @override + void clipRRect(ui.RRect rrect, {bool doAntiAlias = true}) {} + + @override + void clipRect(ui.Rect rect, {ui.ClipOp clipOp = ui.ClipOp.intersect, bool doAntiAlias = true}) {} + + @override + void drawArc(ui.Rect rect, double startAngle, double sweepAngle, bool useCenter, ui.Paint paint) {} + + @override + void drawAtlas(ui.Image atlas, List transforms, List rects, List? colors, ui.BlendMode? blendMode, ui.Rect? cullRect, ui.Paint paint) {} + + @override + void drawCircle(ui.Offset c, double radius, ui.Paint paint) {} + + @override + void drawColor(ui.Color color, ui.BlendMode blendMode) {} + + @override + void drawDRRect(ui.RRect outer, ui.RRect inner, ui.Paint paint) {} + + @override + void drawImage(ui.Image image, ui.Offset offset, ui.Paint paint) {} + + @override + void drawImageNine(ui.Image image, ui.Rect center, ui.Rect dst, ui.Paint paint) {} + + @override + void drawImageRect(ui.Image image, ui.Rect src, ui.Rect dst, ui.Paint paint) {} + + @override + void drawLine(ui.Offset p1, ui.Offset p2, ui.Paint paint) {} + + @override + void drawOval(ui.Rect rect, ui.Paint paint) {} + + @override + void drawPaint(ui.Paint paint) {} + + @override + void drawParagraph(ui.Paragraph paragraph, ui.Offset offset) {} + + @override + void drawPath(ui.Path path, ui.Paint paint) {} + + @override + void drawPoints(ui.PointMode pointMode, List points, ui.Paint paint) {} + + @override + void drawRRect(ui.RRect rrect, ui.Paint paint) {} + + @override + void drawRawAtlas(ui.Image atlas, Float32List rstTransforms, Float32List rects, Int32List? colors, ui.BlendMode? blendMode, ui.Rect? cullRect, ui.Paint paint) {} + + @override + void drawRawPoints(ui.PointMode pointMode, Float32List points, ui.Paint paint) {} + + @override + void drawRect(ui.Rect rect, ui.Paint paint) {} + + @override + void drawShadow(ui.Path path, ui.Color color, double elevation, bool transparentOccluder) {} + + @override + void drawVertices(ui.Vertices vertices, ui.BlendMode blendMode, ui.Paint paint) {} + + @override + ui.Rect getDestinationClipBounds() { + throw UnimplementedError(); + } + + @override + ui.Rect getLocalClipBounds() { + throw UnimplementedError(); + } + + @override + int getSaveCount() { + throw UnimplementedError(); + } + + @override + Float64List getTransform() { + throw UnimplementedError(); + } + + @override + void restore() {} + + @override + void restoreToCount(int count) {} + + @override + void rotate(double radians) {} + + @override + void save() {} + + @override + void saveLayer(ui.Rect? bounds, ui.Paint paint) {} + + @override + void saveLayerWithFilter(ui.Rect? bounds, ui.Paint paint, ui.ImageFilter backdropFilter) {} + + @override + void scale(double sx, [double? sy]) {} + + @override + void skew(double sx, double sy) {} + + @override + void transform(Float64List matrix4) {} + + @override + void translate(double dx, double dy) {} +} From 3870a9b6caa41815aef8321f1c580bbc58126df5 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 22 Aug 2023 14:47:51 -0700 Subject: [PATCH 21/24] Remove profiling flag. --- lib/web_ui/skwasm/BUILD.gn | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/web_ui/skwasm/BUILD.gn b/lib/web_ui/skwasm/BUILD.gn index 5742d0dd1d728..e3c066722a1d8 100644 --- a/lib/web_ui/skwasm/BUILD.gn +++ b/lib/web_ui/skwasm/BUILD.gn @@ -51,7 +51,6 @@ wasm_lib("skwasm") { "-Wno-pthreads-mem-growth", "--js-library", rebase_path("library_skwasm_support.js"), - "--profiling", ] inputs = [ rebase_path("library_skwasm_support.js") ] From 58cb01c3ccf9940c28a65bbd649364b1e6c6e6e6 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 22 Aug 2023 17:48:18 -0700 Subject: [PATCH 22/24] Dispose video frame fixes. --- lib/web_ui/skwasm/BUILD.gn | 2 + lib/web_ui/skwasm/library_skwasm_support.js | 53 ++++++++++++--------- lib/web_ui/skwasm/skwasm_support.h | 3 +- lib/web_ui/skwasm/surface.cpp | 8 ---- lib/web_ui/skwasm/surface.h | 5 +- 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/lib/web_ui/skwasm/BUILD.gn b/lib/web_ui/skwasm/BUILD.gn index e3c066722a1d8..73f93813a0982 100644 --- a/lib/web_ui/skwasm/BUILD.gn +++ b/lib/web_ui/skwasm/BUILD.gn @@ -61,6 +61,8 @@ wasm_lib("skwasm") { "-sASSERTIONS=1", "-sGL_ASSERTIONS=1", ] + } else { + ldflags += [ "--closure=1" ] } deps = [ diff --git a/lib/web_ui/skwasm/library_skwasm_support.js b/lib/web_ui/skwasm/library_skwasm_support.js index fe50a1d0a5918..87dfff43fdb94 100644 --- a/lib/web_ui/skwasm/library_skwasm_support.js +++ b/lib/web_ui/skwasm/library_skwasm_support.js @@ -16,12 +16,12 @@ mergeInto(LibraryManager.library, { pointer, object, }); - } + }; _skwasm_getAssociatedObject = function(pointer) { return associatedObjectsMap.get(pointer); - } + }; _skwasm_registerMessageListener = function(threadId) { - eventListener = function({data}) { + const eventListener = function({data}) { const skwasmMessage = data.skwasmMessage; if (!skwasmMessage) { return; @@ -33,6 +33,12 @@ mergeInto(LibraryManager.library, { case 'setAssociatedObject': associatedObjectsMap.set(data.pointer, data.object); return; + case 'disposeAssociatedObject': + const object = { data }; + if (object.close) { + object.close(); + } + associatedObjectsMap.delete(data.pointer); default: console.warn('unrecognized skwasm message'); } @@ -46,26 +52,26 @@ mergeInto(LibraryManager.library, { _skwasm_createOffscreenCanvas = function(width, height) { const canvas = new OffscreenCanvas(width, height); var contextAttributes = { - 'majorVersion': 2, - 'alpha': true, - 'depth': true, - 'stencil': true, - 'antialias': false, - 'premultipliedAlpha': true, - 'preserveDrawingBuffer': false, - 'powerPreference': 'default', - 'failIfMajorPerformanceCaveat': false, - 'enableExtensionsByDefault': true, + majorVersion: 2, + alpha: true, + depth: true, + stencil: true, + antialias: false, + premultipliedAlpha: true, + preserveDrawingBuffer: false, + powerPreference: 'default', + failIfMajorPerformanceCaveat: false, + enableExtensionsByDefault: true, }; const contextHandle = GL.createContext(canvas, contextAttributes); handleToCanvasMap.set(contextHandle, canvas); return contextHandle; - } + }; _skwasm_resizeCanvas = function(contextHandle, width, height) { const canvas = handleToCanvasMap.get(contextHandle); canvas.width = width; canvas.height = height; - } + }; _skwasm_captureImageBitmap = async function(surfaceHandle, contextHandle, callbackId, width, height) { const canvas = handleToCanvasMap.get(contextHandle); const imageBitmap = await createImageBitmap(canvas, 0, 0, width, height); @@ -75,7 +81,7 @@ mergeInto(LibraryManager.library, { callbackId, imageBitmap, }); - } + }; _skwasm_createGlTextureFromVideoFrame = function(videoFrame, width, height) { const glCtx = GL.currentContext.GLctx; const newTexture = glCtx.createTexture(); @@ -90,15 +96,20 @@ mergeInto(LibraryManager.library, { const textureId = GL.getNewId(GL.textures); GL.textures[textureId] = newTexture; return textureId; - } - _skwasm_disposeVideoFrame = function(videoFrame) { - videoFrame.close(); - } + }; + _skwasm_disposeAssociatedObjectOnThread = function(threadId, object) { + PThread.pthreads[threadId].postMessage({ + skwasmMessage: 'disposeAssociatedObject', + object, + }); + }; }, skwasm_setAssociatedObjectOnThread: function () {}, skwasm_setAssociatedObjectOnThread__deps: ['$skwasm_support_setup'], skwasm_getAssociatedObject: function () {}, skwasm_getAssociatedObject__deps: ['$skwasm_support_setup'], + skwasm_disposeAssociatedObjectOnThread: function () {}, + skwasm_disposeAssociatedObjectOnThread__deps: ['$skwasm_support_setup'], skwasm_registerMessageListener: function() {}, skwasm_registerMessageListener__deps: ['$skwasm_support_setup'], skwasm_createOffscreenCanvas: function () {}, @@ -109,7 +120,5 @@ mergeInto(LibraryManager.library, { skwasm_captureImageBitmap__deps: ['$skwasm_support_setup'], skwasm_createGlTextureFromVideoFrame: function () {}, skwasm_createGlTextureFromVideoFrame__deps: ['$skwasm_support_setup'], - skwasm_disposeVideoFrame: function () {}, - skwasm_disposeVideoFrame__deps: ['$skwasm_support_setup'], }); \ No newline at end of file diff --git a/lib/web_ui/skwasm/skwasm_support.h b/lib/web_ui/skwasm/skwasm_support.h index 5af38a6f2b64b..9547bc29e7a89 100644 --- a/lib/web_ui/skwasm/skwasm_support.h +++ b/lib/web_ui/skwasm/skwasm_support.h @@ -16,6 +16,8 @@ extern void skwasm_setAssociatedObjectOnThread(unsigned long threadId, void* pointer, SkwasmObject object); extern SkwasmObject skwasm_getAssociatedObject(void* pointer); +extern void skwasm_disposeAssociatedObjectOnThread(unsigned long threadId, + void* pointer); extern void skwasm_registerMessageListener(pthread_t threadId); extern uint32_t skwasm_createOffscreenCanvas(int width, int height); extern void skwasm_resizeCanvas(uint32_t contextHandle, int width, int height); @@ -28,5 +30,4 @@ extern unsigned int skwasm_createGlTextureFromVideoFrame( SkwasmObject videoFrame, int width, int height); -extern void skwasm_disposeVideoFrame(SkwasmObject videoFrame); } diff --git a/lib/web_ui/skwasm/surface.cpp b/lib/web_ui/skwasm/surface.cpp index c5098f348d70f..1c22526a633d7 100644 --- a/lib/web_ui/skwasm/surface.cpp +++ b/lib/web_ui/skwasm/surface.cpp @@ -178,10 +178,6 @@ void Surface::_rasterizeImage(SkImage* image, EM_FUNC_SIG_VIII, fOnRasterizeComplete, this, data.release(), callbackId); } -void Surface::_disposeVideoFrame(SkwasmObject videoFrame) { - skwasm_disposeVideoFrame(videoFrame); -} - void Surface::_onRasterizeComplete(SkData* data, uint32_t callbackId) { _callbackHandler(callbackId, data, __builtin_wasm_ref_null_extern()); } @@ -217,10 +213,6 @@ void Surface::fRasterizeImage(Surface* surface, image->unref(); } -void Surface::fDisposeVideoFrame(Surface* surface, SkwasmObject videoFrame) { - surface->_disposeVideoFrame(videoFrame); -} - SKWASM_EXPORT Surface* surface_create() { return new Surface(); } diff --git a/lib/web_ui/skwasm/surface.h b/lib/web_ui/skwasm/surface.h index d93e4a81638b0..22cc57be9876c 100644 --- a/lib/web_ui/skwasm/surface.h +++ b/lib/web_ui/skwasm/surface.h @@ -41,8 +41,7 @@ class VideoFrameWrapper { } ~VideoFrameWrapper() { - skwasm_setAssociatedObjectOnThread(_rasterThreadId, this, - __builtin_wasm_ref_null_extern()); + skwasm_disposeAssociatedObjectOnThread(_rasterThreadId, this); } SkwasmObject getVideoFrame() { return skwasm_getAssociatedObject(this); } @@ -81,7 +80,6 @@ class Surface { void _rasterizeImage(SkImage* image, ImageByteFormat format, uint32_t callbackId); - void _disposeVideoFrame(SkwasmObject videoFrame); void _onRasterizeComplete(SkData* data, uint32_t callbackId); std::string _canvasID; @@ -114,6 +112,5 @@ class Surface { static void fOnRasterizeComplete(Surface* surface, SkData* imageData, uint32_t callbackId); - static void fDisposeVideoFrame(Surface* surface, SkwasmObject videoFrame); }; } // namespace Skwasm From 5974c61418eb0f1f8b028bff2fcdca6abc78d0c4 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Wed, 23 Aug 2023 09:52:09 -0700 Subject: [PATCH 23/24] Round out picture bounds so that we capture all the content. --- lib/web_ui/lib/src/engine/scene_view.dart | 4 ++-- lib/web_ui/skwasm/surface.cpp | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/web_ui/lib/src/engine/scene_view.dart b/lib/web_ui/lib/src/engine/scene_view.dart index c61520fcbaa6e..5604f6f8bd7ec 100644 --- a/lib/web_ui/lib/src/engine/scene_view.dart +++ b/lib/web_ui/lib/src/engine/scene_view.dart @@ -161,8 +161,8 @@ final class PictureSliceContainer extends SliceContainer { style.position = 'absolute'; style.left = '${bounds.left}px'; style.top = '${bounds.top}px'; - canvas.width = bounds.width; - canvas.height = bounds.height; + canvas.width = bounds.width.ceilToDouble(); + canvas.height = bounds.height.ceilToDouble(); } } diff --git a/lib/web_ui/skwasm/surface.cpp b/lib/web_ui/skwasm/surface.cpp index 1c22526a633d7..6ea9da7be9d7a 100644 --- a/lib/web_ui/skwasm/surface.cpp +++ b/lib/web_ui/skwasm/surface.cpp @@ -139,15 +139,18 @@ void Surface::_recreateSurface() { // Worker thread only void Surface::_renderPicture(const SkPicture* picture, uint32_t callbackId) { SkRect pictureRect = picture->cullRect(); - _resizeCanvasToFit(pictureRect.width(), pictureRect.height()); - SkMatrix matrix = SkMatrix::Translate(-pictureRect.fLeft, -pictureRect.fTop); + SkIRect roundedOutRect; + pictureRect.roundOut(&roundedOutRect); + _resizeCanvasToFit(roundedOutRect.width(), roundedOutRect.height()); + SkMatrix matrix = + SkMatrix::Translate(-roundedOutRect.fLeft, -roundedOutRect.fTop); makeCurrent(_glContext); auto canvas = _surface->getCanvas(); canvas->drawColor(SK_ColorTRANSPARENT, SkBlendMode::kSrc); canvas->drawPicture(sk_ref_sp(picture), &matrix, nullptr); _grContext->flush(_surface); - skwasm_captureImageBitmap(this, _glContext, callbackId, pictureRect.width(), - pictureRect.height()); + skwasm_captureImageBitmap(this, _glContext, callbackId, + roundedOutRect.width(), roundedOutRect.height()); } void Surface::_rasterizeImage(SkImage* image, From e72846cdbcb28accf5c0cee46916ff3b0e989ded Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Wed, 23 Aug 2023 10:33:01 -0700 Subject: [PATCH 24/24] Round out the bounds when positioning the canvas too. --- lib/web_ui/lib/src/engine/scene_view.dart | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/web_ui/lib/src/engine/scene_view.dart b/lib/web_ui/lib/src/engine/scene_view.dart index 5604f6f8bd7ec..a8c4a17307a27 100644 --- a/lib/web_ui/lib/src/engine/scene_view.dart +++ b/lib/web_ui/lib/src/engine/scene_view.dart @@ -153,16 +153,23 @@ final class PictureSliceContainer extends SliceContainer { void updateContents() { if (_dirty) { _dirty = false; + + final ui.Rect roundedOutBounds = ui.Rect.fromLTRB( + bounds.left.floorToDouble(), + bounds.top.floorToDouble(), + bounds.right.ceilToDouble(), + bounds.bottom.ceilToDouble() + ); final DomCSSStyleDeclaration style = canvas.style; - final double logicalWidth = bounds.width / window.devicePixelRatio; - final double logicalHeight = bounds.height / window.devicePixelRatio; + final double logicalWidth = roundedOutBounds.width / window.devicePixelRatio; + final double logicalHeight = roundedOutBounds.height / window.devicePixelRatio; style.width = '${logicalWidth}px'; style.height = '${logicalHeight}px'; style.position = 'absolute'; - style.left = '${bounds.left}px'; - style.top = '${bounds.top}px'; - canvas.width = bounds.width.ceilToDouble(); - canvas.height = bounds.height.ceilToDouble(); + style.left = '${roundedOutBounds.left}px'; + style.top = '${roundedOutBounds.top}px'; + canvas.width = roundedOutBounds.width.ceilToDouble(); + canvas.height = roundedOutBounds.height.ceilToDouble(); } }