Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Implement asynchronous texture uploads when using the Metal backend on iOS. #17039

Closed
chinmaygarde wants to merge 2 commits into from
Closed

Implement asynchronous texture uploads when using the Metal backend on iOS. #17039

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0416dbf
Implement asynchronous texture uploads when using the Metal backend o…
chinmaygarde Mar 3, 2020
1cf871f
Licenses.
chinmaygarde Mar 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions ci/licenses_golden/licenses_flutter
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -863,12 +863,18 @@ FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/platform_messa
FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.h
FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.mm
FILE: ../../../flutter/shell/platform/darwin/ios/framework/module.modulemap
FILE: ../../../flutter/shell/platform/darwin/ios/ios_context.h
FILE: ../../../flutter/shell/platform/darwin/ios/ios_context.mm
FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_gl.h
FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_gl.mm
FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_metal.h
FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_metal.mm
FILE: ../../../flutter/shell/platform/darwin/ios/ios_external_texture_gl.h
FILE: ../../../flutter/shell/platform/darwin/ios/ios_external_texture_gl.mm
FILE: ../../../flutter/shell/platform/darwin/ios/ios_gl_context.h
FILE: ../../../flutter/shell/platform/darwin/ios/ios_gl_context.mm
FILE: ../../../flutter/shell/platform/darwin/ios/ios_gl_render_target.h
FILE: ../../../flutter/shell/platform/darwin/ios/ios_gl_render_target.mm
FILE: ../../../flutter/shell/platform/darwin/ios/ios_render_target.h
FILE: ../../../flutter/shell/platform/darwin/ios/ios_render_target.mm
FILE: ../../../flutter/shell/platform/darwin/ios/ios_render_target_gl.h
FILE: ../../../flutter/shell/platform/darwin/ios/ios_render_target_gl.mm
FILE: ../../../flutter/shell/platform/darwin/ios/ios_surface.h
FILE: ../../../flutter/shell/platform/darwin/ios/ios_surface.mm
FILE: ../../../flutter/shell/platform/darwin/ios/ios_surface_gl.h
Expand All @@ -879,6 +885,8 @@ FILE: ../../../flutter/shell/platform/darwin/ios/ios_surface_software.h
FILE: ../../../flutter/shell/platform/darwin/ios/ios_surface_software.mm
FILE: ../../../flutter/shell/platform/darwin/ios/platform_view_ios.h
FILE: ../../../flutter/shell/platform/darwin/ios/platform_view_ios.mm
FILE: ../../../flutter/shell/platform/darwin/ios/rendering_api_selection.h
FILE: ../../../flutter/shell/platform/darwin/ios/rendering_api_selection.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/FlutterMacOS.podspec
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Headers/FlutterAppDelegate.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Headers/FlutterDartProject.h
Expand Down
1 change: 1 addition & 0 deletions flow/compositor_context.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ CompositorContext::ScopedFrame::~ScopedFrame() {
RasterStatus CompositorContext::ScopedFrame::Raster(
flutter::LayerTree& layer_tree,
bool ignore_raster_cache) {
TRACE_EVENT0("flutter", "CompositorContext::ScopedFrame::Raster");
bool root_needs_readback = layer_tree.Preroll(*this, ignore_raster_cache);
bool needs_save_layer = root_needs_readback && !surface_supports_readback();
PostPrerollResult post_preroll_result = PostPrerollResult::kSuccess;
Expand Down
1 change: 1 addition & 0 deletions lib/ui/painting/image_decoder.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@ static SkiaGPUObject<SkImage> UploadRasterImage(
})
.SetIfFalse([&result, context = io_manager->GetResourceContext(),
&pixmap, queue = io_manager->GetSkiaUnrefQueue()] {
TRACE_EVENT0("flutter", "MakeCrossContextImageFromPixmap");
Copy link
Contributor

@dnfield dnfield Mar 9, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit can we split tracing that does not depend on this PR into a separate PR?

We've had some issues recently where adding more tracing has run into timeouts in the devicelab, and if that happens here it'll be nice to know that's all it is and not some other change in this PR.

Also, if this has to get reverted for some reason the tracing will stay.

sk_sp<SkImage> texture_image = SkImage::MakeCrossContextFromPixmap(
context.get(), // context
pixmap, // pixmap
Expand Down
2 changes: 2 additions & 0 deletions shell/common/rasterizer.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,7 @@ RasterStatus Rasterizer::DoDraw(
}

RasterStatus Rasterizer::DrawToSurface(flutter::LayerTree& layer_tree) {
TRACE_EVENT0("flutter", "Rasterizer::DrawToSurface");
FML_DCHECK(surface_);

auto frame = surface_->AcquireFrame(layer_tree.frame_size());
Expand Down Expand Up @@ -350,6 +351,7 @@ RasterStatus Rasterizer::DrawToSurface(flutter::LayerTree& layer_tree) {
FireNextFrameCallbackIfPresent();

if (surface_->GetContext()) {
TRACE_EVENT0("flutter", "PerformDeferredSkiaCleanup");
surface_->GetContext()->performDeferredCleanup(kSkiaCleanupExpiration);
}

Expand Down
1 change: 1 addition & 0 deletions shell/gpu/gpu_surface_gl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ class GPUSurfaceGL : public Surface {
GPUSurfaceGLDelegate* delegate,
bool render_to_surface);

// |Surface|
Copy link
Contributor

@dnfield dnfield Mar 9, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar nit on these.

~GPUSurfaceGL() override;

// |Surface|
Expand Down
7 changes: 4 additions & 3 deletions shell/gpu/gpu_surface_metal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@ namespace flutter {

class GPUSurfaceMetal : public Surface {
public:
GPUSurfaceMetal(GPUSurfaceDelegate* delegate, fml::scoped_nsobject<CAMetalLayer> layer);
GPUSurfaceMetal(GPUSurfaceDelegate* delegate,
sk_sp<GrContext> gr_context,
fml::scoped_nsobject<CAMetalLayer> layer);
fml::scoped_nsobject<CAMetalLayer> layer,
sk_sp<GrContext> context,
fml::scoped_nsprotocol<id<MTLCommandQueue>> command_queue);

// |Surface|
~GPUSurfaceMetal() override;

private:
Expand Down
78 changes: 20 additions & 58 deletions shell/gpu/gpu_surface_metal.mm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

#include <QuartzCore/CAMetalLayer.h>

#include "flutter/fml/trace_event.h"
#include "third_party/skia/include/core/SkSurface.h"
#include "third_party/skia/include/gpu/GrBackendSurface.h"
#include "third_party/skia/include/ports/SkCFObject.h"
Expand All @@ -15,62 +16,17 @@
namespace flutter {

GPUSurfaceMetal::GPUSurfaceMetal(GPUSurfaceDelegate* delegate,
fml::scoped_nsobject<CAMetalLayer> layer)
: delegate_(delegate), layer_(std::move(layer)) {
if (!layer_) {
FML_LOG(ERROR) << "Could not create metal surface because of invalid layer.";
return;
}

layer.get().pixelFormat = MTLPixelFormatBGRA8Unorm;
fml::scoped_nsobject<CAMetalLayer> layer,
sk_sp<GrContext> context,
fml::scoped_nsprotocol<id<MTLCommandQueue>> command_queue)
: delegate_(delegate),
layer_(std::move(layer)),
context_(std::move(context)),
command_queue_(std::move(command_queue)) {
layer_.get().pixelFormat = MTLPixelFormatBGRA8Unorm;
// Flutter needs to read from the color attachment in cases where there are effects such as
// backdrop filters.
layer.get().framebufferOnly = NO;

auto metal_device = fml::scoped_nsprotocol<id<MTLDevice>>([layer_.get().device retain]);
auto metal_queue = fml::scoped_nsprotocol<id<MTLCommandQueue>>([metal_device newCommandQueue]);

if (!metal_device || !metal_queue) {
FML_LOG(ERROR) << "Could not create metal device or queue.";
return;
}

command_queue_ = metal_queue;

// The context creation routine accepts arguments using transfer semantics.
auto context = GrContext::MakeMetal(metal_device.release(), metal_queue.release());
if (!context) {
FML_LOG(ERROR) << "Could not create Skia metal context.";
return;
}

context_ = context;
}

GPUSurfaceMetal::GPUSurfaceMetal(GPUSurfaceDelegate* delegate,
sk_sp<GrContext> gr_context,
fml::scoped_nsobject<CAMetalLayer> layer)
: delegate_(delegate), layer_(std::move(layer)), context_(gr_context) {
if (!layer_) {
FML_LOG(ERROR) << "Could not create metal surface because of invalid layer.";
return;
}
if (!context_) {
FML_LOG(ERROR) << "Could not create metal surface because of invalid Skia metal context.";
return;
}

layer.get().pixelFormat = MTLPixelFormatBGRA8Unorm;

auto metal_device = fml::scoped_nsprotocol<id<MTLDevice>>([layer_.get().device retain]);
auto metal_queue = fml::scoped_nsprotocol<id<MTLCommandQueue>>([metal_device newCommandQueue]);

if (!metal_device || !metal_queue) {
FML_LOG(ERROR) << "Could not create metal device or queue.";
return;
}

command_queue_ = metal_queue;
layer_.get().framebufferOnly = NO;
}

GPUSurfaceMetal::~GPUSurfaceMetal() {
Expand All @@ -95,13 +51,20 @@
}

const auto bounds = layer_.get().bounds.size;
if (bounds.width <= 0.0 || bounds.height <= 0.0) {
const auto scale = layer_.get().contentsScale;
if (bounds.width <= 0.0 || bounds.height <= 0.0 || scale <= 0.0) {
FML_LOG(ERROR) << "Metal layer bounds were invalid.";
return nullptr;
}

const auto scaled_bounds = CGSizeMake(bounds.width * scale, bounds.height * scale);

ReleaseUnusedDrawableIfNecessary();

if (!CGSizeEqualToSize(scaled_bounds, layer_.get().drawableSize)) {
layer_.get().drawableSize = scaled_bounds;
}

auto surface = SkSurface::MakeFromCAMetalLayer(context_.get(), // context
layer_.get(), // layer
kTopLeft_GrSurfaceOrigin, // origin
Expand All @@ -118,6 +81,7 @@
}

auto submit_callback = [this](const SurfaceFrame& surface_frame, SkCanvas* canvas) -> bool {
TRACE_EVENT0("flutter", "GPUSurfaceMetal::Submit");
canvas->flush();

if (next_drawable_ == nullptr) {
Expand Down Expand Up @@ -159,9 +123,7 @@
SkMatrix GPUSurfaceMetal::GetRootTransformation() const {
// This backend does not currently support root surface transformations. Just
// return identity.
SkMatrix matrix;
matrix.reset();
return matrix;
return {};
}

// |Surface|
Expand Down
30 changes: 19 additions & 11 deletions shell/platform/darwin/ios/BUILD.gn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ shared_library("create_flutter_framework_dylib") {

ldflags = [ "-Wl,-install_name,@rpath/Flutter.framework/Flutter" ]

public = _flutter_framework_headers

sources = [
"framework/Source/FlutterAppDelegate.mm",
"framework/Source/FlutterBinaryMessengerRelay.mm",
Expand Down Expand Up @@ -81,12 +83,16 @@ shared_library("create_flutter_framework_dylib") {
"framework/Source/platform_message_router.mm",
"framework/Source/vsync_waiter_ios.h",
"framework/Source/vsync_waiter_ios.mm",
"ios_context.h",
"ios_context.mm",
"ios_context_gl.h",
"ios_context_gl.mm",
"ios_external_texture_gl.h",
"ios_external_texture_gl.mm",
"ios_gl_context.h",
"ios_gl_context.mm",
"ios_gl_render_target.h",
"ios_gl_render_target.mm",
"ios_render_target.h",
"ios_render_target.mm",
"ios_render_target_gl.h",
"ios_render_target_gl.mm",
"ios_surface.h",
"ios_surface.mm",
"ios_surface_gl.h",
Expand All @@ -95,15 +101,10 @@ shared_library("create_flutter_framework_dylib") {
"ios_surface_software.mm",
"platform_view_ios.h",
"platform_view_ios.mm",
"rendering_api_selection.h",
"rendering_api_selection.mm",
]

if (shell_enable_metal) {
sources += [
"ios_surface_metal.h",
"ios_surface_metal.mm",
]
}

sources += _flutter_framework_headers

deps = [
Expand All @@ -126,6 +127,13 @@ shared_library("create_flutter_framework_dylib") {

if (shell_enable_metal) {
defines += [ "FLUTTER_SHELL_ENABLE_METAL=1" ]

sources += [
"ios_context_metal.h",
"ios_context_metal.mm",
"ios_surface_metal.h",
"ios_surface_metal.mm",
]
}

libs = [
Expand Down
6 changes: 4 additions & 2 deletions shell/platform/darwin/ios/framework/Source/FlutterEngine.mm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#import "flutter/shell/platform/darwin/ios/framework/Source/platform_message_response_darwin.h"
#import "flutter/shell/platform/darwin/ios/ios_surface.h"
#import "flutter/shell/platform/darwin/ios/platform_view_ios.h"
#include "flutter/shell/platform/darwin/ios/rendering_api_selection.h"

NSString* const FlutterDefaultDartEntrypoint = nil;

Expand Down Expand Up @@ -427,7 +428,7 @@ - (BOOL)createShell:(NSString*)entrypoint libraryURI:(NSString*)libraryURI {
}

const auto threadLabel = [NSString stringWithFormat:@"%@.%zu", _labelPrefix, shellCount++];
FML_DLOG(INFO) << "Creating threadHost for " << threadLabel.UTF8String;

// The current thread will be used as the platform thread. Ensure that the message loop is
// initialized.
fml::MessageLoop::EnsureInitializedForCurrentThread();
Expand All @@ -441,7 +442,8 @@ - (BOOL)createShell:(NSString*)entrypoint libraryURI:(NSString*)libraryURI {
// synchronous.
flutter::Shell::CreateCallback<flutter::PlatformView> on_create_platform_view =
[](flutter::Shell& shell) {
return std::make_unique<flutter::PlatformViewIOS>(shell, shell.GetTaskRunners());
return std::make_unique<flutter::PlatformViewIOS>(
shell, flutter::GetRenderingAPIForProcess(), shell.GetTaskRunners());
};

flutter::Shell::CreateCallback<flutter::Rasterizer> on_create_rasterizer =
Expand Down
4 changes: 2 additions & 2 deletions shell/platform/darwin/ios/framework/Source/FlutterOverlayView.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

#include "flutter/fml/memory/weak_ptr.h"
#include "flutter/shell/common/shell.h"
#include "flutter/shell/platform/darwin/ios/ios_gl_context.h"
#include "flutter/shell/platform/darwin/ios/ios_context_gl.h"
#include "flutter/shell/platform/darwin/ios/ios_surface.h"
#include "flutter/shell/platform/darwin/ios/ios_surface_gl.h"

Expand All @@ -36,7 +36,7 @@
- (instancetype)init NS_DESIGNATED_INITIALIZER;
- (instancetype)initWithContentsScale:(CGFloat)contentsScale;
- (std::unique_ptr<flutter::IOSSurface>)createSurface:
(std::shared_ptr<flutter::IOSGLContext>)gl_context;
(std::shared_ptr<flutter::IOSContext>)ios_context;

@end

Expand Down
45 changes: 9 additions & 36 deletions shell/platform/darwin/ios/framework/Source/FlutterOverlayView.mm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,6 @@
#include "flutter/shell/common/rasterizer.h"
#include "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h"
#include "flutter/shell/platform/darwin/ios/ios_surface_gl.h"
#if FLUTTER_SHELL_ENABLE_METAL
#include "flutter/shell/platform/darwin/ios/ios_surface_metal.h"
#endif
#include "flutter/shell/platform/darwin/ios/ios_surface_software.h"
#include "third_party/skia/include/utils/mac/SkCGUtils.h"

Expand Down Expand Up @@ -50,18 +47,10 @@ - (instancetype)init {
- (instancetype)initWithContentsScale:(CGFloat)contentsScale {
self = [self init];

if ([self.layer isKindOfClass:[CAEAGLLayer class]]) {
CAEAGLLayer* layer = reinterpret_cast<CAEAGLLayer*>(self.layer);
layer.allowsGroupOpacity = NO;
Copy link
Contributor

@dnfield dnfield Mar 9, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are we dropping this here? According to https://developer.apple.com/documentation/quartzcore/calayer/1621277-allowsgroupopacity?language=objc this defaults to YES on iOS 7.0+.

layer.contentsScale = contentsScale;
layer.rasterizationScale = contentsScale;
#if FLUTTER_SHELL_ENABLE_METAL
} else if ([self.layer isKindOfClass:[CAMetalLayer class]]) {
CAMetalLayer* layer = reinterpret_cast<CAMetalLayer*>(self.layer);
layer.allowsGroupOpacity = NO;
layer.contentsScale = contentsScale;
layer.rasterizationScale = contentsScale;
#endif // FLUTTER_SHELL_ENABLE_METAL
if ([self.layer isKindOfClass:[CAEAGLLayer class]] ||
[self.layer isKindOfClass:[CAMetalLayer class]]) {
self.layer.contentsScale = contentsScale;
self.layer.rasterizationScale = contentsScale;
}

return self;
Expand All @@ -72,27 +61,11 @@ + (Class)layerClass {
}

- (std::unique_ptr<flutter::IOSSurface>)createSurface:
(std::shared_ptr<flutter::IOSGLContext>)gl_context {
if ([self.layer isKindOfClass:[CAEAGLLayer class]]) {
fml::scoped_nsobject<CAEAGLLayer> eagl_layer(
reinterpret_cast<CAEAGLLayer*>([self.layer retain]));
if (@available(iOS 9.0, *)) {
eagl_layer.get().presentsWithTransaction = YES;
Copy link
Contributor

@dnfield dnfield Mar 9, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't this break platform views on iOS?

}
return std::make_unique<flutter::IOSSurfaceGL>(std::move(eagl_layer), gl_context);
#if FLUTTER_SHELL_ENABLE_METAL
} else if ([self.layer isKindOfClass:[CAMetalLayer class]]) {
fml::scoped_nsobject<CAMetalLayer> metalLayer(
reinterpret_cast<CAMetalLayer*>([self.layer retain]));
if (@available(iOS 8.0, *)) {
metalLayer.get().presentsWithTransaction = YES;
}
return std::make_unique<flutter::IOSSurfaceMetal>(std::move(metalLayer));
#endif // FLUTTER_SHELL_ENABLE_METAL
} else {
fml::scoped_nsobject<CALayer> layer(reinterpret_cast<CALayer*>([self.layer retain]));
return std::make_unique<flutter::IOSSurfaceSoftware>(std::move(layer), nullptr);
}
(std::shared_ptr<flutter::IOSContext>)ios_context {
return flutter::IOSSurface::Create(std::move(ios_context), // context
Copy link
Contributor

@dnfield dnfield Mar 9, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will now throw if we're on a simulator, right?

fml::scoped_nsobject<CALayer>{[self.layer retain]}, // layer
nullptr // platform views controller
);
}

// TODO(amirh): implement drawLayer to support snapshotting.
Expand Down
Loading