From c7fc4800870b6a23b070bb80b046285c5933c94d Mon Sep 17 00:00:00 2001 From: ColdPaleLight Date: Wed, 28 Sep 2022 17:58:39 +0800 Subject: [PATCH 01/12] Tweak Texture::Paint API --- common/graphics/BUILD.gn | 1 + common/graphics/texture.h | 17 +++++++++---- flow/layers/texture_layer.cc | 8 +++++-- flow/testing/mock_texture.cc | 12 +++++----- flow/testing/mock_texture.h | 6 ++--- flow/testing/mock_texture_unittests.cc | 16 ++++++++----- impeller/renderer/backend/metal/texture_mtl.h | 9 ++++++- .../renderer/backend/metal/texture_mtl.mm | 16 +++++++++++-- shell/common/shell_unittests.cc | 6 ++--- .../android/android_external_texture_gl.cc | 24 +++++++++---------- .../android/android_external_texture_gl.h | 6 ++--- .../darwin/ios/ios_external_texture_metal.h | 6 ++--- .../darwin/ios/ios_external_texture_metal.mm | 13 +++++----- ...FlutterEmbedderExternalTextureUnittests.mm | 24 +++++++++++++++---- .../embedder/embedder_external_texture_gl.cc | 13 ++++++---- .../embedder/embedder_external_texture_gl.h | 6 ++--- .../embedder_external_texture_metal.h | 6 ++--- .../embedder_external_texture_metal.mm | 13 ++++++---- .../embedder/tests/embedder_unittests_gl.cc | 16 ++++++++----- .../tests/embedder_unittests_metal.mm | 10 +++++--- 20 files changed, 140 insertions(+), 88 deletions(-) diff --git a/common/graphics/BUILD.gn b/common/graphics/BUILD.gn index 772e8711f4a0d..480aefb234342 100644 --- a/common/graphics/BUILD.gn +++ b/common/graphics/BUILD.gn @@ -21,6 +21,7 @@ source_set("graphics") { # additions here could result in added app sizes across embeddings. deps = [ "//flutter/assets", + "//flutter/display_list", "//flutter/fml", "//flutter/shell/version:version", "//third_party/boringssl", diff --git a/common/graphics/texture.h b/common/graphics/texture.h index 0edb84526f994..da17bfadeeca1 100644 --- a/common/graphics/texture.h +++ b/common/graphics/texture.h @@ -7,6 +7,8 @@ #include +#include "flutter/display_list/display_list_builder.h" +#include "flutter/display_list/display_list_paint.h" #include "flutter/fml/macros.h" #include "flutter/fml/synchronization/waitable_event.h" #include "third_party/skia/include/core/SkCanvas.h" @@ -33,16 +35,23 @@ class ContextListener { class Texture : public ContextListener { public: + struct PaintContext { + SkCanvas* canvas = nullptr; + DisplayListBuilder* builder = nullptr; + GrDirectContext* gr_context = nullptr; + const SkPaint* sk_paint = nullptr; + const DlPaint* dl_paint = nullptr; + bool enable_impeller = false; + }; + explicit Texture(int64_t id); // Called from UI or raster thread. virtual ~Texture(); // Called from raster thread. // Called from raster thread. - virtual void Paint(SkCanvas& canvas, + virtual void Paint(PaintContext& context, const SkRect& bounds, bool freeze, - GrDirectContext* context, - const SkSamplingOptions& sampling, - const SkPaint* paint = nullptr) = 0; + const SkSamplingOptions& sampling) = 0; // Called on raster thread. virtual void MarkNewFrameAvailable() = 0; diff --git a/flow/layers/texture_layer.cc b/flow/layers/texture_layer.cc index 7ece96ebd4364..f9c95d7f025ee 100644 --- a/flow/layers/texture_layer.cc +++ b/flow/layers/texture_layer.cc @@ -62,8 +62,12 @@ void TextureLayer::Paint(PaintContext& context) const { return; } AutoCachePaint cache_paint(context); - texture->Paint(*context.leaf_nodes_canvas, paint_bounds(), freeze_, - context.gr_context, ToSk(sampling_), cache_paint.sk_paint()); + Texture::PaintContext ctx{.canvas = context.leaf_nodes_canvas, + .builder = context.leaf_nodes_builder, + .gr_context = context.gr_context, + .sk_paint = cache_paint.sk_paint(), + .dl_paint = cache_paint.dl_paint()}; + texture->Paint(ctx, paint_bounds(), freeze_, ToSk(sampling_)); } } // namespace flutter diff --git a/flow/testing/mock_texture.cc b/flow/testing/mock_texture.cc index ccb11e85437f1..3fe4fd116813c 100644 --- a/flow/testing/mock_texture.cc +++ b/flow/testing/mock_texture.cc @@ -3,6 +3,7 @@ // found in the LICENSE file. #include "flutter/flow/testing/mock_texture.h" +#include "flutter/flow/layers/layer.h" #include "flutter/flow/testing/skia_gpu_object_layer_test.h" namespace flutter { @@ -10,14 +11,13 @@ namespace testing { MockTexture::MockTexture(int64_t textureId) : Texture(textureId) {} -void MockTexture::Paint(SkCanvas& canvas, +void MockTexture::Paint(PaintContext& context, const SkRect& bounds, bool freeze, - GrDirectContext* context, - const SkSamplingOptions& sampling, - const SkPaint* paint) { - paint_calls_.emplace_back( - PaintCall{canvas, bounds, freeze, context, sampling, paint}); + const SkSamplingOptions& sampling) { + paint_calls_.emplace_back(PaintCall{*(context.canvas), bounds, freeze, + context.gr_context, sampling, + context.sk_paint}); } bool operator==(const MockTexture::PaintCall& a, diff --git a/flow/testing/mock_texture.h b/flow/testing/mock_texture.h index ee4ba6d460a61..8b1dfb05b493d 100644 --- a/flow/testing/mock_texture.h +++ b/flow/testing/mock_texture.h @@ -28,12 +28,10 @@ class MockTexture : public Texture { explicit MockTexture(int64_t textureId); // Called from raster thread. - void Paint(SkCanvas& canvas, + void Paint(PaintContext& context, const SkRect& bounds, bool freeze, - GrDirectContext* context, - const SkSamplingOptions& sampling, - const SkPaint* paint = nullptr) override; + const SkSamplingOptions& sampling) override; void OnGrContextCreated() override { gr_context_created_ = true; } void OnGrContextDestroyed() override { gr_context_destroyed_ = true; } diff --git a/flow/testing/mock_texture_unittests.cc b/flow/testing/mock_texture_unittests.cc index 5968928b6cd2c..5dd12f82cfeea 100644 --- a/flow/testing/mock_texture_unittests.cc +++ b/flow/testing/mock_texture_unittests.cc @@ -34,9 +34,11 @@ TEST(MockTextureTest, PaintCalls) { MockTexture::PaintCall{canvas, paint_bounds1, false, nullptr, sampling}, MockTexture::PaintCall{canvas, paint_bounds2, true, nullptr, sampling}}; auto texture = std::make_shared(0); - - texture->Paint(canvas, paint_bounds1, false, nullptr, sampling); - texture->Paint(canvas, paint_bounds2, true, nullptr, sampling); + Texture::PaintContext context{ + .canvas = &canvas, + }; + texture->Paint(context, paint_bounds1, false, sampling); + texture->Paint(context, paint_bounds2, true, sampling); EXPECT_EQ(texture->paint_calls(), expected_paint_calls); } @@ -49,9 +51,11 @@ TEST(MockTextureTest, PaintCallsWithLinearSampling) { MockTexture::PaintCall{canvas, paint_bounds1, false, nullptr, sampling}, MockTexture::PaintCall{canvas, paint_bounds2, true, nullptr, sampling}}; auto texture = std::make_shared(0); - - texture->Paint(canvas, paint_bounds1, false, nullptr, sampling); - texture->Paint(canvas, paint_bounds2, true, nullptr, sampling); + Texture::PaintContext context{ + .canvas = &canvas, + }; + texture->Paint(context, paint_bounds1, false, sampling); + texture->Paint(context, paint_bounds2, true, sampling); EXPECT_EQ(texture->paint_calls(), expected_paint_calls); } diff --git a/impeller/renderer/backend/metal/texture_mtl.h b/impeller/renderer/backend/metal/texture_mtl.h index ef61433449119..9b6170e76e83d 100644 --- a/impeller/renderer/backend/metal/texture_mtl.h +++ b/impeller/renderer/backend/metal/texture_mtl.h @@ -15,16 +15,23 @@ namespace impeller { class TextureMTL final : public Texture, public BackendCast { public: - TextureMTL(TextureDescriptor desc, id texture); + TextureMTL(TextureDescriptor desc, + id texture, + bool wrapped = false); + + static TextureMTL Wrapper(TextureDescriptor desc, id texture); // |Texture| ~TextureMTL() override; id GetMTLTexture() const; + bool IsWrapped() const; + private: id texture_ = nullptr; bool is_valid_ = false; + bool is_wrapped_ = false; // |Texture| void SetLabel(std::string_view label) override; diff --git a/impeller/renderer/backend/metal/texture_mtl.mm b/impeller/renderer/backend/metal/texture_mtl.mm index c220a2967acde..8c295ab65d878 100644 --- a/impeller/renderer/backend/metal/texture_mtl.mm +++ b/impeller/renderer/backend/metal/texture_mtl.mm @@ -5,10 +5,13 @@ #include "impeller/renderer/backend/metal/texture_mtl.h" #include "impeller/base/validation.h" +#include "impeller/renderer/texture_descriptor.h" namespace impeller { -TextureMTL::TextureMTL(TextureDescriptor p_desc, id texture) +TextureMTL::TextureMTL(TextureDescriptor p_desc, + id texture, + bool wrapped) : Texture(p_desc), texture_(texture) { const auto& desc = GetTextureDescriptor(); @@ -21,9 +24,14 @@ return; } + is_wrapped_ = wrapped; is_valid_ = true; } +TextureMTL TextureMTL::Wrapper(TextureDescriptor desc, id texture) { + return TextureMTL(desc, texture, true); +} + TextureMTL::~TextureMTL() = default; void TextureMTL::SetLabel(std::string_view label) { @@ -42,7 +50,7 @@ bool TextureMTL::OnSetContents(const uint8_t* contents, size_t length, size_t slice) { - if (!IsValid() || !contents) { + if (!IsValid() || !contents || is_wrapped_) { return false; } @@ -83,4 +91,8 @@ return is_valid_; } +bool TextureMTL::IsWrapped() const { + return is_wrapped_; +} + } // namespace impeller diff --git a/shell/common/shell_unittests.cc b/shell/common/shell_unittests.cc index 1780a9b7d53e0..9589f0874aefa 100644 --- a/shell/common/shell_unittests.cc +++ b/shell/common/shell_unittests.cc @@ -1850,12 +1850,10 @@ class MockTexture : public Texture { ~MockTexture() override = default; // Called from raster thread. - void Paint(SkCanvas& canvas, + void Paint(PaintContext& context, const SkRect& bounds, bool freeze, - GrDirectContext* context, - const SkSamplingOptions&, - const SkPaint* paint) override {} + const SkSamplingOptions&) override {} void OnGrContextCreated() override {} diff --git a/shell/platform/android/android_external_texture_gl.cc b/shell/platform/android/android_external_texture_gl.cc index 599d6190b3306..a39ec83cc06e9 100644 --- a/shell/platform/android/android_external_texture_gl.cc +++ b/shell/platform/android/android_external_texture_gl.cc @@ -38,12 +38,10 @@ void AndroidExternalTextureGL::MarkNewFrameAvailable() { new_frame_ready_ = true; } -void AndroidExternalTextureGL::Paint(SkCanvas& canvas, +void AndroidExternalTextureGL::Paint(PaintContext& context, const SkRect& bounds, bool freeze, - GrDirectContext* context, - const SkSamplingOptions& sampling, - const SkPaint* paint) { + const SkSamplingOptions& sampling) { if (state_ == AttachmentState::detached) { return; } @@ -60,29 +58,29 @@ void AndroidExternalTextureGL::Paint(SkCanvas& canvas, GL_RGBA8_OES}; GrBackendTexture backendTexture(1, 1, GrMipMapped::kNo, textureInfo); sk_sp image = SkImage::MakeFromTexture( - context, backendTexture, kTopLeft_GrSurfaceOrigin, kRGBA_8888_SkColorType, - kPremul_SkAlphaType, nullptr); + context.gr_context, backendTexture, kTopLeft_GrSurfaceOrigin, + kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr); if (image) { - SkAutoCanvasRestore autoRestore(&canvas, true); + SkAutoCanvasRestore autoRestore(context->canvas, true); // The incoming texture is vertically flipped, so we flip it // back. OpenGL's coordinate system has Positive Y equivalent to up, while // Skia's coordinate system has Negative Y equvalent to up. - canvas.translate(bounds.x(), bounds.y() + bounds.height()); - canvas.scale(bounds.width(), -bounds.height()); + context->canvas->translate(bounds.x(), bounds.y() + bounds.height()); + context->canvas->scale(bounds.width(), -bounds.height()); if (!transform.isIdentity()) { sk_sp shader = image->makeShader( SkTileMode::kRepeat, SkTileMode::kRepeat, sampling, transform); SkPaint paintWithShader; - if (paint) { - paintWithShader = *paint; + if (context->sk_paint) { + paintWithShader = *context->sk_paint; } paintWithShader.setShader(shader); - canvas.drawRect(SkRect::MakeWH(1, 1), paintWithShader); + context->canvas->drawRect(SkRect::MakeWH(1, 1), paintWithShader); } else { - canvas.drawImage(image, 0, 0, sampling, paint); + context->canvas->drawImage(image, 0, 0, sampling, context->sk_paint); } } } diff --git a/shell/platform/android/android_external_texture_gl.h b/shell/platform/android/android_external_texture_gl.h index c9fc07fe6908f..b2cc8d75f85f8 100644 --- a/shell/platform/android/android_external_texture_gl.h +++ b/shell/platform/android/android_external_texture_gl.h @@ -21,12 +21,10 @@ class AndroidExternalTextureGL : public flutter::Texture { ~AndroidExternalTextureGL() override; - void Paint(SkCanvas& canvas, + void Paint(PaintContext& context, const SkRect& bounds, bool freeze, - GrDirectContext* context, - const SkSamplingOptions& sampling, - const SkPaint* paint) override; + const SkSamplingOptions& sampling) override; void OnGrContextCreated() override; diff --git a/shell/platform/darwin/ios/ios_external_texture_metal.h b/shell/platform/darwin/ios/ios_external_texture_metal.h index 158497d336329..ba0153013a907 100644 --- a/shell/platform/darwin/ios/ios_external_texture_metal.h +++ b/shell/platform/darwin/ios/ios_external_texture_metal.h @@ -26,12 +26,10 @@ class IOSExternalTextureMetal final : public Texture { darwin_external_texture_metal_; // |Texture| - void Paint(SkCanvas& canvas, + void Paint(PaintContext& context, const SkRect& bounds, bool freeze, - GrDirectContext* context, - const SkSamplingOptions& sampling, - const SkPaint* paint) override; + const SkSamplingOptions& sampling) override; // |Texture| void OnGrContextCreated() override; diff --git a/shell/platform/darwin/ios/ios_external_texture_metal.mm b/shell/platform/darwin/ios/ios_external_texture_metal.mm index 50d1405a084bf..14d4ab91336f8 100644 --- a/shell/platform/darwin/ios/ios_external_texture_metal.mm +++ b/shell/platform/darwin/ios/ios_external_texture_metal.mm @@ -3,6 +3,7 @@ // found in the LICENSE file. #import "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h" +#include "flow/layers/layer.h" namespace flutter { @@ -13,18 +14,16 @@ IOSExternalTextureMetal::~IOSExternalTextureMetal() = default; -void IOSExternalTextureMetal::Paint(SkCanvas& canvas, +void IOSExternalTextureMetal::Paint(PaintContext& context, const SkRect& bounds, bool freeze, - GrDirectContext* context, - const SkSamplingOptions& sampling, - const SkPaint* paint) { - [darwin_external_texture_metal_ canvas:canvas + const SkSamplingOptions& sampling) { + [darwin_external_texture_metal_ canvas:*context.canvas bounds:bounds freeze:freeze - grContext:context + grContext:context.gr_context sampling:sampling - paint:paint]; + paint:context.sk_paint]; } void IOSExternalTextureMetal::OnGrContextCreated() { diff --git a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm index 144d09e22aad2..305fcb88800b0 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm @@ -111,7 +111,11 @@ - (CVPixelBufferRef)pixelBuffer { std::make_unique(texture_id, callback); SkRect bounds = SkRect::MakeWH(info.width(), info.height()); SkSamplingOptions sampling = SkSamplingOptions(SkFilterMode::kNearest); - texture->Paint(*gpuSurface->getCanvas(), bounds, /*freeze=*/false, grContext, sampling); + flutter::Texture::PaintContext context{ + .canvas = gpuSurface->getCanvas(), + .gr_context = grContext, + }; + texture->Paint(context, bounds, /*freeze=*/false, sampling); ASSERT_TRUE(mtlTexture != nil); @@ -161,7 +165,11 @@ - (CVPixelBufferRef)pixelBuffer { std::make_unique(texture_id, callback); SkRect bounds = SkRect::MakeWH(info.width(), info.height()); SkSamplingOptions sampling = SkSamplingOptions(SkFilterMode::kNearest); - texture->Paint(*gpuSurface->getCanvas(), bounds, /*freeze=*/false, grContext, sampling); + flutter::Texture::PaintContext context{ + .canvas = gpuSurface->getCanvas(), + .gr_context = grContext, + }; + texture->Paint(context, bounds, /*freeze=*/false, sampling); gpuSurface->makeImageSnapshot(); } @@ -209,7 +217,11 @@ - (CVPixelBufferRef)pixelBuffer { std::make_unique(texture_id, callback); SkRect bounds = SkRect::MakeWH(info.width(), info.height()); SkSamplingOptions sampling = SkSamplingOptions(SkFilterMode::kNearest); - texture->Paint(*gpuSurface->getCanvas(), bounds, /*freeze=*/false, grContext, sampling); + flutter::Texture::PaintContext context{ + .canvas = gpuSurface->getCanvas(), + .gr_context = grContext, + }; + texture->Paint(context, bounds, /*freeze=*/false, sampling); gpuSurface->makeImageSnapshot(); } @@ -257,7 +269,11 @@ - (CVPixelBufferRef)pixelBuffer { std::make_unique(texture_id, callback); SkRect bounds = SkRect::MakeWH(info.width(), info.height()); SkSamplingOptions sampling = SkSamplingOptions(SkFilterMode::kNearest); - texture->Paint(*gpuSurface->getCanvas(), bounds, /*freeze=*/false, grContext, sampling); + flutter::Texture::PaintContext context{ + .canvas = gpuSurface->getCanvas(), + .gr_context = grContext, + }; + texture->Paint(context, bounds, /*freeze=*/false, sampling); gpuSurface->makeImageSnapshot(); } diff --git a/shell/platform/embedder/embedder_external_texture_gl.cc b/shell/platform/embedder/embedder_external_texture_gl.cc index 94246797f5175..2454ce8782ba4 100644 --- a/shell/platform/embedder/embedder_external_texture_gl.cc +++ b/shell/platform/embedder/embedder_external_texture_gl.cc @@ -5,6 +5,8 @@ #include "flutter/shell/platform/embedder/embedder_external_texture_gl.h" #include "flutter/fml/logging.h" +#include "include/core/SkCanvas.h" +#include "include/core/SkPaint.h" #include "third_party/skia/include/core/SkAlphaType.h" #include "third_party/skia/include/core/SkColorSpace.h" #include "third_party/skia/include/core/SkColorType.h" @@ -25,20 +27,21 @@ EmbedderExternalTextureGL::EmbedderExternalTextureGL( EmbedderExternalTextureGL::~EmbedderExternalTextureGL() = default; // |flutter::Texture| -void EmbedderExternalTextureGL::Paint(SkCanvas& canvas, +void EmbedderExternalTextureGL::Paint(PaintContext& context, const SkRect& bounds, bool freeze, - GrDirectContext* context, - const SkSamplingOptions& sampling, - const SkPaint* paint) { + const SkSamplingOptions& sampling) { if (last_image_ == nullptr) { last_image_ = ResolveTexture(Id(), // - context, // + context.gr_context, // SkISize::Make(bounds.width(), bounds.height()) // ); } + SkCanvas& canvas = *context.canvas; + const SkPaint* paint = context.sk_paint; + if (last_image_) { if (bounds != SkRect::Make(last_image_->bounds())) { canvas.drawImageRect(last_image_, bounds, sampling, paint); diff --git a/shell/platform/embedder/embedder_external_texture_gl.h b/shell/platform/embedder/embedder_external_texture_gl.h index 887a0e50fe026..6d5585f352493 100644 --- a/shell/platform/embedder/embedder_external_texture_gl.h +++ b/shell/platform/embedder/embedder_external_texture_gl.h @@ -32,12 +32,10 @@ class EmbedderExternalTextureGL : public flutter::Texture { const SkISize& size); // |flutter::Texture| - void Paint(SkCanvas& canvas, + void Paint(PaintContext& context, const SkRect& bounds, bool freeze, - GrDirectContext* context, - const SkSamplingOptions& sampling, - const SkPaint* paint) override; + const SkSamplingOptions& sampling) override; // |flutter::Texture| void OnGrContextCreated() override; diff --git a/shell/platform/embedder/embedder_external_texture_metal.h b/shell/platform/embedder/embedder_external_texture_metal.h index 0587aa0f69ec5..6aa0908aeda75 100644 --- a/shell/platform/embedder/embedder_external_texture_metal.h +++ b/shell/platform/embedder/embedder_external_texture_metal.h @@ -32,12 +32,10 @@ class EmbedderExternalTextureMetal : public flutter::Texture { const SkISize& size); // |flutter::Texture| - void Paint(SkCanvas& canvas, + void Paint(PaintContext& context, const SkRect& bounds, bool freeze, - GrDirectContext* context, - const SkSamplingOptions& sampling, - const SkPaint* paint) override; + const SkSamplingOptions& sampling) override; // |flutter::Texture| void OnGrContextCreated() override; diff --git a/shell/platform/embedder/embedder_external_texture_metal.mm b/shell/platform/embedder/embedder_external_texture_metal.mm index 4c880dc8b8aba..10ea81bda6cb4 100644 --- a/shell/platform/embedder/embedder_external_texture_metal.mm +++ b/shell/platform/embedder/embedder_external_texture_metal.mm @@ -4,6 +4,7 @@ #include "flutter/shell/platform/embedder/embedder_external_texture_metal.h" +#include "flow/layers/layer.h" #include "flutter/fml/logging.h" #import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" #include "third_party/skia/include/core/SkImage.h" @@ -31,16 +32,18 @@ static bool ValidNumTextures(int expected, int actual) { EmbedderExternalTextureMetal::~EmbedderExternalTextureMetal() = default; // |flutter::Texture| -void EmbedderExternalTextureMetal::Paint(SkCanvas& canvas, +void EmbedderExternalTextureMetal::Paint(PaintContext& context, const SkRect& bounds, bool freeze, - GrDirectContext* context, - const SkSamplingOptions& sampling, - const SkPaint* paint) { + const SkSamplingOptions& sampling) { if (last_image_ == nullptr) { - last_image_ = ResolveTexture(Id(), context, SkISize::Make(bounds.width(), bounds.height())); + last_image_ = + ResolveTexture(Id(), context.gr_context, SkISize::Make(bounds.width(), bounds.height())); } + SkCanvas& canvas = *context.canvas; + const SkPaint* paint = context.sk_paint; + if (last_image_) { if (bounds != SkRect::Make(last_image_->bounds())) { canvas.drawImageRect(last_image_, bounds, sampling, paint); diff --git a/shell/platform/embedder/tests/embedder_unittests_gl.cc b/shell/platform/embedder/tests/embedder_unittests_gl.cc index 08f653495f379..a7018859d1d47 100644 --- a/shell/platform/embedder/tests/embedder_unittests_gl.cc +++ b/shell/platform/embedder/tests/embedder_unittests_gl.cc @@ -4047,20 +4047,24 @@ TEST_F(EmbedderTest, ExternalTextureGLRefreshedTooOften) { auto canvas = skia_surface->getCanvas(); Texture* texture_ = &texture; - texture_->Paint(*canvas, SkRect::MakeXYWH(0, 0, 100, 100), false, - context.get(), SkSamplingOptions(SkFilterMode::kLinear)); + Texture::PaintContext ctx{ + .canvas = canvas, + .gr_context = context.get(), + }; + texture_->Paint(ctx, SkRect::MakeXYWH(0, 0, 100, 100), false, + SkSamplingOptions(SkFilterMode::kLinear)); EXPECT_TRUE(resolve_called); resolve_called = false; - texture_->Paint(*canvas, SkRect::MakeXYWH(0, 0, 100, 100), false, - context.get(), SkSamplingOptions(SkFilterMode::kLinear)); + texture_->Paint(ctx, SkRect::MakeXYWH(0, 0, 100, 100), false, + SkSamplingOptions(SkFilterMode::kLinear)); EXPECT_FALSE(resolve_called); texture_->MarkNewFrameAvailable(); - texture_->Paint(*canvas, SkRect::MakeXYWH(0, 0, 100, 100), false, - context.get(), SkSamplingOptions(SkFilterMode::kLinear)); + texture_->Paint(ctx, SkRect::MakeXYWH(0, 0, 100, 100), false, + SkSamplingOptions(SkFilterMode::kLinear)); EXPECT_TRUE(resolve_called); } diff --git a/shell/platform/embedder/tests/embedder_unittests_metal.mm b/shell/platform/embedder/tests/embedder_unittests_metal.mm index 8e2bbf948bbdb..fd221fc736609 100644 --- a/shell/platform/embedder/tests/embedder_unittests_metal.mm +++ b/shell/platform/embedder/tests/embedder_unittests_metal.mm @@ -465,20 +465,24 @@ void Collect() { auto canvas = skia_surface->getCanvas(); Texture* texture_ = &texture; - texture_->Paint(*canvas, SkRect::MakeXYWH(0, 0, 100, 100), false, surface->GetGrContext().get(), + Texture::PaintContext ctx{ + .canvas = canvas, + .gr_context = surface->GetGrContext().get(), + }; + texture_->Paint(ctx, SkRect::MakeXYWH(0, 0, 100, 100), false, SkSamplingOptions(SkFilterMode::kLinear)); EXPECT_TRUE(resolve_called); resolve_called = false; - texture_->Paint(*canvas, SkRect::MakeXYWH(0, 0, 100, 100), false, surface->GetGrContext().get(), + texture_->Paint(ctx, SkRect::MakeXYWH(0, 0, 100, 100), false, SkSamplingOptions(SkFilterMode::kLinear)); EXPECT_FALSE(resolve_called); texture_->MarkNewFrameAvailable(); - texture_->Paint(*canvas, SkRect::MakeXYWH(0, 0, 100, 100), false, surface->GetGrContext().get(), + texture_->Paint(ctx, SkRect::MakeXYWH(0, 0, 100, 100), false, SkSamplingOptions(SkFilterMode::kLinear)); EXPECT_TRUE(resolve_called); From 376fe85bfe3a1096d52c7a95d12888c1b03acd3b Mon Sep 17 00:00:00 2001 From: ColdPaleLight Date: Thu, 29 Sep 2022 18:31:03 +0800 Subject: [PATCH 02/12] Support external textures for iOS --- ci/licenses_golden/licenses_flutter | 2 + flow/layers/texture_layer.cc | 12 +- .../display_list/display_list_dispatcher.cc | 2 +- impeller/renderer/backend/metal/texture_mtl.h | 3 +- .../renderer/backend/metal/texture_mtl.mm | 5 +- impeller/renderer/texture.cc | 4 + impeller/renderer/texture.h | 4 +- shell/platform/darwin/graphics/BUILD.gn | 10 ++ .../graphics/FlutterDarwinContextMetal.mm | 3 +- .../FlutterDarwinContextMetalImpeller.h | 54 ++++++++ .../FlutterDarwinContextMetalImpeller.mm | 73 +++++++++++ .../FlutterDarwinExternalTextureMetal.h | 16 +-- .../FlutterDarwinExternalTextureMetal.mm | 124 +++++++++++++----- .../darwin/ios/ios_context_metal_impeller.h | 3 +- .../darwin/ios/ios_context_metal_impeller.mm | 28 ++-- .../darwin/ios/ios_external_texture_metal.mm | 10 +- 16 files changed, 272 insertions(+), 81 deletions(-) create mode 100644 shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h create mode 100644 shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.mm diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 5d167e125da85..90004845adb5d 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1794,6 +1794,8 @@ FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/flutter_cod FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/flutter_standard_codec_unittest.mm FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm +FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h +FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.mm FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Headers/Flutter.h diff --git a/flow/layers/texture_layer.cc b/flow/layers/texture_layer.cc index f9c95d7f025ee..8a98366f2c254 100644 --- a/flow/layers/texture_layer.cc +++ b/flow/layers/texture_layer.cc @@ -62,11 +62,13 @@ void TextureLayer::Paint(PaintContext& context) const { return; } AutoCachePaint cache_paint(context); - Texture::PaintContext ctx{.canvas = context.leaf_nodes_canvas, - .builder = context.leaf_nodes_builder, - .gr_context = context.gr_context, - .sk_paint = cache_paint.sk_paint(), - .dl_paint = cache_paint.dl_paint()}; + Texture::PaintContext ctx{ + .canvas = context.leaf_nodes_canvas, + .builder = context.leaf_nodes_builder, + .gr_context = context.gr_context, + .sk_paint = cache_paint.sk_paint(), + .dl_paint = cache_paint.dl_paint(), + }; texture->Paint(ctx, paint_bounds(), freeze_, ToSk(sampling_)); } diff --git a/impeller/display_list/display_list_dispatcher.cc b/impeller/display_list/display_list_dispatcher.cc index ad2eb4286d123..d01cd360f87c7 100644 --- a/impeller/display_list/display_list_dispatcher.cc +++ b/impeller/display_list/display_list_dispatcher.cc @@ -1053,7 +1053,7 @@ void DisplayListDispatcher::drawImageRect( std::make_shared(image->impeller_texture()), // image ToRect(src), // source rect ToRect(dst), // destination rect - paint_, // paint + render_with_attributes ? paint_ : Paint(), // paint ToSamplerDescriptor(sampling) // sampling ); } diff --git a/impeller/renderer/backend/metal/texture_mtl.h b/impeller/renderer/backend/metal/texture_mtl.h index 9b6170e76e83d..1aedbeacaa6af 100644 --- a/impeller/renderer/backend/metal/texture_mtl.h +++ b/impeller/renderer/backend/metal/texture_mtl.h @@ -19,7 +19,8 @@ class TextureMTL final : public Texture, id texture, bool wrapped = false); - static TextureMTL Wrapper(TextureDescriptor desc, id texture); + static std::shared_ptr Wrapper(TextureDescriptor desc, + id texture); // |Texture| ~TextureMTL() override; diff --git a/impeller/renderer/backend/metal/texture_mtl.mm b/impeller/renderer/backend/metal/texture_mtl.mm index 8c295ab65d878..6877c1b871ac9 100644 --- a/impeller/renderer/backend/metal/texture_mtl.mm +++ b/impeller/renderer/backend/metal/texture_mtl.mm @@ -28,8 +28,9 @@ is_valid_ = true; } -TextureMTL TextureMTL::Wrapper(TextureDescriptor desc, id texture) { - return TextureMTL(desc, texture, true); +std::shared_ptr TextureMTL::Wrapper(TextureDescriptor desc, + id texture) { + return std::make_shared(desc, texture, true); } TextureMTL::~TextureMTL() = default; diff --git a/impeller/renderer/texture.cc b/impeller/renderer/texture.cc index a2c2079936713..64860c181a038 100644 --- a/impeller/renderer/texture.cc +++ b/impeller/renderer/texture.cc @@ -61,6 +61,10 @@ bool Texture::IsSliceValid(size_t slice) const { FML_UNREACHABLE(); } +void Texture::SetIntent(TextureIntent intent) { + intent_ = intent; +} + TextureIntent Texture::GetIntent() const { return intent_; } diff --git a/impeller/renderer/texture.h b/impeller/renderer/texture.h index df51d7daf9628..8ac7b7b0d1fa9 100644 --- a/impeller/renderer/texture.h +++ b/impeller/renderer/texture.h @@ -35,6 +35,8 @@ class Texture { const TextureDescriptor& GetTextureDescriptor() const; + void SetIntent(TextureIntent intent); + TextureIntent GetIntent() const; virtual Scalar GetYCoordScale() const; @@ -51,7 +53,7 @@ class Texture { size_t slice) = 0; private: - TextureIntent intent_ = TextureIntent::kRenderToTexture; + TextureIntent intent_ = TextureIntent::kUploadFromHost; const TextureDescriptor desc_; bool IsSliceValid(size_t slice) const; diff --git a/shell/platform/darwin/graphics/BUILD.gn b/shell/platform/darwin/graphics/BUILD.gn index c7c151c10d33f..636c02ff7473c 100644 --- a/shell/platform/darwin/graphics/BUILD.gn +++ b/shell/platform/darwin/graphics/BUILD.gn @@ -5,6 +5,7 @@ assert(is_ios || is_mac) import("//flutter/common/config.gni") +import("//flutter/impeller/tools/impeller.gni") source_set("graphics") { cflags_objc = flutter_cflags_objc_arc @@ -19,11 +20,20 @@ source_set("graphics") { deps = [ "//flutter/common/graphics", + "//flutter/display_list", "//flutter/fml", "//flutter/shell/common", "//flutter/shell/platform/darwin/common:framework_shared", ] + if (impeller_supports_rendering) { + sources += [ + "FlutterDarwinContextMetalImpeller.h", + "FlutterDarwinContextMetalImpeller.mm", + ] + deps += [ "//flutter/impeller" ] + } + frameworks = [ "CoreVideo.framework" ] public_deps = [ "//third_party/skia" ] diff --git a/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm b/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm index 0cd3fe98906dc..52f4c466cf0b3 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm +++ b/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm @@ -94,7 +94,8 @@ - (void)dealloc { texture:(NSObject*)texture { return [[FlutterDarwinExternalTextureMetal alloc] initWithTextureCache:_textureCache textureID:textureID - texture:texture]; + texture:texture + enableImpeller:NO]; } @end diff --git a/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h b/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h new file mode 100644 index 0000000000000..ee7154a6dff2a --- /dev/null +++ b/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h @@ -0,0 +1,54 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef SHELL_PLATFORM_DARWIN_GRAPHICS_DARWIN_CONTEXT_METAL_IMPELLER_H_ +#define SHELL_PLATFORM_DARWIN_GRAPHICS_DARWIN_CONTEXT_METAL_IMPELLER_H_ + +#import +#import +#import + +#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h" +#import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" +#include "flutter/impeller/renderer/backend/metal/context_mtl.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Provides skia GrContexts that are shared between iOS and macOS embeddings. + */ +@interface FlutterDarwinContextMetalImpeller : NSObject + +/** + * Initializes a FlutterDarwinContextMetalImpeller. + */ +- (instancetype)init; + +/** + * Creates an external texture with the specified ID and contents. + */ +- (FlutterDarwinExternalTextureMetal*) + createExternalTextureWithIdentifier:(int64_t)textureID + texture:(NSObject*)texture; + +/** + * MTLDevice that is backing this context.s + */ +@property(nonatomic, readonly) id device; + +/** + * Impeller context; +*/ +@property(nonatomic, readonly) std::shared_ptr context; + +/* + * Texture cache for external textures. + */ +@property(nonatomic, readonly) CVMetalTextureCacheRef textureCache; + +@end + +NS_ASSUME_NONNULL_END + +#endif // SHELL_PLATFORM_DARWIN_GRAPHICS_DARWIN_CONTEXT_METAL_IMPELLER_H_ diff --git a/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.mm b/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.mm new file mode 100644 index 0000000000000..2be13f4dcc52b --- /dev/null +++ b/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.mm @@ -0,0 +1,73 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h" + +#include "flutter/common/graphics/persistent_cache.h" +#include "flutter/fml/logging.h" +#include "flutter/shell/common/context_options.h" +#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" +#include "flutter/impeller/entity/mtl/entity_shaders.h" +#include "flutter/impeller/renderer/backend/metal/context_mtl.h" + +FLUTTER_ASSERT_ARC + +static std::shared_ptr CreateImpellerContext() { + std::vector> shader_mappings = { + std::make_shared(impeller_entity_shaders_data, + impeller_entity_shaders_length), + }; + auto context = impeller::ContextMTL::Create(shader_mappings, "Impeller Library"); + if (!context) { + FML_LOG(ERROR) << "Could not create Metal Impeller Context."; + return nullptr; + } + FML_LOG(ERROR) << "Using the Impeller rendering backend."; + + return context; +} + +@implementation FlutterDarwinContextMetalImpeller + +- (instancetype)init { + self = [super init]; + if (self != nil) { + _context = CreateImpellerContext(); + _device = impeller::ContextMTL::Cast(*_context).GetMTLDevice(); + + if (!_device) { + FML_DLOG(ERROR) << "Could not acquire Metal device."; + return nil; + } + + CVReturn cvReturn = CVMetalTextureCacheCreate(kCFAllocatorDefault, // allocator + nil, // cache attributes (nil default) + _device, // metal device + nil, // texture attributes (nil default) + &_textureCache // [out] cache + ); + if (cvReturn != kCVReturnSuccess) { + FML_DLOG(ERROR) << "Could not create Metal texture cache."; + return nil; + } + } + return self; +} + +- (void)dealloc { + if (_textureCache) { + CFRelease(_textureCache); + } +} + +- (FlutterDarwinExternalTextureMetal*) + createExternalTextureWithIdentifier:(int64_t)textureID + texture:(NSObject*)texture { + return [[FlutterDarwinExternalTextureMetal alloc] initWithTextureCache:_textureCache + textureID:textureID + texture:texture + enableImpeller:YES]; +} + +@end diff --git a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h index 042aec510abd8..c3c495870c9ec 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h +++ b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h @@ -5,6 +5,7 @@ #import #import +#include "flutter/common/graphics/texture.h" #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h" #include "third_party/skia/include/core/SkCanvas.h" #include "third_party/skia/include/core/SkImage.h" @@ -28,14 +29,13 @@ - (nullable instancetype)initWithTextureCache:(nonnull CVMetalTextureCacheRef)textureCache textureID:(int64_t)textureID - texture:(nonnull NSObject*)texture; - -- (void)canvas:(SkCanvas&)canvas - bounds:(const SkRect&)bounds - freeze:(BOOL)freeze - grContext:(nonnull GrDirectContext*)grContext - sampling:(const SkSamplingOptions&)sampling - paint:(nullable const SkPaint*)paint; + texture:(nonnull NSObject*)texture + enableImpeller:(BOOL)enableImpeller; + +- (void)paintContext:(flutter::Texture::PaintContext&)context + bounds:(const SkRect&)bounds + freeze:(BOOL)freeze + sampling:(const SkSamplingOptions&)sampling; - (void)onGrContextCreated; diff --git a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm index 6af2b62bdba74..32117058b5589 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm +++ b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm @@ -3,6 +3,13 @@ // found in the LICENSE file. #import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" +#include "flutter/display_list/display_list_image.h" + +#if IMPELLER_SUPPORTS_RENDERING +#include "impeller/base/config.h" +#include "impeller/display_list/display_list_image_impeller.h" +#include "impeller/renderer/backend/metal/texture_mtl.h" +#endif // IMPELLER_SUPPORTS_RENDERING #include "flutter/fml/logging.h" #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" @@ -19,19 +26,22 @@ @implementation FlutterDarwinExternalTextureMetal { CVMetalTextureCacheRef _textureCache; NSObject* _externalTexture; BOOL _textureFrameAvailable; - sk_sp _externalImage; + sk_sp _externalImage; CVPixelBufferRef _lastPixelBuffer; OSType _pixelFormat; + BOOL _enableImpeller; } - (instancetype)initWithTextureCache:(nonnull CVMetalTextureCacheRef)textureCache textureID:(int64_t)textureID - texture:(NSObject*)texture { + texture:(NSObject*)texture + enableImpeller:(BOOL)enableImpeller { if (self = [super init]) { _textureCache = textureCache; CFRetain(_textureCache); _textureID = textureID; _externalTexture = texture; + _enableImpeller = enableImpeller; return self; } return nil; @@ -47,30 +57,43 @@ - (void)dealloc { } } -- (void)canvas:(SkCanvas&)canvas - bounds:(const SkRect&)bounds - freeze:(BOOL)freeze - grContext:(nonnull GrDirectContext*)grContext - sampling:(const SkSamplingOptions&)sampling - paint:(nullable const SkPaint*)paint { +- (void)paintContext:(flutter::Texture::PaintContext&)context + bounds:(const SkRect&)bounds + freeze:(BOOL)freeze + sampling:(const SkSamplingOptions&)sampling { const bool needsUpdatedTexture = (!freeze && _textureFrameAvailable) || !_externalImage; if (needsUpdatedTexture) { - [self onNeedsUpdatedTexture:grContext]; + [self onNeedsUpdatedTexture:context]; } if (_externalImage) { - canvas.drawImageRect(_externalImage, // image - SkRect::Make(_externalImage->bounds()), // source rect - bounds, // destination rect - sampling, // sampling - paint, // paint - SkCanvas::SrcRectConstraint::kFast_SrcRectConstraint // constraint +#if IMPELLER_SUPPORTS_RENDERING + if (_enableImpeller) { + context.builder->drawImageRect( + _externalImage, // image + SkRect::Make(_externalImage->bounds()), // source rect + bounds, // destination rect + flutter::ToDl(sampling), // sampling + context.dl_paint, + SkCanvas::SrcRectConstraint::kFast_SrcRectConstraint // constraint + ); + return; + } +#endif // IMPELLER_SUPPORTS_RENDERING + + context.canvas->drawImageRect( + _externalImage->skia_image(), // image + SkRect::Make(_externalImage->bounds()), // source rect + bounds, // destination rect + sampling, // sampling + context.sk_paint, // paint + SkCanvas::SrcRectConstraint::kFast_SrcRectConstraint // constraint ); } } -- (void)onNeedsUpdatedTexture:(nonnull GrDirectContext*)grContext { +- (void)onNeedsUpdatedTexture:(flutter::Texture::PaintContext&)context { CVPixelBufferRef pixelBuffer = [_externalTexture copyPixelBuffer]; if (pixelBuffer) { CVPixelBufferRelease(_lastPixelBuffer); @@ -80,7 +103,7 @@ - (void)onNeedsUpdatedTexture:(nonnull GrDirectContext*)grContext { // If the application told us there was a texture frame available but did not provide one when // asked for it, reuse the previous texture but make sure to ask again the next time around. - sk_sp image = [self wrapExternalPixelBuffer:_lastPixelBuffer grContext:grContext]; + sk_sp image = [self wrapExternalPixelBuffer:_lastPixelBuffer context:context]; if (image) { _externalImage = image; _textureFrameAvailable = false; @@ -116,29 +139,36 @@ - (void)onTextureUnregistered { #pragma mark - External texture skia wrapper methods. -- (sk_sp)wrapExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer - grContext:(GrDirectContext*)grContext { +- (sk_sp)wrapExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer + context:(flutter::Texture::PaintContext&)context { if (!pixelBuffer) { return nullptr; } - sk_sp image = nullptr; + sk_sp image = nullptr; if (_pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || _pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { - image = [self wrapNV12ExternalPixelBuffer:pixelBuffer grContext:grContext]; + image = [self wrapNV12ExternalPixelBuffer:pixelBuffer context:context]; } else { - image = [self wrapRGBAExternalPixelBuffer:pixelBuffer grContext:grContext]; + image = [self wrapRGBAExternalPixelBuffer:pixelBuffer context:context]; } if (!image) { - FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image."; + FML_DLOG(ERROR) << "Could not wrap Metal texture as a display list image."; } return image; } -- (sk_sp)wrapNV12ExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer - grContext:(GrDirectContext*)grContext { +- (sk_sp)wrapNV12ExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer + context:(flutter::Texture::PaintContext&)context { +#if IMPELLER_SUPPORTS_RENDERING + if (_enableImpeller) { + IMPELLER_UNIMPLEMENTED + return nullptr; + } +#endif // IMPELLER_SUPPORTS_RENDERING + SkISize textureSize = SkISize::Make(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer)); CVMetalTextureRef yMetalTexture = nullptr; @@ -185,15 +215,20 @@ - (void)onTextureUnregistered { id uvTex = CVMetalTextureGetTexture(uvMetalTexture); CVBufferRelease(uvMetalTexture); - return [FlutterDarwinExternalTextureSkImageWrapper wrapYUVATexture:yTex - UVTex:uvTex - grContext:grContext - width:textureSize.width() - height:textureSize.height()]; + auto skImage = [FlutterDarwinExternalTextureSkImageWrapper wrapYUVATexture:yTex + UVTex:uvTex + grContext:context.gr_context + width:textureSize.width() + height:textureSize.height()]; + if (!skImage) { + return nullptr; + } + + return flutter::DlImage::Make(skImage); } -- (sk_sp)wrapRGBAExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer - grContext:(GrDirectContext*)grContext { +- (sk_sp)wrapRGBAExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer + context:(flutter::Texture::PaintContext&)context { SkISize textureSize = SkISize::Make(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer)); CVMetalTextureRef metalTexture = nullptr; @@ -216,10 +251,27 @@ - (void)onTextureUnregistered { id rgbaTex = CVMetalTextureGetTexture(metalTexture); CVBufferRelease(metalTexture); - return [FlutterDarwinExternalTextureSkImageWrapper wrapRGBATexture:rgbaTex - grContext:grContext - width:textureSize.width() - height:textureSize.height()]; +#if IMPELLER_SUPPORTS_RENDERING + if (_enableImpeller) { + impeller::TextureDescriptor desc; + desc.storage_mode = impeller::StorageMode::kHostVisible; + desc.format = impeller::PixelFormat::kB8G8R8A8UNormInt; + desc.size = {textureSize.width(), textureSize.height()}; + desc.mip_count = 1; + auto texture = impeller::TextureMTL::Wrapper(desc, rgbaTex); + texture->SetIntent(impeller::TextureIntent::kUploadFromHost); + return impeller::DlImageImpeller::Make(texture); + } +#endif // IMPELLER_SUPPORTS_RENDERING + + auto skImage = [FlutterDarwinExternalTextureSkImageWrapper wrapRGBATexture:rgbaTex + grContext:context.gr_context + width:textureSize.width() + height:textureSize.height()]; + if (!skImage) { + return nullptr; + } + return flutter::DlImage::Make(skImage); } @end diff --git a/shell/platform/darwin/ios/ios_context_metal_impeller.h b/shell/platform/darwin/ios/ios_context_metal_impeller.h index 4e3dca13e55e8..6efb530728de9 100644 --- a/shell/platform/darwin/ios/ios_context_metal_impeller.h +++ b/shell/platform/darwin/ios/ios_context_metal_impeller.h @@ -7,6 +7,7 @@ #include "flutter/fml/macros.h" #include "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h" +#include "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h" #include "flutter/shell/platform/darwin/ios/ios_context.h" namespace impeller { @@ -33,7 +34,7 @@ class IOSContextMetalImpeller final : public IOSContext { sk_sp GetResourceContext() const; private: - std::shared_ptr context_; + fml::scoped_nsobject darwin_context_metal_impeller_; // |IOSContext| sk_sp CreateResourceContext() override; diff --git a/shell/platform/darwin/ios/ios_context_metal_impeller.mm b/shell/platform/darwin/ios/ios_context_metal_impeller.mm index 6747c522c0f2f..0ad9659fc94e3 100644 --- a/shell/platform/darwin/ios/ios_context_metal_impeller.mm +++ b/shell/platform/darwin/ios/ios_context_metal_impeller.mm @@ -3,28 +3,15 @@ // found in the LICENSE file. #import "flutter/shell/platform/darwin/ios/ios_context_metal_impeller.h" - +#import "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h" #include "flutter/impeller/entity/mtl/entity_shaders.h" -#include "flutter/impeller/renderer/backend/metal/context_mtl.h" namespace flutter { -static std::shared_ptr CreateImpellerContext() { - std::vector> shader_mappings = { - std::make_shared(impeller_entity_shaders_data, - impeller_entity_shaders_length), - }; - auto context = impeller::ContextMTL::Create(shader_mappings, "Impeller Library"); - if (!context) { - FML_LOG(ERROR) << "Could not create Metal Impeller Context."; - return nullptr; - } - FML_LOG(ERROR) << "Using the Impeller rendering backend."; - return context; -} - IOSContextMetalImpeller::IOSContextMetalImpeller() - : IOSContext(MsaaSampleCount::kFour), context_(CreateImpellerContext()) {} + : IOSContext(MsaaSampleCount::kFour), + darwin_context_metal_impeller_(fml::scoped_nsobject{ + [[FlutterDarwinContextMetalImpeller alloc] init]}) {} IOSContextMetalImpeller::~IOSContextMetalImpeller() = default; @@ -51,7 +38,7 @@ // |IOSContext| std::shared_ptr IOSContextMetalImpeller::GetImpellerContext() const { - return context_; + return darwin_context_metal_impeller_.get().context; } // |IOSContext| @@ -64,7 +51,10 @@ std::unique_ptr IOSContextMetalImpeller::CreateExternalTexture( int64_t texture_id, fml::scoped_nsobject> texture) { - return nullptr; + return std::make_unique( + fml::scoped_nsobject{ + [[darwin_context_metal_impeller_ createExternalTextureWithIdentifier:texture_id + texture:texture] retain]}); } } // namespace flutter diff --git a/shell/platform/darwin/ios/ios_external_texture_metal.mm b/shell/platform/darwin/ios/ios_external_texture_metal.mm index 14d4ab91336f8..ccc68aa07a008 100644 --- a/shell/platform/darwin/ios/ios_external_texture_metal.mm +++ b/shell/platform/darwin/ios/ios_external_texture_metal.mm @@ -18,12 +18,10 @@ const SkRect& bounds, bool freeze, const SkSamplingOptions& sampling) { - [darwin_external_texture_metal_ canvas:*context.canvas - bounds:bounds - freeze:freeze - grContext:context.gr_context - sampling:sampling - paint:context.sk_paint]; + [darwin_external_texture_metal_ paintContext:context + bounds:bounds + freeze:freeze + sampling:sampling]; } void IOSExternalTextureMetal::OnGrContextCreated() { From 94b881d430120b3ce0a3e6f0414a503611957493 Mon Sep 17 00:00:00 2001 From: ColdPaleLight Date: Thu, 29 Sep 2022 18:31:47 +0800 Subject: [PATCH 03/12] format code --- .../darwin/graphics/FlutterDarwinContextMetalImpeller.h | 4 ++-- .../darwin/graphics/FlutterDarwinContextMetalImpeller.mm | 6 +++--- shell/platform/darwin/ios/ios_context_metal_impeller.mm | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h b/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h index ee7154a6dff2a..ea56856817fbe 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h +++ b/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h @@ -9,9 +9,9 @@ #import #import +#include "flutter/impeller/renderer/backend/metal/context_mtl.h" #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h" #import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" -#include "flutter/impeller/renderer/backend/metal/context_mtl.h" NS_ASSUME_NONNULL_BEGIN @@ -39,7 +39,7 @@ NS_ASSUME_NONNULL_BEGIN /** * Impeller context; -*/ + */ @property(nonatomic, readonly) std::shared_ptr context; /* diff --git a/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.mm b/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.mm index 2be13f4dcc52b..43e1be235d909 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.mm +++ b/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.mm @@ -6,10 +6,10 @@ #include "flutter/common/graphics/persistent_cache.h" #include "flutter/fml/logging.h" -#include "flutter/shell/common/context_options.h" -#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" #include "flutter/impeller/entity/mtl/entity_shaders.h" #include "flutter/impeller/renderer/backend/metal/context_mtl.h" +#include "flutter/shell/common/context_options.h" +#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" FLUTTER_ASSERT_ARC @@ -24,7 +24,7 @@ return nullptr; } FML_LOG(ERROR) << "Using the Impeller rendering backend."; - + return context; } diff --git a/shell/platform/darwin/ios/ios_context_metal_impeller.mm b/shell/platform/darwin/ios/ios_context_metal_impeller.mm index 0ad9659fc94e3..bab3c96129984 100644 --- a/shell/platform/darwin/ios/ios_context_metal_impeller.mm +++ b/shell/platform/darwin/ios/ios_context_metal_impeller.mm @@ -3,8 +3,8 @@ // found in the LICENSE file. #import "flutter/shell/platform/darwin/ios/ios_context_metal_impeller.h" -#import "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h" #include "flutter/impeller/entity/mtl/entity_shaders.h" +#import "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h" namespace flutter { From 1cfeaa78ded46f1fc2076da8e75e4bf85c485940 Mon Sep 17 00:00:00 2001 From: ColdPaleLight Date: Thu, 29 Sep 2022 21:48:11 +0800 Subject: [PATCH 04/12] Tweak code --- ci/licenses_golden/licenses_flutter | 4 ++-- shell/common/shell_test_platform_view_metal.mm | 8 ++++---- .../android/android_external_texture_gl.cc | 14 +++++++------- shell/platform/darwin/graphics/BUILD.gn | 4 ++-- ...Metal.h => FlutterDarwinContextMetalSkia.h} | 6 +++--- ...tal.mm => FlutterDarwinContextMetalSkia.mm} | 6 +++--- .../darwin/ios/ios_context_metal_impeller.h | 4 ++-- .../darwin/ios/ios_context_metal_impeller.mm | 5 +++-- .../darwin/ios/ios_context_metal_skia.h | 6 +++--- .../darwin/ios/ios_context_metal_skia.mm | 8 ++++---- .../FlutterEmbedderExternalTextureUnittests.mm | 18 +++++++++--------- .../Source/FlutterExternalTextureMetal.h | 4 ++-- .../Source/FlutterExternalTextureMetal.mm | 4 ++-- .../framework/Source/FlutterMetalRenderer.mm | 6 +++--- .../embedder/embedder_surface_metal.mm | 11 ++++++----- 15 files changed, 55 insertions(+), 53 deletions(-) rename shell/platform/darwin/graphics/{FlutterDarwinContextMetal.h => FlutterDarwinContextMetalSkia.h} (90%) rename shell/platform/darwin/graphics/{FlutterDarwinContextMetal.mm => FlutterDarwinContextMetalSkia.mm} (95%) diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 90004845adb5d..b83e91ef8d37d 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1792,8 +1792,8 @@ FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/FlutterStan FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/FlutterStandardCodec_Internal.h FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/flutter_codecs_unittest.mm FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/flutter_standard_codec_unittest.mm -FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h -FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm +FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h +FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.mm FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.mm FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h diff --git a/shell/common/shell_test_platform_view_metal.mm b/shell/common/shell_test_platform_view_metal.mm index 42794260749d1..814e67e6f3933 100644 --- a/shell/common/shell_test_platform_view_metal.mm +++ b/shell/common/shell_test_platform_view_metal.mm @@ -10,7 +10,7 @@ #include "flutter/fml/platform/darwin/scoped_nsobject.h" #include "flutter/shell/gpu/gpu_surface_metal_skia.h" -#include "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h" +#include "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h" namespace flutter { namespace testing { @@ -30,12 +30,12 @@ class DarwinContextMetal { public: DarwinContextMetal() - : context_([[FlutterDarwinContextMetal alloc] initWithDefaultMTLDevice]), + : context_([[FlutterDarwinContextMetalSkia alloc] initWithDefaultMTLDevice]), offscreen_texture_(CreateOffscreenTexture([context_.get() device])) {} ~DarwinContextMetal() = default; - fml::scoped_nsobject context() const { return context_; } + fml::scoped_nsobject context() const { return context_; } fml::scoped_nsprotocol> offscreen_texture() const { return offscreen_texture_; } @@ -47,7 +47,7 @@ GPUMTLTextureInfo offscreen_texture_info() const { } private: - const fml::scoped_nsobject context_; + const fml::scoped_nsobject context_; const fml::scoped_nsprotocol> offscreen_texture_; FML_DISALLOW_COPY_AND_ASSIGN(DarwinContextMetal); diff --git a/shell/platform/android/android_external_texture_gl.cc b/shell/platform/android/android_external_texture_gl.cc index a39ec83cc06e9..2a58de3a96171 100644 --- a/shell/platform/android/android_external_texture_gl.cc +++ b/shell/platform/android/android_external_texture_gl.cc @@ -61,26 +61,26 @@ void AndroidExternalTextureGL::Paint(PaintContext& context, context.gr_context, backendTexture, kTopLeft_GrSurfaceOrigin, kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr); if (image) { - SkAutoCanvasRestore autoRestore(context->canvas, true); + SkAutoCanvasRestore autoRestore(context.canvas, true); // The incoming texture is vertically flipped, so we flip it // back. OpenGL's coordinate system has Positive Y equivalent to up, while // Skia's coordinate system has Negative Y equvalent to up. - context->canvas->translate(bounds.x(), bounds.y() + bounds.height()); - context->canvas->scale(bounds.width(), -bounds.height()); + context.canvas->translate(bounds.x(), bounds.y() + bounds.height()); + context.canvas->scale(bounds.width(), -bounds.height()); if (!transform.isIdentity()) { sk_sp shader = image->makeShader( SkTileMode::kRepeat, SkTileMode::kRepeat, sampling, transform); SkPaint paintWithShader; - if (context->sk_paint) { - paintWithShader = *context->sk_paint; + if (context.sk_paint) { + paintWithShader = *context.sk_paint; } paintWithShader.setShader(shader); - context->canvas->drawRect(SkRect::MakeWH(1, 1), paintWithShader); + context.canvas->drawRect(SkRect::MakeWH(1, 1), paintWithShader); } else { - context->canvas->drawImage(image, 0, 0, sampling, context->sk_paint); + context.canvas->drawImage(image, 0, 0, sampling, context.sk_paint); } } } diff --git a/shell/platform/darwin/graphics/BUILD.gn b/shell/platform/darwin/graphics/BUILD.gn index 636c02ff7473c..fbb5a7a5b1035 100644 --- a/shell/platform/darwin/graphics/BUILD.gn +++ b/shell/platform/darwin/graphics/BUILD.gn @@ -12,8 +12,8 @@ source_set("graphics") { cflags_objcc = flutter_cflags_objcc_arc sources = [ - "FlutterDarwinContextMetal.h", - "FlutterDarwinContextMetal.mm", + "FlutterDarwinContextMetalSkia.h", + "FlutterDarwinContextMetalSkia.mm", "FlutterDarwinExternalTextureMetal.h", "FlutterDarwinExternalTextureMetal.mm", ] diff --git a/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h b/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h similarity index 90% rename from shell/platform/darwin/graphics/FlutterDarwinContextMetal.h rename to shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h index 2c93aff87f8c2..02eb2197332fe 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h +++ b/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h @@ -18,16 +18,16 @@ NS_ASSUME_NONNULL_BEGIN /** * Provides skia GrContexts that are shared between iOS and macOS embeddings. */ -@interface FlutterDarwinContextMetal : NSObject +@interface FlutterDarwinContextMetalSkia : NSObject /** - * Initializes a FlutterDarwinContextMetal with the system default MTLDevice and a new + * Initializes a FlutterDarwinContextMetalSkia with the system default MTLDevice and a new * MTLCommandQueue. */ - (instancetype)initWithDefaultMTLDevice; /** - * Initializes a FlutterDarwinContextMetal with provided MTLDevice and MTLCommandQueue. + * Initializes a FlutterDarwinContextMetalSkia with provided MTLDevice and MTLCommandQueue. */ - (instancetype)initWithMTLDevice:(id)device commandQueue:(id)commandQueue; diff --git a/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm b/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.mm similarity index 95% rename from shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm rename to shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.mm index 52f4c466cf0b3..c6e1321b19f8c 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm +++ b/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.mm @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h" +#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h" #include "flutter/common/graphics/persistent_cache.h" #include "flutter/fml/logging.h" @@ -11,7 +11,7 @@ FLUTTER_ASSERT_ARC -@implementation FlutterDarwinContextMetal +@implementation FlutterDarwinContextMetalSkia - (instancetype)initWithDefaultMTLDevice { id device = MTLCreateSystemDefaultDevice(); @@ -70,7 +70,7 @@ - (instancetype)initWithMTLDevice:(id)device flutter::MakeDefaultContextOptions(flutter::ContextType::kRender, GrBackendApi::kMetal); id device = _device; id commandQueue = _commandQueue; - return [FlutterDarwinContextMetal createGrContext:device commandQueue:commandQueue]; + return [FlutterDarwinContextMetalSkia createGrContext:device commandQueue:commandQueue]; } + (sk_sp)createGrContext:(id)device diff --git a/shell/platform/darwin/ios/ios_context_metal_impeller.h b/shell/platform/darwin/ios/ios_context_metal_impeller.h index 6efb530728de9..a46dc81cfed9c 100644 --- a/shell/platform/darwin/ios/ios_context_metal_impeller.h +++ b/shell/platform/darwin/ios/ios_context_metal_impeller.h @@ -6,8 +6,8 @@ #define FLUTTER_SHELL_PLATFORM_DARWIN_IOS_IOS_CONTEXT_METAL_IMPELER_H_ #include "flutter/fml/macros.h" -#include "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h" #include "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h" +#include "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h" #include "flutter/shell/platform/darwin/ios/ios_context.h" namespace impeller { @@ -24,7 +24,7 @@ class IOSContextMetalImpeller final : public IOSContext { ~IOSContextMetalImpeller(); - fml::scoped_nsobject GetDarwinContext() const; + fml::scoped_nsobject GetDarwinContext() const; IOSRenderingBackend GetBackend() const override; diff --git a/shell/platform/darwin/ios/ios_context_metal_impeller.mm b/shell/platform/darwin/ios/ios_context_metal_impeller.mm index bab3c96129984..fc011f652c4cf 100644 --- a/shell/platform/darwin/ios/ios_context_metal_impeller.mm +++ b/shell/platform/darwin/ios/ios_context_metal_impeller.mm @@ -15,8 +15,9 @@ IOSContextMetalImpeller::~IOSContextMetalImpeller() = default; -fml::scoped_nsobject IOSContextMetalImpeller::GetDarwinContext() const { - return fml::scoped_nsobject{}; +fml::scoped_nsobject IOSContextMetalImpeller::GetDarwinContext() + const { + return fml::scoped_nsobject{}; } IOSRenderingBackend IOSContextMetalImpeller::GetBackend() const { diff --git a/shell/platform/darwin/ios/ios_context_metal_skia.h b/shell/platform/darwin/ios/ios_context_metal_skia.h index 48661f7d18edb..42b6a8285f64b 100644 --- a/shell/platform/darwin/ios/ios_context_metal_skia.h +++ b/shell/platform/darwin/ios/ios_context_metal_skia.h @@ -10,7 +10,7 @@ #include "flutter/fml/macros.h" #include "flutter/fml/platform/darwin/cf_utils.h" #include "flutter/fml/platform/darwin/scoped_nsobject.h" -#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h" +#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h" #import "flutter/shell/platform/darwin/ios/ios_context.h" #include "third_party/skia/include/gpu/GrDirectContext.h" @@ -22,7 +22,7 @@ class IOSContextMetalSkia final : public IOSContext { ~IOSContextMetalSkia(); - fml::scoped_nsobject GetDarwinContext() const; + fml::scoped_nsobject GetDarwinContext() const; // |IOSContext| IOSRenderingBackend GetBackend() const override; @@ -33,7 +33,7 @@ class IOSContextMetalSkia final : public IOSContext { sk_sp GetResourceContext() const; private: - fml::scoped_nsobject darwin_context_metal_; + fml::scoped_nsobject darwin_context_metal_; // |IOSContext| sk_sp CreateResourceContext() override; diff --git a/shell/platform/darwin/ios/ios_context_metal_skia.mm b/shell/platform/darwin/ios/ios_context_metal_skia.mm index fb4ef48762237..1a4d8c8ed358e 100644 --- a/shell/platform/darwin/ios/ios_context_metal_skia.mm +++ b/shell/platform/darwin/ios/ios_context_metal_skia.mm @@ -6,20 +6,20 @@ #include "flutter/common/graphics/persistent_cache.h" #include "flutter/fml/logging.h" -#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h" +#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h" #import "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h" #include "third_party/skia/include/gpu/GrContextOptions.h" namespace flutter { IOSContextMetalSkia::IOSContextMetalSkia(MsaaSampleCount msaa_samples) : IOSContext(msaa_samples) { - darwin_context_metal_ = fml::scoped_nsobject{ - [[FlutterDarwinContextMetal alloc] initWithDefaultMTLDevice]}; + darwin_context_metal_ = fml::scoped_nsobject{ + [[FlutterDarwinContextMetalSkia alloc] initWithDefaultMTLDevice]}; } IOSContextMetalSkia::~IOSContextMetalSkia() = default; -fml::scoped_nsobject IOSContextMetalSkia::GetDarwinContext() const { +fml::scoped_nsobject IOSContextMetalSkia::GetDarwinContext() const { return darwin_context_metal_; } diff --git a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm index 305fcb88800b0..ed2032120f868 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm @@ -8,7 +8,7 @@ #include #include -#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h" +#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h" #import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.h" #include "flutter/shell/platform/embedder/embedder.h" @@ -71,8 +71,8 @@ - (CVPixelBufferRef)pixelBuffer { const int64_t texture_id = 1; // Set up the surface. - FlutterDarwinContextMetal* darwinContextMetal = - [[FlutterDarwinContextMetal alloc] initWithDefaultMTLDevice]; + FlutterDarwinContextMetalSkia* darwinContextMetal = + [[FlutterDarwinContextMetalSkia alloc] initWithDefaultMTLDevice]; SkImageInfo info = SkImageInfo::MakeN32Premul(width, height); GrDirectContext* grContext = darwinContextMetal.mainContext.get(); sk_sp gpuSurface(SkSurface::MakeRenderTarget(grContext, SkBudgeted::kNo, info)); @@ -129,8 +129,8 @@ - (CVPixelBufferRef)pixelBuffer { const int64_t texture_id = 1; // Set up the surface. - FlutterDarwinContextMetal* darwinContextMetal = - [[FlutterDarwinContextMetal alloc] initWithDefaultMTLDevice]; + FlutterDarwinContextMetalSkia* darwinContextMetal = + [[FlutterDarwinContextMetalSkia alloc] initWithDefaultMTLDevice]; SkImageInfo info = SkImageInfo::MakeN32Premul(width, height); GrDirectContext* grContext = darwinContextMetal.mainContext.get(); sk_sp gpuSurface(SkSurface::MakeRenderTarget(grContext, SkBudgeted::kNo, info)); @@ -181,8 +181,8 @@ - (CVPixelBufferRef)pixelBuffer { const int64_t texture_id = 1; // Set up the surface. - FlutterDarwinContextMetal* darwinContextMetal = - [[FlutterDarwinContextMetal alloc] initWithDefaultMTLDevice]; + FlutterDarwinContextMetalSkia* darwinContextMetal = + [[FlutterDarwinContextMetalSkia alloc] initWithDefaultMTLDevice]; SkImageInfo info = SkImageInfo::MakeN32Premul(width, height); GrDirectContext* grContext = darwinContextMetal.mainContext.get(); sk_sp gpuSurface(SkSurface::MakeRenderTarget(grContext, SkBudgeted::kNo, info)); @@ -233,8 +233,8 @@ - (CVPixelBufferRef)pixelBuffer { const int64_t texture_id = 1; // Set up the surface. - FlutterDarwinContextMetal* darwinContextMetal = - [[FlutterDarwinContextMetal alloc] initWithDefaultMTLDevice]; + FlutterDarwinContextMetalSkia* darwinContextMetal = + [[FlutterDarwinContextMetalSkia alloc] initWithDefaultMTLDevice]; SkImageInfo info = SkImageInfo::MakeN32Premul(width, height); GrDirectContext* grContext = darwinContextMetal.mainContext.get(); sk_sp gpuSurface(SkSurface::MakeRenderTarget(grContext, SkBudgeted::kNo, info)); diff --git a/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.h b/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.h index df0db4b1dd95f..8942d1ff71062 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.h +++ b/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.h @@ -4,7 +4,7 @@ #import -#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h" +#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterMacOSExternalTexture.h" /** @@ -17,7 +17,7 @@ * Initializes a texture adapter with |texture|. */ - (nonnull instancetype)initWithFlutterTexture:(nonnull id)texture - darwinMetalContext:(nonnull FlutterDarwinContextMetal*)context; + darwinMetalContext:(nonnull FlutterDarwinContextMetalSkia*)context; /** * Accepts texture buffer copy request from the Flutter engine. diff --git a/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.mm b/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.mm index 1ece011425d3c..6ca826d704f6d 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterExternalTextureMetal.mm @@ -7,7 +7,7 @@ #include "flutter/fml/platform/darwin/cf_utils.h" @implementation FlutterExternalTextureMetal { - FlutterDarwinContextMetal* _darwinMetalContext; + FlutterDarwinContextMetalSkia* _darwinMetalContext; int64_t _textureID; @@ -17,7 +17,7 @@ @implementation FlutterExternalTextureMetal { } - (instancetype)initWithFlutterTexture:(id)texture - darwinMetalContext:(FlutterDarwinContextMetal*)context { + darwinMetalContext:(FlutterDarwinContextMetalSkia*)context { self = [super init]; if (self) { _texture = texture; diff --git a/shell/platform/darwin/macos/framework/Source/FlutterMetalRenderer.mm b/shell/platform/darwin/macos/framework/Source/FlutterMetalRenderer.mm index 2347d06fa098a..c5305c683baa2 100644 --- a/shell/platform/darwin/macos/framework/Source/FlutterMetalRenderer.mm +++ b/shell/platform/darwin/macos/framework/Source/FlutterMetalRenderer.mm @@ -40,7 +40,7 @@ @implementation FlutterMetalRenderer { FlutterView* _flutterView; - FlutterDarwinContextMetal* _darwinMetalContext; + FlutterDarwinContextMetalSkia* _darwinMetalContext; } - (instancetype)initWithFlutterEngine:(nonnull FlutterEngine*)flutterEngine { @@ -60,8 +60,8 @@ - (instancetype)initWithFlutterEngine:(nonnull FlutterEngine*)flutterEngine { return nil; } - _darwinMetalContext = [[FlutterDarwinContextMetal alloc] initWithMTLDevice:_device - commandQueue:_commandQueue]; + _darwinMetalContext = [[FlutterDarwinContextMetalSkia alloc] initWithMTLDevice:_device + commandQueue:_commandQueue]; } return self; } diff --git a/shell/platform/embedder/embedder_surface_metal.mm b/shell/platform/embedder/embedder_surface_metal.mm index afa11d62f610b..b7582d01b5c2c 100644 --- a/shell/platform/embedder/embedder_surface_metal.mm +++ b/shell/platform/embedder/embedder_surface_metal.mm @@ -8,7 +8,7 @@ #include "flutter/fml/logging.h" #include "flutter/shell/gpu/gpu_surface_metal_delegate.h" -#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h" +#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h" #include "third_party/skia/include/gpu/GrDirectContext.h" FLUTTER_ASSERT_NOT_ARC @@ -22,11 +22,12 @@ : GPUSurfaceMetalDelegate(MTLRenderTargetType::kMTLTexture), metal_dispatch_table_(std::move(metal_dispatch_table)), external_view_embedder_(std::move(external_view_embedder)) { - main_context_ = [FlutterDarwinContextMetal createGrContext:(id)device - commandQueue:(id)command_queue]; + main_context_ = + [FlutterDarwinContextMetalSkia createGrContext:(id)device + commandQueue:(id)command_queue]; resource_context_ = - [FlutterDarwinContextMetal createGrContext:(id)device - commandQueue:(id)command_queue]; + [FlutterDarwinContextMetalSkia createGrContext:(id)device + commandQueue:(id)command_queue]; valid_ = main_context_ && resource_context_; } From 134cd0cc4f964b0797f413eeaceb21452900db9a Mon Sep 17 00:00:00 2001 From: ColdPaleLight Date: Fri, 30 Sep 2022 22:20:55 +0800 Subject: [PATCH 05/12] format --- ci/licenses_golden/licenses_flutter | 4 ++-- .../graphics/FlutterDarwinExternalTextureMetal.mm | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index b83e91ef8d37d..baeaec8768873 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1792,10 +1792,10 @@ FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/FlutterStan FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/FlutterStandardCodec_Internal.h FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/flutter_codecs_unittest.mm FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/flutter_standard_codec_unittest.mm -FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h -FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.mm FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.h FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalImpeller.mm +FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.h +FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetalSkia.mm FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Headers/Flutter.h diff --git a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm index 32117058b5589..8b31da85f6b55 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm +++ b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm @@ -71,11 +71,11 @@ - (void)paintContext:(flutter::Texture::PaintContext&)context #if IMPELLER_SUPPORTS_RENDERING if (_enableImpeller) { context.builder->drawImageRect( - _externalImage, // image - SkRect::Make(_externalImage->bounds()), // source rect - bounds, // destination rect - flutter::ToDl(sampling), // sampling - context.dl_paint, + _externalImage, // image + SkRect::Make(_externalImage->bounds()), // source rect + bounds, // destination rect + flutter::ToDl(sampling), // sampling + context.dl_paint, // paint SkCanvas::SrcRectConstraint::kFast_SrcRectConstraint // constraint ); return; From 440769f94e601069df20e03c01b88708ad2fbadd Mon Sep 17 00:00:00 2001 From: ColdPaleLight Date: Fri, 7 Oct 2022 09:10:44 +0800 Subject: [PATCH 06/12] Clean code --- common/graphics/texture.h | 1 - impeller/renderer/texture.h | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/common/graphics/texture.h b/common/graphics/texture.h index da17bfadeeca1..585b3c14bb12d 100644 --- a/common/graphics/texture.h +++ b/common/graphics/texture.h @@ -41,7 +41,6 @@ class Texture : public ContextListener { GrDirectContext* gr_context = nullptr; const SkPaint* sk_paint = nullptr; const DlPaint* dl_paint = nullptr; - bool enable_impeller = false; }; explicit Texture(int64_t id); // Called from UI or raster thread. diff --git a/impeller/renderer/texture.h b/impeller/renderer/texture.h index 8ac7b7b0d1fa9..4491e06f859e9 100644 --- a/impeller/renderer/texture.h +++ b/impeller/renderer/texture.h @@ -53,7 +53,7 @@ class Texture { size_t slice) = 0; private: - TextureIntent intent_ = TextureIntent::kUploadFromHost; + TextureIntent intent_ = TextureIntent::kRenderToTexture; const TextureDescriptor desc_; bool IsSliceValid(size_t slice) const; From da611f656433e866f07c74a9469e56f73cbd1ee0 Mon Sep 17 00:00:00 2001 From: ColdPaleLight Date: Mon, 10 Oct 2022 21:27:24 +0800 Subject: [PATCH 07/12] [Impeller] Support YUV external textures on iOS --- ci/licenses_golden/licenses_flutter | 4 + common/graphics/texture.h | 5 + flow/compositor_context.cc | 9 +- flow/compositor_context.h | 9 +- flow/layers/layer.h | 1 + flow/layers/layer_tree.cc | 1 + flow/layers/layer_tree_unittests.cc | 1 + flow/layers/texture_layer.cc | 1 + impeller/aiks/aiks_context.cc | 4 + impeller/aiks/aiks_context.h | 2 + .../display_list_image_impeller.cc | 20 +++ .../display_list_image_impeller.h | 8 ++ impeller/entity/BUILD.gn | 4 + impeller/entity/contents/content_context.cc | 2 + impeller/entity/contents/content_context.h | 10 ++ .../contents/filters/filter_contents.cc | 12 ++ .../entity/contents/filters/filter_contents.h | 5 + .../filters/yuv_to_rgb_filter_contents.cc | 124 ++++++++++++++++++ .../filters/yuv_to_rgb_filter_contents.h | 33 +++++ impeller/entity/entity_unittests.cc | 94 +++++++++++++ .../entity/shaders/yuv_to_rgb_filter.frag | 35 +++++ .../entity/shaders/yuv_to_rgb_filter.vert | 15 +++ impeller/geometry/color.h | 2 + .../renderer/backend/gles/texture_gles.cc | 26 ++++ impeller/renderer/backend/metal/formats_mtl.h | 4 + impeller/renderer/backend/vulkan/formats_vk.h | 10 ++ impeller/renderer/formats.h | 5 + shell/common/rasterizer.cc | 12 +- .../FlutterDarwinExternalTextureMetal.mm | 37 ++++-- 29 files changed, 476 insertions(+), 19 deletions(-) create mode 100644 impeller/entity/contents/filters/yuv_to_rgb_filter_contents.cc create mode 100644 impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h create mode 100644 impeller/entity/shaders/yuv_to_rgb_filter.frag create mode 100644 impeller/entity/shaders/yuv_to_rgb_filter.vert diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index f7c729b021c1d..b6770b611d486 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1200,6 +1200,8 @@ FILE: ../../../flutter/impeller/entity/contents/filters/morphology_filter_conten FILE: ../../../flutter/impeller/entity/contents/filters/morphology_filter_contents.h FILE: ../../../flutter/impeller/entity/contents/filters/srgb_to_linear_filter_contents.cc FILE: ../../../flutter/impeller/entity/contents/filters/srgb_to_linear_filter_contents.h +FILE: ../../../flutter/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.cc +FILE: ../../../flutter/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h FILE: ../../../flutter/impeller/entity/contents/gradient_generator.cc FILE: ../../../flutter/impeller/entity/contents/gradient_generator.h FILE: ../../../flutter/impeller/entity/contents/linear_gradient_contents.cc @@ -1288,6 +1290,8 @@ FILE: ../../../flutter/impeller/entity/shaders/texture_fill.vert FILE: ../../../flutter/impeller/entity/shaders/tiled_texture_fill.frag FILE: ../../../flutter/impeller/entity/shaders/tiled_texture_fill.vert FILE: ../../../flutter/impeller/entity/shaders/vertices.frag +FILE: ../../../flutter/impeller/entity/shaders/yuv_to_rgb_filter.frag +FILE: ../../../flutter/impeller/entity/shaders/yuv_to_rgb_filter.vert FILE: ../../../flutter/impeller/geometry/color.cc FILE: ../../../flutter/impeller/geometry/color.h FILE: ../../../flutter/impeller/geometry/constants.cc diff --git a/common/graphics/texture.h b/common/graphics/texture.h index 585b3c14bb12d..16a97658d9fed 100644 --- a/common/graphics/texture.h +++ b/common/graphics/texture.h @@ -16,6 +16,10 @@ class GrDirectContext; +namespace impeller { +class AiksContext; +}; + namespace flutter { class ContextListener { @@ -39,6 +43,7 @@ class Texture : public ContextListener { SkCanvas* canvas = nullptr; DisplayListBuilder* builder = nullptr; GrDirectContext* gr_context = nullptr; + impeller::AiksContext* aiks_context = nullptr; const SkPaint* sk_paint = nullptr; const DlPaint* dl_paint = nullptr; }; diff --git a/flow/compositor_context.cc b/flow/compositor_context.cc index b40df6a186dab..5a0a95f206511 100644 --- a/flow/compositor_context.cc +++ b/flow/compositor_context.cc @@ -82,11 +82,12 @@ std::unique_ptr CompositorContext::AcquireFrame( bool instrumentation_enabled, bool surface_supports_readback, fml::RefPtr raster_thread_merger, - DisplayListBuilder* display_list_builder) { + DisplayListBuilder* display_list_builder, + impeller::AiksContext* aiks_context) { return std::make_unique( *this, gr_context, canvas, view_embedder, root_surface_transformation, instrumentation_enabled, surface_supports_readback, raster_thread_merger, - display_list_builder); + display_list_builder, aiks_context); } CompositorContext::ScopedFrame::ScopedFrame( @@ -98,11 +99,13 @@ CompositorContext::ScopedFrame::ScopedFrame( bool instrumentation_enabled, bool surface_supports_readback, fml::RefPtr raster_thread_merger, - DisplayListBuilder* display_list_builder) + DisplayListBuilder* display_list_builder, + impeller::AiksContext* aiks_context) : context_(context), gr_context_(gr_context), canvas_(canvas), display_list_builder_(display_list_builder), + aiks_context_(aiks_context), view_embedder_(view_embedder), root_surface_transformation_(root_surface_transformation), instrumentation_enabled_(instrumentation_enabled), diff --git a/flow/compositor_context.h b/flow/compositor_context.h index 68535a6194061..3e9552bd94f7f 100644 --- a/flow/compositor_context.h +++ b/flow/compositor_context.h @@ -114,7 +114,8 @@ class CompositorContext { bool instrumentation_enabled, bool surface_supports_readback, fml::RefPtr raster_thread_merger, - DisplayListBuilder* display_list_builder); + DisplayListBuilder* display_list_builder, + impeller::AiksContext* aiks_context); virtual ~ScopedFrame(); @@ -136,6 +137,8 @@ class CompositorContext { GrDirectContext* gr_context() const { return gr_context_; } + impeller::AiksContext* aiks_context() const { return aiks_context_; } + virtual RasterStatus Raster(LayerTree& layer_tree, bool ignore_raster_cache, FrameDamage* frame_damage); @@ -145,6 +148,7 @@ class CompositorContext { GrDirectContext* gr_context_; SkCanvas* canvas_; DisplayListBuilder* display_list_builder_; + impeller::AiksContext* aiks_context_; ExternalViewEmbedder* view_embedder_; const SkMatrix& root_surface_transformation_; const bool instrumentation_enabled_; @@ -168,7 +172,8 @@ class CompositorContext { bool instrumentation_enabled, bool surface_supports_readback, fml::RefPtr raster_thread_merger, - DisplayListBuilder* display_list_builder); + DisplayListBuilder* display_list_builder, + impeller::AiksContext* aiks_context); void OnGrContextCreated(); diff --git a/flow/layers/layer.h b/flow/layers/layer.h index 09f47b5de4d50..0e7100f45255b 100644 --- a/flow/layers/layer.h +++ b/flow/layers/layer.h @@ -154,6 +154,7 @@ struct PaintContext { SkScalar inherited_opacity = SK_Scalar1; DisplayListBuilder* leaf_nodes_builder = nullptr; DisplayListBuilderMultiplexer* builder_multiplexer = nullptr; + impeller::AiksContext* aiks_context; }; // Represents a single composited layer. Created on the UI thread but then diff --git a/flow/layers/layer_tree.cc b/flow/layers/layer_tree.cc index 0e5aacc98fe92..6eccffefda3ad 100644 --- a/flow/layers/layer_tree.cc +++ b/flow/layers/layer_tree.cc @@ -158,6 +158,7 @@ void LayerTree::Paint(CompositorContext::ScopedFrame& frame, .inherited_opacity = SK_Scalar1, .leaf_nodes_builder = builder, .builder_multiplexer = builder ? &builder_multiplexer : nullptr, + .aiks_context = frame.aiks_context(), // clang-format on }; diff --git a/flow/layers/layer_tree_unittests.cc b/flow/layers/layer_tree_unittests.cc index 950e542c3a685..882e2039a47a1 100644 --- a/flow/layers/layer_tree_unittests.cc +++ b/flow/layers/layer_tree_unittests.cc @@ -28,6 +28,7 @@ class LayerTreeTest : public CanvasTest { false, true, nullptr, + nullptr, nullptr)) {} LayerTree& layer_tree() { return layer_tree_; } diff --git a/flow/layers/texture_layer.cc b/flow/layers/texture_layer.cc index 8a98366f2c254..dcb099098d0c3 100644 --- a/flow/layers/texture_layer.cc +++ b/flow/layers/texture_layer.cc @@ -66,6 +66,7 @@ void TextureLayer::Paint(PaintContext& context) const { .canvas = context.leaf_nodes_canvas, .builder = context.leaf_nodes_builder, .gr_context = context.gr_context, + .aiks_context = context.aiks_context, .sk_paint = cache_paint.sk_paint(), .dl_paint = cache_paint.dl_paint(), }; diff --git a/impeller/aiks/aiks_context.cc b/impeller/aiks/aiks_context.cc index a1c8b32308e17..fd504cbe03ca3 100644 --- a/impeller/aiks/aiks_context.cc +++ b/impeller/aiks/aiks_context.cc @@ -32,6 +32,10 @@ std::shared_ptr AiksContext::GetContext() const { return context_; } +const ContentContext& AiksContext::GetContentContext() const { + return *content_context_; +} + bool AiksContext::Render(const Picture& picture, RenderTarget& render_target) { if (!IsValid()) { return false; diff --git a/impeller/aiks/aiks_context.h b/impeller/aiks/aiks_context.h index 7f7780ffb69ab..d748f3156fcba 100644 --- a/impeller/aiks/aiks_context.h +++ b/impeller/aiks/aiks_context.h @@ -26,6 +26,8 @@ class AiksContext { std::shared_ptr GetContext() const; + const ContentContext& GetContentContext() const; + bool Render(const Picture& picture, RenderTarget& render_target); private: diff --git a/impeller/display_list/display_list_image_impeller.cc b/impeller/display_list/display_list_image_impeller.cc index f964010b9ae90..ae71d94132578 100644 --- a/impeller/display_list/display_list_image_impeller.cc +++ b/impeller/display_list/display_list_image_impeller.cc @@ -4,6 +4,9 @@ #include "impeller/display_list/display_list_image_impeller.h" +#include "impeller/aiks/aiks_context.h" +#include "impeller/entity/contents/filters/filter_contents.h" + namespace impeller { sk_sp DlImageImpeller::Make(std::shared_ptr texture) { @@ -13,6 +16,23 @@ sk_sp DlImageImpeller::Make(std::shared_ptr texture) { return sk_sp(new DlImageImpeller(std::move(texture))); } +sk_sp DlImageImpeller::MakeFromYUVTextures( + AiksContext* aiks_context, + std::shared_ptr y_texture, + std::shared_ptr uv_texture, + YUVColorSpace yuv_color_space) { + if (!aiks_context || !y_texture || !uv_texture) { + return nullptr; + } + auto yuv_to_rgb_filter_contents = FilterContents::MakeYUVToRGBFilter( + std::move(y_texture), std::move(uv_texture), yuv_color_space); + impeller::Entity entity; + entity.SetBlendMode(impeller::BlendMode::kSource); + auto snapshot = yuv_to_rgb_filter_contents->RenderToSnapshot( + aiks_context->GetContentContext(), entity); + return impeller::DlImageImpeller::Make(snapshot->texture); +} + DlImageImpeller::DlImageImpeller(std::shared_ptr texture) : texture_(std::move(texture)) {} diff --git a/impeller/display_list/display_list_image_impeller.h b/impeller/display_list/display_list_image_impeller.h index cf7ae7501eb69..65b9c2892bc2a 100644 --- a/impeller/display_list/display_list_image_impeller.h +++ b/impeller/display_list/display_list_image_impeller.h @@ -10,10 +10,18 @@ namespace impeller { +class AiksContext; + class DlImageImpeller final : public flutter::DlImage { public: static sk_sp Make(std::shared_ptr texture); + static sk_sp MakeFromYUVTextures( + AiksContext* aiks_context, + std::shared_ptr y_texture, + std::shared_ptr uv_texture, + YUVColorSpace yuv_color_space); + // |DlImage| ~DlImageImpeller() override; diff --git a/impeller/entity/BUILD.gn b/impeller/entity/BUILD.gn index bce02deec6153..71f036b72e8e7 100644 --- a/impeller/entity/BUILD.gn +++ b/impeller/entity/BUILD.gn @@ -60,6 +60,8 @@ impeller_shaders("entity_shaders") { "shaders/position_color.vert", "shaders/position.vert", "shaders/position_uv.vert", + "shaders/yuv_to_rgb_filter.frag", + "shaders/yuv_to_rgb_filter.vert", ] } @@ -103,6 +105,8 @@ impeller_component("entity") { "contents/filters/morphology_filter_contents.h", "contents/filters/srgb_to_linear_filter_contents.cc", "contents/filters/srgb_to_linear_filter_contents.h", + "contents/filters/yuv_to_rgb_filter_contents.cc", + "contents/filters/yuv_to_rgb_filter_contents.h", "contents/gradient_generator.cc", "contents/gradient_generator.h", "contents/linear_gradient_contents.cc", diff --git a/impeller/entity/contents/content_context.cc b/impeller/entity/contents/content_context.cc index 0e514334f358d..b172466ebf8a1 100644 --- a/impeller/entity/contents/content_context.cc +++ b/impeller/entity/contents/content_context.cc @@ -213,6 +213,8 @@ ContentContext::ContentContext(std::shared_ptr context) geometry_position_pipelines_[{}] = CreateDefaultPipeline(*context_); atlas_pipelines_[{}] = CreateDefaultPipeline(*context_); + yuv_to_rgb_filter_pipelines_[{}] = + CreateDefaultPipeline(*context_); // Pipelines that are variants of the base pipelines with custom descriptors. // TODO(98684): Rework this API to allow fetching the descriptor without diff --git a/impeller/entity/contents/content_context.h b/impeller/entity/contents/content_context.h index 8601b4e80c92f..bc9530cc0954d 100644 --- a/impeller/entity/contents/content_context.h +++ b/impeller/entity/contents/content_context.h @@ -61,6 +61,8 @@ #include "impeller/entity/tiled_texture_fill.frag.h" #include "impeller/entity/tiled_texture_fill.vert.h" #include "impeller/entity/vertices.frag.h" +#include "impeller/entity/yuv_to_rgb_filter.frag.h" +#include "impeller/entity/yuv_to_rgb_filter.vert.h" #include "impeller/renderer/formats.h" #include "impeller/renderer/pipeline.h" @@ -158,6 +160,8 @@ using GeometryPositionPipeline = RenderPipelineT; using GeometryColorPipeline = RenderPipelineT; +using YUVToRGBFilterPipeline = + RenderPipelineT; struct ContentContextOptions { SampleCount sample_count = SampleCount::kCount1; @@ -297,6 +301,11 @@ class ContentContext { return GetPipeline(atlas_pipelines_, opts); } + std::shared_ptr> GetYUVToRGBFilterPipeline( + ContentContextOptions opts) const { + return GetPipeline(yuv_to_rgb_filter_pipelines_, opts); + } + // Advanced blends. std::shared_ptr> GetBlendColorPipeline( @@ -417,6 +426,7 @@ class ContentContext { mutable Variants atlas_pipelines_; mutable Variants geometry_position_pipelines_; mutable Variants geometry_color_pipelines_; + mutable Variants yuv_to_rgb_filter_pipelines_; // Advanced blends. mutable Variants blend_color_pipelines_; mutable Variants blend_colorburn_pipelines_; diff --git a/impeller/entity/contents/filters/filter_contents.cc b/impeller/entity/contents/filters/filter_contents.cc index 06f49b2557055..a4d5ce182d496 100644 --- a/impeller/entity/contents/filters/filter_contents.cc +++ b/impeller/entity/contents/filters/filter_contents.cc @@ -24,6 +24,7 @@ #include "impeller/entity/contents/filters/matrix_filter_contents.h" #include "impeller/entity/contents/filters/morphology_filter_contents.h" #include "impeller/entity/contents/filters/srgb_to_linear_filter_contents.h" +#include "impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h" #include "impeller/entity/contents/texture_contents.h" #include "impeller/entity/entity.h" #include "impeller/geometry/path_builder.h" @@ -197,6 +198,17 @@ std::shared_ptr FilterContents::MakeLocalMatrixFilter( return filter; } +std::shared_ptr FilterContents::MakeYUVToRGBFilter( + std::shared_ptr y_texture, + std::shared_ptr uv_texture, + YUVColorSpace yuv_color_space) { + auto filter = std::make_shared(); + filter->SetInputs({impeller::FilterInput::Make(y_texture), + impeller::FilterInput::Make(uv_texture)}); + filter->SetYUVColorSpace(yuv_color_space); + return filter; +} + FilterContents::FilterContents() = default; FilterContents::~FilterContents() = default; diff --git a/impeller/entity/contents/filters/filter_contents.h b/impeller/entity/contents/filters/filter_contents.h index af1f1d4b88b35..fe27e4ec4d38a 100644 --- a/impeller/entity/contents/filters/filter_contents.h +++ b/impeller/entity/contents/filters/filter_contents.h @@ -99,6 +99,11 @@ class FilterContents : public Contents { FilterInput::Ref input, const Matrix& matrix); + static std::shared_ptr MakeYUVToRGBFilter( + std::shared_ptr y_texture, + std::shared_ptr uv_texture, + YUVColorSpace yuv_color_space); + FilterContents(); ~FilterContents() override; diff --git a/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.cc b/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.cc new file mode 100644 index 0000000000000..b00a4b92c8228 --- /dev/null +++ b/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.cc @@ -0,0 +1,124 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h" + +#include "impeller/entity/contents/content_context.h" +#include "impeller/geometry/matrix.h" +#include "impeller/renderer/formats.h" +#include "impeller/renderer/render_pass.h" +#include "impeller/renderer/sampler_library.h" + +namespace impeller { + +// clang-format off +constexpr Matrix kMatrixBT601LimitedRange = { + 1.164, 1.164, 1.164, 0.0, + 0.0, -0.392, 2.017, 0.0, + 1.596, -0.813, 0.0, 0.0, + 0.0, 0.0, 0.0, 1.0 +}; + +constexpr Matrix kMatrixBT601FullRange = { + 1.0, 1.0, 1.0, 0.0, + 0.0, -0.344, 1.772, 0.0, + 1.402, -0.714, 0.0, 0.0, + 0.0, 0.0, 0.0, 1.0 +}; +// clang-format on + +YUVToRGBFilterContents::YUVToRGBFilterContents() = default; + +YUVToRGBFilterContents::~YUVToRGBFilterContents() = default; + +void YUVToRGBFilterContents::SetYUVColorSpace(YUVColorSpace yuv_color_space) { + yuv_color_space_ = yuv_color_space; +} + +std::optional YUVToRGBFilterContents::RenderFilter( + const FilterInput::Vector& inputs, + const ContentContext& renderer, + const Entity& entity, + const Matrix& effect_transform, + const Rect& coverage) const { + if (inputs.size() < 2) { + return std::nullopt; + } + + using VS = YUVToRGBFilterPipeline::VertexShader; + using FS = YUVToRGBFilterPipeline::FragmentShader; + + auto y_input_snapshot = inputs[0]->GetSnapshot(renderer, entity); + auto uv_input_snapshot = inputs[1]->GetSnapshot(renderer, entity); + if (!y_input_snapshot.has_value() || !uv_input_snapshot.has_value()) { + return std::nullopt; + } + + if (y_input_snapshot->texture->GetTextureDescriptor().format != + PixelFormat::kR8UNormInt || + uv_input_snapshot->texture->GetTextureDescriptor().format != + PixelFormat::kR8G8UNormInt) { + return std::nullopt; + } + + ContentContext::SubpassCallback callback = [&](const ContentContext& renderer, + RenderPass& pass) { + Command cmd; + cmd.label = "YUV to RGB Filter"; + + auto options = OptionsFromPass(pass); + options.blend_mode = BlendMode::kSource; + cmd.pipeline = renderer.GetYUVToRGBFilterPipeline(options); + + VertexBufferBuilder vtx_builder; + vtx_builder.AddVertices({ + {Point(0, 0)}, + {Point(1, 0)}, + {Point(1, 1)}, + {Point(0, 0)}, + {Point(1, 1)}, + {Point(0, 1)}, + }); + + auto& host_buffer = pass.GetTransientsBuffer(); + auto vtx_buffer = vtx_builder.CreateVertexBuffer(host_buffer); + cmd.BindVertices(vtx_buffer); + + VS::FrameInfo frame_info; + frame_info.mvp = Matrix::MakeOrthographic(ISize(1, 1)); + + FS::FragInfo frag_info; + frag_info.texture_sampler_y_coord_scale = + y_input_snapshot->texture->GetYCoordScale(); + frag_info.yuv_color_space = static_cast(yuv_color_space_); + switch (yuv_color_space_) { + case YUVColorSpace::kBT601LimitedRange: + frag_info.matrix = kMatrixBT601LimitedRange; + break; + case YUVColorSpace::kBT601FullRange: + frag_info.matrix = kMatrixBT601FullRange; + break; + } + + auto sampler = renderer.GetContext()->GetSamplerLibrary()->GetSampler({}); + FS::BindYTexture(cmd, y_input_snapshot->texture, sampler); + FS::BindUvTexture(cmd, uv_input_snapshot->texture, sampler); + + FS::BindFragInfo(cmd, host_buffer.EmplaceUniform(frag_info)); + VS::BindFrameInfo(cmd, host_buffer.EmplaceUniform(frame_info)); + + return pass.AddCommand(std::move(cmd)); + }; + + auto out_texture = + renderer.MakeSubpass(y_input_snapshot->texture->GetSize(), callback); + if (!out_texture) { + return std::nullopt; + } + out_texture->SetLabel("YUVToRGB Texture"); + + return Snapshot{.texture = out_texture}; +} + +} // namespace impeller diff --git a/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h b/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h new file mode 100644 index 0000000000000..7e15cb5efce97 --- /dev/null +++ b/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h @@ -0,0 +1,33 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include "impeller/entity/contents/filters/filter_contents.h" + +namespace impeller { + +class YUVToRGBFilterContents final : public FilterContents { + public: + YUVToRGBFilterContents(); + + ~YUVToRGBFilterContents() override; + + void SetYUVColorSpace(YUVColorSpace yuv_color_space); + + private: + // |FilterContents| + std::optional RenderFilter( + const FilterInput::Vector& input_textures, + const ContentContext& renderer, + const Entity& entity, + const Matrix& effect_transform, + const Rect& coverage) const override; + + YUVColorSpace yuv_color_space_ = YUVColorSpace::kBT601LimitedRange; + + FML_DISALLOW_COPY_AND_ASSIGN(YUVToRGBFilterContents); +}; + +} // namespace impeller diff --git a/impeller/entity/entity_unittests.cc b/impeller/entity/entity_unittests.cc index b5b2e50a6f91b..f62751d25d26a 100644 --- a/impeller/entity/entity_unittests.cc +++ b/impeller/entity/entity_unittests.cc @@ -2023,5 +2023,99 @@ TEST_P(EntityTest, SdfText) { ASSERT_TRUE(OpenPlaygroundHere(callback)); } +static Vector3 RGBToYUV(Vector3 rgb, YUVColorSpace yuv_color_space) { + Vector3 yuv; + switch (yuv_color_space) { + case YUVColorSpace::kBT601FullRange: + yuv.x = rgb.x * 0.299 + rgb.y * 0.587 + rgb.z * 0.114; + yuv.y = rgb.x * -0.169 + rgb.y * -0.331 + rgb.z * 0.5 + 0.5; + yuv.z = rgb.x * 0.5 + rgb.y * -0.419 + rgb.z * -0.081 + 0.5; + case YUVColorSpace::kBT601LimitedRange: + yuv.x = rgb.x * 0.257 + rgb.y * 0.516 + rgb.z * 0.100 + 0.063; + yuv.y = rgb.x * -0.145 + rgb.y * -0.291 + rgb.z * 0.439 + 0.5; + yuv.z = rgb.x * 0.429 + rgb.y * -0.368 + rgb.z * -0.071 + 0.5; + } + return yuv; +} + +static std::vector> CreateTestYUVTextures( + Context* context, + YUVColorSpace yuv_color_space) { + Vector3 red = {244.0 / 255.0, 67.0 / 255.0, 54.0 / 255.0}; + Vector3 green = {76.0 / 255.0, 175.0 / 255.0, 80.0 / 255.0}; + Vector3 blue = {33.0 / 255.0, 150.0 / 255.0, 243.0 / 255.0}; + Vector3 white = {1.0, 1.0, 1.0}; + Vector3 red_yuv = RGBToYUV(red, yuv_color_space); + Vector3 green_yuv = RGBToYUV(green, yuv_color_space); + Vector3 blue_yuv = RGBToYUV(blue, yuv_color_space); + Vector3 white_yuv = RGBToYUV(white, yuv_color_space); + std::vector yuvs{red_yuv, green_yuv, blue_yuv, white_yuv}; + std::vector y_data; + std::vector uv_data; + for (int i = 0; i < 4; i++) { + auto yuv = yuvs[i]; + uint8_t y = std::round(yuv.x * 255.0); + uint8_t u = std::round(yuv.y * 255.0); + uint8_t v = std::round(yuv.z * 255.0); + for (int j = 0; j < 16; j++) { + y_data.push_back(y); + } + for (int j = 0; j < 8; j++) { + uv_data.push_back(j % 2 == 0 ? u : v); + } + } + impeller::TextureDescriptor y_texture_descriptor; + y_texture_descriptor.storage_mode = impeller::StorageMode::kHostVisible; + y_texture_descriptor.format = PixelFormat::kR8UNormInt; + y_texture_descriptor.size = {8, 8}; + auto y_texture = + context->GetResourceAllocator()->CreateTexture(y_texture_descriptor); + auto y_mapping = std::make_shared(y_data); + if (!y_texture->SetContents(y_mapping)) { + FML_DLOG(ERROR) << "Could not copy contents into Y texture."; + } + + impeller::TextureDescriptor uv_texture_descriptor; + uv_texture_descriptor.storage_mode = impeller::StorageMode::kHostVisible; + uv_texture_descriptor.format = PixelFormat::kR8G8UNormInt; + uv_texture_descriptor.size = {4, 4}; + auto uv_texture = + context->GetResourceAllocator()->CreateTexture(uv_texture_descriptor); + auto uv_mapping = std::make_shared(uv_data); + if (!uv_texture->SetContents(uv_mapping)) { + FML_DLOG(ERROR) << "Could not copy contents into UV texture."; + } + + return {y_texture, uv_texture}; +} + +TEST_P(EntityTest, YUVToRGBFilter) { + auto callback = [&](ContentContext& context, RenderPass& pass) -> bool { + YUVColorSpace yuv_color_space_array[2]{YUVColorSpace::kBT601FullRange, + YUVColorSpace::kBT601LimitedRange}; + for (int i = 0; i < 2; i++) { + auto yuv_color_space = yuv_color_space_array[i]; + auto textures = + CreateTestYUVTextures(GetContext().get(), yuv_color_space); + auto filter_contents = FilterContents::MakeYUVToRGBFilter( + textures[0], textures[1], yuv_color_space); + Entity filter_entity; + filter_entity.SetContents(filter_contents); + auto snapshot = filter_contents->RenderToSnapshot(context, filter_entity); + + Entity entity; + auto contents = TextureContents::MakeRect(Rect::MakeLTRB(0, 0, 256, 256)); + contents->SetTexture(snapshot->texture); + contents->SetSourceRect(Rect::MakeSize(snapshot->texture->GetSize())); + entity.SetContents(contents); + entity.SetTransformation( + Matrix::MakeTranslation({static_cast(100 + 400 * i), 300})); + entity.Render(context, pass); + } + return true; + }; + ASSERT_TRUE(OpenPlaygroundHere(callback)); +} + } // namespace testing } // namespace impeller diff --git a/impeller/entity/shaders/yuv_to_rgb_filter.frag b/impeller/entity/shaders/yuv_to_rgb_filter.frag new file mode 100644 index 0000000000000..b734b486024ab --- /dev/null +++ b/impeller/entity/shaders/yuv_to_rgb_filter.frag @@ -0,0 +1,35 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include + +uniform sampler2D y_texture; +uniform sampler2D uv_texture; + +// These values must correspond to the order of the items in the +// 'YUVColorSpace' enum class. +const float kBT601LimitedRange = 0; +const float kBT601FullRange = 1; + +uniform FragInfo { + float texture_sampler_y_coord_scale; + mat4 matrix; + float yuv_color_space; +} frag_info; + +in vec2 v_position; +out vec4 frag_color; + +void main() { + vec3 yuv; + vec3 yuv_offset = vec3(0.0, 0.5, 0.5); + if (frag_info.yuv_color_space == kBT601LimitedRange) { + yuv_offset.x = 16.0 / 255.0; + } + + yuv.x = IPSample(y_texture, v_position, frag_info.texture_sampler_y_coord_scale).r; + yuv.yz = IPSample(uv_texture, v_position, frag_info.texture_sampler_y_coord_scale).rg; + frag_color = frag_info.matrix * vec4(yuv - yuv_offset, 1); +} diff --git a/impeller/entity/shaders/yuv_to_rgb_filter.vert b/impeller/entity/shaders/yuv_to_rgb_filter.vert new file mode 100644 index 0000000000000..b741b2744ec60 --- /dev/null +++ b/impeller/entity/shaders/yuv_to_rgb_filter.vert @@ -0,0 +1,15 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +uniform FrameInfo { + mat4 mvp; +} frame_info; + +in vec2 position; +out vec2 v_position; + +void main() { + v_position = position; + gl_Position = frame_info.mvp * vec4(position, 0.0, 1.0); +} diff --git a/impeller/geometry/color.h b/impeller/geometry/color.h index 64c1d6b9840cb..6966a3ccc0f6d 100644 --- a/impeller/geometry/color.h +++ b/impeller/geometry/color.h @@ -15,6 +15,8 @@ namespace impeller { struct ColorHSB; struct Vector4; +enum class YUVColorSpace { kBT601LimitedRange, kBT601FullRange }; + /// All blend modes assume that both the source (fragment output) and /// destination (first color attachment) have colors with premultiplied alpha. enum class BlendMode { diff --git a/impeller/renderer/backend/gles/texture_gles.cc b/impeller/renderer/backend/gles/texture_gles.cc index 0c18dab461979..a75a506f41c1e 100644 --- a/impeller/renderer/backend/gles/texture_gles.cc +++ b/impeller/renderer/backend/gles/texture_gles.cc @@ -99,6 +99,16 @@ struct TexImage2DData { external_format = GL_ALPHA; type = GL_UNSIGNED_BYTE; break; + case PixelFormat::kR8UNormInt: + internal_format = GL_RED; + external_format = GL_RED; + type = GL_UNSIGNED_BYTE; + break; + case PixelFormat::kR8G8UNormInt: + internal_format = GL_RG; + external_format = GL_RG; + type = GL_UNSIGNED_BYTE; + break; case PixelFormat::kR8G8B8A8UNormInt: case PixelFormat::kB8G8R8A8UNormInt: case PixelFormat::kR8G8B8A8UNormIntSRGB: @@ -126,6 +136,20 @@ struct TexImage2DData { data = std::move(mapping); break; } + case PixelFormat::kR8UNormInt: { + internal_format = GL_RED; + external_format = GL_RED; + type = GL_UNSIGNED_BYTE; + data = std::move(mapping); + break; + } + case PixelFormat::kR8G8UNormInt: { + internal_format = GL_RG; + external_format = GL_RG; + type = GL_UNSIGNED_BYTE; + data = std::move(mapping); + break; + } case PixelFormat::kR8G8B8A8UNormInt: { internal_format = GL_RGBA; external_format = GL_RGBA; @@ -275,6 +299,8 @@ static std::optional ToRenderBufferFormat(PixelFormat format) { return GL_STENCIL_INDEX8; case PixelFormat::kUnknown: case PixelFormat::kA8UNormInt: + case PixelFormat::kR8UNormInt: + case PixelFormat::kR8G8UNormInt: case PixelFormat::kR8G8B8A8UNormIntSRGB: case PixelFormat::kB8G8R8A8UNormIntSRGB: return std::nullopt; diff --git a/impeller/renderer/backend/metal/formats_mtl.h b/impeller/renderer/backend/metal/formats_mtl.h index c9857f9a3b108..591ecb8d24cc8 100644 --- a/impeller/renderer/backend/metal/formats_mtl.h +++ b/impeller/renderer/backend/metal/formats_mtl.h @@ -43,6 +43,10 @@ constexpr MTLPixelFormat ToMTLPixelFormat(PixelFormat format) { return MTLPixelFormatInvalid; case PixelFormat::kA8UNormInt: return MTLPixelFormatA8Unorm; + case PixelFormat::kR8UNormInt: + return MTLPixelFormatR8Unorm; + case PixelFormat::kR8G8UNormInt: + return MTLPixelFormatRG8Unorm; case PixelFormat::kB8G8R8A8UNormInt: return MTLPixelFormatBGRA8Unorm; case PixelFormat::kB8G8R8A8UNormIntSRGB: diff --git a/impeller/renderer/backend/vulkan/formats_vk.h b/impeller/renderer/backend/vulkan/formats_vk.h index 716f785d69b9f..5f91080644808 100644 --- a/impeller/renderer/backend/vulkan/formats_vk.h +++ b/impeller/renderer/backend/vulkan/formats_vk.h @@ -150,6 +150,10 @@ constexpr vk::Format ToVKImageFormat(PixelFormat format) { return vk::Format::eB8G8R8A8Srgb; case PixelFormat::kS8UInt: return vk::Format::eS8Uint; + case PixelFormat::kR8UNormInt: + return vk::Format::eR8Unorm; + case PixelFormat::kR8G8UNormInt: + return vk::Format::eR8G8Unorm; } } @@ -176,6 +180,12 @@ constexpr PixelFormat ToPixelFormat(vk::Format format) { case vk::Format::eS8Uint: return PixelFormat::kS8UInt; + case vk::Format::eR8Unorm: + return PixelFormat::kR8UNormInt; + + case vk::Format::eR8G8Unorm: + return PixelFormat::kR8G8UNormInt; + default: return PixelFormat::kUnknown; } diff --git a/impeller/renderer/formats.h b/impeller/renderer/formats.h index 118138c03bac1..945b7f87752ed 100644 --- a/impeller/renderer/formats.h +++ b/impeller/renderer/formats.h @@ -81,6 +81,8 @@ enum class StorageMode { enum class PixelFormat { kUnknown, kA8UNormInt, + kR8UNormInt, + kR8G8UNormInt, kR8G8B8A8UNormInt, kR8G8B8A8UNormIntSRGB, kB8G8R8A8UNormInt, @@ -273,8 +275,11 @@ constexpr size_t BytesPerPixelForPixelFormat(PixelFormat format) { case PixelFormat::kUnknown: return 0u; case PixelFormat::kA8UNormInt: + case PixelFormat::kR8UNormInt: case PixelFormat::kS8UInt: return 1u; + case PixelFormat::kR8G8UNormInt: + return 2u; case PixelFormat::kR8G8B8A8UNormInt: case PixelFormat::kR8G8B8A8UNormIntSRGB: case PixelFormat::kB8G8R8A8UNormInt: diff --git a/shell/common/rasterizer.cc b/shell/common/rasterizer.cc index 7f66f93ff8830..cf124ad80476e 100644 --- a/shell/common/rasterizer.cc +++ b/shell/common/rasterizer.cc @@ -528,9 +528,10 @@ RasterStatus Rasterizer::DrawToSurfaceUnsafe( root_surface_transformation, // root surface transformation true, // instrumentation enabled frame->framebuffer_info() - .supports_readback, // surface supports pixel reads - raster_thread_merger_, // thread merger - frame->GetDisplayListBuilder().get() // display list builder + .supports_readback, // surface supports pixel reads + raster_thread_merger_, // thread merger + frame->GetDisplayListBuilder().get(), // display list builder + surface_->GetAiksContext() // aiks context ); if (compositor_frame) { compositor_context_->raster_cache().BeginFrame(); @@ -630,7 +631,7 @@ static sk_sp ScreenshotLayerTreeAsPicture( // https://github.com/flutter/flutter/issues/23435 auto frame = compositor_context.AcquireFrame( nullptr, recorder.getRecordingCanvas(), nullptr, - root_surface_transformation, false, true, nullptr, nullptr); + root_surface_transformation, false, true, nullptr, nullptr, nullptr); frame->Raster(*tree, true, nullptr); #if defined(OS_FUCHSIA) @@ -686,7 +687,8 @@ sk_sp Rasterizer::ScreenshotLayerTreeAsImage( false, // instrumentation enabled true, // render buffer readback supported nullptr, // thread merger - nullptr // display list builder + nullptr, // display list builder + nullptr // aiks context ); canvas->clear(SK_ColorTRANSPARENT); frame->Raster(*tree, true, nullptr); diff --git a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm index 8b31da85f6b55..9bf17b978ee17 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm +++ b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm @@ -4,13 +4,11 @@ #import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" #include "flutter/display_list/display_list_image.h" - #if IMPELLER_SUPPORTS_RENDERING #include "impeller/base/config.h" #include "impeller/display_list/display_list_image_impeller.h" #include "impeller/renderer/backend/metal/texture_mtl.h" #endif // IMPELLER_SUPPORTS_RENDERING - #include "flutter/fml/logging.h" #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" #include "third_party/skia/include/core/SkColorSpace.h" @@ -162,13 +160,6 @@ - (void)onTextureUnregistered { - (sk_sp)wrapNV12ExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer context:(flutter::Texture::PaintContext&)context { -#if IMPELLER_SUPPORTS_RENDERING - if (_enableImpeller) { - IMPELLER_UNIMPLEMENTED - return nullptr; - } -#endif // IMPELLER_SUPPORTS_RENDERING - SkISize textureSize = SkISize::Make(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer)); CVMetalTextureRef yMetalTexture = nullptr; @@ -215,6 +206,34 @@ - (void)onTextureUnregistered { id uvTex = CVMetalTextureGetTexture(uvMetalTexture); CVBufferRelease(uvMetalTexture); +#if IMPELLER_SUPPORTS_RENDERING + if (_enableImpeller) { + impeller::TextureDescriptor yDesc; + yDesc.storage_mode = impeller::StorageMode::kHostVisible; + yDesc.format = impeller::PixelFormat::kR8UNormInt; + yDesc.size = {textureSize.width(), textureSize.height()}; + yDesc.mip_count = 1; + auto yTexture = impeller::TextureMTL::Wrapper(yDesc, yTex); + yTexture->SetIntent(impeller::TextureIntent::kUploadFromHost); + + impeller::TextureDescriptor uvDesc; + uvDesc.storage_mode = impeller::StorageMode::kHostVisible; + uvDesc.format = impeller::PixelFormat::kR8G8UNormInt; + uvDesc.size = {textureSize.width() / 2, textureSize.height() / 2}; + uvDesc.mip_count = 1; + auto uvTexture = impeller::TextureMTL::Wrapper(uvDesc, uvTex); + uvTexture->SetIntent(impeller::TextureIntent::kUploadFromHost); + + impeller::YUVColorSpace yuvColorSpace = + _pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + ? impeller::YUVColorSpace::kBT601LimitedRange + : impeller::YUVColorSpace::kBT601FullRange; + + return impeller::DlImageImpeller::MakeFromYUVTextures(context.aiks_context, yTexture, uvTexture, + yuvColorSpace); + } +#endif // IMPELLER_SUPPORTS_RENDERING + auto skImage = [FlutterDarwinExternalTextureSkImageWrapper wrapYUVATexture:yTex UVTex:uvTex grContext:context.gr_context From 5ae19beb07d485b1a381c16a8fe19936360f5385 Mon Sep 17 00:00:00 2001 From: ColdPaleLight Date: Thu, 20 Oct 2022 11:50:21 +0800 Subject: [PATCH 08/12] Fix test --- impeller/entity/entity_unittests.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/impeller/entity/entity_unittests.cc b/impeller/entity/entity_unittests.cc index f62751d25d26a..6200e63494e84 100644 --- a/impeller/entity/entity_unittests.cc +++ b/impeller/entity/entity_unittests.cc @@ -2030,10 +2030,12 @@ static Vector3 RGBToYUV(Vector3 rgb, YUVColorSpace yuv_color_space) { yuv.x = rgb.x * 0.299 + rgb.y * 0.587 + rgb.z * 0.114; yuv.y = rgb.x * -0.169 + rgb.y * -0.331 + rgb.z * 0.5 + 0.5; yuv.z = rgb.x * 0.5 + rgb.y * -0.419 + rgb.z * -0.081 + 0.5; + break; case YUVColorSpace::kBT601LimitedRange: yuv.x = rgb.x * 0.257 + rgb.y * 0.516 + rgb.z * 0.100 + 0.063; yuv.y = rgb.x * -0.145 + rgb.y * -0.291 + rgb.z * 0.439 + 0.5; yuv.z = rgb.x * 0.429 + rgb.y * -0.368 + rgb.z * -0.071 + 0.5; + break; } return yuv; } From a11f77513c22acf15af26208c6585cd3f2ebe097 Mon Sep 17 00:00:00 2001 From: ColdPaleLight Date: Thu, 20 Oct 2022 15:24:45 +0800 Subject: [PATCH 09/12] Clean code --- .../darwin/graphics/FlutterDarwinExternalTextureMetal.mm | 2 -- 1 file changed, 2 deletions(-) diff --git a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm index 047c295ebd7f2..63552ed4f8bf8 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm +++ b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm @@ -4,8 +4,6 @@ #import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" #include "flutter/display_list/display_list_image.h" -#include "flutter/fml/logging.h" -#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" #include "impeller/base/validation.h" #include "impeller/display_list/display_list_image_impeller.h" #include "impeller/renderer/backend/metal/texture_mtl.h" From 2e552044735136f5b8621a5e5e827097aee4f4a8 Mon Sep 17 00:00:00 2001 From: ColdPaleLight Date: Thu, 3 Nov 2022 15:59:11 +0800 Subject: [PATCH 10/12] Fix compile error --- flow/layers/texture_layer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flow/layers/texture_layer.cc b/flow/layers/texture_layer.cc index a41de9c05f5a3..c141820232441 100644 --- a/flow/layers/texture_layer.cc +++ b/flow/layers/texture_layer.cc @@ -63,9 +63,9 @@ void TextureLayer::Paint(PaintContext& context) const { .canvas = context.leaf_nodes_canvas, .builder = context.leaf_nodes_builder, .gr_context = context.gr_context, + .aiks_context = context.aiks_context, .sk_paint = cache_paint.sk_paint(), .dl_paint = cache_paint.dl_paint(), - .aiks_context = context.aiks_context, }; texture->Paint(ctx, paint_bounds(), freeze_, ToSk(sampling_)); } From 1e63d2f1942dbaf88dde5416f3ba7821f711f2d6 Mon Sep 17 00:00:00 2001 From: ColdPaleLight Date: Thu, 3 Nov 2022 16:04:37 +0800 Subject: [PATCH 11/12] Add todo link --- impeller/entity/entity_unittests.cc | 5 +++ .../renderer/backend/gles/texture_gles.cc | 31 +++++-------------- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/impeller/entity/entity_unittests.cc b/impeller/entity/entity_unittests.cc index f83667af32891..f8c2b07c5fb85 100644 --- a/impeller/entity/entity_unittests.cc +++ b/impeller/entity/entity_unittests.cc @@ -2105,6 +2105,11 @@ static std::vector> CreateTestYUVTextures( } TEST_P(EntityTest, YUVToRGBFilter) { + if (GetParam() == PlaygroundBackend::kOpenGLES) { + // TODO(114588) : Support YUV to RGB filter on OpenGLES backend. + GTEST_SKIP_("YUV to RGB filter is not supported on OpenGLES backend yet."); + } + auto callback = [&](ContentContext& context, RenderPass& pass) -> bool { YUVColorSpace yuv_color_space_array[2]{YUVColorSpace::kBT601FullRange, YUVColorSpace::kBT601LimitedRange}; diff --git a/impeller/renderer/backend/gles/texture_gles.cc b/impeller/renderer/backend/gles/texture_gles.cc index a75a506f41c1e..e58be99263ba8 100644 --- a/impeller/renderer/backend/gles/texture_gles.cc +++ b/impeller/renderer/backend/gles/texture_gles.cc @@ -99,16 +99,7 @@ struct TexImage2DData { external_format = GL_ALPHA; type = GL_UNSIGNED_BYTE; break; - case PixelFormat::kR8UNormInt: - internal_format = GL_RED; - external_format = GL_RED; - type = GL_UNSIGNED_BYTE; - break; - case PixelFormat::kR8G8UNormInt: - internal_format = GL_RG; - external_format = GL_RG; - type = GL_UNSIGNED_BYTE; - break; + case PixelFormat::kR8G8B8A8UNormInt: case PixelFormat::kB8G8R8A8UNormInt: case PixelFormat::kR8G8B8A8UNormIntSRGB: @@ -119,6 +110,8 @@ struct TexImage2DData { break; case PixelFormat::kUnknown: case PixelFormat::kS8UInt: + case PixelFormat::kR8UNormInt: + case PixelFormat::kR8G8UNormInt: return; } is_valid_ = true; @@ -136,20 +129,6 @@ struct TexImage2DData { data = std::move(mapping); break; } - case PixelFormat::kR8UNormInt: { - internal_format = GL_RED; - external_format = GL_RED; - type = GL_UNSIGNED_BYTE; - data = std::move(mapping); - break; - } - case PixelFormat::kR8G8UNormInt: { - internal_format = GL_RG; - external_format = GL_RG; - type = GL_UNSIGNED_BYTE; - data = std::move(mapping); - break; - } case PixelFormat::kR8G8B8A8UNormInt: { internal_format = GL_RGBA; external_format = GL_RGBA; @@ -165,6 +144,10 @@ struct TexImage2DData { return; case PixelFormat::kS8UInt: return; + case PixelFormat::kR8UNormInt: + return; + case PixelFormat::kR8G8UNormInt: + return; } is_valid_ = true; } From a0b1fecfb3637d694a706bee239ab06195052ccc Mon Sep 17 00:00:00 2001 From: ColdPaleLight Date: Thu, 3 Nov 2022 17:20:43 +0800 Subject: [PATCH 12/12] Clean code --- impeller/renderer/backend/gles/texture_gles.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/impeller/renderer/backend/gles/texture_gles.cc b/impeller/renderer/backend/gles/texture_gles.cc index e58be99263ba8..e8a8f9b455721 100644 --- a/impeller/renderer/backend/gles/texture_gles.cc +++ b/impeller/renderer/backend/gles/texture_gles.cc @@ -99,7 +99,6 @@ struct TexImage2DData { external_format = GL_ALPHA; type = GL_UNSIGNED_BYTE; break; - case PixelFormat::kR8G8B8A8UNormInt: case PixelFormat::kB8G8R8A8UNormInt: case PixelFormat::kR8G8B8A8UNormIntSRGB: