diff --git a/shell/platform/darwin/ios/ios_external_texture_gl.h b/shell/platform/darwin/ios/ios_external_texture_gl.h index 5c608506bb82f..6a63357a77381 100644 --- a/shell/platform/darwin/ios/ios_external_texture_gl.h +++ b/shell/platform/darwin/ios/ios_external_texture_gl.h @@ -25,6 +25,9 @@ class IOSExternalTextureGL final : public Texture { fml::CFRef cache_ref_; fml::CFRef texture_ref_; fml::CFRef buffer_ref_; + OSType pixel_format_ = 0; + fml::CFRef y_texture_ref_; + fml::CFRef uv_texture_ref_; // |Texture| void Paint(SkCanvas& canvas, @@ -51,6 +54,16 @@ class IOSExternalTextureGL final : public Texture { bool NeedUpdateTexture(bool freeze); + bool IsTexturesAvailable() const; + + void CreateYUVTexturesFromPixelBuffer(); + + void CreateRGBATextureFromPixelBuffer(); + + sk_sp CreateImageFromYUVTextures(GrContext* context, const SkRect& bounds); + + sk_sp CreateImageFromRGBATexture(GrContext* context, const SkRect& bounds); + FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureGL); }; diff --git a/shell/platform/darwin/ios/ios_external_texture_gl.mm b/shell/platform/darwin/ios/ios_external_texture_gl.mm index bb56fc2849bcf..627da33f09793 100644 --- a/shell/platform/darwin/ios/ios_external_texture_gl.mm +++ b/shell/platform/darwin/ios/ios_external_texture_gl.mm @@ -10,8 +10,10 @@ #include "flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.h" #include "third_party/skia/include/core/SkSurface.h" +#include "third_party/skia/include/core/SkYUVAIndex.h" #include "third_party/skia/include/gpu/GrBackendSurface.h" #include "third_party/skia/include/gpu/GrDirectContext.h" +#include "third_party/skia/src/gpu/gl/GrGLDefines.h" namespace flutter { @@ -42,10 +44,19 @@ if (buffer_ref_ == nullptr) { return; } + if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || + pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { + CreateYUVTexturesFromPixelBuffer(); + } else { + CreateRGBATextureFromPixelBuffer(); + } +} + +void IOSExternalTextureGL::CreateRGBATextureFromPixelBuffer() { CVOpenGLESTextureRef texture; CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage( - kCFAllocatorDefault, cache_ref_, buffer_ref_, nullptr, GL_TEXTURE_2D, GL_RGBA, - static_cast(CVPixelBufferGetWidth(buffer_ref_)), + kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D, + GL_RGBA, static_cast(CVPixelBufferGetWidth(buffer_ref_)), static_cast(CVPixelBufferGetHeight(buffer_ref_)), GL_BGRA, GL_UNSIGNED_BYTE, 0, &texture); if (err != noErr) { @@ -55,10 +66,83 @@ } } +void IOSExternalTextureGL::CreateYUVTexturesFromPixelBuffer() { + size_t width = CVPixelBufferGetWidth(buffer_ref_); + size_t height = CVPixelBufferGetHeight(buffer_ref_); + { + CVOpenGLESTextureRef yTexture; + CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D, + GL_LUMINANCE, width, height, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &yTexture); + if (err != noErr) { + FML_DCHECK(yTexture) << "Could not create texture from pixel buffer: " << err; + } else { + y_texture_ref_.Reset(yTexture); + } + } + + { + CVOpenGLESTextureRef uvTexture; + CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D, + GL_LUMINANCE_ALPHA, width / 2, height / 2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1, + &uvTexture); + if (err != noErr) { + FML_DCHECK(uvTexture) << "Could not create texture from pixel buffer: " << err; + } else { + uv_texture_ref_.Reset(uvTexture); + } + } +} + +sk_sp IOSExternalTextureGL::CreateImageFromRGBATexture(GrContext* context, + const SkRect& bounds) { + GrGLTextureInfo textureInfo = {CVOpenGLESTextureGetTarget(texture_ref_), + CVOpenGLESTextureGetName(texture_ref_), GL_RGBA8_OES}; + GrBackendTexture backendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, textureInfo); + sk_sp image = SkImage::MakeFromTexture(context, backendTexture, kTopLeft_GrSurfaceOrigin, + kRGBA_8888_SkColorType, kPremul_SkAlphaType, + /*imageColorSpace=*/nullptr); + return image; +} + +sk_sp IOSExternalTextureGL::CreateImageFromYUVTextures(GrContext* context, + const SkRect& bounds) { + GrGLTextureInfo yTextureInfo = {CVOpenGLESTextureGetTarget(y_texture_ref_), + CVOpenGLESTextureGetName(y_texture_ref_), GR_GL_LUMINANCE8}; + GrBackendTexture yBackendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, yTextureInfo); + GrGLTextureInfo uvTextureInfo = {CVOpenGLESTextureGetTarget(uv_texture_ref_), + CVOpenGLESTextureGetName(uv_texture_ref_), GR_GL_RGBA8}; + GrBackendTexture uvBackendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, + uvTextureInfo); + GrBackendTexture nv12TextureHandles[] = {yBackendTexture, uvBackendTexture}; + SkYUVAIndex yuvaIndices[4] = { + SkYUVAIndex{0, SkColorChannel::kR}, // Read Y data from the red channel of the first texture + SkYUVAIndex{1, SkColorChannel::kR}, // Read U data from the red channel of the second texture + SkYUVAIndex{ + 1, SkColorChannel::kA}, // Read V data from the alpha channel of the second texture, + // normal NV12 data V should be taken from the green channel, but + // currently only the uv texture created by GL_LUMINANCE_ALPHA + // can be used, so the V value is taken from the alpha channel + SkYUVAIndex{-1, SkColorChannel::kA}}; //-1 means to omit the alpha data of YUVA + SkISize size{yBackendTexture.width(), yBackendTexture.height()}; + sk_sp image = SkImage::MakeFromYUVATextures( + context, kRec601_SkYUVColorSpace, nv12TextureHandles, yuvaIndices, size, + kTopLeft_GrSurfaceOrigin, /*imageColorSpace=*/nullptr); + return image; +} + +bool IOSExternalTextureGL::IsTexturesAvailable() const { + return ((pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || + pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) && + (y_texture_ref_ && uv_texture_ref_)) || + (pixel_format_ == kCVPixelFormatType_32BGRA && texture_ref_); +} + bool IOSExternalTextureGL::NeedUpdateTexture(bool freeze) { // Update texture if `texture_ref_` is reset to `nullptr` when GrContext // is destroyed or new frame is ready. - return (!freeze && new_frame_ready_) || !texture_ref_; + return (!freeze && new_frame_ready_) || !IsTexturesAvailable(); } void IOSExternalTextureGL::Paint(SkCanvas& canvas, @@ -71,19 +155,23 @@ auto pixelBuffer = [external_texture_.get() copyPixelBuffer]; if (pixelBuffer) { buffer_ref_.Reset(pixelBuffer); + pixel_format_ = CVPixelBufferGetPixelFormatType(buffer_ref_); } CreateTextureFromPixelBuffer(); new_frame_ready_ = false; } - if (!texture_ref_) { + if (!IsTexturesAvailable()) { return; } - GrGLTextureInfo textureInfo = {CVOpenGLESTextureGetTarget(texture_ref_), - CVOpenGLESTextureGetName(texture_ref_), GL_RGBA8_OES}; - GrBackendTexture backendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, textureInfo); - sk_sp image = - SkImage::MakeFromTexture(context, backendTexture, kTopLeft_GrSurfaceOrigin, - kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr); + + sk_sp image = nullptr; + if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || + pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { + image = CreateImageFromYUVTextures(context, bounds); + } else { + image = CreateImageFromRGBATexture(context, bounds); + } + FML_DCHECK(image) << "Failed to create SkImage from Texture."; if (image) { SkPaint paint; diff --git a/shell/platform/darwin/ios/ios_external_texture_metal.h b/shell/platform/darwin/ios/ios_external_texture_metal.h index 9cbf84431eb9d..90e9e95746778 100644 --- a/shell/platform/darwin/ios/ios_external_texture_metal.h +++ b/shell/platform/darwin/ios/ios_external_texture_metal.h @@ -33,6 +33,7 @@ class IOSExternalTextureMetal final : public Texture { std::atomic_bool texture_frame_available_; fml::CFRef last_pixel_buffer_; sk_sp external_image_; + OSType pixel_format_ = 0; // |Texture| void Paint(SkCanvas& canvas, @@ -55,6 +56,10 @@ class IOSExternalTextureMetal final : public Texture { sk_sp WrapExternalPixelBuffer(fml::CFRef pixel_buffer, GrDirectContext* context) const; + sk_sp WrapRGBAExternalPixelBuffer(fml::CFRef pixel_buffer, + GrDirectContext* context) const; + sk_sp WrapNV12ExternalPixelBuffer(fml::CFRef pixel_buffer, + GrDirectContext* context) const; FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureMetal); }; diff --git a/shell/platform/darwin/ios/ios_external_texture_metal.mm b/shell/platform/darwin/ios/ios_external_texture_metal.mm index 46eff3415273b..058a4738ce76c 100644 --- a/shell/platform/darwin/ios/ios_external_texture_metal.mm +++ b/shell/platform/darwin/ios/ios_external_texture_metal.mm @@ -5,6 +5,7 @@ #include "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h" #include "flutter/fml/logging.h" +#include "third_party/skia/include/core/SkYUVAIndex.h" #include "third_party/skia/include/gpu/GrBackendSurface.h" #include "third_party/skia/include/gpu/GrDirectContext.h" #include "third_party/skia/include/gpu/mtl/GrMtlTypes.h" @@ -35,6 +36,8 @@ auto pixel_buffer = fml::CFRef([external_texture_ copyPixelBuffer]); if (!pixel_buffer) { pixel_buffer = std::move(last_pixel_buffer_); + } else { + pixel_format_ = CVPixelBufferGetPixelFormatType(pixel_buffer); } // If the application told us there was a texture frame available but did not provide one when @@ -65,21 +68,130 @@ return nullptr; } + sk_sp image = nullptr; + if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || + pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { + image = WrapNV12ExternalPixelBuffer(pixel_buffer, context); + } else { + image = WrapRGBAExternalPixelBuffer(pixel_buffer, context); + } + + if (!image) { + FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image."; + } + + return image; +} + +sk_sp IOSExternalTextureMetal::WrapNV12ExternalPixelBuffer( + fml::CFRef pixel_buffer, + GrDirectContext* context) const { auto texture_size = SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer)); + CVMetalTextureRef y_metal_texture_raw = nullptr; + { + auto cv_return = + CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault, + /*textureCache=*/texture_cache_, + /*sourceImage=*/pixel_buffer, + /*textureAttributes=*/nullptr, + /*pixelFormat=*/MTLPixelFormatR8Unorm, + /*width=*/texture_size.width(), + /*height=*/texture_size.height(), + /*planeIndex=*/0u, + /*texture=*/&y_metal_texture_raw); + + if (cv_return != kCVReturnSuccess) { + FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return; + return nullptr; + } + } + + CVMetalTextureRef uv_metal_texture_raw = nullptr; + { + auto cv_return = + CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault, + /*textureCache=*/texture_cache_, + /*sourceImage=*/pixel_buffer, + /*textureAttributes=*/nullptr, + /*pixelFormat=*/MTLPixelFormatRG8Unorm, + /*width=*/texture_size.width() / 2, + /*height=*/texture_size.height() / 2, + /*planeIndex=*/1u, + /*texture=*/&uv_metal_texture_raw); + + if (cv_return != kCVReturnSuccess) { + FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return; + return nullptr; + } + } + + fml::CFRef y_metal_texture(y_metal_texture_raw); + + GrMtlTextureInfo y_skia_texture_info; + y_skia_texture_info.fTexture = sk_cf_obj{ + [reinterpret_cast(CVMetalTextureGetTexture(y_metal_texture)) retain]}; - CVMetalTextureRef metal_texture_raw = NULL; + GrBackendTexture y_skia_backend_texture(/*width=*/texture_size.width(), + /*height=*/texture_size.height(), + /*mipMapped=*/GrMipMapped ::kNo, + /*textureInfo=*/y_skia_texture_info); + + fml::CFRef uv_metal_texture(uv_metal_texture_raw); + + GrMtlTextureInfo uv_skia_texture_info; + uv_skia_texture_info.fTexture = sk_cf_obj{ + [reinterpret_cast(CVMetalTextureGetTexture(uv_metal_texture)) retain]}; + + GrBackendTexture uv_skia_backend_texture(/*width=*/texture_size.width(), + /*height=*/texture_size.height(), + /*mipMapped=*/GrMipMapped ::kNo, + /*textureInfo=*/uv_skia_texture_info); + GrBackendTexture nv12TextureHandles[] = {y_skia_backend_texture, uv_skia_backend_texture}; + SkYUVAIndex yuvaIndices[4] = { + SkYUVAIndex{0, SkColorChannel::kR}, // Read Y data from the red channel of the first texture + SkYUVAIndex{1, SkColorChannel::kR}, // Read U data from the red channel of the second texture + SkYUVAIndex{1, + SkColorChannel::kG}, // Read V data from the green channel of the second texture + SkYUVAIndex{-1, SkColorChannel::kA}}; //-1 means to omit the alpha data of YUVA + + struct ImageCaptures { + fml::CFRef buffer; + fml::CFRef y_texture; + fml::CFRef uv_texture; + }; + + auto captures = std::make_unique(); + captures->buffer = std::move(pixel_buffer); + captures->y_texture = std::move(y_metal_texture); + captures->uv_texture = std::move(uv_metal_texture); + + SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) { + auto captures = reinterpret_cast(release_context); + delete captures; + }; + sk_sp image = SkImage::MakeFromYUVATextures( + context, kRec601_SkYUVColorSpace, nv12TextureHandles, yuvaIndices, texture_size, + kTopLeft_GrSurfaceOrigin, /*imageColorSpace=*/nullptr, release_proc, captures.release()); + return image; +} + +sk_sp IOSExternalTextureMetal::WrapRGBAExternalPixelBuffer( + fml::CFRef pixel_buffer, + GrDirectContext* context) const { + auto texture_size = + SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer)); + CVMetalTextureRef metal_texture_raw = nullptr; auto cv_return = - CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, // allocator - texture_cache_, // texture cache - pixel_buffer, // source image - NULL, // texture attributes - MTLPixelFormatBGRA8Unorm, // pixel format - texture_size.width(), // width - texture_size.height(), // height - 0u, // plane index - &metal_texture_raw // [out] texture - ); + CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault, + /*textureCache=*/texture_cache_, + /*sourceImage=*/pixel_buffer, + /*textureAttributes=*/nullptr, + /*pixelFormat=*/MTLPixelFormatBGRA8Unorm, + /*width=*/texture_size.width(), + /*height=*/texture_size.height(), + /*planeIndex=*/0u, + /*texture=*/&metal_texture_raw); if (cv_return != kCVReturnSuccess) { FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return; @@ -92,11 +204,10 @@ skia_texture_info.fTexture = sk_cf_obj{ [reinterpret_cast(CVMetalTextureGetTexture(metal_texture)) retain]}; - GrBackendTexture skia_backend_texture(texture_size.width(), // width - texture_size.height(), // height - GrMipMapped ::kNo, // mip-mapped - skia_texture_info // texture info - ); + GrBackendTexture skia_backend_texture(/*width=*/texture_size.width(), + /*height=*/texture_size.height(), + /*mipMapped=*/GrMipMapped ::kNo, + /*textureInfo=*/skia_texture_info); struct ImageCaptures { fml::CFRef buffer; @@ -112,21 +223,12 @@ GrBackendTexture skia_backend_texture(texture_size.width(), // width delete captures; }; - auto image = SkImage::MakeFromTexture(context, // context - skia_backend_texture, // backend texture - kTopLeft_GrSurfaceOrigin, // origin - kBGRA_8888_SkColorType, // color type - kPremul_SkAlphaType, // alpha type - nullptr, // color space - release_proc, // release proc - captures.release() // release context - - ); - - if (!image) { - FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image."; - } + auto image = + SkImage::MakeFromTexture(context, skia_backend_texture, kTopLeft_GrSurfaceOrigin, + kBGRA_8888_SkColorType, kPremul_SkAlphaType, + /*imageColorSpace=*/nullptr, release_proc, captures.release() + ); return image; }