Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Ios nv12 external texture #20082

Merged
gaaclarke merged 1 commit into from
Aug 3, 2020
Merged

Ios nv12 external texture #20082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions shell/platform/darwin/ios/ios_external_texture_gl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ class IOSExternalTextureGL final : public Texture {
fml::CFRef<CVOpenGLESTextureCacheRef> cache_ref_;
fml::CFRef<CVOpenGLESTextureRef> texture_ref_;
fml::CFRef<CVPixelBufferRef> buffer_ref_;
OSType pixel_format_ = 0;
fml::CFRef<CVOpenGLESTextureRef> y_texture_ref_;
fml::CFRef<CVOpenGLESTextureRef> uv_texture_ref_;

// |Texture|
void Paint(SkCanvas& canvas,
Expand All @@ -51,6 +54,16 @@ class IOSExternalTextureGL final : public Texture {

bool NeedUpdateTexture(bool freeze);

bool IsTexturesAvailable() const;

void CreateYUVTexturesFromPixelBuffer();

void CreateRGBATextureFromPixelBuffer();

sk_sp<SkImage> CreateImageFromYUVTextures(GrContext* context, const SkRect& bounds);

sk_sp<SkImage> CreateImageFromRGBATexture(GrContext* context, const SkRect& bounds);

FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureGL);
};

Expand Down
108 changes: 98 additions & 10 deletions shell/platform/darwin/ios/ios_external_texture_gl.mm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@

#include "flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.h"
#include "third_party/skia/include/core/SkSurface.h"
#include "third_party/skia/include/core/SkYUVAIndex.h"
#include "third_party/skia/include/gpu/GrBackendSurface.h"
#include "third_party/skia/include/gpu/GrDirectContext.h"
#include "third_party/skia/src/gpu/gl/GrGLDefines.h"

namespace flutter {

Expand Down Expand Up @@ -42,10 +44,19 @@
if (buffer_ref_ == nullptr) {
return;
}
if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
CreateYUVTexturesFromPixelBuffer();
} else {
CreateRGBATextureFromPixelBuffer();
}
}

void IOSExternalTextureGL::CreateRGBATextureFromPixelBuffer() {
CVOpenGLESTextureRef texture;
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, cache_ref_, buffer_ref_, nullptr, GL_TEXTURE_2D, GL_RGBA,
static_cast<int>(CVPixelBufferGetWidth(buffer_ref_)),
kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D,
GL_RGBA, static_cast<int>(CVPixelBufferGetWidth(buffer_ref_)),
static_cast<int>(CVPixelBufferGetHeight(buffer_ref_)), GL_BGRA, GL_UNSIGNED_BYTE, 0,
&texture);
if (err != noErr) {
Expand All @@ -55,10 +66,83 @@
}
}

void IOSExternalTextureGL::CreateYUVTexturesFromPixelBuffer() {
size_t width = CVPixelBufferGetWidth(buffer_ref_);
size_t height = CVPixelBufferGetHeight(buffer_ref_);
{
CVOpenGLESTextureRef yTexture;
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D,
GL_LUMINANCE, width, height, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &yTexture);
if (err != noErr) {
FML_DCHECK(yTexture) << "Could not create texture from pixel buffer: " << err;
} else {
y_texture_ref_.Reset(yTexture);
}
}

{
CVOpenGLESTextureRef uvTexture;
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D,
GL_LUMINANCE_ALPHA, width / 2, height / 2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1,
&uvTexture);
if (err != noErr) {
FML_DCHECK(uvTexture) << "Could not create texture from pixel buffer: " << err;
} else {
uv_texture_ref_.Reset(uvTexture);
}
}
}

sk_sp<SkImage> IOSExternalTextureGL::CreateImageFromRGBATexture(GrContext* context,
const SkRect& bounds) {
GrGLTextureInfo textureInfo = {CVOpenGLESTextureGetTarget(texture_ref_),
CVOpenGLESTextureGetName(texture_ref_), GL_RGBA8_OES};
GrBackendTexture backendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, textureInfo);
sk_sp<SkImage> image = SkImage::MakeFromTexture(context, backendTexture, kTopLeft_GrSurfaceOrigin,
kRGBA_8888_SkColorType, kPremul_SkAlphaType,
/*imageColorSpace=*/nullptr);
return image;
}

sk_sp<SkImage> IOSExternalTextureGL::CreateImageFromYUVTextures(GrContext* context,
const SkRect& bounds) {
GrGLTextureInfo yTextureInfo = {CVOpenGLESTextureGetTarget(y_texture_ref_),
CVOpenGLESTextureGetName(y_texture_ref_), GR_GL_LUMINANCE8};
GrBackendTexture yBackendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, yTextureInfo);
GrGLTextureInfo uvTextureInfo = {CVOpenGLESTextureGetTarget(uv_texture_ref_),
CVOpenGLESTextureGetName(uv_texture_ref_), GR_GL_RGBA8};
GrBackendTexture uvBackendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo,
uvTextureInfo);
GrBackendTexture nv12TextureHandles[] = {yBackendTexture, uvBackendTexture};
SkYUVAIndex yuvaIndices[4] = {
SkYUVAIndex{0, SkColorChannel::kR}, // Read Y data from the red channel of the first texture
SkYUVAIndex{1, SkColorChannel::kR}, // Read U data from the red channel of the second texture
SkYUVAIndex{
1, SkColorChannel::kA}, // Read V data from the alpha channel of the second texture,
// normal NV12 data V should be taken from the green channel, but
// currently only the uv texture created by GL_LUMINANCE_ALPHA
// can be used, so the V value is taken from the alpha channel
SkYUVAIndex{-1, SkColorChannel::kA}}; //-1 means to omit the alpha data of YUVA
SkISize size{yBackendTexture.width(), yBackendTexture.height()};
sk_sp<SkImage> image = SkImage::MakeFromYUVATextures(
context, kRec601_SkYUVColorSpace, nv12TextureHandles, yuvaIndices, size,
kTopLeft_GrSurfaceOrigin, /*imageColorSpace=*/nullptr);
return image;
}

bool IOSExternalTextureGL::IsTexturesAvailable() const {
return ((pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) &&
(y_texture_ref_ && uv_texture_ref_)) ||
(pixel_format_ == kCVPixelFormatType_32BGRA && texture_ref_);
}

bool IOSExternalTextureGL::NeedUpdateTexture(bool freeze) {
// Update texture if `texture_ref_` is reset to `nullptr` when GrContext
// is destroyed or new frame is ready.
return (!freeze && new_frame_ready_) || !texture_ref_;
return (!freeze && new_frame_ready_) || !IsTexturesAvailable();
}

void IOSExternalTextureGL::Paint(SkCanvas& canvas,
Expand All @@ -71,19 +155,23 @@
auto pixelBuffer = [external_texture_.get() copyPixelBuffer];
if (pixelBuffer) {
buffer_ref_.Reset(pixelBuffer);
pixel_format_ = CVPixelBufferGetPixelFormatType(buffer_ref_);
}
CreateTextureFromPixelBuffer();
new_frame_ready_ = false;
}
if (!texture_ref_) {
if (!IsTexturesAvailable()) {
return;
}
GrGLTextureInfo textureInfo = {CVOpenGLESTextureGetTarget(texture_ref_),
CVOpenGLESTextureGetName(texture_ref_), GL_RGBA8_OES};
GrBackendTexture backendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, textureInfo);
sk_sp<SkImage> image =
SkImage::MakeFromTexture(context, backendTexture, kTopLeft_GrSurfaceOrigin,
kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr);

sk_sp<SkImage> image = nullptr;
if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
image = CreateImageFromYUVTextures(context, bounds);
} else {
image = CreateImageFromRGBATexture(context, bounds);
}

FML_DCHECK(image) << "Failed to create SkImage from Texture.";
if (image) {
SkPaint paint;
Expand Down
5 changes: 5 additions & 0 deletions shell/platform/darwin/ios/ios_external_texture_metal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ class IOSExternalTextureMetal final : public Texture {
std::atomic_bool texture_frame_available_;
fml::CFRef<CVPixelBufferRef> last_pixel_buffer_;
sk_sp<SkImage> external_image_;
OSType pixel_format_ = 0;

// |Texture|
void Paint(SkCanvas& canvas,
Expand All @@ -55,6 +56,10 @@ class IOSExternalTextureMetal final : public Texture {

sk_sp<SkImage> WrapExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const;
sk_sp<SkImage> WrapRGBAExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const;
sk_sp<SkImage> WrapNV12ExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const;

FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureMetal);
};
Expand Down
162 changes: 132 additions & 30 deletions shell/platform/darwin/ios/ios_external_texture_metal.mm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h"

#include "flutter/fml/logging.h"
#include "third_party/skia/include/core/SkYUVAIndex.h"
#include "third_party/skia/include/gpu/GrBackendSurface.h"
#include "third_party/skia/include/gpu/GrDirectContext.h"
#include "third_party/skia/include/gpu/mtl/GrMtlTypes.h"
Expand Down Expand Up @@ -35,6 +36,8 @@
auto pixel_buffer = fml::CFRef<CVPixelBufferRef>([external_texture_ copyPixelBuffer]);
if (!pixel_buffer) {
pixel_buffer = std::move(last_pixel_buffer_);
} else {
pixel_format_ = CVPixelBufferGetPixelFormatType(pixel_buffer);
}

// If the application told us there was a texture frame available but did not provide one when
Expand Down Expand Up @@ -65,21 +68,130 @@
return nullptr;
}

sk_sp<SkImage> image = nullptr;
if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
image = WrapNV12ExternalPixelBuffer(pixel_buffer, context);
} else {
image = WrapRGBAExternalPixelBuffer(pixel_buffer, context);
}

if (!image) {
FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image.";
}

return image;
}

sk_sp<SkImage> IOSExternalTextureMetal::WrapNV12ExternalPixelBuffer(
fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const {
auto texture_size =
SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
CVMetalTextureRef y_metal_texture_raw = nullptr;
{
auto cv_return =
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/texture_cache_,
/*sourceImage=*/pixel_buffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatR8Unorm,
/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*planeIndex=*/0u,
/*texture=*/&y_metal_texture_raw);

if (cv_return != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
return nullptr;
}
}

CVMetalTextureRef uv_metal_texture_raw = nullptr;
{
auto cv_return =
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/texture_cache_,
/*sourceImage=*/pixel_buffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatRG8Unorm,
/*width=*/texture_size.width() / 2,
/*height=*/texture_size.height() / 2,
/*planeIndex=*/1u,
/*texture=*/&uv_metal_texture_raw);

if (cv_return != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
return nullptr;
}
}

fml::CFRef<CVMetalTextureRef> y_metal_texture(y_metal_texture_raw);

GrMtlTextureInfo y_skia_texture_info;
y_skia_texture_info.fTexture = sk_cf_obj<const void*>{
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(y_metal_texture)) retain]};

CVMetalTextureRef metal_texture_raw = NULL;
GrBackendTexture y_skia_backend_texture(/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/y_skia_texture_info);

fml::CFRef<CVMetalTextureRef> uv_metal_texture(uv_metal_texture_raw);

GrMtlTextureInfo uv_skia_texture_info;
uv_skia_texture_info.fTexture = sk_cf_obj<const void*>{
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(uv_metal_texture)) retain]};

GrBackendTexture uv_skia_backend_texture(/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/uv_skia_texture_info);
GrBackendTexture nv12TextureHandles[] = {y_skia_backend_texture, uv_skia_backend_texture};
SkYUVAIndex yuvaIndices[4] = {
SkYUVAIndex{0, SkColorChannel::kR}, // Read Y data from the red channel of the first texture
SkYUVAIndex{1, SkColorChannel::kR}, // Read U data from the red channel of the second texture
SkYUVAIndex{1,
SkColorChannel::kG}, // Read V data from the green channel of the second texture
SkYUVAIndex{-1, SkColorChannel::kA}}; //-1 means to omit the alpha data of YUVA
Copy link
Member

@gaaclarke gaaclarke Jul 30, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we are omitting the alpha data, why is this method referring to yuva?

Copy link
Contributor Author

@cikelengfeng cikelengfeng Jul 31, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because some YUV formats have alpha channels (such as AYUV), skia may need to be compatible with such formats. In addition, I refer to the implementation of SkImage::MakeFromNV12TexturesCopy.

Copy link
Contributor Author

@cikelengfeng cikelengfeng Jul 31, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And NV12 has no alpha channel.

Copy link
Member

@gaaclarke gaaclarke Jul 31, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we change the name of this method then? It's called WrapYUVAExternalPixelBuffer.

Copy link
Contributor Author

@cikelengfeng cikelengfeng Jul 31, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good idea，it should be WrapNV12ExternalPixelBuffer

Copy link
Contributor Author

@cikelengfeng cikelengfeng Aug 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done


struct ImageCaptures {
fml::CFRef<CVPixelBufferRef> buffer;
fml::CFRef<CVMetalTextureRef> y_texture;
fml::CFRef<CVMetalTextureRef> uv_texture;
};

auto captures = std::make_unique<ImageCaptures>();
captures->buffer = std::move(pixel_buffer);
captures->y_texture = std::move(y_metal_texture);
captures->uv_texture = std::move(uv_metal_texture);

SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) {
auto captures = reinterpret_cast<ImageCaptures*>(release_context);
delete captures;
};
sk_sp<SkImage> image = SkImage::MakeFromYUVATextures(
context, kRec601_SkYUVColorSpace, nv12TextureHandles, yuvaIndices, texture_size,
kTopLeft_GrSurfaceOrigin, /*imageColorSpace=*/nullptr, release_proc, captures.release());
return image;
}

sk_sp<SkImage> IOSExternalTextureMetal::WrapRGBAExternalPixelBuffer(
fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const {
auto texture_size =
SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
CVMetalTextureRef metal_texture_raw = nullptr;
auto cv_return =
CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, // allocator
texture_cache_, // texture cache
pixel_buffer, // source image
NULL, // texture attributes
MTLPixelFormatBGRA8Unorm, // pixel format
texture_size.width(), // width
texture_size.height(), // height
0u, // plane index
&metal_texture_raw // [out] texture
);
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/texture_cache_,
/*sourceImage=*/pixel_buffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatBGRA8Unorm,
/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*planeIndex=*/0u,
/*texture=*/&metal_texture_raw);

if (cv_return != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
Expand All @@ -92,11 +204,10 @@
skia_texture_info.fTexture = sk_cf_obj<const void*>{
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(metal_texture)) retain]};

GrBackendTexture skia_backend_texture(texture_size.width(), // width
texture_size.height(), // height
GrMipMapped ::kNo, // mip-mapped
skia_texture_info // texture info
);
GrBackendTexture skia_backend_texture(/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/skia_texture_info);

struct ImageCaptures {
fml::CFRef<CVPixelBufferRef> buffer;
Expand All @@ -112,21 +223,12 @@ GrBackendTexture skia_backend_texture(texture_size.width(), // width
delete captures;
};

auto image = SkImage::MakeFromTexture(context, // context
skia_backend_texture, // backend texture
kTopLeft_GrSurfaceOrigin, // origin
kBGRA_8888_SkColorType, // color type
kPremul_SkAlphaType, // alpha type
nullptr, // color space
release_proc, // release proc
captures.release() // release context

);

if (!image) {
FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image.";
}
auto image =
SkImage::MakeFromTexture(context, skia_backend_texture, kTopLeft_GrSurfaceOrigin,
kBGRA_8888_SkColorType, kPremul_SkAlphaType,
/*imageColorSpace=*/nullptr, release_proc, captures.release()

);
return image;
}

Expand Down