From 8d9a900d7b7819f745e7f213fb25c39c5af4a58f Mon Sep 17 00:00:00 2001 From: Adlai Holler Date: Fri, 28 May 2021 17:08:33 -0400 Subject: [PATCH 1/5] On iOS, decode WebP images directly into a buffer Core Animation likes Without this change, the images we output aren't compatible with the GPU and so CA has to copy them all on the main thread at commit time, which means stutters. Plus memory bloat. The difference here is so substantial that you may seriously want to consider re-running a WebP experiment if you did in the past and bailed on it. --- Source/Classes/Categories/PINImage+WebP.m | 87 +++++++++++++++++++++-- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/Source/Classes/Categories/PINImage+WebP.m b/Source/Classes/Categories/PINImage+WebP.m index a2466a79..d59dec42 100644 --- a/Source/Classes/Categories/PINImage+WebP.m +++ b/Source/Classes/Categories/PINImage+WebP.m @@ -19,6 +19,84 @@ static void releaseData(void *info, const void *data, size_t size) @implementation PINImage (PINWebP) +#if PIN_TARGET_IOS + +static enum WEBP_CSP_MODE webp_cs_mode_from_cg_bitmap_info(CGBitmapInfo info, BOOL *fail) { + CGImageByteOrderInfo byteOrder = info & kCGBitmapByteOrderMask; + BOOL keepByteOrder; + switch (byteOrder) { + case kCGImageByteOrder32Big: + keepByteOrder = YES; + break; + case kCGImageByteOrder32Little: + keepByteOrder = NO; + break; + case kCGImageByteOrder16Big: + case kCGImageByteOrder16Little: + case kCGImageByteOrderDefault: + case kCGImageByteOrderMask: + *fail = YES; + return MODE_RGBA; + } + + CGImageAlphaInfo ai = info & kCGBitmapAlphaInfoMask; + switch (ai) { + case kCGImageAlphaLast: + case kCGImageAlphaNoneSkipLast: + return keepByteOrder ? MODE_RGBA : MODE_ARGB; + case kCGImageAlphaNone: + return keepByteOrder ? MODE_RGB : MODE_BGR; + case kCGImageAlphaFirst: + case kCGImageAlphaNoneSkipFirst: + return keepByteOrder ? MODE_ARGB : MODE_BGRA; + case kCGImageAlphaPremultipliedLast: + return keepByteOrder ? MODE_rgbA : MODE_Argb; + case kCGImageAlphaPremultipliedFirst: + return keepByteOrder ? MODE_Argb : MODE_rgbA; + case kCGImageAlphaOnly: + *fail = YES; + return MODE_RGB; + } +} + +// For iOS we let the system decide all the bitmap options for us, so Core Animation won't have +// to copy our images over to the render server. Use "Color Copied Images" in the iOS simulator to +// detect this case. ++ (PINImage *)pin_imageWithWebPData:(NSData *)webPData +{ + WebPDecoderConfig cfg; + WebPInitDecoderConfig(&cfg); + if (WebPGetFeatures(webPData.bytes, webPData.length, &cfg.input) != VP8_STATUS_OK) { + return nil; + } + CGSize size = CGSizeMake(cfg.input.width, cfg.input.height); + UIGraphicsBeginImageContextWithOptions(size, !cfg.input.has_alpha, 1.0); + CGContextRef ctx = UIGraphicsGetCurrentContext(); + BOOL fail = NO; + cfg.output.colorspace = webp_cs_mode_from_cg_bitmap_info(CGBitmapContextGetBitmapInfo(ctx), + &fail); + if (fail) { + UIGraphicsEndImageContext(); + return nil; + } + cfg.output.width = cfg.input.width; + cfg.output.height = cfg.input.height; + cfg.output.is_external_memory = 1; + cfg.output.u.RGBA.rgba = (uint8_t *)CGBitmapContextGetData(ctx); + cfg.output.u.RGBA.stride = (int)CGBitmapContextGetBytesPerRow(ctx); + cfg.output.u.RGBA.size = cfg.output.u.RGBA.stride * cfg.input.height; + int status = WebPDecode(webPData.bytes, webPData.length, &cfg); + UIImage *image = nil; + if (status == VP8_STATUS_OK) { + image = UIGraphicsGetImageFromCurrentImageContext(); + } + UIGraphicsEndImageContext(); + return image; +} + +#elif PIN_TARGET_MAC + +// TODO: Can we get the optimal bitmap config from macOS like we do for iOS? + (PINImage *)pin_imageWithWebPData:(NSData *)webPData { WebPBitstreamFeatures features; @@ -61,12 +139,7 @@ + (PINImage *)pin_imageWithWebPData:(NSData *)webPData NO, renderingIntent); - PINImage *image = nil; -#if PIN_TARGET_IOS - image = [UIImage imageWithCGImage:imageRef]; -#elif PIN_TARGET_MAC - image = [[self alloc] initWithCGImage:imageRef size:CGSizeZero]; -#endif + PINImage *image = [[self alloc] initWithCGImage:imageRef size:CGSizeZero]; CGImageRelease(imageRef); CGColorSpaceRelease(colorSpaceRef); @@ -78,6 +151,8 @@ + (PINImage *)pin_imageWithWebPData:(NSData *)webPData return nil; } +#endif + @end #endif From fbded90300e9c0dd30c99779c1ed7b9dd764ac82 Mon Sep 17 00:00:00 2001 From: Adlai Holler Date: Sat, 29 May 2021 10:12:47 -0400 Subject: [PATCH 2/5] Try use system decode first if possible --- Source/Classes/Categories/PINImage+DecodedImage.m | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Source/Classes/Categories/PINImage+DecodedImage.m b/Source/Classes/Categories/PINImage+DecodedImage.m index da75e5bc..f7d1e810 100644 --- a/Source/Classes/Categories/PINImage+DecodedImage.m +++ b/Source/Classes/Categories/PINImage+DecodedImage.m @@ -122,12 +122,6 @@ + (PINImage *)pin_decodedImageWithData:(NSData *)data skipDecodeIfPossible:(BOOL return nil; } -#if PIN_WEBP - if ([data pin_isWebP]) { - return [PINImage pin_imageWithWebPData:data]; - } -#endif - PINImage *decodedImage = nil; CGImageSourceRef imageSourceRef = CGImageSourceCreateWithData((CFDataRef)data, NULL); @@ -156,6 +150,12 @@ + (PINImage *)pin_decodedImageWithData:(NSData *)data skipDecodeIfPossible:(BOOL CFRelease(imageSourceRef); } +#if PIN_WEBP + if (!decodedImage && [data pin_isWebP]) { + return [PINImage pin_imageWithWebPData:data]; + } +#endif + return decodedImage; } From 1fe48c855b25c74c11620b29a27dcb53ecb38c23 Mon Sep 17 00:00:00 2001 From: Adlai Holler Date: Wed, 2 Jun 2021 15:00:03 -0400 Subject: [PATCH 3/5] Do it the same on both platforms --- Source/Classes/Categories/PINImage+WebP.m | 96 ++++++----------------- 1 file changed, 23 insertions(+), 73 deletions(-) diff --git a/Source/Classes/Categories/PINImage+WebP.m b/Source/Classes/Categories/PINImage+WebP.m index d59dec42..06be1d73 100644 --- a/Source/Classes/Categories/PINImage+WebP.m +++ b/Source/Classes/Categories/PINImage+WebP.m @@ -12,15 +12,8 @@ #import "webp/decode.h" -static void releaseData(void *info, const void *data, size_t size) -{ - WebPFree((void *)data); -} - @implementation PINImage (PINWebP) -#if PIN_TARGET_IOS - static enum WEBP_CSP_MODE webp_cs_mode_from_cg_bitmap_info(CGBitmapInfo info, BOOL *fail) { CGImageByteOrderInfo byteOrder = info & kCGBitmapByteOrderMask; BOOL keepByteOrder; @@ -59,6 +52,12 @@ static enum WEBP_CSP_MODE webp_cs_mode_from_cg_bitmap_info(CGBitmapInfo info, BO } } +#if PIN_TARGET_IOS +#define PIN_WEBP_DECODE_CLEANUP() UIGraphicsEndImageContext() +#elif PIN_TARGET_MAC +#define PIN_WEBP_DECODE_CLEANUP() [image unlockFocus] +#endif + // For iOS we let the system decide all the bitmap options for us, so Core Animation won't have // to copy our images over to the render server. Use "Color Copied Images" in the iOS simulator to // detect this case. @@ -70,13 +69,22 @@ + (PINImage *)pin_imageWithWebPData:(NSData *)webPData return nil; } CGSize size = CGSizeMake(cfg.input.width, cfg.input.height); + CGContextRef ctx = NULL; + PINImage *image; +#if PIN_TARGET_IOS UIGraphicsBeginImageContextWithOptions(size, !cfg.input.has_alpha, 1.0); - CGContextRef ctx = UIGraphicsGetCurrentContext(); - BOOL fail = NO; + ctx = UIGraphicsGetCurrentContext(); +#elif PIN_TARGET_MAC + image = [[NSImage alloc] initWithSize:NSSizeFromCGSize(size)]; + [image lockFocus]; + ctx = NSGraphicsContext.currentContext.CGContext; +#endif + NSAssert(ctx != NULL, @"Failed to get CG context."); + BOOL getColorspaceFailed = NO; cfg.output.colorspace = webp_cs_mode_from_cg_bitmap_info(CGBitmapContextGetBitmapInfo(ctx), &fail); - if (fail) { - UIGraphicsEndImageContext(); + if (getColorspaceFailed) { + PIN_WEBP_DECODE_CLEANUP(); return nil; } cfg.output.width = cfg.input.width; @@ -86,72 +94,14 @@ + (PINImage *)pin_imageWithWebPData:(NSData *)webPData cfg.output.u.RGBA.stride = (int)CGBitmapContextGetBytesPerRow(ctx); cfg.output.u.RGBA.size = cfg.output.u.RGBA.stride * cfg.input.height; int status = WebPDecode(webPData.bytes, webPData.length, &cfg); - UIImage *image = nil; +#if PIN_TARGET_IOS if (status == VP8_STATUS_OK) { image = UIGraphicsGetImageFromCurrentImageContext(); } - UIGraphicsEndImageContext(); - return image; -} - -#elif PIN_TARGET_MAC - -// TODO: Can we get the optimal bitmap config from macOS like we do for iOS? -+ (PINImage *)pin_imageWithWebPData:(NSData *)webPData -{ - WebPBitstreamFeatures features; - if (WebPGetFeatures([webPData bytes], [webPData length], &features) == VP8_STATUS_OK) { - // Decode the WebP image data into a RGBA value array - int height, width; - uint8_t *data = NULL; - int pixelLength = 0; - - if (features.has_alpha) { - data = WebPDecodeRGBA([webPData bytes], [webPData length], &width, &height); - pixelLength = 4; - } else { - data = WebPDecodeRGB([webPData bytes], [webPData length], &width, &height); - pixelLength = 3; - } - - if (data) { - CGDataProviderRef provider = CGDataProviderCreateWithData(NULL, data, width * height * pixelLength, releaseData); - - CGColorSpaceRef colorSpaceRef = CGColorSpaceCreateDeviceRGB(); - CGBitmapInfo bitmapInfo = kCGBitmapByteOrderDefault; - - if (features.has_alpha) { - bitmapInfo |= kCGImageAlphaLast; - } else { - bitmapInfo |= kCGImageAlphaNone; - } - - CGColorRenderingIntent renderingIntent = kCGRenderingIntentDefault; - CGImageRef imageRef = CGImageCreate(width, - height, - 8, - 8 * pixelLength, - pixelLength * width, - colorSpaceRef, - bitmapInfo, - provider, - NULL, - NO, - renderingIntent); - - PINImage *image = [[self alloc] initWithCGImage:imageRef size:CGSizeZero]; - - CGImageRelease(imageRef); - CGColorSpaceRelease(colorSpaceRef); - CGDataProviderRelease(provider); - - return image; - } - } - return nil; -} - #endif + PIN_WEBP_DECODE_CLEANUP(); + return status == VP8_STATUS_OK ? image : nil; +} @end From 69ba168c113610f30c699fc1fc0905672a3e7333 Mon Sep 17 00:00:00 2001 From: Adlai Holler Date: Wed, 2 Jun 2021 15:01:02 -0400 Subject: [PATCH 4/5] Fix comment --- Source/Classes/Categories/PINImage+WebP.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Classes/Categories/PINImage+WebP.m b/Source/Classes/Categories/PINImage+WebP.m index 06be1d73..dd950644 100644 --- a/Source/Classes/Categories/PINImage+WebP.m +++ b/Source/Classes/Categories/PINImage+WebP.m @@ -58,7 +58,7 @@ static enum WEBP_CSP_MODE webp_cs_mode_from_cg_bitmap_info(CGBitmapInfo info, BO #define PIN_WEBP_DECODE_CLEANUP() [image unlockFocus] #endif -// For iOS we let the system decide all the bitmap options for us, so Core Animation won't have +// We let the system decide all the bitmap options for us, so Core Animation won't have // to copy our images over to the render server. Use "Color Copied Images" in the iOS simulator to // detect this case. + (PINImage *)pin_imageWithWebPData:(NSData *)webPData From fa3da181ac3472e757fd2df6f73189aa9bdd1c81 Mon Sep 17 00:00:00 2001 From: Adlai Holler Date: Wed, 2 Jun 2021 15:02:22 -0400 Subject: [PATCH 5/5] But do it right --- Source/Classes/Categories/PINImage+WebP.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Classes/Categories/PINImage+WebP.m b/Source/Classes/Categories/PINImage+WebP.m index dd950644..f45f1d83 100644 --- a/Source/Classes/Categories/PINImage+WebP.m +++ b/Source/Classes/Categories/PINImage+WebP.m @@ -82,7 +82,7 @@ + (PINImage *)pin_imageWithWebPData:(NSData *)webPData NSAssert(ctx != NULL, @"Failed to get CG context."); BOOL getColorspaceFailed = NO; cfg.output.colorspace = webp_cs_mode_from_cg_bitmap_info(CGBitmapContextGetBitmapInfo(ctx), - &fail); + &getColorspaceFailed); if (getColorspaceFailed) { PIN_WEBP_DECODE_CLEANUP(); return nil;