From 37be5c54685562dd71ba41ab005224e0085c38a0 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Thu, 12 Jan 2023 18:54:23 -0500 Subject: [PATCH 01/35] Fix padding issue --- .../io/flutter/plugins/camera/Camera.java | 60 +++++++++++++++---- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java index 7c592b9c7e99..d597f3e95b5e 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java @@ -63,6 +63,7 @@ import io.flutter.plugins.camera.types.CameraCaptureProperties; import io.flutter.plugins.camera.types.CaptureTimeoutsWrapper; import io.flutter.view.TextureRegistry.SurfaceTextureEntry; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; @@ -1149,22 +1150,31 @@ private void setImageStreamImageAvailableListener(final EventChannel.EventSink i imageStreamReader.setOnImageAvailableListener( reader -> { Image img = reader.acquireNextImage(); + // Use acquireNextImage since image reader is only for one image. if (img == null) return; List> planes = new ArrayList<>(); for (Image.Plane plane : img.getPlanes()) { - ByteBuffer buffer = plane.getBuffer(); - - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes, 0, bytes.length); - - Map planeBuffer = new HashMap<>(); - planeBuffer.put("bytesPerRow", plane.getRowStride()); - planeBuffer.put("bytesPerPixel", plane.getPixelStride()); - planeBuffer.put("bytes", bytes); - - planes.add(planeBuffer); + Map yPlaneBuffer = new HashMap<>(); + yPlaneBuffer.put("bytesPerPixel", plane.getPixelStride()); + + // Now process the plane byte data. We sometimes have to correct the luma channel (Y) + // because some devices have some kind of extra padding included. From testing it seems + // like we do not have to modify the U/V channels, just the Y channel for this case. + if (plane.getRowStride() != img.getWidth() && plane.getPixelStride() == 1) { + // There is some padding we have to handle + yPlaneBuffer.put("bytes", getPixels(plane.getBuffer(), img.getWidth(), img.getHeight(), plane.getRowStride(), plane.getPixelStride())); + yPlaneBuffer.put("bytesPerRow", Math.min(plane.getRowStride(), img.getWidth())); + } else { + // Just use the data as-is + ByteBuffer buffer = plane.getBuffer(); + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes, 0, bytes.length); + yPlaneBuffer.put("bytes", bytes); + yPlaneBuffer.put("bytesPerRow", plane.getRowStride()); + } + planes.add(yPlaneBuffer); } Map imageBuffer = new HashMap<>(); @@ -1185,6 +1195,34 @@ private void setImageStreamImageAvailableListener(final EventChannel.EventSink i backgroundHandler); } + // Adopted from: + // https://github.com/abrenoch/hyperion-android-grabber/blob/c2befd7501c72f93f5c022666266f67142fe8ba3/common/src/main/java/com/abrenoch/hyperiongrabber/common/HyperionScreenEncoder.java#L154-L177 + // + // Will return a new byte buffer with padding accounted for. + private byte[] getPixels(ByteBuffer buffer, int width, int height, int rowStride, int pixelStride){ + int rowPadding = rowStride - width * pixelStride; + int offset = 0; + + ByteArrayOutputStream bao = new ByteArrayOutputStream( + width * height * 3 + ); + + for (int y = 0, compareHeight = height - 1; y < height; y++, offset += rowPadding) { + if ( y > compareHeight) { + offset += width * pixelStride; + continue; + } + + for (int x = 0, compareWidth = width - 1; x < width; x++, offset += pixelStride) { + if (x > compareWidth) continue; + if (offset > buffer.remaining()) continue; + bao.write(buffer.get(offset)); + } + } + + return bao.toByteArray(); + } + private void closeCaptureSession() { if (captureSession != null) { Log.i(TAG, "closeCaptureSession"); From 45d2c58de73bd8a778f8eb44cbfc3317f0ee5aa4 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 13 Jan 2023 09:26:00 -0500 Subject: [PATCH 02/35] New approach that is more efficient --- .../io/flutter/plugins/camera/Camera.java | 97 ++++++++++++------- 1 file changed, 61 insertions(+), 36 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java index d597f3e95b5e..54f6eb76a2b1 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java @@ -1154,27 +1154,48 @@ private void setImageStreamImageAvailableListener(final EventChannel.EventSink i // Use acquireNextImage since image reader is only for one image. if (img == null) return; + List> planes = new ArrayList<>(); - for (Image.Plane plane : img.getPlanes()) { - Map yPlaneBuffer = new HashMap<>(); - yPlaneBuffer.put("bytesPerPixel", plane.getPixelStride()); - - // Now process the plane byte data. We sometimes have to correct the luma channel (Y) - // because some devices have some kind of extra padding included. From testing it seems - // like we do not have to modify the U/V channels, just the Y channel for this case. - if (plane.getRowStride() != img.getWidth() && plane.getPixelStride() == 1) { - // There is some padding we have to handle - yPlaneBuffer.put("bytes", getPixels(plane.getBuffer(), img.getWidth(), img.getHeight(), plane.getRowStride(), plane.getPixelStride())); - yPlaneBuffer.put("bytesPerRow", Math.min(plane.getRowStride(), img.getWidth())); + for (int i=0; i planeBuffer = new HashMap<>(); + planeBuffer.put("bytesPerPixel", plane.getPixelStride()); + + // Sometimes YUV420 has additional padding that must be removed. This is only the case if we are + // streaming YUV420, the row stride does not match the image width, and the pixel stride is 1. + if (reader.getImageFormat() == ImageFormat.YUV_420_888 && + plane.getRowStride() != img.getWidth() && + plane.getPixelStride() == 1) { + // The ordering of planes is guaranteed by Android. It always goes Y, U, V. + int planeWidth; + int planeHeight; + if (i == 0) { + // Y is the image size + planeWidth = img.getWidth(); + planeHeight = img.getHeight(); + } else { + // U and V are guaranteed to be the same size and are half of the image height/width + // in YUV420 + planeWidth = img.getWidth() / 2; + planeHeight = img.getHeight() / 2; + } + + planeBuffer.put("bytes", removePlaneBufferPadding(plane, planeWidth, planeHeight)); + + // Make sure the bytesPerRow matches the image width now that we've removed the padding + planeBuffer.put("bytesPerRow", img.getWidth()); } else { // Just use the data as-is ByteBuffer buffer = plane.getBuffer(); byte[] bytes = new byte[buffer.remaining()]; buffer.get(bytes, 0, bytes.length); - yPlaneBuffer.put("bytes", bytes); - yPlaneBuffer.put("bytesPerRow", plane.getRowStride()); + planeBuffer.put("bytes", bytes); + planeBuffer.put("bytesPerRow", plane.getRowStride()); } - planes.add(yPlaneBuffer); + planes.add(planeBuffer); } Map imageBuffer = new HashMap<>(); @@ -1195,32 +1216,36 @@ private void setImageStreamImageAvailableListener(final EventChannel.EventSink i backgroundHandler); } - // Adopted from: - // https://github.com/abrenoch/hyperion-android-grabber/blob/c2befd7501c72f93f5c022666266f67142fe8ba3/common/src/main/java/com/abrenoch/hyperiongrabber/common/HyperionScreenEncoder.java#L154-L177 + // Copyright (c) 2019 Dmitry Gordin + // Based on: + // https://github.com/gordinmitya/yuv2buf/blob/master/yuv2buf/src/main/java/ru/gordinmitya/yuv2buf/Yuv.java // - // Will return a new byte buffer with padding accounted for. - private byte[] getPixels(ByteBuffer buffer, int width, int height, int rowStride, int pixelStride){ - int rowPadding = rowStride - width * pixelStride; - int offset = 0; - - ByteArrayOutputStream bao = new ByteArrayOutputStream( - width * height * 3 - ); - - for (int y = 0, compareHeight = height - 1; y < height; y++, offset += rowPadding) { - if ( y > compareHeight) { - offset += width * pixelStride; - continue; - } + // Will remove the padding from a given image plane and return the fixed buffer. + private static byte[] removePlaneBufferPadding(Image.Plane plane, int planeWidth, int planeHeight) { + if (plane.getPixelStride() != 1) { + throw new IllegalArgumentException("it's only valid to remove padding when pixelStride == 1"); + } - for (int x = 0, compareWidth = width - 1; x < width; x++, offset += pixelStride) { - if (x > compareWidth) continue; - if (offset > buffer.remaining()) continue; - bao.write(buffer.get(offset)); - } + ByteBuffer dst = ByteBuffer.allocate(planeWidth * planeHeight); + ByteBuffer src = plane.getBuffer(); + int rowStride = plane.getRowStride(); + ByteBuffer row; + for (int i = 0; i < planeHeight; i++) { + row = clipBuffer(src, i * rowStride, planeWidth); + dst.put(row); } - return bao.toByteArray(); + return dst.array(); + } + + // Copyright (c) 2019 Dmitry Gordin + // Based on: + // https://github.com/gordinmitya/yuv2buf/blob/master/yuv2buf/src/main/java/ru/gordinmitya/yuv2buf/Yuv.java + private static ByteBuffer clipBuffer(ByteBuffer buffer, int start, int size) { + ByteBuffer duplicate = buffer.duplicate(); + duplicate.position(start); + duplicate.limit(start + size); + return duplicate.slice(); } private void closeCaptureSession() { From d2492c86feb87e424395112380a4bf3976dae75d Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 13 Jan 2023 10:13:23 -0500 Subject: [PATCH 03/35] Refactored image reader to enable testing --- .../io/flutter/plugins/camera/Camera.java | 80 +------ .../camera/media/ImageStreamReader.java | 208 ++++++++++++++++++ 2 files changed, 215 insertions(+), 73 deletions(-) create mode 100644 packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java index 54f6eb76a2b1..ad0dfa2c7346 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java @@ -59,6 +59,7 @@ import io.flutter.plugins.camera.features.sensororientation.DeviceOrientationManager; import io.flutter.plugins.camera.features.sensororientation.SensorOrientationFeature; import io.flutter.plugins.camera.features.zoomlevel.ZoomLevelFeature; +import io.flutter.plugins.camera.media.ImageStreamReader; import io.flutter.plugins.camera.media.MediaRecorderBuilder; import io.flutter.plugins.camera.types.CameraCaptureProperties; import io.flutter.plugins.camera.types.CaptureTimeoutsWrapper; @@ -136,7 +137,7 @@ class Camera private CameraDeviceWrapper cameraDevice; private CameraCaptureSession captureSession; private ImageReader pictureImageReader; - private ImageReader imageStreamReader; + private ImageStreamReader imageStreamReader; /** {@link CaptureRequest.Builder} for the camera preview */ private CaptureRequest.Builder previewRequestBuilder; @@ -304,12 +305,11 @@ public void open(String imageFormatGroup) throws CameraAccessException { Log.w(TAG, "The selected imageFormatGroup is not supported by Android. Defaulting to yuv420"); imageFormat = ImageFormat.YUV_420_888; } - imageStreamReader = - ImageReader.newInstance( + imageStreamReader = new ImageStreamReader( resolutionFeature.getPreviewSize().getWidth(), resolutionFeature.getPreviewSize().getHeight(), - imageFormat, - 1); + imageFormat + ); // Open the camera. CameraManager cameraManager = CameraUtils.getCameraManager(activity); @@ -1141,79 +1141,13 @@ public void onListen(Object o, EventChannel.EventSink imageStreamSink) { @Override public void onCancel(Object o) { - imageStreamReader.setOnImageAvailableListener(null, backgroundHandler); + imageStreamReader.removeListener(backgroundHandler); } }); } private void setImageStreamImageAvailableListener(final EventChannel.EventSink imageStreamSink) { - imageStreamReader.setOnImageAvailableListener( - reader -> { - Image img = reader.acquireNextImage(); - - // Use acquireNextImage since image reader is only for one image. - if (img == null) return; - - - List> planes = new ArrayList<>(); - for (int i=0; i planeBuffer = new HashMap<>(); - planeBuffer.put("bytesPerPixel", plane.getPixelStride()); - - // Sometimes YUV420 has additional padding that must be removed. This is only the case if we are - // streaming YUV420, the row stride does not match the image width, and the pixel stride is 1. - if (reader.getImageFormat() == ImageFormat.YUV_420_888 && - plane.getRowStride() != img.getWidth() && - plane.getPixelStride() == 1) { - // The ordering of planes is guaranteed by Android. It always goes Y, U, V. - int planeWidth; - int planeHeight; - if (i == 0) { - // Y is the image size - planeWidth = img.getWidth(); - planeHeight = img.getHeight(); - } else { - // U and V are guaranteed to be the same size and are half of the image height/width - // in YUV420 - planeWidth = img.getWidth() / 2; - planeHeight = img.getHeight() / 2; - } - - planeBuffer.put("bytes", removePlaneBufferPadding(plane, planeWidth, planeHeight)); - - // Make sure the bytesPerRow matches the image width now that we've removed the padding - planeBuffer.put("bytesPerRow", img.getWidth()); - } else { - // Just use the data as-is - ByteBuffer buffer = plane.getBuffer(); - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes, 0, bytes.length); - planeBuffer.put("bytes", bytes); - planeBuffer.put("bytesPerRow", plane.getRowStride()); - } - planes.add(planeBuffer); - } - - Map imageBuffer = new HashMap<>(); - imageBuffer.put("width", img.getWidth()); - imageBuffer.put("height", img.getHeight()); - imageBuffer.put("format", img.getFormat()); - imageBuffer.put("planes", planes); - imageBuffer.put("lensAperture", this.captureProps.getLastLensAperture()); - imageBuffer.put("sensorExposureTime", this.captureProps.getLastSensorExposureTime()); - Integer sensorSensitivity = this.captureProps.getLastSensorSensitivity(); - imageBuffer.put( - "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity); - - final Handler handler = new Handler(Looper.getMainLooper()); - handler.post(() -> imageStreamSink.success(imageBuffer)); - img.close(); - }, - backgroundHandler); + imageStreamReader.subscribeListener(this.captureProps, imageStreamSink, backgroundHandler); } // Copyright (c) 2019 Dmitry Gordin diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java new file mode 100644 index 000000000000..88b4390ab57e --- /dev/null +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -0,0 +1,208 @@ +package io.flutter.plugins.camera.media; + +import android.graphics.ImageFormat; +import android.media.Image; +import android.media.ImageReader; +import android.os.Handler; +import android.os.Looper; +import android.view.Surface; + +import androidx.annotation.NonNull; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugins.camera.types.CameraCaptureProperties; + +// Wraps an ImageReader to allow for testing of the image handler. +public class ImageStreamReader { + // The actual im + private final ImageReader imageReader; + + /** + * Creates a new instance of the {@link ImageStreamReader}. + * + * @param width is the integer width of the image stream (preview image) + * @param height is the integer height of the image stream (preview image) + * @param imageFormatGroup is the integer image format group, as a valid {@link ImageFormat}. + */ + public ImageStreamReader( + int width, + int height, + int imageFormatGroup + ) { + this.imageReader = ImageReader.newInstance( + width, + height, + imageFormatGroup, + 1 + ); + } + + /** + * Processes a new frame (image) from the image reader, remove padding if necessary, + * and send the frame to Dart. + * + * @param image is the image which needs processed as an {@link Image} + * @param imageFormat is the image format from the image reader as an int, a valid {@link ImageFormat} + * @param captureProps is the capture props from the camera class as {@link CameraCaptureProperties} + * @param imageStreamSink is the image stream sink from dart as a dart {@link EventChannel.EventSink} + */ + private static void onImageAvailable( + @NonNull Image image, + int imageFormat, + CameraCaptureProperties captureProps, + EventChannel.EventSink imageStreamSink + ) { + List> planes = new ArrayList<>(); + for (int i=0; i planeBuffer = new HashMap<>(); + planeBuffer.put("bytesPerPixel", plane.getPixelStride()); + + // Sometimes YUV420 has additional padding that must be removed. This is only the case if we are + // streaming YUV420, the row stride does not match the image width, and the pixel stride is 1. + if (imageFormat == ImageFormat.YUV_420_888 && + plane.getRowStride() != image.getWidth() && + plane.getPixelStride() == 1) { + // The ordering of planes is guaranteed by Android. It always goes Y, U, V. + int planeWidth; + int planeHeight; + if (i == 0) { + // Y is the image size + planeWidth = image.getWidth(); + planeHeight = image.getHeight(); + } else { + // U and V are guaranteed to be the same size and are half of the image height/width + // in YUV420 + planeWidth = image.getWidth() / 2; + planeHeight = image.getHeight() / 2; + } + + planeBuffer.put("bytes", removePlaneBufferPadding(plane, planeWidth, planeHeight)); + + // Make sure the bytesPerRow matches the image width now that we've removed the padding + planeBuffer.put("bytesPerRow", image.getWidth()); + } else { + // Just use the data as-is + ByteBuffer buffer = plane.getBuffer(); + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes, 0, bytes.length); + planeBuffer.put("bytes", bytes); + planeBuffer.put("bytesPerRow", plane.getRowStride()); + } + planes.add(planeBuffer); + } + + Map imageBuffer = new HashMap<>(); + imageBuffer.put("width", image.getWidth()); + imageBuffer.put("height", image.getHeight()); + imageBuffer.put("format", image.getFormat()); + imageBuffer.put("planes", planes); + imageBuffer.put("lensAperture", captureProps.getLastLensAperture()); + imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime()); + Integer sensorSensitivity = captureProps.getLastSensorSensitivity(); + imageBuffer.put( + "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity); + + final Handler handler = new Handler(Looper.getMainLooper()); + handler.post(() -> imageStreamSink.success(imageBuffer)); + image.close(); + } + + /** + * Copyright (c) 2019 Dmitry Gordin + * Based on: + * https://github.com/gordinmitya/yuv2buf/blob/master/yuv2buf/src/main/java/ru/gordinmitya/yuv2buf/Yuv.java + * + * Will remove the padding from a given image plane and return the fixed buffer. + * + * @param plane is the image plane (buffer) that will be processed as an {@link Image.Plane} + * @param planeWidth is the width of the plane as an int + * @param planeHeight is the height of the plane as an int + */ + private static byte[] removePlaneBufferPadding(Image.Plane plane, int planeWidth, int planeHeight) { + if (plane.getPixelStride() != 1) { + throw new IllegalArgumentException("it's only valid to remove padding when pixelStride == 1"); + } + + ByteBuffer dst = ByteBuffer.allocate(planeWidth * planeHeight); + ByteBuffer src = plane.getBuffer(); + int rowStride = plane.getRowStride(); + ByteBuffer row; + for (int i = 0; i < planeHeight; i++) { + row = clipBuffer(src, i * rowStride, planeWidth); + dst.put(row); + } + + return dst.array(); + } + + /** + * Copyright (c) 2019 Dmitry Gordin + * Based on: + * https://github.com/gordinmitya/yuv2buf/blob/master/yuv2buf/src/main/java/ru/gordinmitya/yuv2buf/Yuv.java + * + * Copies part of a buffer to a new buffer, used to trim the padding. + * + * @param buffer is the source buffer to be modified as a {@link ByteBuffer} + * @param start is the starting offset to read from as an int + * @param size is the length of data to read as an int + */ + private static ByteBuffer clipBuffer(ByteBuffer buffer, int start, int size) { + ByteBuffer duplicate = buffer.duplicate(); + duplicate.position(start); + duplicate.limit(start + size); + return duplicate.slice(); + } + + /** + * Returns the image reader surface. + */ + public Surface getSurface() { + return imageReader.getSurface(); + } + + /** + * Subscribes the image stream reader to handle incoming images using onImageAvailable(). + * + * @param captureProps is the capture props from the camera class as {@link CameraCaptureProperties} + * @param imageStreamSink is the image stream sink from dart as {@link EventChannel.EventSink} + * @param handler is generally the background handler of the camera as {@link Handler} + */ + public void subscribeListener( + CameraCaptureProperties captureProps, + EventChannel.EventSink imageStreamSink, + Handler handler + ) { + imageReader.setOnImageAvailableListener(reader -> { + Image image = reader.acquireNextImage(); + if (image == null) return; + + onImageAvailable(image, imageReader.getImageFormat(), captureProps, imageStreamSink); + }, handler); + } + + /** + * Removes the listener from the image reader. + * + * @param handler is generally the background handler of the camera + */ + public void removeListener(Handler handler) { + imageReader.setOnImageAvailableListener(null, handler); + } + + /** + * Closes the image reader. + */ + public void close() { + imageReader.close(); + } +} From bd559654ccb6b2672e52607161aec20ab37dd409 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Tue, 17 Jan 2023 11:01:18 -0500 Subject: [PATCH 04/35] Added tests for image stream reader --- .../io/flutter/plugins/camera/Camera.java | 7 +- .../camera/media/ImageStreamReader.java | 87 ++++--------- .../camera/media/ImageStreamReaderUtils.java | 53 ++++++++ .../camera/media/ImageStreamReaderTest.java | 114 ++++++++++++++++++ 4 files changed, 195 insertions(+), 66 deletions(-) create mode 100644 packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java create mode 100644 packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java index ad0dfa2c7346..48fd645488c0 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java @@ -305,11 +305,12 @@ public void open(String imageFormatGroup) throws CameraAccessException { Log.w(TAG, "The selected imageFormatGroup is not supported by Android. Defaulting to yuv420"); imageFormat = ImageFormat.YUV_420_888; } - imageStreamReader = new ImageStreamReader( + imageStreamReader = new ImageStreamReader(ImageReader.newInstance( resolutionFeature.getPreviewSize().getWidth(), resolutionFeature.getPreviewSize().getHeight(), - imageFormat - ); + imageFormat, + 1) + ); // Open the camera. CameraManager cameraManager = CameraUtils.getCameraManager(activity); diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index 88b4390ab57e..3f7971234772 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -8,6 +8,7 @@ import android.view.Surface; import androidx.annotation.NonNull; +import androidx.annotation.VisibleForTesting; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -15,32 +16,35 @@ import java.util.List; import java.util.Map; +import io.flutter.Log; import io.flutter.plugin.common.EventChannel; import io.flutter.plugins.camera.types.CameraCaptureProperties; // Wraps an ImageReader to allow for testing of the image handler. public class ImageStreamReader { - // The actual im private final ImageReader imageReader; + private final ImageStreamReaderUtils imageStreamReaderUtils; /** * Creates a new instance of the {@link ImageStreamReader}. * - * @param width is the integer width of the image stream (preview image) - * @param height is the integer height of the image stream (preview image) - * @param imageFormatGroup is the integer image format group, as a valid {@link ImageFormat}. + * @param imageReader is the image reader that will receive frames + * @param imageStreamReaderUtils is an instance of {@link ImageStreamReaderUtils} */ - public ImageStreamReader( - int width, - int height, - int imageFormatGroup - ) { - this.imageReader = ImageReader.newInstance( - width, - height, - imageFormatGroup, - 1 - ); + @VisibleForTesting + public ImageStreamReader(ImageReader imageReader, ImageStreamReaderUtils imageStreamReaderUtils) { + this.imageReader = imageReader; + this.imageStreamReaderUtils = imageStreamReaderUtils; + } + + /** + * Creates a new instance of the {@link ImageStreamReader}. + * + * @param imageReader is the image reader that will receive frames + */ + public ImageStreamReader(ImageReader imageReader) { + this.imageReader = imageReader; + this.imageStreamReaderUtils = new ImageStreamReaderUtils(); } /** @@ -52,12 +56,15 @@ public ImageStreamReader( * @param captureProps is the capture props from the camera class as {@link CameraCaptureProperties} * @param imageStreamSink is the image stream sink from dart as a dart {@link EventChannel.EventSink} */ - private static void onImageAvailable( + @VisibleForTesting + public void onImageAvailable( @NonNull Image image, int imageFormat, CameraCaptureProperties captureProps, EventChannel.EventSink imageStreamSink ) { + Log.i("flutter", "Image planes: " + image.getPlanes().length); + List> planes = new ArrayList<>(); for (int i=0; i Date: Tue, 17 Jan 2023 11:30:29 -0500 Subject: [PATCH 05/35] Added tests for buffer trimming --- .../camera/media/ImageStreamReaderTest.java | 7 -- .../media/ImageStreamReaderUtilsTest.java | 73 +++++++++++++++++++ 2 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java index ccb7eb393c40..f9f74aaf582e 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java @@ -4,20 +4,14 @@ package io.flutter.plugins.camera.media; -import static org.junit.Assert.assertNotNull; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyFloat; import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import android.graphics.ImageFormat; -import android.hardware.camera2.CameraCaptureSession; -import android.hardware.camera2.CaptureRequest; -import android.hardware.camera2.CaptureResult; import android.media.Image; import android.media.ImageReader; @@ -30,7 +24,6 @@ import io.flutter.plugin.common.EventChannel; import io.flutter.plugins.camera.types.CameraCaptureProperties; -import io.flutter.plugins.camera.types.CaptureTimeoutsWrapper; @RunWith(RobolectricTestRunner.class) public class ImageStreamReaderTest { diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java new file mode 100644 index 000000000000..7b6214ca439d --- /dev/null +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java @@ -0,0 +1,73 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package io.flutter.plugins.camera.media; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import android.media.Image; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +@RunWith(RobolectricTestRunner.class) +public class ImageStreamReaderUtilsTest { + private ImageStreamReaderUtils imageStreamReaderUtils; + + @Before + public void setUp() { + this.imageStreamReaderUtils = new ImageStreamReaderUtils(); + } + + @Test(expected = IllegalArgumentException.class) + public void removePlaneBufferPadding_throwsIfPixelStrideInvalid() { + Image.Plane planeU = mock(Image.Plane.class); + when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); + when(planeU.getRowStride()).thenReturn(1536); + when(planeU.getPixelStride()).thenReturn(2); + + imageStreamReaderUtils.removePlaneBufferPadding(planeU, 1280 / 2, 720 / 2); + } + + // Values here are taken from a Vivo V2135 which adds padding in 1280x720 mode + @Test + public void removePlaneBufferPadding_removesPaddingCorrectly() { + Image.Plane planeY = mock(Image.Plane.class); + when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664)); + when(planeY.getRowStride()).thenReturn(1536); + when(planeY.getPixelStride()).thenReturn(1); + + byte[] fixed = imageStreamReaderUtils.removePlaneBufferPadding(planeY, 1280, 720); + + // After trimming the padding, the buffer size should match the image size + Assert.assertEquals(fixed.length, 1280 * 720); + Assert.assertNotEquals(fixed.length, planeY.getBuffer().limit()); + Assert.assertFalse(Arrays.equals(fixed, planeY.getBuffer().array())); + } + + // Values here are taken from a Pixel 6 which does not add any padding. + // We would expect that this code would not remove any data in the event + // the plane row stride equals the image width. + @Test + public void removePlaneBufferPadding_doesNothingIfThereIsNoPadding() { + Image.Plane planeY = mock(Image.Plane.class); + when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(921600)); + when(planeY.getRowStride()).thenReturn(1280); + when(planeY.getPixelStride()).thenReturn(1); + + byte[] fixed = imageStreamReaderUtils.removePlaneBufferPadding(planeY, 1280, 720); + + // After trimming the padding, the buffer size should match the image size + Assert.assertEquals(fixed.length, 1280 * 720); + Assert.assertEquals(fixed.length, planeY.getBuffer().limit()); + Assert.assertArrayEquals(fixed, planeY.getBuffer().array()); + } +} From fee7979a5e824ed0ea6d6ebb5006d2dfa5723ebb Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Tue, 17 Jan 2023 11:47:19 -0500 Subject: [PATCH 06/35] Formatter --- .../plugins/camera/media/ImageStreamReaderUtilsTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java index 7b6214ca439d..822ae076e645 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java @@ -54,8 +54,8 @@ public void removePlaneBufferPadding_removesPaddingCorrectly() { } // Values here are taken from a Pixel 6 which does not add any padding. - // We would expect that this code would not remove any data in the event - // the plane row stride equals the image width. + // In the event we pass a buffer with no padding, the returned buffer + // should be identical to the source one because nothing is trimmed. @Test public void removePlaneBufferPadding_doesNothingIfThereIsNoPadding() { Image.Plane planeY = mock(Image.Plane.class); From f0f064090db59a1f3b8d4f6b4924838c024480cf Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Tue, 17 Jan 2023 11:50:46 -0500 Subject: [PATCH 07/35] Bump pubspec version --- packages/camera/camera_android/CHANGELOG.md | 4 ++++ packages/camera/camera_android/pubspec.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/camera/camera_android/CHANGELOG.md b/packages/camera/camera_android/CHANGELOG.md index 1a0b87e3c29d..b59da3112ce4 100644 --- a/packages/camera/camera_android/CHANGELOG.md +++ b/packages/camera/camera_android/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.10.3 + +* Adds logic to remove buffer padding in YUV frames on some devices / resolutions. + ## 0.10.2+1 * Updates code for stricter lint checks. diff --git a/packages/camera/camera_android/pubspec.yaml b/packages/camera/camera_android/pubspec.yaml index 9d86a00e0e50..4ecf768bfe12 100644 --- a/packages/camera/camera_android/pubspec.yaml +++ b/packages/camera/camera_android/pubspec.yaml @@ -2,7 +2,7 @@ name: camera_android description: Android implementation of the camera plugin. repository: https://github.com/flutter/plugins/tree/main/packages/camera/camera_android issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22 -version: 0.10.2+1 +version: 0.10.3 environment: sdk: ">=2.14.0 <3.0.0" From df99b023fc8bd71a1945e499b02e1ba68d8ce5ea Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Tue, 17 Jan 2023 11:59:25 -0500 Subject: [PATCH 08/35] Optimize imports --- .../main/java/io/flutter/plugins/camera/CameraUtils.java | 4 +++- .../io/flutter/plugins/camera/MethodCallHandlerImpl.java | 9 ++++++--- .../flutter/plugins/camera/features/CameraFeatures.java | 8 +++++--- .../features/exposureoffset/ExposureOffsetFeature.java | 2 ++ .../features/exposurepoint/ExposurePointFeature.java | 2 ++ .../plugins/camera/features/flash/FlashFeature.java | 1 + .../camera/features/fpsrange/FpsRangeFeature.java | 1 + .../camera/features/resolution/ResolutionFeature.java | 5 ++++- 8 files changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraUtils.java index 11b6eeaa5b50..6ba1ae2bccf2 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraUtils.java @@ -10,12 +10,14 @@ import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CameraManager; import android.hardware.camera2.CameraMetadata; -import io.flutter.embedding.engine.systemchannels.PlatformChannel; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import io.flutter.embedding.engine.systemchannels.PlatformChannel; + /** Provides various utilities for camera. */ public final class CameraUtils { diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java index 432344ade8cd..f3d41904de1a 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java @@ -8,8 +8,14 @@ import android.hardware.camera2.CameraAccessException; import android.os.Handler; import android.os.Looper; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugin.common.BinaryMessenger; import io.flutter.plugin.common.EventChannel; @@ -24,9 +30,6 @@ import io.flutter.plugins.camera.features.flash.FlashMode; import io.flutter.plugins.camera.features.resolution.ResolutionPreset; import io.flutter.view.TextureRegistry; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; final class MethodCallHandlerImpl implements MethodChannel.MethodCallHandler { private final Activity activity; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatures.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatures.java index 659fd15963e9..e366ce075717 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatures.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatures.java @@ -5,6 +5,11 @@ package io.flutter.plugins.camera.features; import android.app.Activity; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.DartMessenger; import io.flutter.plugins.camera.features.autofocus.AutoFocusFeature; @@ -19,9 +24,6 @@ import io.flutter.plugins.camera.features.resolution.ResolutionPreset; import io.flutter.plugins.camera.features.sensororientation.SensorOrientationFeature; import io.flutter.plugins.camera.features.zoomlevel.ZoomLevelFeature; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; /** * These are all of our available features in the camera. Used in the Camera to access all features diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposureoffset/ExposureOffsetFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposureoffset/ExposureOffsetFeature.java index d5a9fcd4a38a..86dddf902890 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposureoffset/ExposureOffsetFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposureoffset/ExposureOffsetFeature.java @@ -6,7 +6,9 @@ import android.hardware.camera2.CaptureRequest; import android.util.Range; + import androidx.annotation.NonNull; + import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurepoint/ExposurePointFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurepoint/ExposurePointFeature.java index 336e756e9ed8..3a190c13a670 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurepoint/ExposurePointFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurepoint/ExposurePointFeature.java @@ -7,7 +7,9 @@ import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.params.MeteringRectangle; import android.util.Size; + import androidx.annotation.NonNull; + import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.CameraRegionUtils; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/flash/FlashFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/flash/FlashFeature.java index 054c81f5183b..6e8b64ecd002 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/flash/FlashFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/flash/FlashFeature.java @@ -5,6 +5,7 @@ package io.flutter.plugins.camera.features.flash; import android.hardware.camera2.CaptureRequest; + import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/fpsrange/FpsRangeFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/fpsrange/FpsRangeFeature.java index 500f2aa28dc2..60756bcf775a 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/fpsrange/FpsRangeFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/fpsrange/FpsRangeFeature.java @@ -7,6 +7,7 @@ import android.hardware.camera2.CaptureRequest; import android.os.Build; import android.util.Range; + import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/resolution/ResolutionFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/resolution/ResolutionFeature.java index afbd7c3758a6..ad29f8e97873 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/resolution/ResolutionFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/resolution/ResolutionFeature.java @@ -10,10 +10,13 @@ import android.media.EncoderProfiles; import android.os.Build; import android.util.Size; + import androidx.annotation.VisibleForTesting; + +import java.util.List; + import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; -import java.util.List; /** * Controls the resolutions configuration on the {@link android.hardware.camera2} API. From 90b29932ebf3c2f2b42a7827c8ab82dc1c12a745 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Tue, 17 Jan 2023 12:00:19 -0500 Subject: [PATCH 09/35] Cleanup --- .../io/flutter/plugins/camera/Camera.java | 55 ++++--------------- .../plugins/camera/CameraCaptureCallback.java | 2 + .../plugins/camera/CameraPermissions.java | 1 + .../flutter/plugins/camera/CameraPlugin.java | 2 + .../plugins/camera/CameraProperties.java | 1 + .../plugins/camera/CameraRegionUtils.java | 5 +- .../io/flutter/plugins/camera/CameraZoom.java | 1 + .../flutter/plugins/camera/DartMessenger.java | 7 ++- .../io/flutter/plugins/camera/ImageSaver.java | 2 + .../camera/features/CameraFeature.java | 2 + .../camera/features/CameraFeatureFactory.java | 2 + .../features/CameraFeatureFactoryImpl.java | 2 + .../features/autofocus/AutoFocusFeature.java | 1 + .../exposurelock/ExposureLockFeature.java | 1 + .../focuspoint/FocusPointFeature.java | 2 + .../noisereduction/NoiseReductionFeature.java | 4 +- .../DeviceOrientationManager.java | 2 + .../SensorOrientationFeature.java | 2 + .../features/zoomlevel/ZoomLevelFeature.java | 1 + .../camera/features/zoomlevel/ZoomUtils.java | 1 + .../camera/media/MediaRecorderBuilder.java | 2 + 21 files changed, 51 insertions(+), 47 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java index 48fd645488c0..3ead7af2f453 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java @@ -34,9 +34,21 @@ import android.util.Size; import android.view.Display; import android.view.Surface; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.Executors; + import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; @@ -64,17 +76,6 @@ import io.flutter.plugins.camera.types.CameraCaptureProperties; import io.flutter.plugins.camera.types.CaptureTimeoutsWrapper; import io.flutter.view.TextureRegistry.SurfaceTextureEntry; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.Executors; @FunctionalInterface interface ErrorCallback { @@ -1151,38 +1152,6 @@ private void setImageStreamImageAvailableListener(final EventChannel.EventSink i imageStreamReader.subscribeListener(this.captureProps, imageStreamSink, backgroundHandler); } - // Copyright (c) 2019 Dmitry Gordin - // Based on: - // https://github.com/gordinmitya/yuv2buf/blob/master/yuv2buf/src/main/java/ru/gordinmitya/yuv2buf/Yuv.java - // - // Will remove the padding from a given image plane and return the fixed buffer. - private static byte[] removePlaneBufferPadding(Image.Plane plane, int planeWidth, int planeHeight) { - if (plane.getPixelStride() != 1) { - throw new IllegalArgumentException("it's only valid to remove padding when pixelStride == 1"); - } - - ByteBuffer dst = ByteBuffer.allocate(planeWidth * planeHeight); - ByteBuffer src = plane.getBuffer(); - int rowStride = plane.getRowStride(); - ByteBuffer row; - for (int i = 0; i < planeHeight; i++) { - row = clipBuffer(src, i * rowStride, planeWidth); - dst.put(row); - } - - return dst.array(); - } - - // Copyright (c) 2019 Dmitry Gordin - // Based on: - // https://github.com/gordinmitya/yuv2buf/blob/master/yuv2buf/src/main/java/ru/gordinmitya/yuv2buf/Yuv.java - private static ByteBuffer clipBuffer(ByteBuffer buffer, int start, int size) { - ByteBuffer duplicate = buffer.duplicate(); - duplicate.position(start); - duplicate.limit(start + size); - return duplicate.slice(); - } - private void closeCaptureSession() { if (captureSession != null) { Log.i(TAG, "closeCaptureSession"); diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraCaptureCallback.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraCaptureCallback.java index 805f18298958..100f042dab6e 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraCaptureCallback.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraCaptureCallback.java @@ -10,7 +10,9 @@ import android.hardware.camera2.CaptureResult; import android.hardware.camera2.TotalCaptureResult; import android.util.Log; + import androidx.annotation.NonNull; + import io.flutter.plugins.camera.types.CameraCaptureProperties; import io.flutter.plugins.camera.types.CaptureTimeoutsWrapper; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPermissions.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPermissions.java index ee8fa5a71a16..9648d78f33cb 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPermissions.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPermissions.java @@ -8,6 +8,7 @@ import android.Manifest.permission; import android.app.Activity; import android.content.pm.PackageManager; + import androidx.annotation.VisibleForTesting; import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java index 067ed0295e2e..428fea8d28e0 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java @@ -6,8 +6,10 @@ import android.app.Activity; import android.os.Build; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; + import io.flutter.embedding.engine.plugins.FlutterPlugin; import io.flutter.embedding.engine.plugins.activity.ActivityAware; import io.flutter.embedding.engine.plugins.activity.ActivityPluginBinding; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraProperties.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraProperties.java index 95efebbf6488..ce88fdd4023b 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraProperties.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraProperties.java @@ -12,6 +12,7 @@ import android.util.Range; import android.util.Rational; import android.util.Size; + import androidx.annotation.RequiresApi; /** An interface allowing access to the different characteristics of the device's camera. */ diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraRegionUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraRegionUtils.java index 951a2797d68f..de0483d5816e 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraRegionUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraRegionUtils.java @@ -9,11 +9,14 @@ import android.hardware.camera2.params.MeteringRectangle; import android.os.Build; import android.util.Size; + import androidx.annotation.NonNull; import androidx.annotation.VisibleForTesting; -import io.flutter.embedding.engine.systemchannels.PlatformChannel; + import java.util.Arrays; +import io.flutter.embedding.engine.systemchannels.PlatformChannel; + /** * Utility class offering functions to calculate values regarding the camera boundaries. * diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraZoom.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraZoom.java index 42ad6d76dcfc..e74e015f1016 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraZoom.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraZoom.java @@ -5,6 +5,7 @@ package io.flutter.plugins.camera; import android.graphics.Rect; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.core.math.MathUtils; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/DartMessenger.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/DartMessenger.java index e15078e66afc..9c7f5a476e54 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/DartMessenger.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/DartMessenger.java @@ -6,15 +6,18 @@ import android.os.Handler; import android.text.TextUtils; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; + +import java.util.HashMap; +import java.util.Map; + import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugin.common.BinaryMessenger; import io.flutter.plugin.common.MethodChannel; import io.flutter.plugins.camera.features.autofocus.FocusMode; import io.flutter.plugins.camera.features.exposurelock.ExposureMode; -import java.util.HashMap; -import java.util.Map; /** Utility class that facilitates communication to the Flutter client */ public class DartMessenger { diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/ImageSaver.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/ImageSaver.java index 821c9a50c13f..8d9412186641 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/ImageSaver.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/ImageSaver.java @@ -5,8 +5,10 @@ package io.flutter.plugins.camera; import android.media.Image; + import androidx.annotation.NonNull; import androidx.annotation.VisibleForTesting; + import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeature.java index 92cfd548cd06..8d3eda511a30 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeature.java @@ -5,7 +5,9 @@ package io.flutter.plugins.camera.features; import android.hardware.camera2.CaptureRequest; + import androidx.annotation.NonNull; + import io.flutter.plugins.camera.CameraProperties; /** diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactory.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactory.java index b91f9a1c03f7..21a88c3f83b0 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactory.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactory.java @@ -5,7 +5,9 @@ package io.flutter.plugins.camera.features; import android.app.Activity; + import androidx.annotation.NonNull; + import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.DartMessenger; import io.flutter.plugins.camera.features.autofocus.AutoFocusFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactoryImpl.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactoryImpl.java index 95a8c06caa0a..481022f9a06a 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactoryImpl.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactoryImpl.java @@ -5,7 +5,9 @@ package io.flutter.plugins.camera.features; import android.app.Activity; + import androidx.annotation.NonNull; + import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.DartMessenger; import io.flutter.plugins.camera.features.autofocus.AutoFocusFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/autofocus/AutoFocusFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/autofocus/AutoFocusFeature.java index 1789a964253b..7500829740ae 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/autofocus/AutoFocusFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/autofocus/AutoFocusFeature.java @@ -6,6 +6,7 @@ import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CaptureRequest; + import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurelock/ExposureLockFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurelock/ExposureLockFeature.java index df08cd9a3c77..11aee728bc88 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurelock/ExposureLockFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurelock/ExposureLockFeature.java @@ -5,6 +5,7 @@ package io.flutter.plugins.camera.features.exposurelock; import android.hardware.camera2.CaptureRequest; + import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/focuspoint/FocusPointFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/focuspoint/FocusPointFeature.java index a3a0172d3c37..7cb461233eea 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/focuspoint/FocusPointFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/focuspoint/FocusPointFeature.java @@ -7,7 +7,9 @@ import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.params.MeteringRectangle; import android.util.Size; + import androidx.annotation.NonNull; + import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.CameraRegionUtils; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/noisereduction/NoiseReductionFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/noisereduction/NoiseReductionFeature.java index 408575b375e6..60dcdffefde6 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/noisereduction/NoiseReductionFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/noisereduction/NoiseReductionFeature.java @@ -8,9 +8,11 @@ import android.os.Build.VERSION; import android.os.Build.VERSION_CODES; import android.util.Log; + +import java.util.HashMap; + import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; -import java.util.HashMap; /** * This can either be enabled or disabled. Only full capability devices can set this to off. Legacy diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/DeviceOrientationManager.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/DeviceOrientationManager.java index ec6fa13dbd1d..cabdc61bf357 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/DeviceOrientationManager.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/DeviceOrientationManager.java @@ -13,8 +13,10 @@ import android.view.Display; import android.view.Surface; import android.view.WindowManager; + import androidx.annotation.NonNull; import androidx.annotation.VisibleForTesting; + import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.embedding.engine.systemchannels.PlatformChannel.DeviceOrientation; import io.flutter.plugins.camera.DartMessenger; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/SensorOrientationFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/SensorOrientationFeature.java index 9e316f741805..d31b7b0d40b9 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/SensorOrientationFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/SensorOrientationFeature.java @@ -7,7 +7,9 @@ import android.app.Activity; import android.hardware.camera2.CameraMetadata; import android.hardware.camera2.CaptureRequest; + import androidx.annotation.NonNull; + import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.DartMessenger; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomLevelFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomLevelFeature.java index 736fad4d92dc..68e607c8b1cc 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomLevelFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomLevelFeature.java @@ -6,6 +6,7 @@ import android.graphics.Rect; import android.hardware.camera2.CaptureRequest; + import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomUtils.java index a4890b952cff..024b026d38f7 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomUtils.java @@ -5,6 +5,7 @@ package io.flutter.plugins.camera.features.zoomlevel; import android.graphics.Rect; + import androidx.annotation.NonNull; import androidx.core.math.MathUtils; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/MediaRecorderBuilder.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/MediaRecorderBuilder.java index 0aebfee39e0a..403a1d67efe8 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/MediaRecorderBuilder.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/MediaRecorderBuilder.java @@ -8,7 +8,9 @@ import android.media.EncoderProfiles; import android.media.MediaRecorder; import android.os.Build; + import androidx.annotation.NonNull; + import java.io.IOException; public class MediaRecorderBuilder { From f5c7f8f4ea9717dd61ee01a6601e9b716dbff129 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Tue, 17 Jan 2023 12:09:24 -0500 Subject: [PATCH 10/35] Imports cleanup --- .../src/main/java/io/flutter/plugins/camera/Camera.java | 5 ----- .../java/io/flutter/plugins/camera/CameraProperties.java | 1 - .../src/main/java/io/flutter/plugins/camera/CameraZoom.java | 1 - .../src/main/java/io/flutter/plugins/camera/ImageSaver.java | 2 -- .../io/flutter/plugins/camera/MethodCallHandlerImpl.java | 3 --- .../io/flutter/plugins/camera/features/CameraFeatures.java | 2 -- .../plugins/camera/features/autofocus/AutoFocusFeature.java | 1 - .../camera/features/exposurelock/ExposureLockFeature.java | 1 - .../features/exposureoffset/ExposureOffsetFeature.java | 2 -- .../camera/features/resolution/ResolutionFeature.java | 3 --- .../plugins/camera/features/zoomlevel/ZoomLevelFeature.java | 1 - .../flutter/plugins/camera/features/zoomlevel/ZoomUtils.java | 1 - .../flutter/plugins/camera/media/ImageStreamReaderUtils.java | 1 - 13 files changed, 24 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java index 3ead7af2f453..6fd09c3116c4 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java @@ -21,7 +21,6 @@ import android.hardware.camera2.params.SessionConfiguration; import android.media.CamcorderProfile; import android.media.EncoderProfiles; -import android.media.Image; import android.media.ImageReader; import android.media.MediaRecorder; import android.os.Build; @@ -34,21 +33,17 @@ import android.util.Size; import android.view.Display; import android.view.Surface; - import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; - import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.concurrent.Executors; - import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraProperties.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraProperties.java index ce88fdd4023b..95efebbf6488 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraProperties.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraProperties.java @@ -12,7 +12,6 @@ import android.util.Range; import android.util.Rational; import android.util.Size; - import androidx.annotation.RequiresApi; /** An interface allowing access to the different characteristics of the device's camera. */ diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraZoom.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraZoom.java index e74e015f1016..42ad6d76dcfc 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraZoom.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraZoom.java @@ -5,7 +5,6 @@ package io.flutter.plugins.camera; import android.graphics.Rect; - import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.core.math.MathUtils; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/ImageSaver.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/ImageSaver.java index 8d9412186641..821c9a50c13f 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/ImageSaver.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/ImageSaver.java @@ -5,10 +5,8 @@ package io.flutter.plugins.camera; import android.media.Image; - import androidx.annotation.NonNull; import androidx.annotation.VisibleForTesting; - import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java index f3d41904de1a..3f0c7a283593 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java @@ -8,14 +8,11 @@ import android.hardware.camera2.CameraAccessException; import android.os.Handler; import android.os.Looper; - import androidx.annotation.NonNull; import androidx.annotation.Nullable; - import java.util.HashMap; import java.util.Map; import java.util.Objects; - import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugin.common.BinaryMessenger; import io.flutter.plugin.common.EventChannel; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatures.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatures.java index e366ce075717..aa922ad1ec63 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatures.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatures.java @@ -5,11 +5,9 @@ package io.flutter.plugins.camera.features; import android.app.Activity; - import java.util.Collection; import java.util.HashMap; import java.util.Map; - import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.DartMessenger; import io.flutter.plugins.camera.features.autofocus.AutoFocusFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/autofocus/AutoFocusFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/autofocus/AutoFocusFeature.java index 7500829740ae..1789a964253b 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/autofocus/AutoFocusFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/autofocus/AutoFocusFeature.java @@ -6,7 +6,6 @@ import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CaptureRequest; - import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurelock/ExposureLockFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurelock/ExposureLockFeature.java index 11aee728bc88..df08cd9a3c77 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurelock/ExposureLockFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurelock/ExposureLockFeature.java @@ -5,7 +5,6 @@ package io.flutter.plugins.camera.features.exposurelock; import android.hardware.camera2.CaptureRequest; - import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposureoffset/ExposureOffsetFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposureoffset/ExposureOffsetFeature.java index 86dddf902890..d5a9fcd4a38a 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposureoffset/ExposureOffsetFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposureoffset/ExposureOffsetFeature.java @@ -6,9 +6,7 @@ import android.hardware.camera2.CaptureRequest; import android.util.Range; - import androidx.annotation.NonNull; - import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/resolution/ResolutionFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/resolution/ResolutionFeature.java index ad29f8e97873..d794273a3a79 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/resolution/ResolutionFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/resolution/ResolutionFeature.java @@ -10,11 +10,8 @@ import android.media.EncoderProfiles; import android.os.Build; import android.util.Size; - import androidx.annotation.VisibleForTesting; - import java.util.List; - import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomLevelFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomLevelFeature.java index 68e607c8b1cc..736fad4d92dc 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomLevelFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomLevelFeature.java @@ -6,7 +6,6 @@ import android.graphics.Rect; import android.hardware.camera2.CaptureRequest; - import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomUtils.java index 024b026d38f7..a4890b952cff 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/zoomlevel/ZoomUtils.java @@ -5,7 +5,6 @@ package io.flutter.plugins.camera.features.zoomlevel; import android.graphics.Rect; - import androidx.annotation.NonNull; import androidx.core.math.MathUtils; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java index f414d8af90b7..002029db84e3 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java @@ -1,7 +1,6 @@ package io.flutter.plugins.camera.media; import android.media.Image; - import java.nio.ByteBuffer; public class ImageStreamReaderUtils { From fd858eba608c5390118aa7e6081055eeabebe793 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Tue, 17 Jan 2023 12:09:30 -0500 Subject: [PATCH 11/35] Imports --- .../java/io/flutter/plugins/camera/CameraCaptureCallback.java | 2 -- .../main/java/io/flutter/plugins/camera/CameraPermissions.java | 1 - .../src/main/java/io/flutter/plugins/camera/CameraPlugin.java | 2 -- .../main/java/io/flutter/plugins/camera/CameraRegionUtils.java | 3 --- .../src/main/java/io/flutter/plugins/camera/CameraUtils.java | 2 -- .../src/main/java/io/flutter/plugins/camera/DartMessenger.java | 3 --- .../java/io/flutter/plugins/camera/features/CameraFeature.java | 2 -- .../flutter/plugins/camera/features/CameraFeatureFactory.java | 2 -- .../plugins/camera/features/CameraFeatureFactoryImpl.java | 2 -- .../camera/features/exposurepoint/ExposurePointFeature.java | 2 -- .../io/flutter/plugins/camera/features/flash/FlashFeature.java | 1 - .../plugins/camera/features/focuspoint/FocusPointFeature.java | 2 -- .../plugins/camera/features/fpsrange/FpsRangeFeature.java | 1 - .../camera/features/noisereduction/NoiseReductionFeature.java | 2 -- .../features/sensororientation/DeviceOrientationManager.java | 2 -- .../features/sensororientation/SensorOrientationFeature.java | 2 -- .../io/flutter/plugins/camera/media/ImageStreamReader.java | 3 --- .../io/flutter/plugins/camera/media/MediaRecorderBuilder.java | 2 -- 18 files changed, 36 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraCaptureCallback.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraCaptureCallback.java index 100f042dab6e..805f18298958 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraCaptureCallback.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraCaptureCallback.java @@ -10,9 +10,7 @@ import android.hardware.camera2.CaptureResult; import android.hardware.camera2.TotalCaptureResult; import android.util.Log; - import androidx.annotation.NonNull; - import io.flutter.plugins.camera.types.CameraCaptureProperties; import io.flutter.plugins.camera.types.CaptureTimeoutsWrapper; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPermissions.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPermissions.java index 9648d78f33cb..ee8fa5a71a16 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPermissions.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPermissions.java @@ -8,7 +8,6 @@ import android.Manifest.permission; import android.app.Activity; import android.content.pm.PackageManager; - import androidx.annotation.VisibleForTesting; import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java index 428fea8d28e0..067ed0295e2e 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraPlugin.java @@ -6,10 +6,8 @@ import android.app.Activity; import android.os.Build; - import androidx.annotation.NonNull; import androidx.annotation.Nullable; - import io.flutter.embedding.engine.plugins.FlutterPlugin; import io.flutter.embedding.engine.plugins.activity.ActivityAware; import io.flutter.embedding.engine.plugins.activity.ActivityPluginBinding; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraRegionUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraRegionUtils.java index de0483d5816e..b4e7262e4a20 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraRegionUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraRegionUtils.java @@ -9,12 +9,9 @@ import android.hardware.camera2.params.MeteringRectangle; import android.os.Build; import android.util.Size; - import androidx.annotation.NonNull; import androidx.annotation.VisibleForTesting; - import java.util.Arrays; - import io.flutter.embedding.engine.systemchannels.PlatformChannel; /** diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraUtils.java index 6ba1ae2bccf2..43af8dcecdbb 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraUtils.java @@ -10,12 +10,10 @@ import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CameraManager; import android.hardware.camera2.CameraMetadata; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; - import io.flutter.embedding.engine.systemchannels.PlatformChannel; /** Provides various utilities for camera. */ diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/DartMessenger.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/DartMessenger.java index 9c7f5a476e54..5f90a8a08533 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/DartMessenger.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/DartMessenger.java @@ -6,13 +6,10 @@ import android.os.Handler; import android.text.TextUtils; - import androidx.annotation.NonNull; import androidx.annotation.Nullable; - import java.util.HashMap; import java.util.Map; - import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugin.common.BinaryMessenger; import io.flutter.plugin.common.MethodChannel; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeature.java index 8d3eda511a30..92cfd548cd06 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeature.java @@ -5,9 +5,7 @@ package io.flutter.plugins.camera.features; import android.hardware.camera2.CaptureRequest; - import androidx.annotation.NonNull; - import io.flutter.plugins.camera.CameraProperties; /** diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactory.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactory.java index 21a88c3f83b0..b91f9a1c03f7 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactory.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactory.java @@ -5,9 +5,7 @@ package io.flutter.plugins.camera.features; import android.app.Activity; - import androidx.annotation.NonNull; - import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.DartMessenger; import io.flutter.plugins.camera.features.autofocus.AutoFocusFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactoryImpl.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactoryImpl.java index 481022f9a06a..95a8c06caa0a 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactoryImpl.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatureFactoryImpl.java @@ -5,9 +5,7 @@ package io.flutter.plugins.camera.features; import android.app.Activity; - import androidx.annotation.NonNull; - import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.DartMessenger; import io.flutter.plugins.camera.features.autofocus.AutoFocusFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurepoint/ExposurePointFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurepoint/ExposurePointFeature.java index 3a190c13a670..336e756e9ed8 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurepoint/ExposurePointFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/exposurepoint/ExposurePointFeature.java @@ -7,9 +7,7 @@ import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.params.MeteringRectangle; import android.util.Size; - import androidx.annotation.NonNull; - import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.CameraRegionUtils; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/flash/FlashFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/flash/FlashFeature.java index 6e8b64ecd002..054c81f5183b 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/flash/FlashFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/flash/FlashFeature.java @@ -5,7 +5,6 @@ package io.flutter.plugins.camera.features.flash; import android.hardware.camera2.CaptureRequest; - import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/focuspoint/FocusPointFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/focuspoint/FocusPointFeature.java index 7cb461233eea..a3a0172d3c37 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/focuspoint/FocusPointFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/focuspoint/FocusPointFeature.java @@ -7,9 +7,7 @@ import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.params.MeteringRectangle; import android.util.Size; - import androidx.annotation.NonNull; - import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.CameraRegionUtils; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/fpsrange/FpsRangeFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/fpsrange/FpsRangeFeature.java index 60756bcf775a..500f2aa28dc2 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/fpsrange/FpsRangeFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/fpsrange/FpsRangeFeature.java @@ -7,7 +7,6 @@ import android.hardware.camera2.CaptureRequest; import android.os.Build; import android.util.Range; - import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/noisereduction/NoiseReductionFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/noisereduction/NoiseReductionFeature.java index 60dcdffefde6..360a574d8500 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/noisereduction/NoiseReductionFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/noisereduction/NoiseReductionFeature.java @@ -8,9 +8,7 @@ import android.os.Build.VERSION; import android.os.Build.VERSION_CODES; import android.util.Log; - import java.util.HashMap; - import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/DeviceOrientationManager.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/DeviceOrientationManager.java index cabdc61bf357..ec6fa13dbd1d 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/DeviceOrientationManager.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/DeviceOrientationManager.java @@ -13,10 +13,8 @@ import android.view.Display; import android.view.Surface; import android.view.WindowManager; - import androidx.annotation.NonNull; import androidx.annotation.VisibleForTesting; - import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.embedding.engine.systemchannels.PlatformChannel.DeviceOrientation; import io.flutter.plugins.camera.DartMessenger; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/SensorOrientationFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/SensorOrientationFeature.java index d31b7b0d40b9..9e316f741805 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/SensorOrientationFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/sensororientation/SensorOrientationFeature.java @@ -7,9 +7,7 @@ import android.app.Activity; import android.hardware.camera2.CameraMetadata; import android.hardware.camera2.CaptureRequest; - import androidx.annotation.NonNull; - import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.DartMessenger; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index 3f7971234772..2a3c5ce08dff 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -6,16 +6,13 @@ import android.os.Handler; import android.os.Looper; import android.view.Surface; - import androidx.annotation.NonNull; import androidx.annotation.VisibleForTesting; - import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; - import io.flutter.Log; import io.flutter.plugin.common.EventChannel; import io.flutter.plugins.camera.types.CameraCaptureProperties; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/MediaRecorderBuilder.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/MediaRecorderBuilder.java index 403a1d67efe8..0aebfee39e0a 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/MediaRecorderBuilder.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/MediaRecorderBuilder.java @@ -8,9 +8,7 @@ import android.media.EncoderProfiles; import android.media.MediaRecorder; import android.os.Build; - import androidx.annotation.NonNull; - import java.io.IOException; public class MediaRecorderBuilder { From b1f292e71b27d0f0305653e1eeca0ccbfdd108fe Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Tue, 17 Jan 2023 12:11:33 -0500 Subject: [PATCH 12/35] Imports --- .../main/java/io/flutter/plugins/camera/CameraUtils.java | 2 +- .../main/java/io/flutter/plugins/camera/DartMessenger.java | 4 ++-- .../io/flutter/plugins/camera/MethodCallHandlerImpl.java | 6 +++--- .../io/flutter/plugins/camera/features/CameraFeatures.java | 6 +++--- .../features/noisereduction/NoiseReductionFeature.java | 2 +- .../camera/features/resolution/ResolutionFeature.java | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraUtils.java index 43af8dcecdbb..11b6eeaa5b50 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraUtils.java @@ -10,11 +10,11 @@ import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CameraManager; import android.hardware.camera2.CameraMetadata; +import io.flutter.embedding.engine.systemchannels.PlatformChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import io.flutter.embedding.engine.systemchannels.PlatformChannel; /** Provides various utilities for camera. */ public final class CameraUtils { diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/DartMessenger.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/DartMessenger.java index 5f90a8a08533..e15078e66afc 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/DartMessenger.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/DartMessenger.java @@ -8,13 +8,13 @@ import android.text.TextUtils; import androidx.annotation.NonNull; import androidx.annotation.Nullable; -import java.util.HashMap; -import java.util.Map; import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugin.common.BinaryMessenger; import io.flutter.plugin.common.MethodChannel; import io.flutter.plugins.camera.features.autofocus.FocusMode; import io.flutter.plugins.camera.features.exposurelock.ExposureMode; +import java.util.HashMap; +import java.util.Map; /** Utility class that facilitates communication to the Flutter client */ public class DartMessenger { diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java index 3f0c7a283593..432344ade8cd 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java @@ -10,9 +10,6 @@ import android.os.Looper; import androidx.annotation.NonNull; import androidx.annotation.Nullable; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugin.common.BinaryMessenger; import io.flutter.plugin.common.EventChannel; @@ -27,6 +24,9 @@ import io.flutter.plugins.camera.features.flash.FlashMode; import io.flutter.plugins.camera.features.resolution.ResolutionPreset; import io.flutter.view.TextureRegistry; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; final class MethodCallHandlerImpl implements MethodChannel.MethodCallHandler { private final Activity activity; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatures.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatures.java index aa922ad1ec63..659fd15963e9 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatures.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/CameraFeatures.java @@ -5,9 +5,6 @@ package io.flutter.plugins.camera.features; import android.app.Activity; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.DartMessenger; import io.flutter.plugins.camera.features.autofocus.AutoFocusFeature; @@ -22,6 +19,9 @@ import io.flutter.plugins.camera.features.resolution.ResolutionPreset; import io.flutter.plugins.camera.features.sensororientation.SensorOrientationFeature; import io.flutter.plugins.camera.features.zoomlevel.ZoomLevelFeature; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; /** * These are all of our available features in the camera. Used in the Camera to access all features diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/noisereduction/NoiseReductionFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/noisereduction/NoiseReductionFeature.java index 360a574d8500..408575b375e6 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/noisereduction/NoiseReductionFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/noisereduction/NoiseReductionFeature.java @@ -8,9 +8,9 @@ import android.os.Build.VERSION; import android.os.Build.VERSION_CODES; import android.util.Log; -import java.util.HashMap; import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; +import java.util.HashMap; /** * This can either be enabled or disabled. Only full capability devices can set this to off. Legacy diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/resolution/ResolutionFeature.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/resolution/ResolutionFeature.java index d794273a3a79..afbd7c3758a6 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/resolution/ResolutionFeature.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/features/resolution/ResolutionFeature.java @@ -11,9 +11,9 @@ import android.os.Build; import android.util.Size; import androidx.annotation.VisibleForTesting; -import java.util.List; import io.flutter.plugins.camera.CameraProperties; import io.flutter.plugins.camera.features.CameraFeature; +import java.util.List; /** * Controls the resolutions configuration on the {@link android.hardware.camera2} API. From e6e1f58b06707b23ee1156fb620889058c2bfc89 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Tue, 17 Jan 2023 12:12:12 -0500 Subject: [PATCH 13/35] Update Camera.java --- .../java/io/flutter/plugins/camera/Camera.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java index 6fd09c3116c4..6f55b5e8cadc 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java @@ -36,14 +36,6 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.concurrent.Executors; import io.flutter.embedding.engine.systemchannels.PlatformChannel; import io.flutter.plugin.common.EventChannel; import io.flutter.plugin.common.MethodChannel; @@ -71,6 +63,14 @@ import io.flutter.plugins.camera.types.CameraCaptureProperties; import io.flutter.plugins.camera.types.CaptureTimeoutsWrapper; import io.flutter.view.TextureRegistry.SurfaceTextureEntry; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.Executors; @FunctionalInterface interface ErrorCallback { From 800878f58475dafb9b9607c487dbf1b28f39c207 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Tue, 17 Jan 2023 12:12:51 -0500 Subject: [PATCH 14/35] Update CameraRegionUtils.java --- .../main/java/io/flutter/plugins/camera/CameraRegionUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraRegionUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraRegionUtils.java index b4e7262e4a20..951a2797d68f 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraRegionUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/CameraRegionUtils.java @@ -11,8 +11,8 @@ import android.util.Size; import androidx.annotation.NonNull; import androidx.annotation.VisibleForTesting; -import java.util.Arrays; import io.flutter.embedding.engine.systemchannels.PlatformChannel; +import java.util.Arrays; /** * Utility class offering functions to calculate values regarding the camera boundaries. From 56995bbdbcdfd6bad1ead73202a93895f7e8af34 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Tue, 17 Jan 2023 12:16:02 -0500 Subject: [PATCH 15/35] Update ImageStreamReader.java --- .../io/flutter/plugins/camera/media/ImageStreamReader.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index 2a3c5ce08dff..d72951cae2dc 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -13,7 +13,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import io.flutter.Log; import io.flutter.plugin.common.EventChannel; import io.flutter.plugins.camera.types.CameraCaptureProperties; @@ -60,8 +59,6 @@ public void onImageAvailable( CameraCaptureProperties captureProps, EventChannel.EventSink imageStreamSink ) { - Log.i("flutter", "Image planes: " + image.getPlanes().length); - List> planes = new ArrayList<>(); for (int i=0; i Date: Tue, 17 Jan 2023 12:18:25 -0500 Subject: [PATCH 16/35] Format imports on tests --- .../flutter/plugins/camera/media/ImageStreamReaderTest.java | 4 ---- .../plugins/camera/media/ImageStreamReaderUtilsTest.java | 3 --- 2 files changed, 7 deletions(-) diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java index f9f74aaf582e..a55c6a3aac23 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java @@ -10,18 +10,14 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; - import android.graphics.ImageFormat; import android.media.Image; import android.media.ImageReader; - import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; - import java.nio.ByteBuffer; - import io.flutter.plugin.common.EventChannel; import io.flutter.plugins.camera.types.CameraCaptureProperties; diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java index 822ae076e645..50f611112496 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java @@ -6,15 +6,12 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; - import android.media.Image; - import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; - import java.nio.ByteBuffer; import java.util.Arrays; From 666718334c0f6f0fa3dfddb41ce4659a82cdb6c8 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Wed, 18 Jan 2023 10:31:37 -0500 Subject: [PATCH 17/35] Formatters --- .../io/flutter/plugins/camera/Camera.java | 13 +- .../camera/media/ImageStreamReader.java | 282 +++++++++--------- .../camera/media/ImageStreamReaderUtils.java | 82 +++-- .../camera/media/ImageStreamReaderTest.java | 11 +- .../media/ImageStreamReaderUtilsTest.java | 87 +++--- 5 files changed, 239 insertions(+), 236 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java index 6f55b5e8cadc..886c9ce12a48 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java @@ -301,12 +301,13 @@ public void open(String imageFormatGroup) throws CameraAccessException { Log.w(TAG, "The selected imageFormatGroup is not supported by Android. Defaulting to yuv420"); imageFormat = ImageFormat.YUV_420_888; } - imageStreamReader = new ImageStreamReader(ImageReader.newInstance( - resolutionFeature.getPreviewSize().getWidth(), - resolutionFeature.getPreviewSize().getHeight(), - imageFormat, - 1) - ); + imageStreamReader = + new ImageStreamReader( + ImageReader.newInstance( + resolutionFeature.getPreviewSize().getWidth(), + resolutionFeature.getPreviewSize().getHeight(), + imageFormat, + 1)); // Open the camera. CameraManager cameraManager = CameraUtils.getCameraManager(activity); diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index d72951cae2dc..14d8cc7c62dd 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -8,156 +8,158 @@ import android.view.Surface; import androidx.annotation.NonNull; import androidx.annotation.VisibleForTesting; +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugins.camera.types.CameraCaptureProperties; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import io.flutter.plugin.common.EventChannel; -import io.flutter.plugins.camera.types.CameraCaptureProperties; // Wraps an ImageReader to allow for testing of the image handler. public class ImageStreamReader { - private final ImageReader imageReader; - private final ImageStreamReaderUtils imageStreamReaderUtils; - - /** - * Creates a new instance of the {@link ImageStreamReader}. - * - * @param imageReader is the image reader that will receive frames - * @param imageStreamReaderUtils is an instance of {@link ImageStreamReaderUtils} - */ - @VisibleForTesting - public ImageStreamReader(ImageReader imageReader, ImageStreamReaderUtils imageStreamReaderUtils) { - this.imageReader = imageReader; - this.imageStreamReaderUtils = imageStreamReaderUtils; - } - - /** - * Creates a new instance of the {@link ImageStreamReader}. - * - * @param imageReader is the image reader that will receive frames - */ - public ImageStreamReader(ImageReader imageReader) { - this.imageReader = imageReader; - this.imageStreamReaderUtils = new ImageStreamReaderUtils(); - } - - /** - * Processes a new frame (image) from the image reader, remove padding if necessary, - * and send the frame to Dart. - * - * @param image is the image which needs processed as an {@link Image} - * @param imageFormat is the image format from the image reader as an int, a valid {@link ImageFormat} - * @param captureProps is the capture props from the camera class as {@link CameraCaptureProperties} - * @param imageStreamSink is the image stream sink from dart as a dart {@link EventChannel.EventSink} - */ - @VisibleForTesting - public void onImageAvailable( - @NonNull Image image, - int imageFormat, - CameraCaptureProperties captureProps, - EventChannel.EventSink imageStreamSink - ) { - List> planes = new ArrayList<>(); - for (int i=0; i planeBuffer = new HashMap<>(); - planeBuffer.put("bytesPerPixel", plane.getPixelStride()); - - // Sometimes YUV420 has additional padding that must be removed. This is only the case if we are - // streaming YUV420, the row stride does not match the image width, and the pixel stride is 1. - if (imageFormat == ImageFormat.YUV_420_888 && - plane.getRowStride() != image.getWidth() && - plane.getPixelStride() == 1) { - // The ordering of planes is guaranteed by Android. It always goes Y, U, V. - int planeWidth; - int planeHeight; - if (i == 0) { - // Y is the image size - planeWidth = image.getWidth(); - planeHeight = image.getHeight(); - } else { - // U and V are guaranteed to be the same size and are half of the image height/width - // in YUV420 - planeWidth = image.getWidth() / 2; - planeHeight = image.getHeight() / 2; - } - - planeBuffer.put("bytes", imageStreamReaderUtils.removePlaneBufferPadding(plane, planeWidth, planeHeight)); - - // Make sure the bytesPerRow matches the image width now that we've removed the padding - planeBuffer.put("bytesPerRow", image.getWidth()); - } else { - // Just use the data as-is - ByteBuffer buffer = plane.getBuffer(); - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes, 0, bytes.length); - planeBuffer.put("bytes", bytes); - planeBuffer.put("bytesPerRow", plane.getRowStride()); - } - planes.add(planeBuffer); + private final ImageReader imageReader; + private final ImageStreamReaderUtils imageStreamReaderUtils; + + /** + * Creates a new instance of the {@link ImageStreamReader}. + * + * @param imageReader is the image reader that will receive frames + * @param imageStreamReaderUtils is an instance of {@link ImageStreamReaderUtils} + */ + @VisibleForTesting + public ImageStreamReader(ImageReader imageReader, ImageStreamReaderUtils imageStreamReaderUtils) { + this.imageReader = imageReader; + this.imageStreamReaderUtils = imageStreamReaderUtils; + } + + /** + * Creates a new instance of the {@link ImageStreamReader}. + * + * @param imageReader is the image reader that will receive frames + */ + public ImageStreamReader(ImageReader imageReader) { + this.imageReader = imageReader; + this.imageStreamReaderUtils = new ImageStreamReaderUtils(); + } + + /** + * Processes a new frame (image) from the image reader, remove padding if necessary, and send the + * frame to Dart. + * + * @param image is the image which needs processed as an {@link Image} + * @param imageFormat is the image format from the image reader as an int, a valid {@link + * ImageFormat} + * @param captureProps is the capture props from the camera class as {@link + * CameraCaptureProperties} + * @param imageStreamSink is the image stream sink from dart as a dart {@link + * EventChannel.EventSink} + */ + @VisibleForTesting + public void onImageAvailable( + @NonNull Image image, + int imageFormat, + CameraCaptureProperties captureProps, + EventChannel.EventSink imageStreamSink) { + List> planes = new ArrayList<>(); + for (int i = 0; i < image.getPlanes().length; i++) { + // Current plane + Image.Plane plane = image.getPlanes()[i]; + + // The metadata to be returned to dart + Map planeBuffer = new HashMap<>(); + planeBuffer.put("bytesPerPixel", plane.getPixelStride()); + + // Sometimes YUV420 has additional padding that must be removed. This is only the case if we are + // streaming YUV420, the row stride does not match the image width, and the pixel stride is 1. + if (imageFormat == ImageFormat.YUV_420_888 + && plane.getRowStride() != image.getWidth() + && plane.getPixelStride() == 1) { + // The ordering of planes is guaranteed by Android. It always goes Y, U, V. + int planeWidth; + int planeHeight; + if (i == 0) { + // Y is the image size + planeWidth = image.getWidth(); + planeHeight = image.getHeight(); + } else { + // U and V are guaranteed to be the same size and are half of the image height/width + // in YUV420 + planeWidth = image.getWidth() / 2; + planeHeight = image.getHeight() / 2; } - Map imageBuffer = new HashMap<>(); - imageBuffer.put("width", image.getWidth()); - imageBuffer.put("height", image.getHeight()); - imageBuffer.put("format", image.getFormat()); - imageBuffer.put("planes", planes); - imageBuffer.put("lensAperture", captureProps.getLastLensAperture()); - imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime()); - Integer sensorSensitivity = captureProps.getLastSensorSensitivity(); - imageBuffer.put( - "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity); - - final Handler handler = new Handler(Looper.getMainLooper()); - handler.post(() -> imageStreamSink.success(imageBuffer)); - image.close(); + planeBuffer.put( + "bytes", + imageStreamReaderUtils.removePlaneBufferPadding(plane, planeWidth, planeHeight)); + + // Make sure the bytesPerRow matches the image width now that we've removed the padding + planeBuffer.put("bytesPerRow", image.getWidth()); + } else { + // Just use the data as-is + ByteBuffer buffer = plane.getBuffer(); + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes, 0, bytes.length); + planeBuffer.put("bytes", bytes); + planeBuffer.put("bytesPerRow", plane.getRowStride()); + } + planes.add(planeBuffer); } - /** - * Returns the image reader surface. - */ - public Surface getSurface() { - return imageReader.getSurface(); - } - - /** - * Subscribes the image stream reader to handle incoming images using onImageAvailable(). - * - * @param captureProps is the capture props from the camera class as {@link CameraCaptureProperties} - * @param imageStreamSink is the image stream sink from dart as {@link EventChannel.EventSink} - * @param handler is generally the background handler of the camera as {@link Handler} - */ - public void subscribeListener( - CameraCaptureProperties captureProps, - EventChannel.EventSink imageStreamSink, - Handler handler - ) { - imageReader.setOnImageAvailableListener(reader -> { - Image image = reader.acquireNextImage(); - if (image == null) return; - - onImageAvailable(image, imageReader.getImageFormat(), captureProps, imageStreamSink); - }, handler); - } - - /** - * Removes the listener from the image reader. - * - * @param handler is generally the background handler of the camera - */ - public void removeListener(Handler handler) { - imageReader.setOnImageAvailableListener(null, handler); - } - - /** - * Closes the image reader. - */ - public void close() { - imageReader.close(); - } + Map imageBuffer = new HashMap<>(); + imageBuffer.put("width", image.getWidth()); + imageBuffer.put("height", image.getHeight()); + imageBuffer.put("format", image.getFormat()); + imageBuffer.put("planes", planes); + imageBuffer.put("lensAperture", captureProps.getLastLensAperture()); + imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime()); + Integer sensorSensitivity = captureProps.getLastSensorSensitivity(); + imageBuffer.put( + "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity); + + final Handler handler = new Handler(Looper.getMainLooper()); + handler.post(() -> imageStreamSink.success(imageBuffer)); + image.close(); + } + + /** Returns the image reader surface. */ + public Surface getSurface() { + return imageReader.getSurface(); + } + + /** + * Subscribes the image stream reader to handle incoming images using onImageAvailable(). + * + * @param captureProps is the capture props from the camera class as {@link + * CameraCaptureProperties} + * @param imageStreamSink is the image stream sink from dart as {@link EventChannel.EventSink} + * @param handler is generally the background handler of the camera as {@link Handler} + */ + public void subscribeListener( + CameraCaptureProperties captureProps, + EventChannel.EventSink imageStreamSink, + Handler handler) { + imageReader.setOnImageAvailableListener( + reader -> { + Image image = reader.acquireNextImage(); + if (image == null) return; + + onImageAvailable(image, imageReader.getImageFormat(), captureProps, imageStreamSink); + }, + handler); + } + + /** + * Removes the listener from the image reader. + * + * @param handler is generally the background handler of the camera + */ + public void removeListener(Handler handler) { + imageReader.setOnImageAvailableListener(null, handler); + } + + /** Closes the image reader. */ + public void close() { + imageReader.close(); + } } diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java index 002029db84e3..088eabbebe80 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java @@ -4,49 +4,47 @@ import java.nio.ByteBuffer; public class ImageStreamReaderUtils { - /** - * Copyright (c) 2019 Dmitry Gordin - * Based on: - * https://github.com/gordinmitya/yuv2buf/blob/master/yuv2buf/src/main/java/ru/gordinmitya/yuv2buf/Yuv.java - * - * Will remove the padding from a given image plane and return the fixed buffer. - * - * @param plane is the image plane (buffer) that will be processed as an {@link Image.Plane} - * @param planeWidth is the width of the plane as an int - * @param planeHeight is the height of the plane as an int - */ - public byte[] removePlaneBufferPadding(Image.Plane plane, int planeWidth, int planeHeight) { - if (plane.getPixelStride() != 1) { - throw new IllegalArgumentException("it's only valid to remove padding when pixelStride == 1"); - } - - ByteBuffer dst = ByteBuffer.allocate(planeWidth * planeHeight); - ByteBuffer src = plane.getBuffer(); - int rowStride = plane.getRowStride(); - ByteBuffer row; - for (int i = 0; i < planeHeight; i++) { - row = clipBuffer(src, i * rowStride, planeWidth); - dst.put(row); - } - - return dst.array(); + /** + * Copyright (c) 2019 Dmitry Gordin Based on: + * https://github.com/gordinmitya/yuv2buf/blob/master/yuv2buf/src/main/java/ru/gordinmitya/yuv2buf/Yuv.java + * + *

Will remove the padding from a given image plane and return the fixed buffer. + * + * @param plane is the image plane (buffer) that will be processed as an {@link Image.Plane} + * @param planeWidth is the width of the plane as an int + * @param planeHeight is the height of the plane as an int + */ + public byte[] removePlaneBufferPadding(Image.Plane plane, int planeWidth, int planeHeight) { + if (plane.getPixelStride() != 1) { + throw new IllegalArgumentException("it's only valid to remove padding when pixelStride == 1"); } - /** - * Copyright (c) 2019 Dmitry Gordin - * Based on: - * https://github.com/gordinmitya/yuv2buf/blob/master/yuv2buf/src/main/java/ru/gordinmitya/yuv2buf/Yuv.java - * - * Copies part of a buffer to a new buffer, used to trim the padding. - * - * @param buffer is the source buffer to be modified as a {@link ByteBuffer} - * @param start is the starting offset to read from as an int - * @param size is the length of data to read as an int - */ - public ByteBuffer clipBuffer(ByteBuffer buffer, int start, int size) { - ByteBuffer duplicate = buffer.duplicate(); - duplicate.position(start); - duplicate.limit(start + size); - return duplicate.slice(); + ByteBuffer dst = ByteBuffer.allocate(planeWidth * planeHeight); + ByteBuffer src = plane.getBuffer(); + int rowStride = plane.getRowStride(); + ByteBuffer row; + for (int i = 0; i < planeHeight; i++) { + row = clipBuffer(src, i * rowStride, planeWidth); + dst.put(row); } + + return dst.array(); + } + + /** + * Copyright (c) 2019 Dmitry Gordin Based on: + * https://github.com/gordinmitya/yuv2buf/blob/master/yuv2buf/src/main/java/ru/gordinmitya/yuv2buf/Yuv.java + * + *

Copies part of a buffer to a new buffer, used to trim the padding. + * + * @param buffer is the source buffer to be modified as a {@link ByteBuffer} + * @param start is the starting offset to read from as an int + * @param size is the length of data to read as an int + */ + public ByteBuffer clipBuffer(ByteBuffer buffer, int start, int size) { + ByteBuffer duplicate = buffer.duplicate(); + duplicate.position(start); + duplicate.limit(start + size); + return duplicate.slice(); + } } diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java index a55c6a3aac23..a695e597f918 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java @@ -10,16 +10,17 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; + import android.graphics.ImageFormat; import android.media.Image; import android.media.ImageReader; +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugins.camera.types.CameraCaptureProperties; +import java.nio.ByteBuffer; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; -import java.nio.ByteBuffer; -import io.flutter.plugin.common.EventChannel; -import io.flutter.plugins.camera.types.CameraCaptureProperties; @RunWith(RobolectricTestRunner.class) public class ImageStreamReaderTest { @@ -32,7 +33,8 @@ public void setUp() { ImageStreamReaderUtils mockImageStreamReaderUtils = mock(ImageStreamReaderUtils.class); this.mockImageStreamReaderUtils = mockImageStreamReaderUtils; - this.imageStreamReader = new ImageStreamReader(mockImageReader, this.mockImageStreamReaderUtils); + this.imageStreamReader = + new ImageStreamReader(mockImageReader, this.mockImageStreamReaderUtils); } @Test @@ -50,7 +52,6 @@ public void onImageAvailable_doesNotTryToFixPaddingOnNonYuvImage() { Image.Plane[] planes = {plane0}; when(mockImage.getPlanes()).thenReturn(planes); - CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class); EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class); imageStreamReader.onImageAvailable(mockImage, imageFormat, mockCaptureProps, mockEventSink); diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java index 50f611112496..a94dfdb12110 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java @@ -6,65 +6,66 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; + import android.media.Image; +import java.nio.ByteBuffer; +import java.util.Arrays; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; -import java.nio.ByteBuffer; -import java.util.Arrays; @RunWith(RobolectricTestRunner.class) public class ImageStreamReaderUtilsTest { - private ImageStreamReaderUtils imageStreamReaderUtils; + private ImageStreamReaderUtils imageStreamReaderUtils; - @Before - public void setUp() { - this.imageStreamReaderUtils = new ImageStreamReaderUtils(); - } + @Before + public void setUp() { + this.imageStreamReaderUtils = new ImageStreamReaderUtils(); + } - @Test(expected = IllegalArgumentException.class) - public void removePlaneBufferPadding_throwsIfPixelStrideInvalid() { - Image.Plane planeU = mock(Image.Plane.class); - when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); - when(planeU.getRowStride()).thenReturn(1536); - when(planeU.getPixelStride()).thenReturn(2); + @Test(expected = IllegalArgumentException.class) + public void removePlaneBufferPadding_throwsIfPixelStrideInvalid() { + Image.Plane planeU = mock(Image.Plane.class); + when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); + when(planeU.getRowStride()).thenReturn(1536); + when(planeU.getPixelStride()).thenReturn(2); - imageStreamReaderUtils.removePlaneBufferPadding(planeU, 1280 / 2, 720 / 2); - } + imageStreamReaderUtils.removePlaneBufferPadding(planeU, 1280 / 2, 720 / 2); + } - // Values here are taken from a Vivo V2135 which adds padding in 1280x720 mode - @Test - public void removePlaneBufferPadding_removesPaddingCorrectly() { - Image.Plane planeY = mock(Image.Plane.class); - when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664)); - when(planeY.getRowStride()).thenReturn(1536); - when(planeY.getPixelStride()).thenReturn(1); + // Values here are taken from a Vivo V2135 which adds padding in 1280x720 mode + @Test + public void removePlaneBufferPadding_removesPaddingCorrectly() { + Image.Plane planeY = mock(Image.Plane.class); + when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664)); + when(planeY.getRowStride()).thenReturn(1536); + when(planeY.getPixelStride()).thenReturn(1); - byte[] fixed = imageStreamReaderUtils.removePlaneBufferPadding(planeY, 1280, 720); + byte[] fixed = imageStreamReaderUtils.removePlaneBufferPadding(planeY, 1280, 720); - // After trimming the padding, the buffer size should match the image size - Assert.assertEquals(fixed.length, 1280 * 720); - Assert.assertNotEquals(fixed.length, planeY.getBuffer().limit()); - Assert.assertFalse(Arrays.equals(fixed, planeY.getBuffer().array())); - } + // After trimming the padding, the buffer size should match the image size + Assert.assertEquals(fixed.length, 1280 * 720); + Assert.assertNotEquals(fixed.length, planeY.getBuffer().limit()); + Assert.assertFalse(Arrays.equals(fixed, planeY.getBuffer().array())); + } - // Values here are taken from a Pixel 6 which does not add any padding. - // In the event we pass a buffer with no padding, the returned buffer - // should be identical to the source one because nothing is trimmed. - @Test - public void removePlaneBufferPadding_doesNothingIfThereIsNoPadding() { - Image.Plane planeY = mock(Image.Plane.class); - when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(921600)); - when(planeY.getRowStride()).thenReturn(1280); - when(planeY.getPixelStride()).thenReturn(1); + // Values here are taken from a Pixel 6 which does not add any padding. + // In the event we pass a buffer with no padding, the returned buffer + // should be identical to the source one because nothing is trimmed. + @Test + public void removePlaneBufferPadding_doesNothingIfThereIsNoPadding() { + Image.Plane planeY = mock(Image.Plane.class); + when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(921600)); + when(planeY.getRowStride()).thenReturn(1280); + when(planeY.getPixelStride()).thenReturn(1); - byte[] fixed = imageStreamReaderUtils.removePlaneBufferPadding(planeY, 1280, 720); + byte[] fixed = imageStreamReaderUtils.removePlaneBufferPadding(planeY, 1280, 720); - // After trimming the padding, the buffer size should match the image size - Assert.assertEquals(fixed.length, 1280 * 720); - Assert.assertEquals(fixed.length, planeY.getBuffer().limit()); - Assert.assertArrayEquals(fixed, planeY.getBuffer().array()); - } + // After trimming the padding, the buffer size should match the image size + Assert.assertEquals(fixed.length, 1280 * 720); + Assert.assertEquals(fixed.length, planeY.getBuffer().limit()); + Assert.assertArrayEquals(fixed, planeY.getBuffer().array()); + } } From b7dec7f5cdb37e3af558600fdcfd216a66e5b55c Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Wed, 18 Jan 2023 10:37:42 -0500 Subject: [PATCH 18/35] Licenses --- .../io/flutter/plugins/camera/media/ImageStreamReader.java | 4 ++++ .../flutter/plugins/camera/media/ImageStreamReaderUtils.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index 14d8cc7c62dd..9b747bc26c97 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -1,3 +1,7 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + package io.flutter.plugins.camera.media; import android.graphics.ImageFormat; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java index 088eabbebe80..dee11bec0776 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java @@ -1,3 +1,7 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + package io.flutter.plugins.camera.media; import android.media.Image; From ce2a3b9b2a9e588dcf6f21d1816484b38f301129 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 20 Jan 2023 10:42:55 -0500 Subject: [PATCH 19/35] Added support for NV21 streaming on android as a new image format --- .../io/flutter/plugins/camera/Camera.java | 4 +- .../camera/media/ImageStreamReader.java | 164 +++++++++++------- .../camera/media/ImageStreamReaderUtils.java | 159 +++++++++++++---- .../lib/src/types/image_format_group.dart | 8 + 4 files changed, 246 insertions(+), 89 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java index 886c9ce12a48..ee3543b66cda 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java @@ -107,6 +107,7 @@ class Camera supportedImageFormats = new HashMap<>(); supportedImageFormats.put("yuv420", ImageFormat.YUV_420_888); supportedImageFormats.put("jpeg", ImageFormat.JPEG); + supportedImageFormats.put("nv21", ImageFormat.NV21); } /** @@ -303,11 +304,10 @@ public void open(String imageFormatGroup) throws CameraAccessException { } imageStreamReader = new ImageStreamReader( - ImageReader.newInstance( resolutionFeature.getPreviewSize().getWidth(), resolutionFeature.getPreviewSize().getHeight(), imageFormat, - 1)); + 1); // Open the camera. CameraManager cameraManager = CameraUtils.getCameraManager(activity); diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index 9b747bc26c97..16e005ab7d67 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -22,9 +22,25 @@ // Wraps an ImageReader to allow for testing of the image handler. public class ImageStreamReader { + + /** + * The image format we are going to send back to dart. Usually it's the + * same as streamImageFormat but in the case of NV21 we will actually + * request YUV frames but convert it to NV21 before sending to dart. + */ + private final int dartImageFormat; + private final ImageReader imageReader; private final ImageStreamReaderUtils imageStreamReaderUtils; + // For NV21, we will be streaming YUV420 but converting the frames + // before sending back to dart. + public static ImageReader createImageReader(int width, int height, int imageFormat, int maxImages) { + final int _imageFormat = imageFormat == ImageFormat.NV21 ? ImageFormat.YUV_420_888 : imageFormat; + + return ImageReader.newInstance(width, height, _imageFormat, maxImages); + } + /** * Creates a new instance of the {@link ImageStreamReader}. * @@ -34,19 +50,43 @@ public class ImageStreamReader { @VisibleForTesting public ImageStreamReader(ImageReader imageReader, ImageStreamReaderUtils imageStreamReaderUtils) { this.imageReader = imageReader; + this.dartImageFormat = imageReader.getImageFormat(); this.imageStreamReaderUtils = imageStreamReaderUtils; } /** * Creates a new instance of the {@link ImageStreamReader}. * - * @param imageReader is the image reader that will receive frames + * @param width is the image width + * @param height is the image height + * @param imageFormat is the {@link ImageFormat} that should be returned to dart. + * @param maxImages is how many images can be acquired at one time, usually 1. */ - public ImageStreamReader(ImageReader imageReader) { - this.imageReader = imageReader; + public ImageStreamReader(int width, int height, int imageFormat, int maxImages) { + this.dartImageFormat = imageFormat; + this.imageReader = ImageReader.newInstance( + width, + height, + computeStreamImageFormat(imageFormat), + maxImages); this.imageStreamReaderUtils = new ImageStreamReaderUtils(); } + /** + * Returns the image format to stream based on a requested input format. + * Usually it's the same except when dart is requesting NV21. In that case + * we stream YUV420 and process it into NV21 before sending the frames over. + * @param dartImageFormat + * @return + */ + private static int computeStreamImageFormat(int dartImageFormat) { + if (dartImageFormat == ImageFormat.NV21) { + return ImageFormat.YUV_420_888; + } else { + return dartImageFormat; + } + } + /** * Processes a new frame (image) from the image reader, remove padding if necessary, and send the * frame to Dart. @@ -61,69 +101,77 @@ public ImageStreamReader(ImageReader imageReader) { */ @VisibleForTesting public void onImageAvailable( - @NonNull Image image, - int imageFormat, - CameraCaptureProperties captureProps, - EventChannel.EventSink imageStreamSink) { + @NonNull Image image, + int imageFormat, + CameraCaptureProperties captureProps, + EventChannel.EventSink imageStreamSink) { + try { + Map imageBuffer = new HashMap<>(); + + // Get plane data ready + if (dartImageFormat == ImageFormat.NV21) { + imageBuffer.put("planes", parsePlanesForNv21(image)); + } else { + imageBuffer.put("planes", parsePlanesForYuvOrJpeg(image)); + } + + imageBuffer.put("width", image.getWidth()); + imageBuffer.put("height", image.getHeight()); + imageBuffer.put("format", dartImageFormat); + imageBuffer.put("lensAperture", captureProps.getLastLensAperture()); + imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime()); + Integer sensorSensitivity = captureProps.getLastSensorSensitivity(); + imageBuffer.put( + "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity); + + final Handler handler = new Handler(Looper.getMainLooper()); + handler.post(() -> imageStreamSink.success(imageBuffer)); + image.close(); + + } catch (IllegalStateException e) { + // Handle "buffer is inaccessible" errors that can happen on some devices from ImageStreamReaderUtils.yuv420ThreePlanesToNV21() + final Handler handler = new Handler(Looper.getMainLooper()); + handler.post(() -> imageStreamSink.error("IllegalStateException", "Caught IllegalStateException: " + e.getMessage(), null)); + image.close(); + } + } + + public List> parsePlanesForYuvOrJpeg(Image image) { List> planes = new ArrayList<>(); - for (int i = 0; i < image.getPlanes().length; i++) { - // Current plane - Image.Plane plane = image.getPlanes()[i]; - // The metadata to be returned to dart + // For YUV420 and JPEG, just send the data as-is for each plane. + for (Image.Plane plane : image.getPlanes()) { + ByteBuffer buffer = plane.getBuffer(); + + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes, 0, bytes.length); + Map planeBuffer = new HashMap<>(); + planeBuffer.put("bytesPerRow", plane.getRowStride()); planeBuffer.put("bytesPerPixel", plane.getPixelStride()); + planeBuffer.put("bytes", bytes); - // Sometimes YUV420 has additional padding that must be removed. This is only the case if we are - // streaming YUV420, the row stride does not match the image width, and the pixel stride is 1. - if (imageFormat == ImageFormat.YUV_420_888 - && plane.getRowStride() != image.getWidth() - && plane.getPixelStride() == 1) { - // The ordering of planes is guaranteed by Android. It always goes Y, U, V. - int planeWidth; - int planeHeight; - if (i == 0) { - // Y is the image size - planeWidth = image.getWidth(); - planeHeight = image.getHeight(); - } else { - // U and V are guaranteed to be the same size and are half of the image height/width - // in YUV420 - planeWidth = image.getWidth() / 2; - planeHeight = image.getHeight() / 2; - } - - planeBuffer.put( - "bytes", - imageStreamReaderUtils.removePlaneBufferPadding(plane, planeWidth, planeHeight)); - - // Make sure the bytesPerRow matches the image width now that we've removed the padding - planeBuffer.put("bytesPerRow", image.getWidth()); - } else { - // Just use the data as-is - ByteBuffer buffer = plane.getBuffer(); - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes, 0, bytes.length); - planeBuffer.put("bytes", bytes); - planeBuffer.put("bytesPerRow", plane.getRowStride()); - } planes.add(planeBuffer); } + return planes; + } + + public List> parsePlanesForNv21(Image image) { + List> planes = new ArrayList<>(); - Map imageBuffer = new HashMap<>(); - imageBuffer.put("width", image.getWidth()); - imageBuffer.put("height", image.getHeight()); - imageBuffer.put("format", image.getFormat()); - imageBuffer.put("planes", planes); - imageBuffer.put("lensAperture", captureProps.getLastLensAperture()); - imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime()); - Integer sensorSensitivity = captureProps.getLastSensorSensitivity(); - imageBuffer.put( - "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity); - - final Handler handler = new Handler(Looper.getMainLooper()); - handler.post(() -> imageStreamSink.success(imageBuffer)); - image.close(); + // We will convert the YUV data to NV21 which is a single-plane image + ByteBuffer bytes = imageStreamReaderUtils.yuv420ThreePlanesToNV21( + image.getPlanes(), + image.getWidth(), + image.getHeight() + ); + + Map planeBuffer = new HashMap<>(); + planeBuffer.put("bytesPerRow", image.getWidth()); + planeBuffer.put("bytesPerPixel", 1); + planeBuffer.put("bytes", bytes.array()); + planes.add(planeBuffer); + return planes; } /** Returns the image reader surface. */ diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java index dee11bec0776..1fa56a2eaa1d 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java @@ -6,49 +6,150 @@ import android.media.Image; import java.nio.ByteBuffer; +import io.flutter.Log; public class ImageStreamReaderUtils { /** - * Copyright (c) 2019 Dmitry Gordin Based on: - * https://github.com/gordinmitya/yuv2buf/blob/master/yuv2buf/src/main/java/ru/gordinmitya/yuv2buf/Yuv.java + * Converts YUV_420_888 to NV21 bytebuffer. * - *

Will remove the padding from a given image plane and return the fixed buffer. + *

The NV21 format consists of a single byte array containing the Y, U and V values. For an + * image of size S, the first S positions of the array contain all the Y values. The remaining + * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both + * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain + * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU * - * @param plane is the image plane (buffer) that will be processed as an {@link Image.Plane} - * @param planeWidth is the width of the plane as an int - * @param planeHeight is the height of the plane as an int + *

YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled + * by a factor of 2 in both dimensions. {@link Image#getPlanes} returns an array with the Y, U and + * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into + * the first part of the NV21 array. The U and V planes may already have the representation in the + * NV21 format. This happens if the planes share the same buffer, the V buffer is one position + * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy + * them to the NV21 array. + * + * https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java */ - public byte[] removePlaneBufferPadding(Image.Plane plane, int planeWidth, int planeHeight) { - if (plane.getPixelStride() != 1) { - throw new IllegalArgumentException("it's only valid to remove padding when pixelStride == 1"); - } + public ByteBuffer yuv420ThreePlanesToNV21( + Image.Plane[] yuv420888planes, int width, int height) { + int imageSize = width * height; + byte[] out = new byte[imageSize + 2 * (imageSize / 4)]; + + if (areUVPlanesNV21(yuv420888planes, width, height)) { + Log.i("flutter", "planes are NV21"); + // Copy the Y values. + yuv420888planes[0].getBuffer().get(out, 0, imageSize); + + ByteBuffer uBuffer = yuv420888planes[1].getBuffer(); + ByteBuffer vBuffer = yuv420888planes[2].getBuffer(); + // Get the first V value from the V buffer, since the U buffer does not contain it. + vBuffer.get(out, imageSize, 1); + // Copy the first U value and the remaining VU values from the U buffer. + uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1); + } else {Log.i("flutter", "planes are not NV21"); - ByteBuffer dst = ByteBuffer.allocate(planeWidth * planeHeight); - ByteBuffer src = plane.getBuffer(); - int rowStride = plane.getRowStride(); - ByteBuffer row; - for (int i = 0; i < planeHeight; i++) { - row = clipBuffer(src, i * rowStride, planeWidth); - dst.put(row); + // Fallback to copying the UV values one by one, which is slower but also works. + // Unpack Y. + unpackPlane(yuv420888planes[0], width, height, out, 0, 1); + // Unpack U. + unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2); + // Unpack V. + unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2); } - return dst.array(); + return ByteBuffer.wrap(out); } /** - * Copyright (c) 2019 Dmitry Gordin Based on: - * https://github.com/gordinmitya/yuv2buf/blob/master/yuv2buf/src/main/java/ru/gordinmitya/yuv2buf/Yuv.java + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * - *

Copies part of a buffer to a new buffer, used to trim the padding. + * Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format. * - * @param buffer is the source buffer to be modified as a {@link ByteBuffer} - * @param start is the starting offset to read from as an int - * @param size is the length of data to read as an int + * https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java */ - public ByteBuffer clipBuffer(ByteBuffer buffer, int start, int size) { - ByteBuffer duplicate = buffer.duplicate(); - duplicate.position(start); - duplicate.limit(start + size); - return duplicate.slice(); + private static boolean areUVPlanesNV21(Image.Plane[] planes, int width, int height) { + int imageSize = width * height; + + ByteBuffer uBuffer = planes[1].getBuffer(); + ByteBuffer vBuffer = planes[2].getBuffer(); + + // Backup buffer properties. + int vBufferPosition = vBuffer.position(); + int uBufferLimit = uBuffer.limit(); + + // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value. + vBuffer.position(vBufferPosition + 1); + // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value. + uBuffer.limit(uBufferLimit - 1); + + // Check that the buffers are equal and have the expected number of elements. + boolean areNV21 = + (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); + + // Restore buffers to their initial state. + vBuffer.position(vBufferPosition); + uBuffer.limit(uBufferLimit); + + return areNV21; + } + + /** + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Unpack an image plane into a byte array. + * + *

The input plane data will be copied in 'out', starting at 'offset' and every pixel will be + * spaced by 'pixelStride'. Note that there is no row padding on the output. + * + * https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java + */ + private static void unpackPlane( + Image.Plane plane, int width, int height, byte[] out, int offset, int pixelStride) throws IllegalStateException{ + ByteBuffer buffer = plane.getBuffer(); + buffer.rewind(); + + // Compute the size of the current plane. + // We assume that it has the aspect ratio as the original image. + int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride(); + if (numRow == 0) { + return; + } + int scaleFactor = height / numRow; + int numCol = width / scaleFactor; + + // Extract the data in the output buffer. + int outputPos = offset; + int rowStart = 0; + for (int row = 0; row < numRow; row++) { + int inputPos = rowStart; + for (int col = 0; col < numCol; col++) { + out[outputPos] = buffer.get(inputPos); + outputPos += pixelStride; + inputPos += plane.getPixelStride(); + } + rowStart += plane.getRowStride(); + } } } diff --git a/packages/camera/camera_platform_interface/lib/src/types/image_format_group.dart b/packages/camera/camera_platform_interface/lib/src/types/image_format_group.dart index edbf7d24098c..833dcd771820 100644 --- a/packages/camera/camera_platform_interface/lib/src/types/image_format_group.dart +++ b/packages/camera/camera_platform_interface/lib/src/types/image_format_group.dart @@ -31,6 +31,12 @@ enum ImageFormatGroup { /// On Android, this is `android.graphics.ImageFormat.JPEG`. See /// https://developer.android.com/reference/android/graphics/ImageFormat#JPEG jpeg, + + /// YCrCb format used for images, which uses the NV21 encoding format. + /// + /// On Android, this is `android.graphics.ImageFormat.NV21`. See + /// https://developer.android.com/reference/android/graphics/ImageFormat#NV21 + nv21, } /// Extension on [ImageFormatGroup] to stringify the enum @@ -46,6 +52,8 @@ extension ImageFormatGroupName on ImageFormatGroup { return 'yuv420'; case ImageFormatGroup.jpeg: return 'jpeg'; + case ImageFormatGroup.nv21: + return 'nv21'; case ImageFormatGroup.unknown: default: return 'unknown'; From 404a954e965c3591bdb5c90d06cc31aacd762a1f Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 20 Jan 2023 14:52:44 -0500 Subject: [PATCH 20/35] Remove unused parameter --- .../io/flutter/plugins/camera/media/ImageStreamReader.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index 16e005ab7d67..69f2db8b5adb 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -102,7 +102,6 @@ private static int computeStreamImageFormat(int dartImageFormat) { @VisibleForTesting public void onImageAvailable( @NonNull Image image, - int imageFormat, CameraCaptureProperties captureProps, EventChannel.EventSink imageStreamSink) { try { @@ -196,7 +195,7 @@ public void subscribeListener( Image image = reader.acquireNextImage(); if (image == null) return; - onImageAvailable(image, imageReader.getImageFormat(), captureProps, imageStreamSink); + onImageAvailable(image, captureProps, imageStreamSink); }, handler); } From 3ec602e903f988a9da76d0690de0995d116e8086 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 20 Jan 2023 17:16:35 -0500 Subject: [PATCH 21/35] Add NV21 type conversion --- packages/camera/camera_android/lib/src/type_conversion.dart | 2 ++ packages/camera/camera_android/pubspec.yaml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/packages/camera/camera_android/lib/src/type_conversion.dart b/packages/camera/camera_android/lib/src/type_conversion.dart index 754a5a032715..7691f8e9988b 100644 --- a/packages/camera/camera_android/lib/src/type_conversion.dart +++ b/packages/camera/camera_android/lib/src/type_conversion.dart @@ -34,6 +34,8 @@ ImageFormatGroup _imageFormatGroupFromPlatformData(dynamic data) { return ImageFormatGroup.yuv420; case 256: // android.graphics.ImageFormat.JPEG return ImageFormatGroup.jpeg; + case 17: // android.graphics.ImageFormat.NV21 + return ImageFormatGroup.nv21; } return ImageFormatGroup.unknown; diff --git a/packages/camera/camera_android/pubspec.yaml b/packages/camera/camera_android/pubspec.yaml index 4ecf768bfe12..c6c3d551cac5 100644 --- a/packages/camera/camera_android/pubspec.yaml +++ b/packages/camera/camera_android/pubspec.yaml @@ -30,3 +30,7 @@ dev_dependencies: sdk: flutter flutter_test: sdk: flutter + +dependency_overrides: + camera_platform_interface: + path: ../camera_platform_interface From a010585980c694a8812ed31ec5ec32fcbf553d40 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Sun, 22 Jan 2023 12:24:11 -0500 Subject: [PATCH 22/35] Add legacy code path for nv21 to camera package --- packages/camera/camera/lib/src/camera_image.dart | 3 +++ packages/camera/camera/pubspec.yaml | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/packages/camera/camera/lib/src/camera_image.dart b/packages/camera/camera/lib/src/camera_image.dart index bfcad6626dd6..e1303f6d16e0 100644 --- a/packages/camera/camera/lib/src/camera_image.dart +++ b/packages/camera/camera/lib/src/camera_image.dart @@ -90,6 +90,9 @@ ImageFormatGroup _asImageFormatGroup(dynamic rawFormat) { // android.graphics.ImageFormat.JPEG case 256: return ImageFormatGroup.jpeg; + // android.graphics.ImageFormat.NV21 + case 17: + return ImageFormatGroup.nv21; } } diff --git a/packages/camera/camera/pubspec.yaml b/packages/camera/camera/pubspec.yaml index f8b23bf09db8..38367fdb2bcd 100644 --- a/packages/camera/camera/pubspec.yaml +++ b/packages/camera/camera/pubspec.yaml @@ -38,3 +38,7 @@ dev_dependencies: mockito: ^5.0.0 plugin_platform_interface: ^2.0.0 video_player: ^2.0.0 + +dependency_overrides: + camera_platform_interface: + path: ../camera_platform_interface From 8bd8710324d5797efed2b40b200730e8d229d4b3 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 06:03:32 -0500 Subject: [PATCH 23/35] Clean up log messages --- .../plugins/camera/media/ImageStreamReaderUtils.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java index 1fa56a2eaa1d..164430940813 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java @@ -34,7 +34,8 @@ public ByteBuffer yuv420ThreePlanesToNV21( byte[] out = new byte[imageSize + 2 * (imageSize / 4)]; if (areUVPlanesNV21(yuv420888planes, width, height)) { - Log.i("flutter", "planes are NV21"); +// Log.i("flutter", "planes are NV21"); + // Copy the Y values. yuv420888planes[0].getBuffer().get(out, 0, imageSize); @@ -44,7 +45,8 @@ public ByteBuffer yuv420ThreePlanesToNV21( vBuffer.get(out, imageSize, 1); // Copy the first U value and the remaining VU values from the U buffer. uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1); - } else {Log.i("flutter", "planes are not NV21"); + } else { +// Log.i("flutter", "planes are not NV21"); // Fallback to copying the UV values one by one, which is slower but also works. // Unpack Y. From 7b7708bfbea1b3c1f1a4322fddaf89dc76e1c5a7 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 06:05:06 -0500 Subject: [PATCH 24/35] Formatters --- .../io/flutter/plugins/camera/Camera.java | 8 +-- .../camera/media/ImageStreamReader.java | 49 +++++++++-------- .../camera/media/ImageStreamReaderUtils.java | 53 +++++++++---------- 3 files changed, 54 insertions(+), 56 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java index ee3543b66cda..80ce71eaf857 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java @@ -304,10 +304,10 @@ public void open(String imageFormatGroup) throws CameraAccessException { } imageStreamReader = new ImageStreamReader( - resolutionFeature.getPreviewSize().getWidth(), - resolutionFeature.getPreviewSize().getHeight(), - imageFormat, - 1); + resolutionFeature.getPreviewSize().getWidth(), + resolutionFeature.getPreviewSize().getHeight(), + imageFormat, + 1); // Open the camera. CameraManager cameraManager = CameraUtils.getCameraManager(activity); diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index 69f2db8b5adb..90468767af15 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -24,9 +24,9 @@ public class ImageStreamReader { /** - * The image format we are going to send back to dart. Usually it's the - * same as streamImageFormat but in the case of NV21 we will actually - * request YUV frames but convert it to NV21 before sending to dart. + * The image format we are going to send back to dart. Usually it's the same as streamImageFormat + * but in the case of NV21 we will actually request YUV frames but convert it to NV21 before + * sending to dart. */ private final int dartImageFormat; @@ -35,8 +35,10 @@ public class ImageStreamReader { // For NV21, we will be streaming YUV420 but converting the frames // before sending back to dart. - public static ImageReader createImageReader(int width, int height, int imageFormat, int maxImages) { - final int _imageFormat = imageFormat == ImageFormat.NV21 ? ImageFormat.YUV_420_888 : imageFormat; + public static ImageReader createImageReader( + int width, int height, int imageFormat, int maxImages) { + final int _imageFormat = + imageFormat == ImageFormat.NV21 ? ImageFormat.YUV_420_888 : imageFormat; return ImageReader.newInstance(width, height, _imageFormat, maxImages); } @@ -64,18 +66,16 @@ public ImageStreamReader(ImageReader imageReader, ImageStreamReaderUtils imageSt */ public ImageStreamReader(int width, int height, int imageFormat, int maxImages) { this.dartImageFormat = imageFormat; - this.imageReader = ImageReader.newInstance( - width, - height, - computeStreamImageFormat(imageFormat), - maxImages); + this.imageReader = + ImageReader.newInstance(width, height, computeStreamImageFormat(imageFormat), maxImages); this.imageStreamReaderUtils = new ImageStreamReaderUtils(); } /** - * Returns the image format to stream based on a requested input format. - * Usually it's the same except when dart is requesting NV21. In that case - * we stream YUV420 and process it into NV21 before sending the frames over. + * Returns the image format to stream based on a requested input format. Usually it's the same + * except when dart is requesting NV21. In that case we stream YUV420 and process it into NV21 + * before sending the frames over. + * * @param dartImageFormat * @return */ @@ -101,9 +101,9 @@ private static int computeStreamImageFormat(int dartImageFormat) { */ @VisibleForTesting public void onImageAvailable( - @NonNull Image image, - CameraCaptureProperties captureProps, - EventChannel.EventSink imageStreamSink) { + @NonNull Image image, + CameraCaptureProperties captureProps, + EventChannel.EventSink imageStreamSink) { try { Map imageBuffer = new HashMap<>(); @@ -121,7 +121,7 @@ public void onImageAvailable( imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime()); Integer sensorSensitivity = captureProps.getLastSensorSensitivity(); imageBuffer.put( - "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity); + "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity); final Handler handler = new Handler(Looper.getMainLooper()); handler.post(() -> imageStreamSink.success(imageBuffer)); @@ -130,7 +130,12 @@ public void onImageAvailable( } catch (IllegalStateException e) { // Handle "buffer is inaccessible" errors that can happen on some devices from ImageStreamReaderUtils.yuv420ThreePlanesToNV21() final Handler handler = new Handler(Looper.getMainLooper()); - handler.post(() -> imageStreamSink.error("IllegalStateException", "Caught IllegalStateException: " + e.getMessage(), null)); + handler.post( + () -> + imageStreamSink.error( + "IllegalStateException", + "Caught IllegalStateException: " + e.getMessage(), + null)); image.close(); } } @@ -159,11 +164,9 @@ public List> parsePlanesForNv21(Image image) { List> planes = new ArrayList<>(); // We will convert the YUV data to NV21 which is a single-plane image - ByteBuffer bytes = imageStreamReaderUtils.yuv420ThreePlanesToNV21( - image.getPlanes(), - image.getWidth(), - image.getHeight() - ); + ByteBuffer bytes = + imageStreamReaderUtils.yuv420ThreePlanesToNV21( + image.getPlanes(), image.getWidth(), image.getHeight()); Map planeBuffer = new HashMap<>(); planeBuffer.put("bytesPerRow", image.getWidth()); diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java index 164430940813..cc87aa2f389d 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java @@ -6,7 +6,6 @@ import android.media.Image; import java.nio.ByteBuffer; -import io.flutter.Log; public class ImageStreamReaderUtils { /** @@ -26,16 +25,15 @@ public class ImageStreamReaderUtils { * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy * them to the NV21 array. * - * https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java + *

https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java */ - public ByteBuffer yuv420ThreePlanesToNV21( - Image.Plane[] yuv420888planes, int width, int height) { + public ByteBuffer yuv420ThreePlanesToNV21(Image.Plane[] yuv420888planes, int width, int height) { int imageSize = width * height; byte[] out = new byte[imageSize + 2 * (imageSize / 4)]; if (areUVPlanesNV21(yuv420888planes, width, height)) { -// Log.i("flutter", "planes are NV21"); - + // Log.i("flutter", "planes are NV21"); + // Copy the Y values. yuv420888planes[0].getBuffer().get(out, 0, imageSize); @@ -46,7 +44,7 @@ public ByteBuffer yuv420ThreePlanesToNV21( // Copy the first U value and the remaining VU values from the U buffer. uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1); } else { -// Log.i("flutter", "planes are not NV21"); + // Log.i("flutter", "planes are not NV21"); // Fallback to copying the UV values one by one, which is slower but also works. // Unpack Y. @@ -63,21 +61,19 @@ public ByteBuffer yuv420ThreePlanesToNV21( /** * Copyright 2020 Google LLC. All rights reserved. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and * limitations under the License. * - * Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format. + *

Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format. * - * https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java + *

https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java */ private static boolean areUVPlanesNV21(Image.Plane[] planes, int width, int height) { int imageSize = width * height; @@ -96,7 +92,7 @@ private static boolean areUVPlanesNV21(Image.Plane[] planes, int width, int heig // Check that the buffers are equal and have the expected number of elements. boolean areNV21 = - (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); + (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); // Restore buffers to their initial state. vBuffer.position(vBufferPosition); @@ -108,27 +104,26 @@ private static boolean areUVPlanesNV21(Image.Plane[] planes, int width, int heig /** * Copyright 2020 Google LLC. All rights reserved. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and * limitations under the License. * - * Unpack an image plane into a byte array. + *

Unpack an image plane into a byte array. * *

The input plane data will be copied in 'out', starting at 'offset' and every pixel will be * spaced by 'pixelStride'. Note that there is no row padding on the output. * - * https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java + *

https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java */ private static void unpackPlane( - Image.Plane plane, int width, int height, byte[] out, int offset, int pixelStride) throws IllegalStateException{ + Image.Plane plane, int width, int height, byte[] out, int offset, int pixelStride) + throws IllegalStateException { ByteBuffer buffer = plane.getBuffer(); buffer.rewind(); From d18a997ff4fc85bf317f0642eab2a1248399dfc7 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 06:08:05 -0500 Subject: [PATCH 25/35] Dependency overrides as per contributing guidelines --- packages/camera/camera/pubspec.yaml | 6 +++++- packages/camera/camera_android/example/pubspec.yaml | 8 ++++++++ packages/camera/camera_android/pubspec.yaml | 4 +++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/camera/camera/pubspec.yaml b/packages/camera/camera/pubspec.yaml index 38367fdb2bcd..4ea96f95204f 100644 --- a/packages/camera/camera/pubspec.yaml +++ b/packages/camera/camera/pubspec.yaml @@ -39,6 +39,10 @@ dev_dependencies: plugin_platform_interface: ^2.0.0 video_player: ^2.0.0 + +# FOR TESTING ONLY. DO NOT MERGE. dependency_overrides: + camera_android: + path: ../../camera/camera_android camera_platform_interface: - path: ../camera_platform_interface + path: ../../camera/camera_platform_interface diff --git a/packages/camera/camera_android/example/pubspec.yaml b/packages/camera/camera_android/example/pubspec.yaml index 8c985d94fd5a..c760b027525b 100644 --- a/packages/camera/camera_android/example/pubspec.yaml +++ b/packages/camera/camera_android/example/pubspec.yaml @@ -32,3 +32,11 @@ dev_dependencies: flutter: uses-material-design: true + + +# FOR TESTING ONLY. DO NOT MERGE. +dependency_overrides: + camera_android: + path: ../../../camera/camera_android + camera_platform_interface: + path: ../../../camera/camera_platform_interface diff --git a/packages/camera/camera_android/pubspec.yaml b/packages/camera/camera_android/pubspec.yaml index c6c3d551cac5..7a059dc6144f 100644 --- a/packages/camera/camera_android/pubspec.yaml +++ b/packages/camera/camera_android/pubspec.yaml @@ -31,6 +31,8 @@ dev_dependencies: flutter_test: sdk: flutter + +# FOR TESTING ONLY. DO NOT MERGE. dependency_overrides: camera_platform_interface: - path: ../camera_platform_interface + path: ../../camera/camera_platform_interface From 37fa30daca4aa79cf09e1eea77e122af827378f0 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 06:52:01 -0500 Subject: [PATCH 26/35] Fix tests for image stream reader --- packages/camera/camera/example/pubspec.yaml | 7 + .../camera/media/ImageStreamReader.java | 27 +--- .../camera/media/ImageStreamReaderTest.java | 110 +++++++++++--- .../media/ImageStreamReaderUtilsTest.java | 142 +++++++++--------- 4 files changed, 173 insertions(+), 113 deletions(-) diff --git a/packages/camera/camera/example/pubspec.yaml b/packages/camera/camera/example/pubspec.yaml index e63024076fef..77b1ff58d4cd 100644 --- a/packages/camera/camera/example/pubspec.yaml +++ b/packages/camera/camera/example/pubspec.yaml @@ -30,3 +30,10 @@ dev_dependencies: flutter: uses-material-design: true + +# FOR TESTING ONLY. DO NOT MERGE. +dependency_overrides: + camera_android: + path: ../../../camera/camera_android + camera_platform_interface: + path: ../../../camera/camera_platform_interface diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index 90468767af15..87da373ec309 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -33,16 +33,6 @@ public class ImageStreamReader { private final ImageReader imageReader; private final ImageStreamReaderUtils imageStreamReaderUtils; - // For NV21, we will be streaming YUV420 but converting the frames - // before sending back to dart. - public static ImageReader createImageReader( - int width, int height, int imageFormat, int maxImages) { - final int _imageFormat = - imageFormat == ImageFormat.NV21 ? ImageFormat.YUV_420_888 : imageFormat; - - return ImageReader.newInstance(width, height, _imageFormat, maxImages); - } - /** * Creates a new instance of the {@link ImageStreamReader}. * @@ -50,9 +40,9 @@ public static ImageReader createImageReader( * @param imageStreamReaderUtils is an instance of {@link ImageStreamReaderUtils} */ @VisibleForTesting - public ImageStreamReader(ImageReader imageReader, ImageStreamReaderUtils imageStreamReaderUtils) { + public ImageStreamReader(ImageReader imageReader, int dartImageFormat, ImageStreamReaderUtils imageStreamReaderUtils) { this.imageReader = imageReader; - this.dartImageFormat = imageReader.getImageFormat(); + this.dartImageFormat = dartImageFormat; this.imageStreamReaderUtils = imageStreamReaderUtils; } @@ -76,10 +66,11 @@ public ImageStreamReader(int width, int height, int imageFormat, int maxImages) * except when dart is requesting NV21. In that case we stream YUV420 and process it into NV21 * before sending the frames over. * - * @param dartImageFormat - * @return + * @param dartImageFormat is the image format dart is requesting. + * @return the image format that should be streamed from the camera. */ - private static int computeStreamImageFormat(int dartImageFormat) { + @VisibleForTesting + public static int computeStreamImageFormat(int dartImageFormat) { if (dartImageFormat == ImageFormat.NV21) { return ImageFormat.YUV_420_888; } else { @@ -92,8 +83,6 @@ private static int computeStreamImageFormat(int dartImageFormat) { * frame to Dart. * * @param image is the image which needs processed as an {@link Image} - * @param imageFormat is the image format from the image reader as an int, a valid {@link - * ImageFormat} * @param captureProps is the capture props from the camera class as {@link * CameraCaptureProperties} * @param imageStreamSink is the image stream sink from dart as a dart {@link @@ -140,7 +129,7 @@ public void onImageAvailable( } } - public List> parsePlanesForYuvOrJpeg(Image image) { + public List> parsePlanesForYuvOrJpeg(@NonNull Image image) { List> planes = new ArrayList<>(); // For YUV420 and JPEG, just send the data as-is for each plane. @@ -160,7 +149,7 @@ public List> parsePlanesForYuvOrJpeg(Image image) { return planes; } - public List> parsePlanesForNv21(Image image) { + public List> parsePlanesForNv21(@NonNull Image image) { List> planes = new ArrayList<>(); // We will convert the YUV data to NV21 which is a single-plane image diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java index a695e597f918..834cdcb03512 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java @@ -4,6 +4,7 @@ package io.flutter.plugins.camera.media; +import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; @@ -24,43 +25,105 @@ @RunWith(RobolectricTestRunner.class) public class ImageStreamReaderTest { - private ImageStreamReader imageStreamReader; - private ImageStreamReaderUtils mockImageStreamReaderUtils; + /** + * If we request YUV42 we should stream in YUV420. + */ + @Test + public void computeStreamImageFormat_computesCorrectStreamFormatYuv() { + int requestedStreamFormat = ImageFormat.YUV_420_888; + int result = ImageStreamReader.computeStreamImageFormat(requestedStreamFormat); + assertEquals(result, ImageFormat.YUV_420_888); + } + + /** + * When we want to stream in NV21, we should still request YUV420 from the + * camera because we will convert it to NV21 before sending it to dart. + */ + @Test + public void computeStreamImageFormat_computesCorrectStreamFormatNv21() { + int requestedStreamFormat = ImageFormat.NV21; + int result = ImageStreamReader.computeStreamImageFormat(requestedStreamFormat); + assertEquals(result, ImageFormat.YUV_420_888); + } + + /** + * If we are requesting NV21, then the planes should be processed and + * converted to NV21 before being sent to dart. We make sure yuv420ThreePlanesToNV21 + * is called when we are requesting + */ + @Test + public void onImageAvailable_parsesPlanesForNv21() { + // Dart wants NV21 frames + int dartImageFormat = ImageFormat.NV21; - @Before - public void setUp() { ImageReader mockImageReader = mock(ImageReader.class); ImageStreamReaderUtils mockImageStreamReaderUtils = mock(ImageStreamReaderUtils.class); + ImageStreamReader imageStreamReader = + new ImageStreamReader(mockImageReader, dartImageFormat, mockImageStreamReaderUtils); - this.mockImageStreamReaderUtils = mockImageStreamReaderUtils; - this.imageStreamReader = - new ImageStreamReader(mockImageReader, this.mockImageStreamReaderUtils); - } + ByteBuffer mockBytes = ByteBuffer.allocate(0); + when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt())).thenReturn(mockBytes); - @Test - public void onImageAvailable_doesNotTryToFixPaddingOnNonYuvImage() { - // Mock JPEG image - int imageFormat = ImageFormat.JPEG; + // The image format as streamed from the camera + int imageFormat = ImageFormat.YUV_420_888; + + // Mock YUV image Image mockImage = mock(Image.class); + when(mockImage.getWidth()).thenReturn(1280); + when(mockImage.getHeight()).thenReturn(720); when(mockImage.getFormat()).thenReturn(imageFormat); - // Mock plane. JPEG images have only one plane - Image.Plane plane0 = mock(Image.Plane.class); - when(plane0.getBuffer()).thenReturn(ByteBuffer.allocate(497950)); - when(plane0.getRowStride()).thenReturn(0); - when(plane0.getPixelStride()).thenReturn(0); - Image.Plane[] planes = {plane0}; + // Mock planes. YUV images have 3 planes (Y, U, V). + Image.Plane planeY = mock(Image.Plane.class); + Image.Plane planeU = mock(Image.Plane.class); + Image.Plane planeV = mock(Image.Plane.class); + + // Y plane is width*height + // Row stride is generally == width but when there is padding it will + // be larger. The numbers in this example are from a Vivo V2135 on 'high' + // setting (1280x720). + when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664)); + when(planeY.getRowStride()).thenReturn(1536); + when(planeY.getPixelStride()).thenReturn(1); + + // U and V planes are always the same sizes/values. + // https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888 + when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); + when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); + when(planeU.getRowStride()).thenReturn(1536); + when(planeV.getRowStride()).thenReturn(1536); + when(planeU.getPixelStride()).thenReturn(2); + when(planeV.getPixelStride()).thenReturn(2); + + // Add planes to image + Image.Plane[] planes = {planeY, planeU, planeV}; when(mockImage.getPlanes()).thenReturn(planes); CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class); EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class); - imageStreamReader.onImageAvailable(mockImage, imageFormat, mockCaptureProps, mockEventSink); + imageStreamReader.onImageAvailable(mockImage, mockCaptureProps, mockEventSink); - verify(mockImageStreamReaderUtils, never()).removePlaneBufferPadding(any(), anyInt(), anyInt()); + // Make sure we processed the frame with parsePlanesForNv21 + verify(mockImageStreamReaderUtils).yuv420ThreePlanesToNV21(any(), anyInt(), anyInt()); } + /** + * If we are requesting YUV420, then we should send the 3-plane image as it is. + */ @Test - public void onImageAvailable_shouldTryToFixPaddingOnYuvImageWithExtraPadding() { + public void onImageAvailable_parsesPlanesForYuv420() { + // Dart wants NV21 frames + int dartImageFormat = ImageFormat.YUV_420_888; + + ImageReader mockImageReader = mock(ImageReader.class); + ImageStreamReaderUtils mockImageStreamReaderUtils = mock(ImageStreamReaderUtils.class); + ImageStreamReader imageStreamReader = + new ImageStreamReader(mockImageReader, dartImageFormat, mockImageStreamReaderUtils); + + ByteBuffer mockBytes = ByteBuffer.allocate(0); + when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt())).thenReturn(mockBytes); + + // The image format as streamed from the camera int imageFormat = ImageFormat.YUV_420_888; // Mock YUV image @@ -97,8 +160,9 @@ public void onImageAvailable_shouldTryToFixPaddingOnYuvImageWithExtraPadding() { CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class); EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class); - imageStreamReader.onImageAvailable(mockImage, imageFormat, mockCaptureProps, mockEventSink); + imageStreamReader.onImageAvailable(mockImage, mockCaptureProps, mockEventSink); - verify(mockImageStreamReaderUtils).removePlaneBufferPadding(any(), anyInt(), anyInt()); + // Make sure we processed the frame with parsePlanesForYuvOrJpeg + verify(mockImageStreamReaderUtils, never()).yuv420ThreePlanesToNV21(any(), anyInt(), anyInt()); } } diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java index a94dfdb12110..3e8313f9d197 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java @@ -1,71 +1,71 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package io.flutter.plugins.camera.media; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import android.media.Image; -import java.nio.ByteBuffer; -import java.util.Arrays; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.robolectric.RobolectricTestRunner; - -@RunWith(RobolectricTestRunner.class) -public class ImageStreamReaderUtilsTest { - private ImageStreamReaderUtils imageStreamReaderUtils; - - @Before - public void setUp() { - this.imageStreamReaderUtils = new ImageStreamReaderUtils(); - } - - @Test(expected = IllegalArgumentException.class) - public void removePlaneBufferPadding_throwsIfPixelStrideInvalid() { - Image.Plane planeU = mock(Image.Plane.class); - when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); - when(planeU.getRowStride()).thenReturn(1536); - when(planeU.getPixelStride()).thenReturn(2); - - imageStreamReaderUtils.removePlaneBufferPadding(planeU, 1280 / 2, 720 / 2); - } - - // Values here are taken from a Vivo V2135 which adds padding in 1280x720 mode - @Test - public void removePlaneBufferPadding_removesPaddingCorrectly() { - Image.Plane planeY = mock(Image.Plane.class); - when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664)); - when(planeY.getRowStride()).thenReturn(1536); - when(planeY.getPixelStride()).thenReturn(1); - - byte[] fixed = imageStreamReaderUtils.removePlaneBufferPadding(planeY, 1280, 720); - - // After trimming the padding, the buffer size should match the image size - Assert.assertEquals(fixed.length, 1280 * 720); - Assert.assertNotEquals(fixed.length, planeY.getBuffer().limit()); - Assert.assertFalse(Arrays.equals(fixed, planeY.getBuffer().array())); - } - - // Values here are taken from a Pixel 6 which does not add any padding. - // In the event we pass a buffer with no padding, the returned buffer - // should be identical to the source one because nothing is trimmed. - @Test - public void removePlaneBufferPadding_doesNothingIfThereIsNoPadding() { - Image.Plane planeY = mock(Image.Plane.class); - when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(921600)); - when(planeY.getRowStride()).thenReturn(1280); - when(planeY.getPixelStride()).thenReturn(1); - - byte[] fixed = imageStreamReaderUtils.removePlaneBufferPadding(planeY, 1280, 720); - - // After trimming the padding, the buffer size should match the image size - Assert.assertEquals(fixed.length, 1280 * 720); - Assert.assertEquals(fixed.length, planeY.getBuffer().limit()); - Assert.assertArrayEquals(fixed, planeY.getBuffer().array()); - } -} +//// Copyright 2013 The Flutter Authors. All rights reserved. +//// Use of this source code is governed by a BSD-style license that can be +//// found in the LICENSE file. +// +//package io.flutter.plugins.camera.media; +// +//import static org.mockito.Mockito.mock; +//import static org.mockito.Mockito.when; +// +//import android.media.Image; +//import java.nio.ByteBuffer; +//import java.util.Arrays; +//import org.junit.Assert; +//import org.junit.Before; +//import org.junit.Test; +//import org.junit.runner.RunWith; +//import org.robolectric.RobolectricTestRunner; +// +//@RunWith(RobolectricTestRunner.class) +//public class ImageStreamReaderUtilsTest { +// private ImageStreamReaderUtils imageStreamReaderUtils; +// +// @Before +// public void setUp() { +// this.imageStreamReaderUtils = new ImageStreamReaderUtils(); +// } +// +// @Test(expected = IllegalArgumentException.class) +// public void removePlaneBufferPadding_throwsIfPixelStrideInvalid() { +// Image.Plane planeU = mock(Image.Plane.class); +// when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); +// when(planeU.getRowStride()).thenReturn(1536); +// when(planeU.getPixelStride()).thenReturn(2); +// +// imageStreamReaderUtils.removePlaneBufferPadding(planeU, 1280 / 2, 720 / 2); +// } +// +// // Values here are taken from a Vivo V2135 which adds padding in 1280x720 mode +// @Test +// public void removePlaneBufferPadding_removesPaddingCorrectly() { +// Image.Plane planeY = mock(Image.Plane.class); +// when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664)); +// when(planeY.getRowStride()).thenReturn(1536); +// when(planeY.getPixelStride()).thenReturn(1); +// +// byte[] fixed = imageStreamReaderUtils.removePlaneBufferPadding(planeY, 1280, 720); +// +// // After trimming the padding, the buffer size should match the image size +// Assert.assertEquals(fixed.length, 1280 * 720); +// Assert.assertNotEquals(fixed.length, planeY.getBuffer().limit()); +// Assert.assertFalse(Arrays.equals(fixed, planeY.getBuffer().array())); +// } +// +// // Values here are taken from a Pixel 6 which does not add any padding. +// // In the event we pass a buffer with no padding, the returned buffer +// // should be identical to the source one because nothing is trimmed. +// @Test +// public void removePlaneBufferPadding_doesNothingIfThereIsNoPadding() { +// Image.Plane planeY = mock(Image.Plane.class); +// when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(921600)); +// when(planeY.getRowStride()).thenReturn(1280); +// when(planeY.getPixelStride()).thenReturn(1); +// +// byte[] fixed = imageStreamReaderUtils.removePlaneBufferPadding(planeY, 1280, 720); +// +// // After trimming the padding, the buffer size should match the image size +// Assert.assertEquals(fixed.length, 1280 * 720); +// Assert.assertEquals(fixed.length, planeY.getBuffer().limit()); +// Assert.assertArrayEquals(fixed, planeY.getBuffer().array()); +// } +//} From c4c171976fb527aa07af1c26bfcd1a77911ce06e Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 07:08:12 -0500 Subject: [PATCH 27/35] Test cleanup --- .../camera/media/ImageStreamReaderUtils.java | 5 +- .../media/ImageStreamReaderUtilsTest.java | 150 +++++++++--------- 2 files changed, 82 insertions(+), 73 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java index cc87aa2f389d..b5a4b1b09990 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java @@ -5,6 +5,7 @@ package io.flutter.plugins.camera.media; import android.media.Image; +import androidx.annotation.NonNull; import java.nio.ByteBuffer; public class ImageStreamReaderUtils { @@ -75,7 +76,7 @@ public ByteBuffer yuv420ThreePlanesToNV21(Image.Plane[] yuv420888planes, int wid * *

https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java */ - private static boolean areUVPlanesNV21(Image.Plane[] planes, int width, int height) { + private static boolean areUVPlanesNV21(@NonNull Image.Plane[] planes, int width, int height) { int imageSize = width * height; ByteBuffer uBuffer = planes[1].getBuffer(); @@ -122,7 +123,7 @@ private static boolean areUVPlanesNV21(Image.Plane[] planes, int width, int heig *

https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java */ private static void unpackPlane( - Image.Plane plane, int width, int height, byte[] out, int offset, int pixelStride) + @NonNull Image.Plane plane, int width, int height, byte[] out, int offset, int pixelStride) throws IllegalStateException { ByteBuffer buffer = plane.getBuffer(); buffer.rewind(); diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java index 3e8313f9d197..4f6b0bcae00d 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java @@ -1,71 +1,79 @@ -//// Copyright 2013 The Flutter Authors. All rights reserved. -//// Use of this source code is governed by a BSD-style license that can be -//// found in the LICENSE file. -// -//package io.flutter.plugins.camera.media; -// -//import static org.mockito.Mockito.mock; -//import static org.mockito.Mockito.when; -// -//import android.media.Image; -//import java.nio.ByteBuffer; -//import java.util.Arrays; -//import org.junit.Assert; -//import org.junit.Before; -//import org.junit.Test; -//import org.junit.runner.RunWith; -//import org.robolectric.RobolectricTestRunner; -// -//@RunWith(RobolectricTestRunner.class) -//public class ImageStreamReaderUtilsTest { -// private ImageStreamReaderUtils imageStreamReaderUtils; -// -// @Before -// public void setUp() { -// this.imageStreamReaderUtils = new ImageStreamReaderUtils(); -// } -// -// @Test(expected = IllegalArgumentException.class) -// public void removePlaneBufferPadding_throwsIfPixelStrideInvalid() { -// Image.Plane planeU = mock(Image.Plane.class); -// when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); -// when(planeU.getRowStride()).thenReturn(1536); -// when(planeU.getPixelStride()).thenReturn(2); -// -// imageStreamReaderUtils.removePlaneBufferPadding(planeU, 1280 / 2, 720 / 2); -// } -// -// // Values here are taken from a Vivo V2135 which adds padding in 1280x720 mode -// @Test -// public void removePlaneBufferPadding_removesPaddingCorrectly() { -// Image.Plane planeY = mock(Image.Plane.class); -// when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664)); -// when(planeY.getRowStride()).thenReturn(1536); -// when(planeY.getPixelStride()).thenReturn(1); -// -// byte[] fixed = imageStreamReaderUtils.removePlaneBufferPadding(planeY, 1280, 720); -// -// // After trimming the padding, the buffer size should match the image size -// Assert.assertEquals(fixed.length, 1280 * 720); -// Assert.assertNotEquals(fixed.length, planeY.getBuffer().limit()); -// Assert.assertFalse(Arrays.equals(fixed, planeY.getBuffer().array())); -// } -// -// // Values here are taken from a Pixel 6 which does not add any padding. -// // In the event we pass a buffer with no padding, the returned buffer -// // should be identical to the source one because nothing is trimmed. -// @Test -// public void removePlaneBufferPadding_doesNothingIfThereIsNoPadding() { -// Image.Plane planeY = mock(Image.Plane.class); -// when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(921600)); -// when(planeY.getRowStride()).thenReturn(1280); -// when(planeY.getPixelStride()).thenReturn(1); -// -// byte[] fixed = imageStreamReaderUtils.removePlaneBufferPadding(planeY, 1280, 720); -// -// // After trimming the padding, the buffer size should match the image size -// Assert.assertEquals(fixed.length, 1280 * 720); -// Assert.assertEquals(fixed.length, planeY.getBuffer().limit()); -// Assert.assertArrayEquals(fixed, planeY.getBuffer().array()); -// } -//} +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package io.flutter.plugins.camera.media; + +import android.graphics.ImageFormat; +import android.media.Image; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; +import java.nio.ByteBuffer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(RobolectricTestRunner.class) +public class ImageStreamReaderUtilsTest { + private ImageStreamReaderUtils imageStreamReaderUtils; + + @Before + public void setUp() { + this.imageStreamReaderUtils = new ImageStreamReaderUtils(); + } + + /** + * Ensure that passing in an image with padding returns one without padding + */ + @Test + public void yuv420ThreePlanesToNV21_trimsPaddingWhenPresent() { + int imageWidth = 640; + int imageHeight = 480; + int padding = 256; + int rowStride =640 + padding; + + int ySize = (rowStride * imageHeight) - padding; + int uSize = (ySize / 2) - (padding / 2); + int vSize = uSize; + + // Mock YUV image + Image mockImage = mock(Image.class); + when(mockImage.getWidth()).thenReturn(imageWidth); + when(mockImage.getHeight()).thenReturn(imageHeight); + when(mockImage.getFormat()).thenReturn(ImageFormat.YUV_420_888); + + + + // Mock planes. YUV images have 3 planes (Y, U, V). + Image.Plane planeY = mock(Image.Plane.class); + Image.Plane planeU = mock(Image.Plane.class); + Image.Plane planeV = mock(Image.Plane.class); + + // Y plane is width*height + // Row stride is generally == width but when there is padding it will + // be larger. + // Here we are adding 256 padding. + when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(ySize)); + when(planeY.getRowStride()).thenReturn(rowStride); + when(planeY.getPixelStride()).thenReturn(1); + + // U and V planes are always the same sizes/values. + // https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888 + when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(uSize)); + when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(vSize)); + when(planeU.getRowStride()).thenReturn(rowStride); + when(planeV.getRowStride()).thenReturn(rowStride); + when(planeU.getPixelStride()).thenReturn(2); + when(planeV.getPixelStride()).thenReturn(2); + + // Add planes to image + Image.Plane[] planes = {planeY, planeU, planeV}; + when(mockImage.getPlanes()).thenReturn(planes); + + // TODO: find correct size for result here + ByteBuffer result = imageStreamReaderUtils.yuv420ThreePlanesToNV21(planes, mockImage.getWidth(), mockImage.getHeight()); + Assert.assertEquals(result.limit(), imageWidth * imageHeight); + } +} From bf18e06e0922d5989ca40d9a506145274942b29a Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 07:43:18 -0500 Subject: [PATCH 28/35] Added tests for removing padding --- .../media/ImageStreamReaderUtilsTest.java | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java index 4f6b0bcae00d..e3caf9ad66d7 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java @@ -24,15 +24,8 @@ public void setUp() { this.imageStreamReaderUtils = new ImageStreamReaderUtils(); } - /** - * Ensure that passing in an image with padding returns one without padding - */ - @Test - public void yuv420ThreePlanesToNV21_trimsPaddingWhenPresent() { - int imageWidth = 640; - int imageHeight = 480; - int padding = 256; - int rowStride =640 + padding; + Image getImage(int imageWidth, int imageHeight, int padding) { + int rowStride =imageWidth + padding; int ySize = (rowStride * imageHeight) - padding; int uSize = (ySize / 2) - (padding / 2); @@ -72,8 +65,32 @@ public void yuv420ThreePlanesToNV21_trimsPaddingWhenPresent() { Image.Plane[] planes = {planeY, planeU, planeV}; when(mockImage.getPlanes()).thenReturn(planes); - // TODO: find correct size for result here - ByteBuffer result = imageStreamReaderUtils.yuv420ThreePlanesToNV21(planes, mockImage.getWidth(), mockImage.getHeight()); - Assert.assertEquals(result.limit(), imageWidth * imageHeight); + return mockImage; + } + + /** + * Ensure that passing in an image with padding returns one without padding + */ + @Test + public void yuv420ThreePlanesToNV21_trimsPaddingWhenPresent() { + Image mockImage = getImage(640, 480, 256); + int imageWidth = mockImage.getWidth(); + int imageHeight = mockImage.getHeight(); + + ByteBuffer result = imageStreamReaderUtils.yuv420ThreePlanesToNV21(mockImage.getPlanes(), mockImage.getWidth(), mockImage.getHeight()); + Assert.assertEquals(((long) imageWidth * imageHeight) + (2 * ((long) (imageWidth / 2) * (imageHeight / 2))), result.limit()); + } + + /** + * Ensure that passing in an image without padding returns the same size + */ + @Test + public void yuv420ThreePlanesToNV21_trimsPaddingWhenAbsent() { + Image mockImage = getImage(640, 480, 0); + int imageWidth = mockImage.getWidth(); + int imageHeight = mockImage.getHeight(); + + ByteBuffer result = imageStreamReaderUtils.yuv420ThreePlanesToNV21(mockImage.getPlanes(), mockImage.getWidth(), mockImage.getHeight()); + Assert.assertEquals(((long) imageWidth * imageHeight) + (2 * ((long) (imageWidth / 2) * (imageHeight / 2))), result.limit()); } } From 628a16ad2736bc1461b8c88c73cd5b51d99662f1 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 07:44:13 -0500 Subject: [PATCH 29/35] Formatter --- .../camera/media/ImageStreamReader.java | 3 +- .../camera/media/ImageStreamReaderUtils.java | 2 +- .../camera/media/ImageStreamReaderTest.java | 28 ++- .../media/ImageStreamReaderUtilsTest.java | 161 +++++++++--------- 4 files changed, 97 insertions(+), 97 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index 87da373ec309..4153288b73e6 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -40,7 +40,8 @@ public class ImageStreamReader { * @param imageStreamReaderUtils is an instance of {@link ImageStreamReaderUtils} */ @VisibleForTesting - public ImageStreamReader(ImageReader imageReader, int dartImageFormat, ImageStreamReaderUtils imageStreamReaderUtils) { + public ImageStreamReader( + ImageReader imageReader, int dartImageFormat, ImageStreamReaderUtils imageStreamReaderUtils) { this.imageReader = imageReader; this.dartImageFormat = dartImageFormat; this.imageStreamReaderUtils = imageStreamReaderUtils; diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java index b5a4b1b09990..aa5976593c74 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java @@ -123,7 +123,7 @@ private static boolean areUVPlanesNV21(@NonNull Image.Plane[] planes, int width, *

https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java */ private static void unpackPlane( - @NonNull Image.Plane plane, int width, int height, byte[] out, int offset, int pixelStride) + @NonNull Image.Plane plane, int width, int height, byte[] out, int offset, int pixelStride) throws IllegalStateException { ByteBuffer buffer = plane.getBuffer(); buffer.rewind(); diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java index 834cdcb03512..f8e3f3b33c92 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java @@ -18,16 +18,13 @@ import io.flutter.plugin.common.EventChannel; import io.flutter.plugins.camera.types.CameraCaptureProperties; import java.nio.ByteBuffer; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; @RunWith(RobolectricTestRunner.class) public class ImageStreamReaderTest { - /** - * If we request YUV42 we should stream in YUV420. - */ + /** If we request YUV42 we should stream in YUV420. */ @Test public void computeStreamImageFormat_computesCorrectStreamFormatYuv() { int requestedStreamFormat = ImageFormat.YUV_420_888; @@ -36,8 +33,8 @@ public void computeStreamImageFormat_computesCorrectStreamFormatYuv() { } /** - * When we want to stream in NV21, we should still request YUV420 from the - * camera because we will convert it to NV21 before sending it to dart. + * When we want to stream in NV21, we should still request YUV420 from the camera because we will + * convert it to NV21 before sending it to dart. */ @Test public void computeStreamImageFormat_computesCorrectStreamFormatNv21() { @@ -47,9 +44,8 @@ public void computeStreamImageFormat_computesCorrectStreamFormatNv21() { } /** - * If we are requesting NV21, then the planes should be processed and - * converted to NV21 before being sent to dart. We make sure yuv420ThreePlanesToNV21 - * is called when we are requesting + * If we are requesting NV21, then the planes should be processed and converted to NV21 before + * being sent to dart. We make sure yuv420ThreePlanesToNV21 is called when we are requesting */ @Test public void onImageAvailable_parsesPlanesForNv21() { @@ -59,10 +55,11 @@ public void onImageAvailable_parsesPlanesForNv21() { ImageReader mockImageReader = mock(ImageReader.class); ImageStreamReaderUtils mockImageStreamReaderUtils = mock(ImageStreamReaderUtils.class); ImageStreamReader imageStreamReader = - new ImageStreamReader(mockImageReader, dartImageFormat, mockImageStreamReaderUtils); + new ImageStreamReader(mockImageReader, dartImageFormat, mockImageStreamReaderUtils); ByteBuffer mockBytes = ByteBuffer.allocate(0); - when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt())).thenReturn(mockBytes); + when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt())) + .thenReturn(mockBytes); // The image format as streamed from the camera int imageFormat = ImageFormat.YUV_420_888; @@ -107,9 +104,7 @@ public void onImageAvailable_parsesPlanesForNv21() { verify(mockImageStreamReaderUtils).yuv420ThreePlanesToNV21(any(), anyInt(), anyInt()); } - /** - * If we are requesting YUV420, then we should send the 3-plane image as it is. - */ + /** If we are requesting YUV420, then we should send the 3-plane image as it is. */ @Test public void onImageAvailable_parsesPlanesForYuv420() { // Dart wants NV21 frames @@ -118,10 +113,11 @@ public void onImageAvailable_parsesPlanesForYuv420() { ImageReader mockImageReader = mock(ImageReader.class); ImageStreamReaderUtils mockImageStreamReaderUtils = mock(ImageStreamReaderUtils.class); ImageStreamReader imageStreamReader = - new ImageStreamReader(mockImageReader, dartImageFormat, mockImageStreamReaderUtils); + new ImageStreamReader(mockImageReader, dartImageFormat, mockImageStreamReaderUtils); ByteBuffer mockBytes = ByteBuffer.allocate(0); - when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt())).thenReturn(mockBytes); + when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt())) + .thenReturn(mockBytes); // The image format as streamed from the camera int imageFormat = ImageFormat.YUV_420_888; diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java index e3caf9ad66d7..c130f226f8e8 100644 --- a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java @@ -4,93 +4,96 @@ package io.flutter.plugins.camera.media; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import android.graphics.ImageFormat; import android.media.Image; +import java.nio.ByteBuffer; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; -import java.nio.ByteBuffer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; @RunWith(RobolectricTestRunner.class) public class ImageStreamReaderUtilsTest { - private ImageStreamReaderUtils imageStreamReaderUtils; - - @Before - public void setUp() { - this.imageStreamReaderUtils = new ImageStreamReaderUtils(); - } - - Image getImage(int imageWidth, int imageHeight, int padding) { - int rowStride =imageWidth + padding; - - int ySize = (rowStride * imageHeight) - padding; - int uSize = (ySize / 2) - (padding / 2); - int vSize = uSize; - - // Mock YUV image - Image mockImage = mock(Image.class); - when(mockImage.getWidth()).thenReturn(imageWidth); - when(mockImage.getHeight()).thenReturn(imageHeight); - when(mockImage.getFormat()).thenReturn(ImageFormat.YUV_420_888); - - - - // Mock planes. YUV images have 3 planes (Y, U, V). - Image.Plane planeY = mock(Image.Plane.class); - Image.Plane planeU = mock(Image.Plane.class); - Image.Plane planeV = mock(Image.Plane.class); - - // Y plane is width*height - // Row stride is generally == width but when there is padding it will - // be larger. - // Here we are adding 256 padding. - when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(ySize)); - when(planeY.getRowStride()).thenReturn(rowStride); - when(planeY.getPixelStride()).thenReturn(1); - - // U and V planes are always the same sizes/values. - // https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888 - when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(uSize)); - when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(vSize)); - when(planeU.getRowStride()).thenReturn(rowStride); - when(planeV.getRowStride()).thenReturn(rowStride); - when(planeU.getPixelStride()).thenReturn(2); - when(planeV.getPixelStride()).thenReturn(2); - - // Add planes to image - Image.Plane[] planes = {planeY, planeU, planeV}; - when(mockImage.getPlanes()).thenReturn(planes); - - return mockImage; - } - - /** - * Ensure that passing in an image with padding returns one without padding - */ - @Test - public void yuv420ThreePlanesToNV21_trimsPaddingWhenPresent() { - Image mockImage = getImage(640, 480, 256); - int imageWidth = mockImage.getWidth(); - int imageHeight = mockImage.getHeight(); - - ByteBuffer result = imageStreamReaderUtils.yuv420ThreePlanesToNV21(mockImage.getPlanes(), mockImage.getWidth(), mockImage.getHeight()); - Assert.assertEquals(((long) imageWidth * imageHeight) + (2 * ((long) (imageWidth / 2) * (imageHeight / 2))), result.limit()); - } - - /** - * Ensure that passing in an image without padding returns the same size - */ - @Test - public void yuv420ThreePlanesToNV21_trimsPaddingWhenAbsent() { - Image mockImage = getImage(640, 480, 0); - int imageWidth = mockImage.getWidth(); - int imageHeight = mockImage.getHeight(); - - ByteBuffer result = imageStreamReaderUtils.yuv420ThreePlanesToNV21(mockImage.getPlanes(), mockImage.getWidth(), mockImage.getHeight()); - Assert.assertEquals(((long) imageWidth * imageHeight) + (2 * ((long) (imageWidth / 2) * (imageHeight / 2))), result.limit()); - } + private ImageStreamReaderUtils imageStreamReaderUtils; + + @Before + public void setUp() { + this.imageStreamReaderUtils = new ImageStreamReaderUtils(); + } + + Image getImage(int imageWidth, int imageHeight, int padding) { + int rowStride = imageWidth + padding; + + int ySize = (rowStride * imageHeight) - padding; + int uSize = (ySize / 2) - (padding / 2); + int vSize = uSize; + + // Mock YUV image + Image mockImage = mock(Image.class); + when(mockImage.getWidth()).thenReturn(imageWidth); + when(mockImage.getHeight()).thenReturn(imageHeight); + when(mockImage.getFormat()).thenReturn(ImageFormat.YUV_420_888); + + // Mock planes. YUV images have 3 planes (Y, U, V). + Image.Plane planeY = mock(Image.Plane.class); + Image.Plane planeU = mock(Image.Plane.class); + Image.Plane planeV = mock(Image.Plane.class); + + // Y plane is width*height + // Row stride is generally == width but when there is padding it will + // be larger. + // Here we are adding 256 padding. + when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(ySize)); + when(planeY.getRowStride()).thenReturn(rowStride); + when(planeY.getPixelStride()).thenReturn(1); + + // U and V planes are always the same sizes/values. + // https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888 + when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(uSize)); + when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(vSize)); + when(planeU.getRowStride()).thenReturn(rowStride); + when(planeV.getRowStride()).thenReturn(rowStride); + when(planeU.getPixelStride()).thenReturn(2); + when(planeV.getPixelStride()).thenReturn(2); + + // Add planes to image + Image.Plane[] planes = {planeY, planeU, planeV}; + when(mockImage.getPlanes()).thenReturn(planes); + + return mockImage; + } + + /** Ensure that passing in an image with padding returns one without padding */ + @Test + public void yuv420ThreePlanesToNV21_trimsPaddingWhenPresent() { + Image mockImage = getImage(640, 480, 256); + int imageWidth = mockImage.getWidth(); + int imageHeight = mockImage.getHeight(); + + ByteBuffer result = + imageStreamReaderUtils.yuv420ThreePlanesToNV21( + mockImage.getPlanes(), mockImage.getWidth(), mockImage.getHeight()); + Assert.assertEquals( + ((long) imageWidth * imageHeight) + (2 * ((long) (imageWidth / 2) * (imageHeight / 2))), + result.limit()); + } + + /** Ensure that passing in an image without padding returns the same size */ + @Test + public void yuv420ThreePlanesToNV21_trimsPaddingWhenAbsent() { + Image mockImage = getImage(640, 480, 0); + int imageWidth = mockImage.getWidth(); + int imageHeight = mockImage.getHeight(); + + ByteBuffer result = + imageStreamReaderUtils.yuv420ThreePlanesToNV21( + mockImage.getPlanes(), mockImage.getWidth(), mockImage.getHeight()); + Assert.assertEquals( + ((long) imageWidth * imageHeight) + (2 * ((long) (imageWidth / 2) * (imageHeight / 2))), + result.limit()); + } } From 651d20227e7a703efc52d3d536250563054fda58 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 07:47:56 -0500 Subject: [PATCH 30/35] Update CHANGELOG.md --- packages/camera/camera_android/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/camera/camera_android/CHANGELOG.md b/packages/camera/camera_android/CHANGELOG.md index a9ab72e696f9..4dfefa073d93 100644 --- a/packages/camera/camera_android/CHANGELOG.md +++ b/packages/camera/camera_android/CHANGELOG.md @@ -1,6 +1,7 @@ ## 0.10.5 -* Adds logic to remove buffer padding in YUV frames on some devices / resolutions. +* Adds support for NV21 as a new streaming format in Android which includes correct handling of image padding when present. + ## 0.10.4 * Temporarily fixes issue with requested video profiles being null by falling back to deprecated behavior in that case. From cbfb447d78063276a8d40feeb25799e6bd9c936c Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 07:50:00 -0500 Subject: [PATCH 31/35] Changelogs --- packages/camera/camera/CHANGELOG.md | 3 +++ packages/camera/camera/pubspec.yaml | 3 +-- packages/camera/camera_platform_interface/CHANGELOG.md | 3 +++ packages/camera/camera_platform_interface/pubspec.yaml | 4 ++-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/camera/camera/CHANGELOG.md b/packages/camera/camera/CHANGELOG.md index 13c00402449a..0be72364e975 100644 --- a/packages/camera/camera/CHANGELOG.md +++ b/packages/camera/camera/CHANGELOG.md @@ -1,3 +1,6 @@ +## 0.10.4 +* Adds NV21 as an image streaming option for Android. + ## 0.10.3 * Adds back use of Optional type. diff --git a/packages/camera/camera/pubspec.yaml b/packages/camera/camera/pubspec.yaml index 7da3803645f1..59d72c109f1c 100644 --- a/packages/camera/camera/pubspec.yaml +++ b/packages/camera/camera/pubspec.yaml @@ -4,7 +4,7 @@ description: A Flutter plugin for controlling the camera. Supports previewing Dart. repository: https://github.com/flutter/plugins/tree/main/packages/camera/camera issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22 -version: 0.10.3 +version: 0.10.4 environment: sdk: ">=2.14.0 <3.0.0" @@ -39,7 +39,6 @@ dev_dependencies: plugin_platform_interface: ^2.0.0 video_player: ^2.0.0 - # FOR TESTING ONLY. DO NOT MERGE. dependency_overrides: camera_android: diff --git a/packages/camera/camera_platform_interface/CHANGELOG.md b/packages/camera/camera_platform_interface/CHANGELOG.md index b51eb9c78a43..65eeb07fdc8f 100644 --- a/packages/camera/camera_platform_interface/CHANGELOG.md +++ b/packages/camera/camera_platform_interface/CHANGELOG.md @@ -1,3 +1,6 @@ +## 2.5.0 +* Adds NV21 as an image stream format (suitable for Android). + ## 2.4.0 * Allows camera to be switched while video recording. diff --git a/packages/camera/camera_platform_interface/pubspec.yaml b/packages/camera/camera_platform_interface/pubspec.yaml index 4cdb2855a156..59e72d0ce0a0 100644 --- a/packages/camera/camera_platform_interface/pubspec.yaml +++ b/packages/camera/camera_platform_interface/pubspec.yaml @@ -4,10 +4,10 @@ repository: https://github.com/flutter/plugins/tree/main/packages/camera/camera_ issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22 # NOTE: We strongly prefer non-breaking changes, even at the expense of a # less-clean API. See https://flutter.dev/go/platform-interface-breaking-changes -version: 2.4.0 +version: 2.5.0 environment: - sdk: '>=2.12.0 <3.0.0' + sdk: ">=2.12.0 <3.0.0" flutter: ">=3.0.0" dependencies: From 1d1ecbf8a5c6094ee93fbfccffbc6cdb08a8511b Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 07:50:27 -0500 Subject: [PATCH 32/35] Update CHANGELOG.md --- packages/camera/camera_android/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/camera/camera_android/CHANGELOG.md b/packages/camera/camera_android/CHANGELOG.md index 4dfefa073d93..fce627ce0576 100644 --- a/packages/camera/camera_android/CHANGELOG.md +++ b/packages/camera/camera_android/CHANGELOG.md @@ -2,6 +2,7 @@ * Adds support for NV21 as a new streaming format in Android which includes correct handling of image padding when present. + ## 0.10.4 * Temporarily fixes issue with requested video profiles being null by falling back to deprecated behavior in that case. From 804ddc7cfaad9b64f5261f5824b12ddaaaf0d855 Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 08:33:26 -0500 Subject: [PATCH 33/35] Update ImageStreamReader.java --- .../io/flutter/plugins/camera/media/ImageStreamReader.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index 4153288b73e6..fc586c9164e0 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -80,8 +80,7 @@ public static int computeStreamImageFormat(int dartImageFormat) { } /** - * Processes a new frame (image) from the image reader, remove padding if necessary, and send the - * frame to Dart. + * Processes a new frame (image) from the image reader and send the frame to Dart. * * @param image is the image which needs processed as an {@link Image} * @param captureProps is the capture props from the camera class as {@link From 5a10847b819dbe1b91c000bf597f5d4b1e7e386a Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 08:38:50 -0500 Subject: [PATCH 34/35] Comments --- .../plugins/camera/media/ImageStreamReader.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index fc586c9164e0..e6bc65f1af5d 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -129,6 +129,15 @@ public void onImageAvailable( } } + /** + * Given an input image, will return a list of maps suitable to send back to dart where + * each map describes the image plane. + * + * For Yuv / Jpeg, we do no further processing on the frame so we simply send it as-is. + * + * @param image - the image to process. + * @return parsed map describing the image planes to be sent to dart. + */ public List> parsePlanesForYuvOrJpeg(@NonNull Image image) { List> planes = new ArrayList<>(); @@ -149,6 +158,12 @@ public List> parsePlanesForYuvOrJpeg(@NonNull Image image) { return planes; } + /** + * Given an input image, will return a single-plane NV21 image. Assumes YUV420 as an input type. + * + * @param image - the image to process. + * @return parsed map describing the image planes to be sent to dart. + */ public List> parsePlanesForNv21(@NonNull Image image) { List> planes = new ArrayList<>(); From 198c55d59400736b46e46c1bc57a74841bce7dbd Mon Sep 17 00:00:00 2001 From: Andrew Coutts Date: Fri, 17 Feb 2023 08:40:23 -0500 Subject: [PATCH 35/35] comments --- .../io/flutter/plugins/camera/media/ImageStreamReader.java | 2 +- .../flutter/plugins/camera/media/ImageStreamReaderUtils.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java index e6bc65f1af5d..b730208cf9c2 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -134,7 +134,7 @@ public void onImageAvailable( * each map describes the image plane. * * For Yuv / Jpeg, we do no further processing on the frame so we simply send it as-is. - * + * * @param image - the image to process. * @return parsed map describing the image planes to be sent to dart. */ diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java index aa5976593c74..1356dc910c21 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java @@ -1,6 +1,9 @@ // Copyright 2013 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// +// Note: the code in this file is taken directly from the official Google MLKit example: +// https://github.com/googlesamples/mlkit package io.flutter.plugins.camera.media;