diff --git a/packages/camera/camera_android/CHANGELOG.md b/packages/camera/camera_android/CHANGELOG.md index 0aec0be2b442..096acca16e7a 100644 --- a/packages/camera/camera_android/CHANGELOG.md +++ b/packages/camera/camera_android/CHANGELOG.md @@ -1,5 +1,8 @@ ## NEXT +## 0.10.3 + +* Adds support for NV21 image format in Android. * Updates minimum Flutter version to 3.0. ## 0.10.2+3 diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java index 7c592b9c7e99..ee3543b66cda 100644 --- a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/Camera.java @@ -21,7 +21,6 @@ import android.hardware.camera2.params.SessionConfiguration; import android.media.CamcorderProfile; import android.media.EncoderProfiles; -import android.media.Image; import android.media.ImageReader; import android.media.MediaRecorder; import android.os.Build; @@ -59,19 +58,18 @@ import io.flutter.plugins.camera.features.sensororientation.DeviceOrientationManager; import io.flutter.plugins.camera.features.sensororientation.SensorOrientationFeature; import io.flutter.plugins.camera.features.zoomlevel.ZoomLevelFeature; +import io.flutter.plugins.camera.media.ImageStreamReader; import io.flutter.plugins.camera.media.MediaRecorderBuilder; import io.flutter.plugins.camera.types.CameraCaptureProperties; import io.flutter.plugins.camera.types.CaptureTimeoutsWrapper; import io.flutter.view.TextureRegistry.SurfaceTextureEntry; import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.concurrent.Executors; @FunctionalInterface @@ -109,6 +107,7 @@ class Camera supportedImageFormats = new HashMap<>(); supportedImageFormats.put("yuv420", ImageFormat.YUV_420_888); supportedImageFormats.put("jpeg", ImageFormat.JPEG); + supportedImageFormats.put("nv21", ImageFormat.NV21); } /** @@ -135,7 +134,7 @@ class Camera private CameraDeviceWrapper cameraDevice; private CameraCaptureSession captureSession; private ImageReader pictureImageReader; - private ImageReader imageStreamReader; + private ImageStreamReader imageStreamReader; /** {@link CaptureRequest.Builder} for the camera preview */ private CaptureRequest.Builder previewRequestBuilder; @@ -304,11 +303,11 @@ public void open(String imageFormatGroup) throws CameraAccessException { imageFormat = ImageFormat.YUV_420_888; } imageStreamReader = - ImageReader.newInstance( - resolutionFeature.getPreviewSize().getWidth(), - resolutionFeature.getPreviewSize().getHeight(), - imageFormat, - 1); + new ImageStreamReader( + resolutionFeature.getPreviewSize().getWidth(), + resolutionFeature.getPreviewSize().getHeight(), + imageFormat, + 1); // Open the camera. CameraManager cameraManager = CameraUtils.getCameraManager(activity); @@ -1140,49 +1139,13 @@ public void onListen(Object o, EventChannel.EventSink imageStreamSink) { @Override public void onCancel(Object o) { - imageStreamReader.setOnImageAvailableListener(null, backgroundHandler); + imageStreamReader.removeListener(backgroundHandler); } }); } private void setImageStreamImageAvailableListener(final EventChannel.EventSink imageStreamSink) { - imageStreamReader.setOnImageAvailableListener( - reader -> { - Image img = reader.acquireNextImage(); - // Use acquireNextImage since image reader is only for one image. - if (img == null) return; - - List> planes = new ArrayList<>(); - for (Image.Plane plane : img.getPlanes()) { - ByteBuffer buffer = plane.getBuffer(); - - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes, 0, bytes.length); - - Map planeBuffer = new HashMap<>(); - planeBuffer.put("bytesPerRow", plane.getRowStride()); - planeBuffer.put("bytesPerPixel", plane.getPixelStride()); - planeBuffer.put("bytes", bytes); - - planes.add(planeBuffer); - } - - Map imageBuffer = new HashMap<>(); - imageBuffer.put("width", img.getWidth()); - imageBuffer.put("height", img.getHeight()); - imageBuffer.put("format", img.getFormat()); - imageBuffer.put("planes", planes); - imageBuffer.put("lensAperture", this.captureProps.getLastLensAperture()); - imageBuffer.put("sensorExposureTime", this.captureProps.getLastSensorExposureTime()); - Integer sensorSensitivity = this.captureProps.getLastSensorSensitivity(); - imageBuffer.put( - "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity); - - final Handler handler = new Handler(Looper.getMainLooper()); - handler.post(() -> imageStreamSink.success(imageBuffer)); - img.close(); - }, - backgroundHandler); + imageStreamReader.subscribeListener(this.captureProps, imageStreamSink, backgroundHandler); } private void closeCaptureSession() { diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java new file mode 100644 index 000000000000..c28a3d2ba992 --- /dev/null +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReader.java @@ -0,0 +1,224 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package io.flutter.plugins.camera.media; + +import android.graphics.ImageFormat; +import android.media.Image; +import android.media.ImageReader; +import android.os.Handler; +import android.os.Looper; +import android.view.Surface; +import androidx.annotation.NonNull; +import androidx.annotation.VisibleForTesting; +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugins.camera.types.CameraCaptureProperties; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +// Wraps an ImageReader to allow for testing of the image handler. +public class ImageStreamReader { + + /** + * The image format we are going to send back to dart. Usually it's the + * same as streamImageFormat but in the case of NV21 we will actually + * request YUV frames but convert it to NV21 before sending to dart. + */ + private final int dartImageFormat; + + private final ImageReader imageReader; + private final ImageStreamReaderUtils imageStreamReaderUtils; + + /** + * Creates a new instance of the {@link ImageStreamReader}. + * Used for testing so we can provide a mock ImageReader. + * + * @param imageReader is the image reader that will receive frames + * @param imageStreamReaderUtils is an instance of {@link ImageStreamReaderUtils} + */ + @VisibleForTesting + public ImageStreamReader(ImageReader imageReader, int imageFormat, ImageStreamReaderUtils imageStreamReaderUtils) { + this.imageReader = imageReader; + this.dartImageFormat = imageFormat; + this.imageStreamReaderUtils = imageStreamReaderUtils; + } + + /** + * Creates a new instance of the {@link ImageStreamReader}. + * + * @param width is the image width + * @param height is the image height + * @param imageFormat is the {@link ImageFormat} that should be returned to dart. + * @param maxImages is how many images can be acquired at one time, usually 1. + */ + public ImageStreamReader(int width, int height, int imageFormat, int maxImages) { + this.dartImageFormat = imageFormat; + this.imageReader = ImageReader.newInstance( + width, + height, + computeStreamImageFormat(imageFormat), + maxImages); + this.imageStreamReaderUtils = new ImageStreamReaderUtils(); + } + + /** + * Returns the streaming image format to stream based on a requested input format. + * Usually it's the same except when dart is requesting NV21. In that case + * we stream YUV420 and process it into NV21 before sending the frames over. + * @param dartImageFormat is the format we want to send to dart. + * @return is the image format that will be requested from the camera. + */ + @VisibleForTesting + public static int computeStreamImageFormat(int dartImageFormat) { + if (dartImageFormat == ImageFormat.NV21) { + return ImageFormat.YUV_420_888; + } else { + return dartImageFormat; + } + } + + /** + * Processes a new frame (image) from the image reader, remove padding if necessary, and send the + * frame to Dart. + * + * @param image is the image which needs processed as an {@link Image} + * @param captureProps is the capture props from the camera class as {@link + * CameraCaptureProperties} + * @param imageStreamSink is the image stream sink from dart as a dart {@link + * EventChannel.EventSink} + */ + @VisibleForTesting + public void onImageAvailable( + @NonNull Image image, + CameraCaptureProperties captureProps, + EventChannel.EventSink imageStreamSink) { + try { + Map imageBuffer = new HashMap<>(); + + // Get plane data ready + if (dartImageFormat == ImageFormat.NV21) { + imageBuffer.put("planes", parsePlanesForNv21(image)); + } else { + imageBuffer.put("planes", parsePlanesForYuvOrJpeg(image)); + } + + imageBuffer.put("width", image.getWidth()); + imageBuffer.put("height", image.getHeight()); + imageBuffer.put("format", dartImageFormat); + imageBuffer.put("lensAperture", captureProps.getLastLensAperture()); + imageBuffer.put("sensorExposureTime", captureProps.getLastSensorExposureTime()); + Integer sensorSensitivity = captureProps.getLastSensorSensitivity(); + imageBuffer.put( + "sensorSensitivity", sensorSensitivity == null ? null : (double) sensorSensitivity); + + final Handler handler = new Handler(Looper.getMainLooper()); + handler.post(() -> imageStreamSink.success(imageBuffer)); + image.close(); + + } catch (IllegalStateException e) { + // Handle "buffer is inaccessible" errors that can happen on some devices from ImageStreamReaderUtils.yuv420ThreePlanesToNV21() + final Handler handler = new Handler(Looper.getMainLooper()); + handler.post(() -> imageStreamSink.error("IllegalStateException", "Caught IllegalStateException: " + e.getMessage(), null)); + image.close(); + } + } + + /** + * Should be used when we do not want to alter the camera image data + * and just want to send it to dart exactly as received from the sensor. + * + * Generally used for YUV420 and Jpeg streaming formats. + * @param image is an {@link Image} from the camera. + * @return is the planes array list of parsed data that will be sent to dart. + */ + @VisibleForTesting + public List> parsePlanesForYuvOrJpeg(Image image) { + List> planes = new ArrayList<>(); + + // For YUV420 and JPEG, just send the data as-is for each plane. + for (Image.Plane plane : image.getPlanes()) { + ByteBuffer buffer = plane.getBuffer(); + + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes, 0, bytes.length); + + Map planeBuffer = new HashMap<>(); + planeBuffer.put("bytesPerRow", plane.getRowStride()); + planeBuffer.put("bytesPerPixel", plane.getPixelStride()); + planeBuffer.put("bytes", bytes); + + planes.add(planeBuffer); + } + return planes; + } + + /** + * Should be used when we want to convert the image data from 3-plane YUV + * to a single plane NV21 image. + * @param image is an {@link Image} from the camera. + * @return is the planes array list of parsed data that will be sent to dart. + */ + @VisibleForTesting + public List> parsePlanesForNv21(Image image) { + List> planes = new ArrayList<>(); + + // We will convert the YUV data to NV21 which is a single-plane image + ByteBuffer bytes = imageStreamReaderUtils.yuv420ThreePlanesToNV21( + image.getPlanes(), + image.getWidth(), + image.getHeight() + ); + + Map planeBuffer = new HashMap<>(); + planeBuffer.put("bytesPerRow", image.getWidth()); + planeBuffer.put("bytesPerPixel", 1); + planeBuffer.put("bytes", bytes.array()); + planes.add(planeBuffer); + return planes; + } + + /** Returns the image reader surface. */ + public Surface getSurface() { + return imageReader.getSurface(); + } + + /** + * Subscribes the image stream reader to handle incoming images using onImageAvailable(). + * + * @param captureProps is the capture props from the camera class as {@link + * CameraCaptureProperties} + * @param imageStreamSink is the image stream sink from dart as {@link EventChannel.EventSink} + * @param handler is generally the background handler of the camera as {@link Handler} + */ + public void subscribeListener( + CameraCaptureProperties captureProps, + EventChannel.EventSink imageStreamSink, + Handler handler) { + imageReader.setOnImageAvailableListener( + reader -> { + Image image = reader.acquireNextImage(); + if (image == null) return; + + onImageAvailable(image, captureProps, imageStreamSink); + }, + handler); + } + + /** + * Removes the listener from the image reader. + * + * @param handler is generally the background handler of the camera + */ + public void removeListener(Handler handler) { + imageReader.setOnImageAvailableListener(null, handler); + } + + /** Closes the image reader. */ + public void close() { + imageReader.close(); + } +} diff --git a/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java new file mode 100644 index 000000000000..ca7630c04faf --- /dev/null +++ b/packages/camera/camera_android/android/src/main/java/io/flutter/plugins/camera/media/ImageStreamReaderUtils.java @@ -0,0 +1,158 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package io.flutter.plugins.camera.media; + +import android.media.Image; +import androidx.annotation.VisibleForTesting; +import java.nio.ByteBuffer; +import io.flutter.Log; + +public class ImageStreamReaderUtils { + /** + * Converts YUV_420_888 to NV21 bytebuffer. + * + *

The NV21 format consists of a single byte array containing the Y, U and V values. For an + * image of size S, the first S positions of the array contain all the Y values. The remaining + * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both + * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain + * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU + * + *

YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled + * by a factor of 2 in both dimensions. {@link Image#getPlanes} returns an array with the Y, U and + * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into + * the first part of the NV21 array. The U and V planes may already have the representation in the + * NV21 format. This happens if the planes share the same buffer, the V buffer is one position + * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy + * them to the NV21 array. + * + * https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java + */ + public ByteBuffer yuv420ThreePlanesToNV21( + Image.Plane[] yuv420888planes, int width, int height) { + int imageSize = width * height; + byte[] out = new byte[imageSize + 2 * (imageSize / 4)]; + + if (areUVPlanesNV21(yuv420888planes, width, height)) { + Log.i("flutter", "planes are NV21"); + // Copy the Y values. + yuv420888planes[0].getBuffer().get(out, 0, imageSize); + + ByteBuffer uBuffer = yuv420888planes[1].getBuffer(); + ByteBuffer vBuffer = yuv420888planes[2].getBuffer(); + // Get the first V value from the V buffer, since the U buffer does not contain it. + vBuffer.get(out, imageSize, 1); + // Copy the first U value and the remaining VU values from the U buffer. + uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1); + } else {Log.i("flutter", "planes are not NV21"); + + // Fallback to copying the UV values one by one, which is slower but also works. + // Unpack Y. + unpackPlane(yuv420888planes[0], width, height, out, 0, 1); + // Unpack U. + unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2); + // Unpack V. + unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2); + } + + return ByteBuffer.wrap(out); + } + + /** + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format. + * + * https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java + */ + @VisibleForTesting + public static boolean areUVPlanesNV21(Image.Plane[] planes, int width, int height) { + int imageSize = width * height; + + ByteBuffer uBuffer = planes[1].getBuffer(); + ByteBuffer vBuffer = planes[2].getBuffer(); + + // Backup buffer properties. + int vBufferPosition = vBuffer.position(); + int uBufferLimit = uBuffer.limit(); + + // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value. + vBuffer.position(vBufferPosition + 1); + // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value. + uBuffer.limit(uBufferLimit - 1); + + // Check that the buffers are equal and have the expected number of elements. + boolean areNV21 = + (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); + + // Restore buffers to their initial state. + vBuffer.position(vBufferPosition); + uBuffer.limit(uBufferLimit); + + return areNV21; + } + + /** + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Unpack an image plane into a byte array. + * + *

The input plane data will be copied in 'out', starting at 'offset' and every pixel will be + * spaced by 'pixelStride'. Note that there is no row padding on the output. + * + * https://github.com/googlesamples/mlkit/blob/master/android/vision-quickstart/app/src/main/java/com/google/mlkit/vision/demo/BitmapUtils.java + */ + @VisibleForTesting + public static void unpackPlane( + Image.Plane plane, int width, int height, byte[] out, int offset, int pixelStride) throws IllegalStateException{ + ByteBuffer buffer = plane.getBuffer(); + buffer.rewind(); + + // Compute the size of the current plane. + // We assume that it has the aspect ratio as the original image. + int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride(); + if (numRow == 0) { + return; + } + int scaleFactor = height / numRow; + int numCol = width / scaleFactor; + + // Extract the data in the output buffer. + int outputPos = offset; + int rowStart = 0; + for (int row = 0; row < numRow; row++) { + int inputPos = rowStart; + for (int col = 0; col < numCol; col++) { + out[outputPos] = buffer.get(inputPos); + outputPos += pixelStride; + inputPos += plane.getPixelStride(); + } + rowStart += plane.getRowStride(); + } + } +} diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java new file mode 100644 index 000000000000..7adcdd327cb6 --- /dev/null +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderTest.java @@ -0,0 +1,125 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package io.flutter.plugins.camera.media; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import android.graphics.ImageFormat; +import android.media.Image; +import android.media.ImageReader; +import io.flutter.plugin.common.EventChannel; +import io.flutter.plugins.camera.types.CameraCaptureProperties; +import java.nio.ByteBuffer; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; + +@RunWith(RobolectricTestRunner.class) +public class ImageStreamReaderTest { + private ImageStreamReader imageStreamReader; + private ImageStreamReaderUtils mockImageStreamReaderUtils; + + @Before + public void setUp() { + ImageReader mockImageReader = mock(ImageReader.class); + ImageStreamReaderUtils mockImageStreamReaderUtils = mock(ImageStreamReaderUtils.class); + + this.mockImageStreamReaderUtils = mockImageStreamReaderUtils; + this.imageStreamReader = + new ImageStreamReader(mockImageReader, ImageFormat.YUV_420_888, this.mockImageStreamReaderUtils); + } + + @Test + public void computeStreamImageFormat_forNv21() { + int result = ImageStreamReader.computeStreamImageFormat(ImageFormat.NV21); + Assert.assertEquals(ImageFormat.YUV_420_888, result); + } + + @Test + public void computeStreamImageFormat_forJpeg() { + int result = ImageStreamReader.computeStreamImageFormat(ImageFormat.JPEG); + Assert.assertEquals(ImageFormat.JPEG, result); + } + + + @Test + public void onImageAvailable_doesNotTryToFixConvertForNonNv21Format() { + // Mock JPEG image + int imageFormat = ImageFormat.JPEG; + Image mockImage = mock(Image.class); + when(mockImage.getFormat()).thenReturn(imageFormat); + + // Mock plane. JPEG images have only one plane + Image.Plane plane0 = mock(Image.Plane.class); + when(plane0.getBuffer()).thenReturn(ByteBuffer.allocate(497950)); + when(plane0.getRowStride()).thenReturn(0); + when(plane0.getPixelStride()).thenReturn(0); + Image.Plane[] planes = {plane0}; + when(mockImage.getPlanes()).thenReturn(planes); + + CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class); + EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class); + imageStreamReader.onImageAvailable(mockImage, mockCaptureProps, mockEventSink); + + verify(mockImageStreamReaderUtils, never()).yuv420ThreePlanesToNV21(any(), anyInt(), anyInt()); + } + + @Test + public void onImageAvailable_doesConvertPlanesForNv21() { + ImageReader mockImageReader = mock(ImageReader.class); + ImageStreamReaderUtils mockImageStreamReaderUtils = mock(ImageStreamReaderUtils.class); + + // Make sure we setup the correct dartImageFormat + imageStreamReader = + new ImageStreamReader(mockImageReader, ImageFormat.NV21, mockImageStreamReaderUtils); + + // Mock YUV image + Image mockImage = mock(Image.class); + when(mockImage.getWidth()).thenReturn(1280); + when(mockImage.getHeight()).thenReturn(720); + when(mockImage.getFormat()).thenReturn(ImageFormat.YUV_420_888); + + // Mock planes. YUV images have 3 planes (Y, U, V). + Image.Plane planeY = mock(Image.Plane.class); + Image.Plane planeU = mock(Image.Plane.class); + Image.Plane planeV = mock(Image.Plane.class); + + // Y plane is width*height + // Row stride is generally == width but when there is padding it will + // be larger. The numbers in this example are from a Vivo V2135 on 'high' + // setting (1280x720). + when(planeY.getBuffer()).thenReturn(ByteBuffer.allocate(1105664)); + when(planeY.getRowStride()).thenReturn(1536); + when(planeY.getPixelStride()).thenReturn(1); + + // U and V planes are always the same sizes/values. + // https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888 + when(planeU.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); + when(planeV.getBuffer()).thenReturn(ByteBuffer.allocate(552703)); + when(planeU.getRowStride()).thenReturn(1536); + when(planeV.getRowStride()).thenReturn(1536); + when(planeU.getPixelStride()).thenReturn(2); + when(planeV.getPixelStride()).thenReturn(2); + + // Add planes to image + Image.Plane[] planes = {planeY, planeU, planeV}; + when(mockImage.getPlanes()).thenReturn(planes); + + when(mockImageStreamReaderUtils.yuv420ThreePlanesToNV21(any(), anyInt(), anyInt())).thenReturn(ByteBuffer.allocate(1280*720)); + + CameraCaptureProperties mockCaptureProps = mock(CameraCaptureProperties.class); + EventChannel.EventSink mockEventSink = mock(EventChannel.EventSink.class); + imageStreamReader.onImageAvailable(mockImage, mockCaptureProps, mockEventSink); + + verify(mockImageStreamReaderUtils).yuv420ThreePlanesToNV21(any(), anyInt(), anyInt()); + } +} diff --git a/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java new file mode 100644 index 000000000000..c31094339e66 --- /dev/null +++ b/packages/camera/camera_android/android/src/test/java/io/flutter/plugins/camera/media/ImageStreamReaderUtilsTest.java @@ -0,0 +1,138 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package io.flutter.plugins.camera.media; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import android.media.Image; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; + +@RunWith(RobolectricTestRunner.class) +public class ImageStreamReaderUtilsTest { + private ImageStreamReaderUtils imageStreamReaderUtils; + + @Before + public void setUp() { + this.imageStreamReaderUtils = new ImageStreamReaderUtils(); + } + + @Test + public void areUVPlanesNV21_shouldDetectYuvDataThatIsNv21() { + Image.Plane planeY = mock(Image.Plane.class); + Image.Plane planeU = mock(Image.Plane.class); + Image.Plane planeV = mock(Image.Plane.class); + Image.Plane[] planes = {planeY, planeU, planeV}; + + // We construct the buffers to be valid NV21 split into its 3 planes. + // This is generally described as a Y buffer with length of the image size + // followed by two identical size U and V buffers which are sub-sampled by 2. + ByteBuffer yBuffer = ByteBuffer.allocate(640*360); + ByteBuffer uBuffer = ByteBuffer.allocate(2 * 640 * 360 / 4 - 1); + ByteBuffer vBuffer = ByteBuffer.allocate(2 * 640 * 360 / 4 - 1); + when(planeY.getBuffer()).thenReturn(yBuffer); + when(planeU.getBuffer()).thenReturn(uBuffer); + when(planeV.getBuffer()).thenReturn(vBuffer); + + boolean result = ImageStreamReaderUtils.areUVPlanesNV21(planes, 640, 360); + Assert.assertEquals(true, result); + + yBuffer.clear(); + uBuffer.clear(); + vBuffer.clear(); + } + + @Test + public void areUVPlanesNV21_shouldDetectYuvDataThatIsNotNv21() { + Image.Plane planeY = mock(Image.Plane.class); + Image.Plane planeU = mock(Image.Plane.class); + Image.Plane planeV = mock(Image.Plane.class); + Image.Plane[] planes = {planeY, planeU, planeV}; + + // We construct the buffers to be a 3-buffer YUV420 image. + ByteBuffer yBuffer = ByteBuffer.allocate(640*360); + ByteBuffer uBuffer = ByteBuffer.allocate(640 / 2 * 360 / 2); + ByteBuffer vBuffer = ByteBuffer.allocate(640 / 2 * 360 / 2); + when(planeY.getBuffer()).thenReturn(yBuffer); + when(planeU.getBuffer()).thenReturn(uBuffer); + when(planeV.getBuffer()).thenReturn(vBuffer); + + boolean result = ImageStreamReaderUtils.areUVPlanesNV21(planes, 640, 360); + Assert.assertEquals(false, result); + + yBuffer.clear(); + uBuffer.clear(); + vBuffer.clear(); + } + + // Feed it YUV data that has no padding and it should create a valid NV21 image. + @Test + public void yuv420ThreePlanesToNV21_shouldCreateValidNv21FromYuvDataNoPadding() { + Image.Plane planeY = mock(Image.Plane.class); + Image.Plane planeU = mock(Image.Plane.class); + Image.Plane planeV = mock(Image.Plane.class); + Image.Plane[] planes = {planeY, planeU, planeV}; + + when(planeY.getRowStride()).thenReturn(640); + when(planeU.getRowStride()).thenReturn(640); + when(planeV.getRowStride()).thenReturn(640); + when(planeY.getPixelStride()).thenReturn(1); + when(planeU.getPixelStride()).thenReturn(2); + when(planeV.getPixelStride()).thenReturn(2); + + // We construct the buffers to be a 3-buffer YUV420 image. + ByteBuffer yBuffer = ByteBuffer.allocate(640*360); + ByteBuffer uBuffer = ByteBuffer.allocate(640 / 2 * 360 / 2); + ByteBuffer vBuffer = ByteBuffer.allocate(640 / 2 * 360 / 2); + when(planeY.getBuffer()).thenReturn(yBuffer); + when(planeU.getBuffer()).thenReturn(uBuffer); + when(planeV.getBuffer()).thenReturn(vBuffer); + + ByteBuffer result = imageStreamReaderUtils.yuv420ThreePlanesToNV21(planes, 640, 360); + Assert.assertEquals(yBuffer.limit() + yBuffer.limit() / 4 + yBuffer.limit() / 4, result.limit()); + + yBuffer.clear(); + uBuffer.clear(); + vBuffer.clear(); + } + + // Feed it YUV data that has padding and it should create a valid NV21 image. + // The result NV21 will have the padding trimmed away. + @Test + public void yuv420ThreePlanesToNV21_shouldCreateValidNv21FromYuvDataWithPadding() { + Image.Plane planeY = mock(Image.Plane.class); + Image.Plane planeU = mock(Image.Plane.class); + Image.Plane planeV = mock(Image.Plane.class); + Image.Plane[] planes = {planeY, planeU, planeV}; + + when(planeY.getRowStride()).thenReturn(640); + when(planeU.getRowStride()).thenReturn(1536); + when(planeV.getRowStride()).thenReturn(1536); + when(planeY.getPixelStride()).thenReturn(1); + when(planeU.getPixelStride()).thenReturn(2); + when(planeV.getPixelStride()).thenReturn(2); + + // We construct the buffers to be a 3-buffer YUV420 image. + ByteBuffer yBuffer = ByteBuffer.allocate(640*360); + ByteBuffer uBuffer = ByteBuffer.allocate(1536 / 2 * 360 / 2); + ByteBuffer vBuffer = ByteBuffer.allocate(1536 / 2 * 360 / 2); + when(planeY.getBuffer()).thenReturn(yBuffer); + when(planeU.getBuffer()).thenReturn(uBuffer); + when(planeV.getBuffer()).thenReturn(vBuffer); + + ByteBuffer result = imageStreamReaderUtils.yuv420ThreePlanesToNV21(planes, 640, 360); + Assert.assertEquals(yBuffer.limit() + yBuffer.limit() / 4 + yBuffer.limit() / 4, result.limit()); + + yBuffer.clear(); + uBuffer.clear(); + vBuffer.clear(); + } +} diff --git a/packages/camera/camera_android/lib/src/type_conversion.dart b/packages/camera/camera_android/lib/src/type_conversion.dart index 754a5a032715..7691f8e9988b 100644 --- a/packages/camera/camera_android/lib/src/type_conversion.dart +++ b/packages/camera/camera_android/lib/src/type_conversion.dart @@ -34,6 +34,8 @@ ImageFormatGroup _imageFormatGroupFromPlatformData(dynamic data) { return ImageFormatGroup.yuv420; case 256: // android.graphics.ImageFormat.JPEG return ImageFormatGroup.jpeg; + case 17: // android.graphics.ImageFormat.NV21 + return ImageFormatGroup.nv21; } return ImageFormatGroup.unknown; diff --git a/packages/camera/camera_android/pubspec.yaml b/packages/camera/camera_android/pubspec.yaml index 30d6153cece6..f52006bfdcc6 100644 --- a/packages/camera/camera_android/pubspec.yaml +++ b/packages/camera/camera_android/pubspec.yaml @@ -2,7 +2,7 @@ name: camera_android description: Android implementation of the camera plugin. repository: https://github.com/flutter/plugins/tree/main/packages/camera/camera_android issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22 -version: 0.10.2+3 +version: 0.10.3 environment: sdk: ">=2.14.0 <3.0.0" @@ -18,7 +18,7 @@ flutter: dartPluginClass: AndroidCamera dependencies: - camera_platform_interface: ^2.3.1 + camera_platform_interface: ^2.4.0 flutter: sdk: flutter flutter_plugin_android_lifecycle: ^2.0.2