From 78ab3f6e9fdf92093a34e1724751902dfc1fa4b8 Mon Sep 17 00:00:00 2001 From: Sergejs Luhmirins Date: Thu, 12 Sep 2024 15:49:33 +0300 Subject: [PATCH 1/2] MS-71 Offload image rotation to camera API --- .../screens/livefeedback/FrameProcessor.kt | 61 ++----------------- .../livefeedback/LiveFeedbackFragment.kt | 22 +++---- .../usecases/ImageProxyToBitmapUseCase.kt | 4 +- 3 files changed, 14 insertions(+), 73 deletions(-) diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/FrameProcessor.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/FrameProcessor.kt index 295b87712c..818f294397 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/FrameProcessor.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/FrameProcessor.kt @@ -58,64 +58,22 @@ internal class FrameProcessor @Inject constructor( val cameraWidth = image.width val cameraHeight = image.height - val (rotatedCameraWidth, rotatedCameraHeight) = getCameraRotatedPair(image) - val newRectSize = getRectSizeBasedOnCameraCropping( previewViewWidth, previewViewHeight, - rotatedCameraWidth, - rotatedCameraHeight, + cameraWidth, + cameraHeight, boxOnTheScreen ) - val newBoundingBox = CameraTargetOverlay.rectForPlane( - width = rotatedCameraWidth, - height = rotatedCameraHeight, + return CameraTargetOverlay.rectForPlane( + width = cameraWidth, + height = cameraHeight, rectSize = newRectSize, screenOrientation = screenOrientation - ) - - return getRotatedBoundingBox( - image.imageInfo.rotationDegrees, - newBoundingBox, - cameraWidth, - cameraHeight ).toRect() } - private fun getRotatedBoundingBox( - rotation: Int, - newBoundingBox: RectF, - cameraWidth: Int, - cameraHeight: Int, - ): RectF { - return when (360 - rotation) { - 0, 360 -> newBoundingBox - 90 -> RectF( - cameraWidth - newBoundingBox.bottom, - newBoundingBox.left, - cameraWidth - newBoundingBox.top, - newBoundingBox.right - ) - - 180 -> RectF( - cameraWidth - newBoundingBox.right, - cameraHeight - newBoundingBox.bottom, - cameraWidth - newBoundingBox.left, - cameraHeight - newBoundingBox.top - ) - - 270 -> RectF( - newBoundingBox.top, - cameraHeight - newBoundingBox.right, - newBoundingBox.bottom, - cameraHeight - newBoundingBox.left - ) - - else -> throw IllegalArgumentException("Unsupported rotation angle: ${rotation}") - } - } - private fun getRectSizeBasedOnCameraCropping( screenWidth: Int, screenHeight: Int, @@ -151,15 +109,6 @@ internal class FrameProcessor @Inject constructor( } } - private fun getCameraRotatedPair(image: ImageProxy): Pair = - if (cameraRotatedPortrait(image.imageInfo.rotationDegrees)) { - Pair(image.height, image.width) - } else { - Pair(image.width, image.height) - } - - private fun cameraRotatedPortrait(rotation: Int) = rotation in arrayOf(90, 270) - private fun sizeFromMinRatio( cameraWidth: Int, screenWidth: Int, diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt index 78fc8497a2..1fa462d43b 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt @@ -73,7 +73,6 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) } } - override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) initFragment() @@ -115,9 +114,14 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) if (!::targetResolution.isInitialized) { targetResolution = Size(binding.captureOverlay.width, binding.captureOverlay.height) } - val imageAnalyzer = ImageAnalysis.Builder().setTargetResolution(targetResolution) - .setOutputImageFormat(OUTPUT_IMAGE_FORMAT_RGBA_8888).build() + + val imageAnalyzer = ImageAnalysis.Builder() + .setTargetResolution(targetResolution) + .setOutputImageRotationEnabled(true) + .setOutputImageFormat(OUTPUT_IMAGE_FORMAT_RGBA_8888) + .build() imageAnalyzer.setAnalyzer(cameraExecutor, ::analyze) + // Preview val preview = Preview.Builder().setTargetResolution(targetResolution).build() val cameraProvider = ProcessCameraProvider.getInstance(requireContext()).await() @@ -143,19 +147,9 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) launchPermissionRequest.launch(Manifest.permission.CAMERA) } } + else -> mainVm.shouldCheckCameraPermissions.set(true) } -// if (mainVm.shouldCheckCameraPermissions.getAndSet(false)) { -// // Check permission in onResume() so that if user left the app to go to Settings -// // and give the permission, it's reflected when they come back to SID -// if (requireActivity().hasPermission(Manifest.permission.CAMERA)) { -// setUpCamera() -// } else { -// launchPermissionRequest.launch(Manifest.permission.CAMERA) -// } -// } else { -// mainVm.shouldCheckCameraPermissions.set(true) -// } } override fun onStop() { diff --git a/face/capture/src/main/java/com/simprints/face/capture/usecases/ImageProxyToBitmapUseCase.kt b/face/capture/src/main/java/com/simprints/face/capture/usecases/ImageProxyToBitmapUseCase.kt index ae4ddb7277..89b99ea34f 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/usecases/ImageProxyToBitmapUseCase.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/usecases/ImageProxyToBitmapUseCase.kt @@ -25,8 +25,6 @@ internal class ImageProxyToBitmapUseCase @Inject constructor() { imageProxy.width + rowPadding / pixelStride, imageProxy.height, Bitmap.Config.ARGB_8888 ) bitmap.copyPixelsFromBuffer(buffer) - val rotationMatrix = Matrix() - rotationMatrix.postRotate(imageProxy.imageInfo.rotationDegrees.toFloat()) if (cropRect.isEmpty) { return null @@ -38,7 +36,7 @@ internal class ImageProxyToBitmapUseCase @Inject constructor() { cropRect.top, cropRect.width(), cropRect.height(), - rotationMatrix, + null, true ) bitmap.recycle() From 984cc33d827b27ec230de28650c7779a0cdb4284 Mon Sep 17 00:00:00 2001 From: Sergejs Luhmirins Date: Tue, 17 Sep 2024 16:29:25 +0300 Subject: [PATCH 2/2] MS-71 Move image cropping closer to cameraX APIs to simplify processing pipeline --- .../CropToTargetOverlayAnalyzer.kt | 64 ++++++ .../screens/livefeedback/FrameProcessor.kt | 124 ----------- .../livefeedback/LiveFeedbackFragment.kt | 24 +-- .../LiveFeedbackFragmentViewModel.kt | 20 +- .../usecases/ImageProxyToBitmapUseCase.kt | 46 ----- .../CropToTargetOverlayAnalyzerTest.kt | 120 +++++++++++ .../livefeedback/FrameProcessorTest.kt | 192 ------------------ .../LiveFeedbackFragmentViewModelTest.kt | 64 +++--- .../usecases/ImageProxyToBitmapUseCaseTest.kt | 103 ---------- 9 files changed, 216 insertions(+), 541 deletions(-) create mode 100644 face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/CropToTargetOverlayAnalyzer.kt delete mode 100644 face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/FrameProcessor.kt delete mode 100644 face/capture/src/main/java/com/simprints/face/capture/usecases/ImageProxyToBitmapUseCase.kt create mode 100644 face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/CropToTargetOverlayAnalyzerTest.kt delete mode 100644 face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/FrameProcessorTest.kt delete mode 100644 face/capture/src/test/java/com/simprints/face/capture/usecases/ImageProxyToBitmapUseCaseTest.kt diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/CropToTargetOverlayAnalyzer.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/CropToTargetOverlayAnalyzer.kt new file mode 100644 index 0000000000..a7d37cf13a --- /dev/null +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/CropToTargetOverlayAnalyzer.kt @@ -0,0 +1,64 @@ +package com.simprints.face.capture.screens.livefeedback + +import android.graphics.Bitmap +import android.graphics.RectF +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.ImageProxy +import com.simprints.face.capture.screens.livefeedback.views.CameraTargetOverlay +import kotlin.also +import kotlin.math.max +import kotlin.math.min +import kotlin.takeUnless + +internal class CropToTargetOverlayAnalyzer( + private val targetOverlay: CameraTargetOverlay, + private val onImageCropped: (Bitmap) -> Unit +) : ImageAnalysis.Analyzer { + + private var cachedRect: RectF? = null + + override fun analyze(image: ImageProxy) { + val previewRect = cachedRect + ?: targetOverlay.rectInCanvas.takeUnless { it.isEmpty }?.also { cachedRect = it } + ?: return + + // Adjust overlay size to be fit-center with the image size + val scale = getSmallerRatio( + image.width, image.height, + targetOverlay.width, targetOverlay.height, + ) + val scaledWidth = (targetOverlay.width * scale).toInt() + val scaledHeight = (targetOverlay.height * scale).toInt() + + // Find the offsets caused by fit-center scaling + val offsetX = (max(image.width, scaledWidth) - min(image.width, scaledWidth)) / 2 + val offsetY = (max(image.height, scaledHeight) - min(image.height, scaledHeight)) / 2 + + // Scale the preview target to the new scale and offset + val cropLeft = offsetX + (previewRect.left * scale).toInt() + val cropWidth = (previewRect.width() * scale).toInt() + val cropTop = offsetY + (previewRect.top * scale).toInt() + val cropHeight = (previewRect.height() * scale).toInt() + + onImageCropped(image.use { + Bitmap.createBitmap( + it.toBitmap(), + cropLeft, + cropTop, + cropWidth, + cropHeight + ) + }) + } + + private fun getSmallerRatio( + cameraWidth: Int, + cameraHeight: Int, + screenWidth: Int, + screenHeight: Int, + ): Float { + val widthRatio = cameraWidth / screenWidth.toFloat() + val heightRatio = cameraHeight / screenHeight.toFloat() + return min(widthRatio, heightRatio) + } +} diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/FrameProcessor.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/FrameProcessor.kt deleted file mode 100644 index 818f294397..0000000000 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/FrameProcessor.kt +++ /dev/null @@ -1,124 +0,0 @@ -package com.simprints.face.capture.screens.livefeedback - -import android.graphics.Bitmap -import android.graphics.Rect -import android.graphics.RectF -import android.util.Size -import androidx.camera.core.ImageProxy -import androidx.core.graphics.toRect -import com.simprints.face.capture.models.ScreenOrientation -import com.simprints.face.capture.screens.livefeedback.views.CameraTargetOverlay -import com.simprints.face.capture.usecases.ImageProxyToBitmapUseCase -import java.lang.Float.min -import javax.inject.Inject - - -internal class FrameProcessor @Inject constructor( - private val imageProxyToBitmap: ImageProxyToBitmapUseCase, -) { - - private var previewViewWidth: Int = 0 - private var previewViewHeight: Int = 0 - - private lateinit var boxOnTheScreen: RectF - var cropRect: Rect? = null - private set - - /** - * Init the frame processor - * - * @param previewViewSize the camera preview view size - * @param boxOnTheScreen the circle target indicator coordinates. - * we will use this coordinates to compute the area to be cropped for processing - */ - fun init(previewViewSize: Size, boxOnTheScreen: RectF) { - previewViewWidth = previewViewSize.width - previewViewHeight = previewViewSize.height - this.boxOnTheScreen = boxOnTheScreen - } - - fun clear() { - cropRect = null - } - - /** - * Extracts part of the image that lays inside - * the cropRect - * - * @param image - * @return Bitmap - */ - fun cropRotateFrame(image: ImageProxy, screenOrientation: ScreenOrientation): Bitmap? { - val cropRect = this.cropRect - ?: calcRotatedCropRect(image, screenOrientation).also { this.cropRect = it } - return imageProxyToBitmap(image, cropRect) - } - - private fun calcRotatedCropRect(image: ImageProxy, screenOrientation: ScreenOrientation): Rect { - val cameraWidth = image.width - val cameraHeight = image.height - - val newRectSize = getRectSizeBasedOnCameraCropping( - previewViewWidth, - previewViewHeight, - cameraWidth, - cameraHeight, - boxOnTheScreen - ) - - return CameraTargetOverlay.rectForPlane( - width = cameraWidth, - height = cameraHeight, - rectSize = newRectSize, - screenOrientation = screenOrientation - ).toRect() - } - - private fun getRectSizeBasedOnCameraCropping( - screenWidth: Int, - screenHeight: Int, - cameraWidth: Int, - cameraHeight: Int, - boxOnTheScreen: RectF, - ): Float { - return if (screenWidth == cameraWidth || screenHeight == cameraHeight) { - val cameraArea = cameraHeight * cameraWidth - val screenArea = screenHeight * screenWidth - // This means center crop is tight - if (cameraArea > screenArea) { - boxOnTheScreen.width() - } else { - // This means the camera is zooming in to center crop - sizeFromMinRatio( - cameraWidth, - screenWidth, - cameraHeight, - screenHeight, - boxOnTheScreen.width() - ) - } - } else { - // This means the camera is zooming out to center crop - sizeFromMinRatio( - cameraWidth, - screenWidth, - cameraHeight, - screenHeight, - boxOnTheScreen.width() - ) - } - } - - private fun sizeFromMinRatio( - cameraWidth: Int, - screenWidth: Int, - cameraHeight: Int, - screenHeight: Int, - currentWidth: Float, - ): Float { - val widthRatio = cameraWidth / screenWidth.toFloat() - val heightRatio = cameraHeight / screenHeight.toFloat() - val minRatio = min(widthRatio, heightRatio) - return currentWidth * minRatio - } -} diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt index 1fa462d43b..a1075b172a 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt @@ -2,6 +2,7 @@ package com.simprints.face.capture.screens.livefeedback import android.Manifest import android.content.Intent +import android.graphics.Bitmap import android.net.Uri import android.os.Bundle import android.provider.Settings @@ -11,7 +12,6 @@ import androidx.activity.result.contract.ActivityResultContracts import androidx.camera.core.CameraSelector.DEFAULT_BACK_CAMERA import androidx.camera.core.ImageAnalysis import androidx.camera.core.ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888 -import androidx.camera.core.ImageProxy import androidx.camera.core.Preview import androidx.camera.lifecycle.ProcessCameraProvider import androidx.core.content.ContextCompat @@ -78,11 +78,6 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) initFragment() } - override fun onDestroyView() { - vm.clearFrameProcessor() - super.onDestroyView() - } - private fun initFragment() { screenSize = with(resources.displayMetrics) { Size(widthPixels, widthPixels) } bindViewModel() @@ -93,11 +88,7 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) //Wait till the views gets its final size then init frame processor and setup the camera binding.faceCaptureCamera.post { if (view != null) { - vm.initFrameProcessor( - mainVm.samplesToCapture, mainVm.attemptNumber, - binding.captureOverlay.rectInCanvas, - Size(binding.captureOverlay.width, binding.captureOverlay.height), - ) + vm.initCapture(mainVm.samplesToCapture, mainVm.attemptNumber) } } } @@ -120,7 +111,9 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) .setOutputImageRotationEnabled(true) .setOutputImageFormat(OUTPUT_IMAGE_FORMAT_RGBA_8888) .build() - imageAnalyzer.setAnalyzer(cameraExecutor, ::analyze) + val cropAnalyzer = CropToTargetOverlayAnalyzer(binding.captureOverlay, ::analyze) + + imageAnalyzer.setAnalyzer(cameraExecutor, cropAnalyzer) // Preview val preview = Preview.Builder().setTargetResolution(targetResolution).build() @@ -183,12 +176,9 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) } } - private fun analyze(image: ImageProxy) { + private fun analyze(image: Bitmap) { try { - vm.process( - image = image, - screenOrientation = ScreenOrientation.getCurrentOrientation(resources) - ) + vm.process(croppedBitmap = image) } catch (t: Throwable) { Simber.e(t) // Image analysis is running in bg thread diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragmentViewModel.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragmentViewModel.kt index 13c76dc432..4e4d9194f6 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragmentViewModel.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragmentViewModel.kt @@ -1,9 +1,6 @@ package com.simprints.face.capture.screens.livefeedback import android.graphics.Bitmap -import android.graphics.RectF -import android.util.Size -import androidx.camera.core.ImageProxy import androidx.lifecycle.MutableLiveData import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope @@ -11,7 +8,6 @@ import com.simprints.core.tools.extentions.area import com.simprints.core.tools.time.TimeHelper import com.simprints.face.capture.models.FaceDetection import com.simprints.face.capture.models.FaceTarget -import com.simprints.face.capture.models.ScreenOrientation import com.simprints.face.capture.models.SymmetricTarget import com.simprints.face.capture.usecases.SimpleCaptureEventReporter import com.simprints.face.infra.basebiosdk.detection.Face @@ -28,7 +24,6 @@ import javax.inject.Inject @HiltViewModel internal class LiveFeedbackFragmentViewModel @Inject constructor( - private val frameProcessor: FrameProcessor, private val resolveFaceBioSdk: ResolveFaceBioSdkUseCase, private val configManager: ConfigManager, private val eventReporter: SimpleCaptureEventReporter, @@ -58,13 +53,8 @@ internal class LiveFeedbackFragmentViewModel @Inject constructor( * * @param image is the camera frame */ - fun process(image: ImageProxy, screenOrientation: ScreenOrientation) { + fun process(croppedBitmap: Bitmap) { val captureStartTime = timeHelper.now() - val croppedBitmap = frameProcessor.cropRotateFrame(image, screenOrientation) - if (croppedBitmap == null) { - image.close() - return - } val potentialFace = faceDetector.analyze(croppedBitmap) val faceDetection = getFaceDetectionFromPotentialFace(croppedBitmap, potentialFace) @@ -85,25 +75,19 @@ internal class LiveFeedbackFragmentViewModel @Inject constructor( else -> {//no-op } } - image.close() } - fun initFrameProcessor( + fun initCapture( samplesToCapture: Int, attemptNumber: Int, - cropRect: RectF, - previewSize: Size, ) { this.samplesToCapture = samplesToCapture this.attemptNumber = attemptNumber viewModelScope.launch { faceDetector = resolveFaceBioSdk().detector - frameProcessor.init(previewSize, cropRect) } } - fun clearFrameProcessor() = frameProcessor.clear() - fun startCapture() { capturingState.value = CapturingState.CAPTURING } diff --git a/face/capture/src/main/java/com/simprints/face/capture/usecases/ImageProxyToBitmapUseCase.kt b/face/capture/src/main/java/com/simprints/face/capture/usecases/ImageProxyToBitmapUseCase.kt deleted file mode 100644 index 89b99ea34f..0000000000 --- a/face/capture/src/main/java/com/simprints/face/capture/usecases/ImageProxyToBitmapUseCase.kt +++ /dev/null @@ -1,46 +0,0 @@ -package com.simprints.face.capture.usecases - -import android.graphics.Bitmap -import android.graphics.Matrix -import android.graphics.PixelFormat -import android.graphics.Rect -import androidx.camera.core.ImageProxy -import javax.inject.Inject - -/** - * Convert ImageProxy image To bitmap then crop and rotate it. - * Image format should be RGBA_8888 - */ -internal class ImageProxyToBitmapUseCase @Inject constructor() { - - operator fun invoke(imageProxy: ImageProxy, cropRect: Rect): Bitmap? { - require(imageProxy.format == PixelFormat.RGBA_8888) { - "${imageProxy.format} is not supported. RGBA_8888 is the only supported image format" - } - val buffer = imageProxy.planes[0].buffer - val pixelStride = imageProxy.planes[0].pixelStride - val rowStride = imageProxy.planes[0].rowStride - val rowPadding = rowStride - pixelStride * imageProxy.width - val bitmap = Bitmap.createBitmap( - imageProxy.width + rowPadding / pixelStride, imageProxy.height, Bitmap.Config.ARGB_8888 - ) - bitmap.copyPixelsFromBuffer(buffer) - - if (cropRect.isEmpty) { - return null - } - - val croppedRotatedBitmap = Bitmap.createBitmap( - bitmap, - cropRect.left, - cropRect.top, - cropRect.width(), - cropRect.height(), - null, - true - ) - bitmap.recycle() - return croppedRotatedBitmap - } - -} diff --git a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/CropToTargetOverlayAnalyzerTest.kt b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/CropToTargetOverlayAnalyzerTest.kt new file mode 100644 index 0000000000..139b9f6790 --- /dev/null +++ b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/CropToTargetOverlayAnalyzerTest.kt @@ -0,0 +1,120 @@ +package com.simprints.face.capture.screens.livefeedback + +import android.graphics.Bitmap +import android.graphics.RectF +import androidx.camera.core.ImageProxy +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.google.common.truth.Truth.assertThat +import com.simprints.face.capture.screens.livefeedback.views.CameraTargetOverlay +import io.mockk.MockKAnnotations +import io.mockk.every +import io.mockk.impl.annotations.MockK +import io.mockk.justRun +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +internal class CropToTargetOverlayAnalyzerTest { + + @MockK + lateinit var targetOverlay: CameraTargetOverlay + + @MockK + lateinit var imageProxy: ImageProxy + + lateinit var analyzer: CropToTargetOverlayAnalyzer + var capturedBitmap: Bitmap? = null + + @Before + fun setUp() { + MockKAnnotations.init(this) + justRun { imageProxy.close() } + + capturedBitmap = null + analyzer = CropToTargetOverlayAnalyzer(targetOverlay) { capturedBitmap = it } + } + + @Test + fun `Skip cropping when target is empty`() { + // Target is a square 600x600px with 200px from top bounds + setupScreenSize(1000, 2000) + every { targetOverlay.rectInCanvas } returns RectF(200f, 200f, 200f, 200f) + setupImageSize(1000, 1000) + + analyzer.analyze(imageProxy) + + // Cropped should be still square and half the side length of original + assertThat(capturedBitmap?.width).isNull() + assertThat(capturedBitmap?.height).isNull() + } + + @Test + fun `Correctly crops when camera resolution is smaller than preview in portrait`() { + // Target is a square 600x600px with 200px from top bounds + setupScreenSize(1000, 2000) + every { targetOverlay.rectInCanvas } returns RectF(200f, 200f, 800f, 800f) + setupImageSize(1000, 1000) + + analyzer.analyze(imageProxy) + + // Cropped should be still square and half the side length of original + assertThat(capturedBitmap?.width).isEqualTo(300) + assertThat(capturedBitmap?.height).isEqualTo(300) + } + + @Test + fun `Correctly crops when camera resolution is smaller than preview in landscape`() { + // Target is a square 600x600px with 200px from top bounds + setupScreenSize(2000, 1000) + every { targetOverlay.rectInCanvas } returns RectF(700f, 200f, 1300f, 800f) + setupImageSize(1000, 1000) + + analyzer.analyze(imageProxy) + + // Cropped should be still square and half the side length of original + assertThat(capturedBitmap?.width).isEqualTo(300) + assertThat(capturedBitmap?.height).isEqualTo(300) + } + + @Test + fun `Correctly crops when camera resolution is larger than preview in portrait`() { + // Target is a square 600x600px with 200px from top bounds + setupScreenSize(1000, 2000) + every { targetOverlay.rectInCanvas } returns RectF(200f, 200f, 800f, 800f) + setupImageSize(2000, 2000) + + analyzer.analyze(imageProxy) + + // Cropped should be still square and half the side length of original + assertThat(capturedBitmap?.width).isEqualTo(600) + assertThat(capturedBitmap?.height).isEqualTo(600) + } + + + @Test + fun `Correctly crops when camera resolution is larger than preview in landscape`() { + // Target is a square 600x600px with 200px from top bounds + setupScreenSize(2000, 1000) + every { targetOverlay.rectInCanvas } returns RectF(700f, 200f, 1300f, 800f) + setupImageSize(2000, 2000) + + analyzer.analyze(imageProxy) + + // Cropped should be still square and half the side length of original + assertThat(capturedBitmap?.width).isEqualTo(600) + assertThat(capturedBitmap?.height).isEqualTo(600) + } + + private fun setupScreenSize(width: Int, height: Int) { + every { targetOverlay.width } returns width + every { targetOverlay.height } returns height + } + + private fun setupImageSize(width: Int, height: Int) { + every { imageProxy.toBitmap() } returns Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + every { imageProxy.width } returns width + every { imageProxy.height } returns height + } + +} diff --git a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/FrameProcessorTest.kt b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/FrameProcessorTest.kt deleted file mode 100644 index eeab24831a..0000000000 --- a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/FrameProcessorTest.kt +++ /dev/null @@ -1,192 +0,0 @@ -package com.simprints.face.capture.screens.livefeedback - - -import android.graphics.Rect -import android.graphics.RectF -import android.util.Size -import androidx.camera.core.ImageProxy -import com.google.common.truth.Truth.assertThat -import com.simprints.face.capture.models.ScreenOrientation -import com.simprints.face.capture.usecases.ImageProxyToBitmapUseCase -import io.mockk.CapturingSlot -import io.mockk.MockKAnnotations -import io.mockk.every -import io.mockk.impl.annotations.MockK -import io.mockk.mockk -import io.mockk.slot -import org.junit.Before -import org.junit.Test - -internal class FrameProcessorTest { - - @MockK - private lateinit var imageToBitmap: ImageProxyToBitmapUseCase - - @MockK - private lateinit var image: ImageProxy - - private lateinit var frameProcessor: FrameProcessor - - private lateinit var cropRectCapture: CapturingSlot - private val boxOnTheScreen = RectF(100f, 100f, 200f, 200f) - - @Before - fun setUp() { - MockKAnnotations.init(this, relaxed = true) - - cropRectCapture = slot() - every { imageToBitmap.invoke(any(), capture(cropRectCapture)) } returns mockk() - - frameProcessor = FrameProcessor(imageToBitmap) - } - - - @Test - fun `test cropRotateFrame with portrait orientation and cameraWidth equal to screenWidth`() { - // Given - every { image.width } returns 1000 - every { image.height } returns 1000 - every { image.imageInfo.rotationDegrees } returns 0 - - val screenWidth = 1000 - val screenHeight = 500 - - frameProcessor.init(Size(screenWidth, screenHeight), boxOnTheScreen) - // When - frameProcessor.cropRotateFrame(image, ScreenOrientation.Portrait) - - // Then - assertThat(cropRectCapture.captured.toString()) - .isEqualTo(Rect(100, 100, 200, 200).toString()) - } - - @Test - fun `test cropRotateFrame with flipped portrait orientation and cameraWidth equal to screenWidth`() { - // Given - every { image.width } returns 1000 - every { image.height } returns 1000 - every { image.imageInfo.rotationDegrees } returns 180 - - val screenWidth = 1000 - val screenHeight = 500 - - frameProcessor.init(Size(screenWidth, screenHeight), boxOnTheScreen) - // When - frameProcessor.cropRotateFrame(image, ScreenOrientation.Portrait) - - // Then - assertThat(cropRectCapture.captured.toString()) - .isEqualTo(Rect(100, 100, 200, 200).toString()) - } - - @Test - fun `test cropRotateFrame with landscape orientation and cameraWidth equal to screenWidth`() { - // Given - every { image.width } returns 1000 - every { image.height } returns 1000 - every { image.imageInfo.rotationDegrees } returns 90 - - val screenWidth = 1000 - val screenHeight = 500 - - frameProcessor.init(Size(screenWidth, screenHeight), boxOnTheScreen) - // When - frameProcessor.cropRotateFrame(image, ScreenOrientation.Portrait) - - // Then - assertThat(cropRectCapture.captured.toString()) - .isEqualTo(Rect(200, 100, 100, 200).toString()) - } - - @Test - fun `test cropRotateFrame with flipped landscape orientation and cameraWidth equal to screenWidth`() { - // Given - every { image.width } returns 1000 - every { image.height } returns 1000 - every { image.imageInfo.rotationDegrees } returns 270 - - val screenWidth = 1000 - val screenHeight = 500 - - frameProcessor.init(Size(screenWidth, screenHeight), boxOnTheScreen) - // When - frameProcessor.cropRotateFrame(image, ScreenOrientation.Portrait) - - // Then - assertThat(cropRectCapture.captured.toString()) - .isEqualTo(Rect(200, 100, 100, 200).toString()) - } - - @Test(expected = IllegalArgumentException::class) - fun `test cropRotateFrame with unsupported orientation and cameraWidth equal to screenWidth`() { - // Given - every { image.width } returns 1000 - every { image.height } returns 1000 - every { image.imageInfo.rotationDegrees } returns 10 - - val screenWidth = 1000 - val screenHeight = 500 - - frameProcessor.init(Size(screenWidth, screenHeight), boxOnTheScreen) - // When - frameProcessor.cropRotateFrame(image, ScreenOrientation.Portrait) - // Then throw IllegalArgumentException - - } - - @Test - fun `test cropRotateFrame with portrait orientation and cameraWidth greater than screenWidth`() { - // Given - every { image.width } returns 2000 - every { image.height } returns 1000 - every { image.imageInfo.rotationDegrees } returns 0 - - val screenWidth = 1000 - val screenHeight = 500 - - frameProcessor.init(Size(screenWidth, screenHeight), boxOnTheScreen) - // When - frameProcessor.cropRotateFrame(image, ScreenOrientation.Portrait) - // Then - assertThat(cropRectCapture.captured.toString()) - .isEqualTo(Rect(100, 100, 200, 200).toString()) - } - - @Test - fun `test cropRotateFrame with landscape orientation and cameraWidth greater than screenWidth`() { - // Given - every { image.width } returns 2000 - every { image.height } returns 1000 - every { image.imageInfo.rotationDegrees } returns 90 - - val screenWidth = 1000 - val screenHeight = 500 - - frameProcessor.init(Size(screenWidth, screenHeight), boxOnTheScreen) - // When - frameProcessor.cropRotateFrame(image, ScreenOrientation.Portrait) - // Then - assertThat(cropRectCapture.captured.toString()) - .isEqualTo(Rect(100, 100, 200, 200).toString()) - } - - @Test - fun `when clear is called, cropRect becomes null`() { - every { image.width } returns 2000 - every { image.height } returns 1000 - every { image.imageInfo.rotationDegrees } returns 90 - - val screenWidth = 1000 - val screenHeight = 500 - - frameProcessor.init(Size(screenWidth, screenHeight), boxOnTheScreen) - frameProcessor.cropRotateFrame(image, ScreenOrientation.Portrait) - - assertThat(frameProcessor.cropRect).isNotNull() - - frameProcessor.clear() - - assertThat(frameProcessor.cropRect).isNull() - } - -} diff --git a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragmentViewModelTest.kt b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragmentViewModelTest.kt index c3d4e72b68..8f3dd14b75 100644 --- a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragmentViewModelTest.kt +++ b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragmentViewModelTest.kt @@ -2,15 +2,12 @@ package com.simprints.face.capture.screens.livefeedback import android.graphics.Bitmap import android.graphics.Rect -import android.graphics.RectF -import android.util.Size import androidx.arch.core.executor.testing.InstantTaskExecutorRule -import androidx.camera.core.ImageProxy +import androidx.test.ext.junit.runners.AndroidJUnit4 import com.google.common.truth.Truth.assertThat import com.simprints.core.tools.time.TimeHelper import com.simprints.core.tools.time.Timestamp import com.simprints.face.capture.models.FaceDetection -import com.simprints.face.capture.models.ScreenOrientation import com.simprints.face.capture.usecases.SimpleCaptureEventReporter import com.simprints.face.infra.basebiosdk.detection.Face import com.simprints.face.infra.basebiosdk.detection.FaceDetector @@ -30,10 +27,9 @@ import org.junit.Before import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith -import org.robolectric.RobolectricTestRunner import kotlin.random.Random -@RunWith(RobolectricTestRunner::class) +@RunWith(AndroidJUnit4::class) internal class LiveFeedbackFragmentViewModelTest { @get:Rule @@ -42,21 +38,15 @@ internal class LiveFeedbackFragmentViewModelTest { @get:Rule val testCoroutineRule = TestCoroutineRule() - @MockK - lateinit var frameProcessor: FrameProcessor - @MockK lateinit var faceDetector: FaceDetector @MockK - lateinit var frame: ImageProxy + lateinit var frame: Bitmap @MockK lateinit var previewFrame: Bitmap - @MockK - lateinit var rectF: RectF - @MockK lateinit var configManager: ConfigManager @@ -66,9 +56,6 @@ internal class LiveFeedbackFragmentViewModelTest { @MockK lateinit var timeHelper: TimeHelper - private val previewViewSize: Size = Size(100, 100) - private val screenOrientation = ScreenOrientation.Portrait - private lateinit var viewModel: LiveFeedbackFragmentViewModel @Before @@ -77,10 +64,7 @@ internal class LiveFeedbackFragmentViewModelTest { coEvery { configManager.getProjectConfiguration().face?.qualityThreshold } returns QUALITY_THRESHOLD every { timeHelper.now() } returnsMany (0..100L).map { Timestamp(it) } - justRun { frameProcessor.init(any(), any()) } - justRun { frame.close() } justRun { previewFrame.recycle() } - every { frameProcessor.cropRotateFrame(frame, screenOrientation) } returns previewFrame val resolveFaceBioSdkUseCase = mockk { coEvery { this@mockk.invoke() } returns mockk { every { detector } returns faceDetector @@ -89,16 +73,16 @@ internal class LiveFeedbackFragmentViewModelTest { viewModel = LiveFeedbackFragmentViewModel( - frameProcessor, resolveFaceBioSdkUseCase, configManager, eventReporter, timeHelper + resolveFaceBioSdkUseCase, configManager, eventReporter, timeHelper ) } @Test fun `Process fallback image when valid face correctly but not started capture`() = runTest { - coEvery { faceDetector.analyze(previewFrame) } returns getFace() + coEvery { faceDetector.analyze(frame) } returns getFace() - viewModel.initFrameProcessor(1, 0, rectF, previewViewSize) - viewModel.process(frame, screenOrientation) + viewModel.initCapture(1, 0) + viewModel.process(frame) val currentDetection = viewModel.currentDetection.testObserver() assertThat(currentDetection.observedValues.last()?.status).isEqualTo(FaceDetection.Status.VALID) @@ -108,12 +92,12 @@ internal class LiveFeedbackFragmentViewModelTest { @Test fun `Process valid face correctly`() = runTest { - coEvery { faceDetector.analyze(previewFrame) } returns getFace() + coEvery { faceDetector.analyze(frame) } returns getFace() - viewModel.initFrameProcessor(1, 0, rectF, previewViewSize) - viewModel.process(frame, screenOrientation) + viewModel.initCapture(1, 0) + viewModel.process(frame) viewModel.startCapture() - viewModel.process(frame, screenOrientation) + viewModel.process(frame) val currentDetection = viewModel.currentDetection.testObserver() assertThat(currentDetection.observedValues.last()?.status).isEqualTo(FaceDetection.Status.VALID_CAPTURING) @@ -129,8 +113,7 @@ internal class LiveFeedbackFragmentViewModelTest { val rolledFace: Face = getFace(roll = 45f) val noFace = null - every { frameProcessor.cropRotateFrame(frame, screenOrientation) } returns previewFrame - every { faceDetector.analyze(previewFrame) } returnsMany listOf( + every { faceDetector.analyze(frame) } returnsMany listOf( smallFace, bigFace, yawedFace, @@ -139,13 +122,13 @@ internal class LiveFeedbackFragmentViewModelTest { ) val detections = viewModel.currentDetection.testObserver() - viewModel.initFrameProcessor(2, 0, rectF, previewViewSize) + viewModel.initCapture(2, 0) - viewModel.process(frame, screenOrientation) - viewModel.process(frame, screenOrientation) - viewModel.process(frame, screenOrientation) - viewModel.process(frame, screenOrientation) - viewModel.process(frame, screenOrientation) + viewModel.process(frame) + viewModel.process(frame) + viewModel.process(frame) + viewModel.process(frame) + viewModel.process(frame) detections.observedValues.let { assertThat(it[0]?.status).isEqualTo(FaceDetection.Status.TOOFAR) @@ -161,17 +144,16 @@ internal class LiveFeedbackFragmentViewModelTest { @Test fun `Save all valid captures without fallback image`() = runTest { val validFace: Face = getFace() - every { frameProcessor.cropRotateFrame(frame, screenOrientation) } returns previewFrame - every { faceDetector.analyze(previewFrame) } returns validFace + every { faceDetector.analyze(frame) } returns validFace every { timeHelper.now() } returnsMany (0..100L).map { Timestamp(it) } val currentDetectionObserver = viewModel.currentDetection.testObserver() val capturingStateObserver = viewModel.capturingState.testObserver() - viewModel.initFrameProcessor(2, 0, rectF, previewViewSize) - viewModel.process(frame, screenOrientation) + viewModel.initCapture(2, 0) + viewModel.process(frame) viewModel.startCapture() - viewModel.process(frame, screenOrientation) - viewModel.process(frame, screenOrientation) + viewModel.process(frame) + viewModel.process(frame) currentDetectionObserver.observedValues.let { assertThat(it[0]?.status).isEqualTo(FaceDetection.Status.VALID) diff --git a/face/capture/src/test/java/com/simprints/face/capture/usecases/ImageProxyToBitmapUseCaseTest.kt b/face/capture/src/test/java/com/simprints/face/capture/usecases/ImageProxyToBitmapUseCaseTest.kt deleted file mode 100644 index 94e0eacde8..0000000000 --- a/face/capture/src/test/java/com/simprints/face/capture/usecases/ImageProxyToBitmapUseCaseTest.kt +++ /dev/null @@ -1,103 +0,0 @@ -package com.simprints.face.capture.usecases - -import android.graphics.Bitmap -import android.graphics.ImageFormat -import android.graphics.Rect -import androidx.camera.core.ImageProxy -import androidx.test.ext.junit.runners.AndroidJUnit4 -import com.google.common.truth.Truth.assertThat -import io.mockk.every -import io.mockk.justRun -import io.mockk.mockk -import io.mockk.mockkStatic -import io.mockk.unmockkStatic -import org.junit.After -import org.junit.Before -import org.junit.Test -import org.junit.runner.RunWith - -@RunWith(AndroidJUnit4::class) -class ImageProxyToBitmapUseCaseTest { - - private var imageBytes: ByteArray = ByteArray(BYTEARRAY_SIZE) - private val imageProxy = mockk { - every { width } returns IMAGE_WIDTH - every { height } returns IMAGE_HEIGHT - every { format } returns android.graphics.PixelFormat.RGBA_8888 - every { imageInfo.rotationDegrees } returns 0 - every { planes } returns arrayOf(mockk { - every { buffer } returns java.nio.ByteBuffer.wrap(imageBytes) - every { pixelStride } returns 1 - every { rowStride } returns IMAGE_WIDTH - }) - } - - private lateinit var useCase: ImageProxyToBitmapUseCase - - private val bitmapMock = mockk { - justRun { copyPixelsFromBuffer(any()) } - justRun { recycle() } - } - - @Before - fun setup() { - mockkStatic(Bitmap::class) - every { Bitmap.createBitmap(any(), any(), any()) } returns bitmapMock - every { Bitmap.createBitmap(any(), any(), any(), any(), any(), any(), true) } returns bitmapMock - - useCase = ImageProxyToBitmapUseCase() - } - - @After - fun tearDown() { - unmockkStatic(Bitmap::class) - } - - @Test - fun `Should return a Bitmap with the same pixel format as the ImageProxy`() { - // When - val bitmap = useCase.invoke( - imageProxy, - Rect(CROP_RECT_LEFT, CROP_RECT_TOP, CROP_RECT_RIGHT, CROP_RECT_BOTTOM) - ) - // Then - assertThat(bitmap).isNotNull() - } - - @Test - fun `Should return a null if cropRect is empty`() { - - val rect = Rect(CROP_RECT_LEFT, CROP_RECT_TOP, CROP_RECT_LEFT, CROP_RECT_TOP) - // When - val bitmap = useCase.invoke(imageProxy, rect) - // Then - assertThat(bitmap).isNull() - } - - @Test(expected = IllegalArgumentException::class) - fun `Should throw if the pixel format is not RGBA_8888`() { - // Given - every { imageProxy.format } returns ImageFormat.YUV_420_888 - - // When - useCase.invoke( - imageProxy, - Rect(CROP_RECT_LEFT, CROP_RECT_TOP, CROP_RECT_RIGHT, CROP_RECT_BOTTOM) - ) - // Then throws IllegalArgumentException - } - - companion object { - - const val IMAGE_WIDTH = 150 - const val IMAGE_HEIGHT = 150 - - const val CROP_RECT_LEFT = 0 - const val CROP_RECT_TOP = 0 - const val CROP_RECT_RIGHT = 100 - const val CROP_RECT_BOTTOM = 100 - - const val BYTEARRAY_SIZE = IMAGE_WIDTH * IMAGE_HEIGHT * 4 - - } -}