From fd1a4ba2aea5433ebc885d77a1558c357666d22f Mon Sep 17 00:00:00 2001 From: Milen Marinov Date: Wed, 5 Mar 2025 17:11:16 +0200 Subject: [PATCH 1/2] [MS-929] Count the number of loaded candidates to provide an accurate pool count in CoSync 1:N --- .../matcher/usecases/FaceMatcherUseCase.kt | 17 ++-- .../usecases/FingerprintMatcherUseCase.kt | 17 ++-- .../usecases/FaceMatcherUseCaseTest.kt | 94 ++++++++++--------- 3 files changed, 74 insertions(+), 54 deletions(-) diff --git a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt index a151aca006..82d70f9a32 100644 --- a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt +++ b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt @@ -45,19 +45,23 @@ internal class FaceMatcherUseCase @Inject constructor( val queryWithSupportedFormat = matchParams.queryForCandidates.copy( faceSampleFormat = faceMatcher.supportedTemplateFormat, ) - val totalCandidates = enrolmentRecordRepository.count( + val expectedCandidates = enrolmentRecordRepository.count( queryWithSupportedFormat, dataSource = matchParams.biometricDataSource, ) - if (totalCandidates == 0) { + if (expectedCandidates == 0) { send(MatcherState.Success(emptyList(), 0, faceMatcher.matcherName)) return@channelFlow } Simber.i("Matching candidates", tag = crashReportTag) - send(MatcherState.LoadingStarted(totalCandidates)) + send(MatcherState.LoadingStarted(expectedCandidates)) + // When using local DB loadedCandidates = expectedCandidates + // However, when using CommCare as data source, loadedCandidates < expectedCandidates + // as it's count function does not take into account filtering criteria + var loadedCandidates = 0 val resultItems = coroutineScope { - createRanges(totalCandidates) + createRanges(expectedCandidates) .map { range -> async(dispatcher) { val batchCandidates = getCandidates( @@ -67,6 +71,7 @@ internal class FaceMatcherUseCase @Inject constructor( dataSource = matchParams.biometricDataSource, ) { // When a candidate is loaded + loadedCandidates++ trySend(MatcherState.CandidateLoaded) } match(batchCandidates, samples) @@ -76,9 +81,9 @@ internal class FaceMatcherUseCase @Inject constructor( .toList() } - Simber.i("Matched $totalCandidates candidates", tag = crashReportTag) + Simber.i("Matched $loadedCandidates candidates", tag = crashReportTag) - send(MatcherState.Success(resultItems, totalCandidates, faceMatcher.matcherName)) + send(MatcherState.Success(resultItems, loadedCandidates, faceMatcher.matcherName)) } private fun mapSamples(probes: List) = probes.map { FaceSample(it.faceId, it.template) } diff --git a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt index 0153fd45d7..4050f5db81 100644 --- a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt +++ b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt @@ -53,19 +53,24 @@ internal class FingerprintMatcherUseCase @Inject constructor( matchParams.queryForCandidates.copy( fingerprintSampleFormat = bioSdkWrapper.supportedTemplateFormat, ) - val totalCandidates = enrolmentRecordRepository.count(queryWithSupportedFormat, dataSource = matchParams.biometricDataSource) - if (totalCandidates == 0) { + val expectedCandidates = enrolmentRecordRepository.count(queryWithSupportedFormat, dataSource = matchParams.biometricDataSource) + if (expectedCandidates == 0) { send(MatcherState.Success(emptyList(), 0, bioSdkWrapper.matcherName)) return@channelFlow } Simber.i("Matching candidates", tag = crashReportTag) - send(MatcherState.LoadingStarted(totalCandidates)) - val resultItems = createRanges(totalCandidates) + send(MatcherState.LoadingStarted(expectedCandidates)) + // When using local DB loadedCandidates = expectedCandidates + // However, when using CommCare as data source, loadedCandidates < expectedCandidates + // as it's count function does not take into account filtering criteria + var loadedCandidates = 0 + val resultItems = createRanges(expectedCandidates) .map { range -> async(dispatcher) { val batchCandidates = getCandidates(queryWithSupportedFormat, range, matchParams.biometricDataSource, project) { // When a candidate is loaded + loadedCandidates++ trySend(MatcherState.CandidateLoaded) } match(samples, batchCandidates, matchParams.flowType, bioSdkWrapper, bioSdk = matchParams.fingerprintSDK) @@ -77,8 +82,8 @@ internal class FingerprintMatcherUseCase @Inject constructor( .reduce { acc, subSet -> acc.addAll(subSet) } .toList() - Simber.i("Matched $totalCandidates candidates", tag = crashReportTag) - send(MatcherState.Success(resultItems, totalCandidates, bioSdkWrapper.matcherName)) + Simber.i("Matched $loadedCandidates candidates", tag = crashReportTag) + send(MatcherState.Success(resultItems, loadedCandidates, bioSdkWrapper.matcherName)) } private fun mapSamples(probes: List) = probes diff --git a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt index 11fe0458b7..d9021cf83e 100644 --- a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt +++ b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt @@ -63,15 +63,16 @@ internal class FaceMatcherUseCaseTest { fun `Skips matching if there are no probes`() = runTest { coEvery { faceMatcher.getHighestComparisonScoreForCandidate(any(), any()) } returns 1f - val results = useCase.invoke( - MatchParams( - probeReferenceId = "referenceId", - flowType = FlowType.VERIFY, - queryForCandidates = SubjectQuery(), - biometricDataSource = BiometricDataSource.Simprints, - ), - project - ).toList() + val results = useCase + .invoke( + MatchParams( + probeReferenceId = "referenceId", + flowType = FlowType.VERIFY, + queryForCandidates = SubjectQuery(), + biometricDataSource = BiometricDataSource.Simprints, + ), + project, + ).toList() coVerify(exactly = 0) { faceMatcher.getHighestComparisonScoreForCandidate(any(), any()) } @@ -79,8 +80,8 @@ internal class FaceMatcherUseCaseTest { MatcherUseCase.MatcherState.Success( matchResultItems = emptyList(), totalCandidates = 0, - matcherName = "" - ) + matcherName = "", + ), ) } @@ -88,18 +89,19 @@ internal class FaceMatcherUseCaseTest { fun `Skips matching if there are no candidates`() = runTest { coEvery { enrolmentRecordRepository.count(any()) } returns 0 - val results = useCase.invoke( - MatchParams( - probeReferenceId = "referenceId", - probeFaceSamples = listOf( - MatchParams.FaceSample("faceId", byteArrayOf(1, 2, 3)), + val results = useCase + .invoke( + MatchParams( + probeReferenceId = "referenceId", + probeFaceSamples = listOf( + MatchParams.FaceSample("faceId", byteArrayOf(1, 2, 3)), + ), + flowType = FlowType.VERIFY, + queryForCandidates = SubjectQuery(), + biometricDataSource = BiometricDataSource.Simprints, ), - flowType = FlowType.VERIFY, - queryForCandidates = SubjectQuery(), - biometricDataSource = BiometricDataSource.Simprints, - ), - project - ).toList() + project, + ).toList() coVerify(exactly = 0) { faceMatcher.getHighestComparisonScoreForCandidate(any(), any()) } @@ -107,48 +109,56 @@ internal class FaceMatcherUseCaseTest { MatcherUseCase.MatcherState.Success( matchResultItems = emptyList(), totalCandidates = 0, - matcherName = "" - ) + matcherName = "", + ), ) } @Test fun `Correctly calls SDK matcher`() = runTest { - val totalCandidates = 100 + val totalCandidates = 1 val faceIdentities = listOf( FaceIdentity( "subjectId", listOf(FaceSample(byteArrayOf(1, 2, 3), "format", "faceTemplate")), - ) + ), ) - coEvery { enrolmentRecordRepository.count(any(), any()) } returns 100 + coEvery { enrolmentRecordRepository.count(any(), any()) } returns 1 coEvery { createRangesUseCase(any()) } returns listOf(0..99) - coEvery { enrolmentRecordRepository.loadFaceIdentities(any(), any(), any(), any(), any()) } returns faceIdentities + coEvery { enrolmentRecordRepository.loadFaceIdentities(any(), any(), any(), any(), any()) } coAnswers { + // Call the onCandidateLoaded callback (5th parameter) + val onCandidateLoaded = arg<() -> Unit>(4) + onCandidateLoaded() + + // Return the face identities + faceIdentities + } coEvery { faceMatcher.getHighestComparisonScoreForCandidate(any(), any()) } returns 42f - - val results = useCase.invoke( - matchParams = MatchParams( - probeReferenceId = "referenceId", - probeFaceSamples = listOf( - MatchParams.FaceSample("faceId", byteArrayOf(1, 2, 3)), + val results = useCase + .invoke( + matchParams = MatchParams( + probeReferenceId = "referenceId", + probeFaceSamples = listOf( + MatchParams.FaceSample("faceId", byteArrayOf(1, 2, 3)), + ), + flowType = FlowType.VERIFY, + queryForCandidates = SubjectQuery(), + biometricDataSource = BiometricDataSource.Simprints, ), - flowType = FlowType.VERIFY, - queryForCandidates = SubjectQuery(), - biometricDataSource = BiometricDataSource.Simprints, - ), - project - ).toList() + project, + ).toList() coVerify { faceMatcher.getHighestComparisonScoreForCandidate(any(), any()) } assertThat(results).containsExactly( MatcherUseCase.MatcherState.LoadingStarted(totalCandidates), + MatcherUseCase.MatcherState.CandidateLoaded, MatcherUseCase.MatcherState.Success( matchResultItems = listOf(FaceMatchResult.Item("subjectId", 42f)), totalCandidates = totalCandidates, - matcherName = "" - ) + matcherName = "", + ), ) } } From c496c9317001ae3524580fe86de5b2b27a296e4e Mon Sep 17 00:00:00 2001 From: Milen Marinov Date: Wed, 5 Mar 2025 17:12:27 +0200 Subject: [PATCH 2/2] [MS-929] Move onCandidateLoaded() callback after filtering to provide accurate count of loaded candidates --- .../repository/commcare/CommCareIdentityDataSource.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt index cee89e791b..2569b6db23 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt @@ -52,10 +52,11 @@ internal class CommCareIdentityDataSource @Inject constructor( project: Project, onCandidateLoaded: () -> Unit, ): List = withContext(dispatcher) { - loadEnrolmentRecordCreationEvents(range, dataSource.callerPackageName(), query, project, onCandidateLoaded) + loadEnrolmentRecordCreationEvents(range, dataSource.callerPackageName(), query, project) .filter { erce -> erce.payload.biometricReferences.any { it is FingerprintReference && it.format == query.fingerprintSampleFormat } }.map { + onCandidateLoaded() FingerprintIdentity( it.payload.subjectId, it.payload.biometricReferences @@ -80,7 +81,6 @@ internal class CommCareIdentityDataSource @Inject constructor( callerPackageName: String, query: SubjectQuery, project: Project, - onCandidateLoaded: () -> Unit, ): List { val enrolmentRecordCreationEvents: MutableList = mutableListOf() try { @@ -103,7 +103,6 @@ internal class CommCareIdentityDataSource @Inject constructor( enrolmentRecordCreationEvents.addAll( loadEnrolmentRecordCreationEvents(caseId, callerPackageName, query, project), ) - onCandidateLoaded() } } while (caseMetadataCursor.moveToNext() && caseMetadataCursor.position < range.last) } @@ -132,10 +131,11 @@ internal class CommCareIdentityDataSource @Inject constructor( project: Project, onCandidateLoaded: () -> Unit, ): List = withContext(dispatcher) { - loadEnrolmentRecordCreationEvents(range, dataSource.callerPackageName(), query, project, onCandidateLoaded) + loadEnrolmentRecordCreationEvents(range, dataSource.callerPackageName(), query, project) .filter { erce -> erce.payload.biometricReferences.any { it is FaceReference && it.format == query.faceSampleFormat } }.map { + onCandidateLoaded() FaceIdentity( it.payload.subjectId, it.payload.biometricReferences