diff --git a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt index 054a85ff92..5077c81e0e 100644 --- a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt +++ b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt @@ -4,32 +4,38 @@ import com.simprints.core.DispatcherBG import com.simprints.face.infra.basebiosdk.matching.FaceIdentity import com.simprints.face.infra.basebiosdk.matching.FaceMatcher import com.simprints.face.infra.basebiosdk.matching.FaceSample +import com.simprints.face.infra.biosdkresolver.FaceBioSDK import com.simprints.face.infra.biosdkresolver.ResolveFaceBioSdkUseCase import com.simprints.infra.config.store.models.Project import com.simprints.infra.enrolment.records.repository.EnrolmentRecordRepository -import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource -import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery import com.simprints.infra.logging.LoggingConstants import com.simprints.infra.logging.Simber import com.simprints.matcher.FaceMatchResult import com.simprints.matcher.MatchParams import com.simprints.matcher.usecases.MatcherUseCase.MatcherState import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.launch import javax.inject.Inject +import kotlin.math.min +import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity as DomainFaceIdentity internal class FaceMatcherUseCase @Inject constructor( private val enrolmentRecordRepository: EnrolmentRecordRepository, private val resolveFaceBioSdk: ResolveFaceBioSdkUseCase, private val createRanges: CreateRangesUseCase, - @DispatcherBG private val dispatcher: CoroutineDispatcher, + @DispatcherBG private val dispatcherBG: CoroutineDispatcher, ) : MatcherUseCase { override val crashReportTag = LoggingConstants.CrashReportTag.FACE_MATCHING + // When using local DB loadedCandidates = expectedCandidates + // However, when using CommCare as data source, loadedCandidates < expectedCandidates + // as it's count function does not take into account filtering criteria + // This var is not thread safe + var loadedCandidates = 0 + override suspend operator fun invoke( matchParams: MatchParams, project: Project, @@ -53,60 +59,57 @@ internal class FaceMatcherUseCase @Inject constructor( send(MatcherState.Success(emptyList(), 0, bioSdk.matcherName)) return@channelFlow } - + loadedCandidates = 0 Simber.i("Matching candidates", tag = crashReportTag) send(MatcherState.LoadingStarted(expectedCandidates)) - // When using local DB loadedCandidates = expectedCandidates - // However, when using CommCare as data source, loadedCandidates < expectedCandidates - // as it's count function does not take into account filtering criteria - var loadedCandidates = 0 - val resultItems = coroutineScope { + val ranges = createRanges(expectedCandidates) + // if number of ranges less than the number of cores then use the number of ranges + val numConsumers = min(Runtime.getRuntime().availableProcessors(), ranges.size) - createRanges(expectedCandidates) - .map { range -> - async(dispatcher) { - val batchCandidates = getCandidates( - queryWithSupportedFormat, - range, - project = project, - dataSource = matchParams.biometricDataSource, - ) { - // When a candidate is loaded - loadedCandidates++ - trySend(MatcherState.CandidateLoaded) - } - bioSdk.createMatcher(samples).use { match(it, batchCandidates) } - } - }.awaitAll() - .reduce { acc, subSet -> acc.addAll(subSet) } - .toList() - } + val resultSet = MatchResultSet() + val candidatesChannel = enrolmentRecordRepository + .loadFaceIdentities( + query = queryWithSupportedFormat, + ranges = ranges, + dataSource = matchParams.biometricDataSource, + project = project, + scope = this, + onCandidateLoaded = { + loadedCandidates++ + this.trySend(MatcherState.CandidateLoaded) + }, + ) - Simber.i("Matched $loadedCandidates candidates", tag = crashReportTag) + // Start Consumers in BG thread + val consumerJobs = List(numConsumers) { + launch(dispatcherBG) { + consumeAndMatch(candidatesChannel, samples, resultSet, bioSdk) + } + } + // Wait for all to complete + consumerJobs.forEach { it.join() } + send(MatcherState.Success(resultSet.toList(), loadedCandidates, bioSdk.matcherName)) + } - send(MatcherState.Success(resultItems, loadedCandidates, bioSdk.matcherName)) + suspend fun consumeAndMatch( + candidatesChannel: ReceiveChannel>, + samples: List, + resultSet: MatchResultSet, + bioSdk: FaceBioSDK, + ) { + for (batch in candidatesChannel) { + val results = bioSdk.createMatcher(samples).use { matcher -> + match(matcher, batch.mapToFaceIdentities()) + } + resultSet.addAll(results) + } } private fun mapSamples(probes: List) = probes.map { FaceSample(it.faceId, it.template) } - private suspend fun getCandidates( - query: SubjectQuery, - range: IntRange, - dataSource: BiometricDataSource = BiometricDataSource.Simprints, - project: Project, - onCandidateLoaded: () -> Unit, - ) = enrolmentRecordRepository - .loadFaceIdentities(query, range, dataSource, project, onCandidateLoaded) - .map { - FaceIdentity( - it.subjectId, - it.faces.map { face -> FaceSample(face.id, face.template) }, - ) - } - private suspend fun match( matcher: FaceMatcher, - batchCandidates: List + batchCandidates: List, ) = batchCandidates.fold(MatchResultSet()) { acc, candidate -> acc.add( FaceMatchResult.Item( @@ -115,4 +118,16 @@ internal class FaceMatcherUseCase @Inject constructor( ), ) } + + private fun List.mapToFaceIdentities(): List = map { + FaceIdentity( + it.subjectId, + it.faces.map { + FaceSample( + it.referenceId, + it.template, + ) + }, + ) + } } diff --git a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt index 4050f5db81..5d6317487e 100644 --- a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt +++ b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt @@ -13,29 +13,35 @@ import com.simprints.infra.config.store.models.FingerprintConfiguration.FingerCo import com.simprints.infra.config.store.models.Project import com.simprints.infra.config.sync.ConfigManager import com.simprints.infra.enrolment.records.repository.EnrolmentRecordRepository -import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource -import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery import com.simprints.infra.logging.LoggingConstants import com.simprints.infra.logging.Simber import com.simprints.matcher.FingerprintMatchResult import com.simprints.matcher.MatchParams import com.simprints.matcher.usecases.MatcherUseCase.MatcherState import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.launch import javax.inject.Inject +import kotlin.math.min +import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity as DomainFingerprintIdentity internal class FingerprintMatcherUseCase @Inject constructor( private val enrolmentRecordRepository: EnrolmentRecordRepository, private val resolveBioSdkWrapper: ResolveBioSdkWrapperUseCase, private val configManager: ConfigManager, private val createRanges: CreateRangesUseCase, - @DispatcherBG private val dispatcher: CoroutineDispatcher, + @DispatcherBG private val dispatcherBG: CoroutineDispatcher, ) : MatcherUseCase { override val crashReportTag = LoggingConstants.CrashReportTag.FINGER_MATCHING + // When using local DB loadedCandidates = expectedCandidates + // However, when using CommCare as data source, loadedCandidates < expectedCandidates + // as it's count function does not take into account filtering criteria + // This var is not thread safe + var loadedCandidates = 0 + override suspend operator fun invoke( matchParams: MatchParams, project: Project, @@ -61,55 +67,56 @@ internal class FingerprintMatcherUseCase @Inject constructor( Simber.i("Matching candidates", tag = crashReportTag) send(MatcherState.LoadingStarted(expectedCandidates)) - // When using local DB loadedCandidates = expectedCandidates - // However, when using CommCare as data source, loadedCandidates < expectedCandidates - // as it's count function does not take into account filtering criteria - var loadedCandidates = 0 - val resultItems = createRanges(expectedCandidates) - .map { range -> - async(dispatcher) { - val batchCandidates = getCandidates(queryWithSupportedFormat, range, matchParams.biometricDataSource, project) { - // When a candidate is loaded - loadedCandidates++ - trySend(MatcherState.CandidateLoaded) - } - match(samples, batchCandidates, matchParams.flowType, bioSdkWrapper, bioSdk = matchParams.fingerprintSDK) - .fold(MatchResultSet()) { acc, item -> - acc.add(FingerprintMatchResult.Item(item.id, item.score)) - } - } - }.awaitAll() - .reduce { acc, subSet -> acc.addAll(subSet) } - .toList() + loadedCandidates = 0 + val ranges = createRanges(expectedCandidates) + // if number of ranges less than the number of cores then use the number of ranges + val numConsumers = min(Runtime.getRuntime().availableProcessors(), ranges.size) + val channel = enrolmentRecordRepository.loadFingerprintIdentities( + query = queryWithSupportedFormat, + ranges = ranges, + dataSource = matchParams.biometricDataSource, + scope = this, + project = project, + ) { + loadedCandidates++ + trySend(MatcherState.CandidateLoaded) + } + + val resultSet = MatchResultSet() + + // Start Consumers in BG thread + val consumerJobs = List(numConsumers) { + launch(dispatcherBG) { + consumeAndMatch(channel, samples, resultSet, bioSdkWrapper, matchParams) + } + } + // Wait for all to complete + consumerJobs.forEach { it.join() } Simber.i("Matched $loadedCandidates candidates", tag = crashReportTag) - send(MatcherState.Success(resultItems, loadedCandidates, bioSdkWrapper.matcherName)) + send(MatcherState.Success(resultSet.toList(), loadedCandidates, bioSdkWrapper.matcherName)) + } + + private suspend fun consumeAndMatch( + channel: ReceiveChannel>, + samples: List, + resultSet: MatchResultSet, + bioSdkWrapper: BioSdkWrapper, + matchParams: MatchParams, + ) { + for (batch in channel) { + val matchResults = + match(samples, batch.mapToFingerprintIdentity(), matchParams.flowType, bioSdkWrapper, bioSdk = matchParams.fingerprintSDK!!) + .fold(MatchResultSet()) { acc, item -> + acc.add(FingerprintMatchResult.Item(item.id, item.score)) + } + resultSet.addAll(matchResults) + } } private fun mapSamples(probes: List) = probes .map { Fingerprint(it.fingerId.toMatcherDomain(), it.template, it.format) } - private suspend fun getCandidates( - query: SubjectQuery, - range: IntRange, - dataSource: BiometricDataSource = BiometricDataSource.Simprints, - project: Project, - onCandidateLoaded: () -> Unit, - ) = enrolmentRecordRepository - .loadFingerprintIdentities(query, range, dataSource, project, onCandidateLoaded) - .map { - FingerprintIdentity( - it.subjectId, - it.fingerprints.map { finger -> - Fingerprint( - finger.fingerIdentifier.toMatcherDomain(), - finger.template, - finger.format, - ) - }, - ) - } - private suspend fun match( probes: List, candidates: List, @@ -144,4 +151,17 @@ internal class FingerprintMatcherUseCase @Inject constructor( IFingerIdentifier.LEFT_4TH_FINGER -> FingerIdentifier.LEFT_4TH_FINGER IFingerIdentifier.LEFT_5TH_FINGER -> FingerIdentifier.LEFT_5TH_FINGER } + + private fun List.mapToFingerprintIdentity() = map { + FingerprintIdentity( + it.subjectId, + it.fingerprints.map { finger -> + Fingerprint( + finger.fingerIdentifier.toMatcherDomain(), + finger.template, + finger.format, + ) + }, + ) + } } diff --git a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt index 255e25cf9d..d2f06f5fcd 100644 --- a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt +++ b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt @@ -123,13 +123,13 @@ internal class FaceMatcherUseCaseTest { ) coEvery { enrolmentRecordRepository.count(any(), any()) } returns 1 coEvery { createRangesUseCase(any()) } returns listOf(0..99) - coEvery { enrolmentRecordRepository.loadFaceIdentities(any(), any(), any(), any(), any()) } coAnswers { + coEvery { enrolmentRecordRepository.loadFaceIdentities(any(), any(), any(), any(), any(), any()) } coAnswers { // Call the onCandidateLoaded callback (5th parameter) - val onCandidateLoaded = arg<() -> Unit>(4) + val onCandidateLoaded = arg<() -> Unit>(5) onCandidateLoaded() // Return the face identities - faceIdentities + createTestChannel(faceIdentities) } coEvery { faceMatcher.getHighestComparisonScoreForCandidate(any()) } returns 42f diff --git a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FingerprintMatcherUseCaseTest.kt b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FingerprintMatcherUseCaseTest.kt index c727447c15..8fb25993a1 100644 --- a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FingerprintMatcherUseCaseTest.kt +++ b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FingerprintMatcherUseCaseTest.kt @@ -1,7 +1,7 @@ package com.simprints.matcher.usecases import androidx.arch.core.executor.testing.InstantTaskExecutorRule -import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.* import com.simprints.core.domain.common.FlowType import com.simprints.core.domain.fingerprint.FingerprintSample import com.simprints.core.domain.fingerprint.IFingerIdentifier @@ -16,11 +16,12 @@ import com.simprints.infra.enrolment.records.repository.domain.models.Fingerprin import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery import com.simprints.matcher.MatchParams import com.simprints.testtools.common.coroutines.TestCoroutineRule -import io.mockk.MockKAnnotations -import io.mockk.coEvery -import io.mockk.coVerify +import io.mockk.* import io.mockk.impl.annotations.MockK +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import org.junit.Before import org.junit.Rule @@ -50,7 +51,6 @@ internal class FingerprintMatcherUseCaseTest { @MockK lateinit var createRangesUseCase: CreateRangesUseCase - private val onCandidateLoaded: () -> Unit = {} private lateinit var useCase: FingerprintMatcherUseCase @Before @@ -99,7 +99,8 @@ internal class FingerprintMatcherUseCaseTest { @Test fun `Skips matching if there are no candidates`() = runTest { coEvery { enrolmentRecordRepository.count(any()) } returns 0 - coEvery { enrolmentRecordRepository.loadFaceIdentities(any(), any(), any(), project, onCandidateLoaded) } returns emptyList() + coEvery { enrolmentRecordRepository.loadFingerprintIdentities(any(), any(), any(), project, any(), any()) } returns + createTestChannel(emptyList()) coEvery { bioSdkWrapper.match(any(), any(), any()) } returns listOf() val results = useCase @@ -142,25 +143,29 @@ internal class FingerprintMatcherUseCaseTest { any(), any(), project, - onCandidateLoaded, + any(), + any(), ) - } returns listOf( - FingerprintIdentity( - "personId", + } returns + createTestChannel( listOf( - fingerprintSample(IFingerIdentifier.RIGHT_5TH_FINGER), - fingerprintSample(IFingerIdentifier.RIGHT_4TH_FINGER), - fingerprintSample(IFingerIdentifier.RIGHT_3RD_FINGER), - fingerprintSample(IFingerIdentifier.RIGHT_INDEX_FINGER), - fingerprintSample(IFingerIdentifier.RIGHT_THUMB), - fingerprintSample(IFingerIdentifier.LEFT_THUMB), - fingerprintSample(IFingerIdentifier.LEFT_INDEX_FINGER), - fingerprintSample(IFingerIdentifier.LEFT_3RD_FINGER), - fingerprintSample(IFingerIdentifier.LEFT_4TH_FINGER), - fingerprintSample(IFingerIdentifier.LEFT_5TH_FINGER), + FingerprintIdentity( + "personId", + listOf( + fingerprintSample(IFingerIdentifier.RIGHT_5TH_FINGER), + fingerprintSample(IFingerIdentifier.RIGHT_4TH_FINGER), + fingerprintSample(IFingerIdentifier.RIGHT_3RD_FINGER), + fingerprintSample(IFingerIdentifier.RIGHT_INDEX_FINGER), + fingerprintSample(IFingerIdentifier.RIGHT_THUMB), + fingerprintSample(IFingerIdentifier.LEFT_THUMB), + fingerprintSample(IFingerIdentifier.LEFT_INDEX_FINGER), + fingerprintSample(IFingerIdentifier.LEFT_3RD_FINGER), + fingerprintSample(IFingerIdentifier.LEFT_4TH_FINGER), + fingerprintSample(IFingerIdentifier.LEFT_5TH_FINGER), + ), + ), ), - ), - ) + ) coEvery { bioSdkWrapper.match(any(), any(), any()) } returns listOf() useCase @@ -181,9 +186,19 @@ internal class FingerprintMatcherUseCaseTest { ), project, ).toList() - coVerify { bioSdkWrapper.match(any(), any(), any()) } } private fun fingerprintSample(finger: IFingerIdentifier) = FingerprintSample(finger, byteArrayOf(1), "format", "referenceId") } + +fun createTestChannel(vararg lists: List): ReceiveChannel> { + val channel = Channel>(lists.size) + runBlocking { + for (list in lists) { + channel.send(list) + } + channel.close() + } + return channel +} diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepository.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepository.kt index 5db9c23c68..8684140537 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepository.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepository.kt @@ -3,8 +3,6 @@ package com.simprints.infra.enrolment.records.repository import com.simprints.core.ExcludedFromGeneratedTestCoverageReports import com.simprints.infra.config.store.models.Project import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource -import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity -import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource @@ -18,20 +16,4 @@ interface EnrolmentRecordRepository : EnrolmentRecordLocalDataSource { query: SubjectQuery, dataSource: BiometricDataSource, ): Int - - override suspend fun loadFingerprintIdentities( - query: SubjectQuery, - range: IntRange, - dataSource: BiometricDataSource, - project: Project, - onCandidateLoaded: () -> Unit, - ): List - - override suspend fun loadFaceIdentities( - query: SubjectQuery, - range: IntRange, - dataSource: BiometricDataSource, - project: Project, - onCandidateLoaded: () -> Unit, - ): List } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt index 0c89d95022..d5f64e4d66 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt @@ -1,6 +1,7 @@ package com.simprints.infra.enrolment.records.repository import android.content.Context +import androidx.core.content.edit import com.simprints.core.DispatcherIO import com.simprints.core.domain.tokenization.TokenizableString import com.simprints.infra.config.store.models.Project @@ -8,8 +9,6 @@ import com.simprints.infra.config.store.models.TokenKeyType import com.simprints.infra.config.store.tokenization.TokenizationProcessor import com.simprints.infra.enrolment.records.realm.store.exceptions.RealmUninitialisedException import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource -import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity -import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAction import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource @@ -17,6 +16,7 @@ import com.simprints.infra.enrolment.records.repository.remote.EnrolmentRecordRe import com.simprints.infra.logging.Simber import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.withContext import javax.inject.Inject @@ -66,36 +66,32 @@ internal class EnrolmentRecordRepositoryImpl( afterSubjectId = lastUploadedRecord, ) } - localDataSource - .load(query) - .chunked(batchSize) - .forEach { subjects -> - remoteDataSource.uploadRecords(subjects) - prefs.edit().putString(PROGRESS_KEY, subjects.last().subjectId).apply() - } - prefs.edit().remove(PROGRESS_KEY).apply() + localDataSource.load(query).chunked(batchSize).forEach { subjects -> + remoteDataSource.uploadRecords(subjects) + prefs.edit { putString(PROGRESS_KEY, subjects.last().subjectId) } + } + prefs.edit { remove(PROGRESS_KEY) } } override suspend fun tokenizeExistingRecords(project: Project) { try { val query = SubjectQuery(projectId = project.id, hasUntokenizedFields = true) - val tokenizedSubjectsCreateAction = - localDataSource - .load(query) - .mapNotNull { subject -> - if (subject.projectId != project.id) return@mapNotNull null - val moduleId = tokenizeIfNecessary( - value = subject.moduleId, - tokenKeyType = TokenKeyType.ModuleId, - project = project, - ) - val attendantId = tokenizeIfNecessary( - value = subject.attendantId, - tokenKeyType = TokenKeyType.AttendantId, - project = project, - ) - return@mapNotNull subject.copy(moduleId = moduleId, attendantId = attendantId) - }.map(SubjectAction::Creation) + val tokenizedSubjectsCreateAction = localDataSource + .load(query) + .mapNotNull { subject -> + if (subject.projectId != project.id) return@mapNotNull null + val moduleId = tokenizeIfNecessary( + value = subject.moduleId, + tokenKeyType = TokenKeyType.ModuleId, + project = project, + ) + val attendantId = tokenizeIfNecessary( + value = subject.attendantId, + tokenKeyType = TokenKeyType.AttendantId, + project = project, + ) + return@mapNotNull subject.copy(moduleId = moduleId, attendantId = attendantId) + }.map(SubjectAction::Creation) localDataSource.performActions(tokenizedSubjectsCreateAction, project) } catch (e: Exception) { when (e) { @@ -123,23 +119,37 @@ internal class EnrolmentRecordRepositoryImpl( dataSource: BiometricDataSource, ): Int = fromIdentityDataSource(dataSource).count(query, dataSource) - override suspend fun loadFingerprintIdentities( + override fun loadFingerprintIdentities( query: SubjectQuery, - range: IntRange, + ranges: List, dataSource: BiometricDataSource, project: Project, + scope: CoroutineScope, onCandidateLoaded: () -> Unit, - ): List = fromIdentityDataSource(dataSource) - .loadFingerprintIdentities(query, range, dataSource, project, onCandidateLoaded) + ) = fromIdentityDataSource(dataSource).loadFingerprintIdentities( + query = query, + ranges = ranges, + dataSource = dataSource, + project = project, + scope = scope, + onCandidateLoaded = onCandidateLoaded, + ) - override suspend fun loadFaceIdentities( + override fun loadFaceIdentities( query: SubjectQuery, - range: IntRange, + ranges: List, dataSource: BiometricDataSource, project: Project, + scope: CoroutineScope, onCandidateLoaded: () -> Unit, - ): List = fromIdentityDataSource(dataSource) - .loadFaceIdentities(query, range, dataSource, project, onCandidateLoaded) + ) = fromIdentityDataSource(dataSource).loadFaceIdentities( + query = query, + ranges = ranges, + dataSource = dataSource, + project = project, + scope = scope, + onCandidateLoaded = onCandidateLoaded, + ) private fun fromIdentityDataSource(dataSource: BiometricDataSource) = when (dataSource) { is BiometricDataSource.Simprints -> localDataSource diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/IdentityDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/IdentityDataSource.kt index 32c6c7dd12..7aceac3deb 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/IdentityDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/IdentityDataSource.kt @@ -5,6 +5,14 @@ import com.simprints.infra.enrolment.records.repository.domain.models.BiometricD import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.ReceiveChannel +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit interface IdentityDataSource { suspend fun count( @@ -12,19 +20,48 @@ interface IdentityDataSource { dataSource: BiometricDataSource = BiometricDataSource.Simprints, ): Int - suspend fun loadFingerprintIdentities( + fun loadFingerprintIdentities( query: SubjectQuery, - range: IntRange, - dataSource: BiometricDataSource = BiometricDataSource.Simprints, + ranges: List, + dataSource: BiometricDataSource, project: Project, + scope: CoroutineScope, onCandidateLoaded: () -> Unit, - ): List + ): ReceiveChannel> - suspend fun loadFaceIdentities( + fun loadFaceIdentities( query: SubjectQuery, - range: IntRange, - dataSource: BiometricDataSource = BiometricDataSource.Simprints, + ranges: List, + dataSource: BiometricDataSource, project: Project, + scope: CoroutineScope, onCandidateLoaded: () -> Unit, - ): List + ): ReceiveChannel> + + /** + * Loads identities concurrently using the provided dispatcher and parallelism level. + * + */ + fun loadIdentitiesConcurrently( + ranges: List, + dispatcher: CoroutineDispatcher, + parallelism: Int, + scope: CoroutineScope, + load: suspend (IntRange) -> List, + ): ReceiveChannel> { + val channel = Channel>(parallelism) + val semaphore = Semaphore(parallelism) + scope.launch { + ranges + .map { range -> + launch { + semaphore.withPermit { + channel.send(load(range)) + } + } + }.joinAll() + channel.close() + } + return channel + } } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt index 15ea4c74fe..1a2b99cefc 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt @@ -6,7 +6,7 @@ import android.net.Uri import androidx.core.net.toUri import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.module.SimpleModule -import com.simprints.core.DispatcherIO +import com.simprints.core.DispatcherBG import com.simprints.core.domain.face.FaceSample import com.simprints.core.domain.fingerprint.FingerprintSample import com.simprints.core.domain.tokenization.TokenizableString @@ -31,6 +31,8 @@ import com.simprints.infra.logging.Simber import com.simprints.libsimprints.Constants.SIMPRINTS_COSYNC_SUBJECT_ACTIONS import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.withContext import org.json.JSONException import javax.inject.Inject @@ -40,41 +42,37 @@ internal class CommCareIdentityDataSource @Inject constructor( private val jsonHelper: JsonHelper, private val compareImplicitTokenizedStringsUseCase: CompareImplicitTokenizedStringsUseCase, @ApplicationContext private val context: Context, - @DispatcherIO private val dispatcher: CoroutineDispatcher, + @DispatcherBG private val dispatcher: CoroutineDispatcher, ) : IdentityDataSource { private fun getCaseMetadataUri(packageName: String): Uri = "content://$packageName.case/casedb/case".toUri() private fun getCaseDataUri(packageName: String): Uri = "content://$packageName.case/casedb/data".toUri() - override suspend fun loadFingerprintIdentities( + private fun loadFingerprintIdentities( query: SubjectQuery, range: IntRange, dataSource: BiometricDataSource, project: Project, onCandidateLoaded: () -> Unit, - ): List = withContext(dispatcher) { - loadEnrolmentRecordCreationEvents(range, dataSource.callerPackageName(), query, project) - .filter { erce -> - erce.payload.biometricReferences.any { it is FingerprintReference && it.format == query.fingerprintSampleFormat } - }.map { - onCandidateLoaded() - FingerprintIdentity( - it.payload.subjectId, - it.payload.biometricReferences - .filterIsInstance() - .flatMap { fingerprintReference -> - fingerprintReference.templates.map { fingerprintTemplate -> - FingerprintSample( - fingerIdentifier = fingerprintTemplate.finger, - template = encoder.base64ToBytes(fingerprintTemplate.template), - format = fingerprintReference.format, - referenceId = fingerprintReference.id, - ) - } - }, - ) - } - } + ): List = loadEnrolmentRecordCreationEvents(range, dataSource.callerPackageName(), query, project) + .filter { erce -> + erce.payload.biometricReferences.any { it is FingerprintReference && it.format == query.fingerprintSampleFormat } + }.map { + onCandidateLoaded() + FingerprintIdentity( + it.payload.subjectId, + it.payload.biometricReferences.filterIsInstance().flatMap { fingerprintReference -> + fingerprintReference.templates.map { fingerprintTemplate -> + FingerprintSample( + fingerIdentifier = fingerprintTemplate.finger, + template = encoder.base64ToBytes(fingerprintTemplate.template), + format = fingerprintReference.format, + referenceId = fingerprintReference.id, + ) + } + }, + ) + } private fun loadEnrolmentRecordCreationEvents( range: IntRange, @@ -114,44 +112,38 @@ internal class CommCareIdentityDataSource @Inject constructor( return enrolmentRecordCreationEvents } - private fun attemptExtractingCaseId(metadata: String?) = metadata - ?.takeUnless { it.isEmpty() } - ?.let { - try { - JsonHelper.fromJson>(it)[ARG_CASE_ID] as? String - } catch (_: JSONException) { - null - } + private fun attemptExtractingCaseId(metadata: String?) = metadata?.takeUnless { it.isEmpty() }?.let { + try { + JsonHelper.fromJson>(it)[ARG_CASE_ID] as? String + } catch (_: JSONException) { + null } + } - override suspend fun loadFaceIdentities( + private fun loadFaceIdentities( query: SubjectQuery, range: IntRange, dataSource: BiometricDataSource, project: Project, onCandidateLoaded: () -> Unit, - ): List = withContext(dispatcher) { - loadEnrolmentRecordCreationEvents(range, dataSource.callerPackageName(), query, project) - .filter { erce -> - erce.payload.biometricReferences.any { it is FaceReference && it.format == query.faceSampleFormat } - }.map { - onCandidateLoaded() - FaceIdentity( - it.payload.subjectId, - it.payload.biometricReferences - .filterIsInstance() - .flatMap { faceReference -> - faceReference.templates.map { faceTemplate -> - FaceSample( - template = encoder.base64ToBytes(faceTemplate.template), - format = faceReference.format, - referenceId = faceReference.id, - ) - } - }, - ) - } - } + ): List = loadEnrolmentRecordCreationEvents(range, dataSource.callerPackageName(), query, project) + .filter { erce -> + erce.payload.biometricReferences.any { it is FaceReference && it.format == query.faceSampleFormat } + }.map { + onCandidateLoaded() + FaceIdentity( + it.payload.subjectId, + it.payload.biometricReferences.filterIsInstance().flatMap { faceReference -> + faceReference.templates.map { faceTemplate -> + FaceSample( + template = encoder.base64ToBytes(faceTemplate.template), + format = faceReference.format, + referenceId = faceReference.id, + ) + } + }, + ) + } private fun loadEnrolmentRecordCreationEvents( caseId: String, @@ -169,16 +161,17 @@ internal class CommCareIdentityDataSource @Inject constructor( Simber.d(subjectActions) val coSyncEnrolmentRecordEvents = parseRecordEvents(subjectActions) - coSyncEnrolmentRecordEvents - ?.events - ?.filterIsInstance() - ?.filter { event -> - // [MS-852] Plain strings from CommCare might be tokenized or untokenized. The only way to properly compare them - // is by trying to decrypt the values to check if already tokenized, and then compare the values - isSubjectIdNullOrMatching(query, event) && - isAttendantIdNullOrMatching(query, event, project) && - isModuleIdNullOrMatching(query, event, project) - } + coSyncEnrolmentRecordEvents?.events?.filterIsInstance()?.filter { event -> + // [MS-852] Plain strings from CommCare might be tokenized or untokenized. The only way to properly compare them + // is by trying to decrypt the values to check if already tokenized, and then compare the values + isSubjectIdNullOrMatching(query, event) && + isAttendantIdNullOrMatching( + query, + event, + project, + ) && + isModuleIdNullOrMatching(query, event, project) + } }.orEmpty() } @@ -265,6 +258,52 @@ internal class CommCareIdentityDataSource @Inject constructor( count } + private val parallelism = Runtime.getRuntime().availableProcessors() + + override fun loadFaceIdentities( + query: SubjectQuery, + ranges: List, + dataSource: BiometricDataSource, + project: Project, + scope: CoroutineScope, + onCandidateLoaded: () -> Unit, + ): ReceiveChannel> = loadIdentitiesConcurrently( + ranges = ranges, + dispatcher = dispatcher, + parallelism = parallelism, + scope = scope, + ) { range -> + loadFaceIdentities( + query = query, + range = range, + project = project, + dataSource = dataSource, + onCandidateLoaded = onCandidateLoaded, + ) + } + + override fun loadFingerprintIdentities( + query: SubjectQuery, + ranges: List, + dataSource: BiometricDataSource, + project: Project, + scope: CoroutineScope, + onCandidateLoaded: () -> Unit, + ): ReceiveChannel> = loadIdentitiesConcurrently( + ranges = ranges, + dispatcher = dispatcher, + parallelism = parallelism, + scope = scope, + ) { range -> + loadFingerprintIdentities( + query = query, + range = range, + project = project, + dataSource = dataSource, + onCandidateLoaded = onCandidateLoaded, + ) + } + companion object { const val COLUMN_CASE_ID = "case_id" const val COLUMN_DATUM_ID = "datum_id" diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImpl.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImpl.kt index 086bceca59..5e1261ff67 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImpl.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImpl.kt @@ -1,5 +1,6 @@ package com.simprints.infra.enrolment.records.repository.local +import com.simprints.core.DispatcherIO import com.simprints.core.domain.tokenization.TokenizableString import com.simprints.infra.config.store.models.Project import com.simprints.infra.config.store.models.TokenKeyType @@ -25,11 +26,15 @@ import io.realm.kotlin.query.RealmQuery import io.realm.kotlin.query.Sort import io.realm.kotlin.query.find import io.realm.kotlin.types.RealmUUID +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.ReceiveChannel import javax.inject.Inject internal class EnrolmentRecordLocalDataSourceImpl @Inject constructor( private val realmWrapper: RealmWrapper, private val tokenizationProcessor: TokenizationProcessor, + @DispatcherIO private val dispatcher: CoroutineDispatcher, ) : EnrolmentRecordLocalDataSource { companion object { const val PROJECT_ID_FIELD = "projectId" @@ -41,6 +46,7 @@ internal class EnrolmentRecordLocalDataSourceImpl @Inject constructor( const val FINGERPRINT_SAMPLES_FIELD = "fingerprintSamples" const val FACE_SAMPLES_FIELD = "faceSamples" const val FORMAT_FIELD = "format" + const val PARALLELISM = 1 } override suspend fun load(query: SubjectQuery): List = realmWrapper.readRealm { @@ -51,11 +57,49 @@ internal class EnrolmentRecordLocalDataSourceImpl @Inject constructor( .map { dbSubject -> dbSubject.fromDbToDomain() } } - override suspend fun loadFingerprintIdentities( + override fun loadFaceIdentities( query: SubjectQuery, - range: IntRange, + ranges: List, + dataSource: BiometricDataSource, + project: Project, + scope: CoroutineScope, + onCandidateLoaded: () -> Unit, + ): ReceiveChannel> = loadIdentitiesConcurrently( + ranges = ranges, + dispatcher = dispatcher, + parallelism = PARALLELISM, + scope = scope, + ) { range -> + loadFaceIdentities( + query = query, + range = range, + onCandidateLoaded = onCandidateLoaded, + ) + } + + override fun loadFingerprintIdentities( + query: SubjectQuery, + ranges: List, dataSource: BiometricDataSource, project: Project, + scope: CoroutineScope, + onCandidateLoaded: () -> Unit, + ): ReceiveChannel> = loadIdentitiesConcurrently( + ranges = ranges, + dispatcher = dispatcher, + parallelism = PARALLELISM, + scope = scope, + ) { range -> + loadFingerprintIdentities( + query = query, + range = range, + onCandidateLoaded = onCandidateLoaded, + ) + } + + private suspend fun loadFingerprintIdentities( + query: SubjectQuery, + range: IntRange, onCandidateLoaded: () -> Unit, ): List = realmWrapper.readRealm { realm -> realm @@ -71,11 +115,9 @@ internal class EnrolmentRecordLocalDataSourceImpl @Inject constructor( } } - override suspend fun loadFaceIdentities( + private suspend fun loadFaceIdentities( query: SubjectQuery, range: IntRange, - dataSource: BiometricDataSource, - project: Project, onCandidateLoaded: () -> Unit, ): List = realmWrapper.readRealm { realm -> realm diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImplTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImplTest.kt index 24484216ad..ff0ba31bbc 100644 --- a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImplTest.kt +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImplTest.kt @@ -15,10 +15,12 @@ import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAct import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource import com.simprints.infra.enrolment.records.repository.remote.EnrolmentRecordRemoteDataSource -import io.mockk.coEvery -import io.mockk.coVerify -import io.mockk.every -import io.mockk.mockk +import io.mockk.* +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.ReceiveChannel +import kotlinx.coroutines.channels.consumeEach +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.UnconfinedTestDispatcher import kotlinx.coroutines.test.runTest import org.junit.Before @@ -65,6 +67,7 @@ class EnrolmentRecordRepositoryImplTest { private lateinit var repository: EnrolmentRecordRepositoryImpl private val project = mockk() + @OptIn(ExperimentalCoroutinesApi::class) @Before fun setup() { every { prefsEditor.putString(any(), any()) } returns prefsEditor @@ -276,7 +279,7 @@ class EnrolmentRecordRepositoryImplTest { @Test fun `should forward the call to the local data source when loading fingerprint identities and dataSource is Simprints`() = runTest { val expectedSubjectQuery = SubjectQuery() - val expectedRange = 0..10 + val expectedRange = listOf(0..10) val expectedFingerprintIdentities = listOf() coEvery { localDataSource.loadFingerprintIdentities( @@ -284,17 +287,23 @@ class EnrolmentRecordRepositoryImplTest { expectedRange, any(), project, + this@runTest, onCandidateLoaded, ) - } returns expectedFingerprintIdentities - - val fingerprintIdentities = repository.loadFingerprintIdentities( - query = expectedSubjectQuery, - range = expectedRange, - dataSource = BiometricDataSource.Simprints, - project = project, - onCandidateLoaded = onCandidateLoaded, - ) + } returns createTestChannel(expectedFingerprintIdentities) + + val fingerprintIdentities = mutableListOf() + repository + .loadFingerprintIdentities( + query = expectedSubjectQuery, + ranges = expectedRange, + dataSource = BiometricDataSource.Simprints, + project = project, + scope = this, + onCandidateLoaded = onCandidateLoaded, + ).consumeEach { + fingerprintIdentities.addAll(it) + } assert(fingerprintIdentities == expectedFingerprintIdentities) coVerify(exactly = 1) { @@ -303,6 +312,7 @@ class EnrolmentRecordRepositoryImplTest { expectedRange, any(), project, + this@runTest, onCandidateLoaded, ) } @@ -311,7 +321,7 @@ class EnrolmentRecordRepositoryImplTest { @Test fun `should forward the call to the commcare data source when loading fingerprint identities and dataSource is CommCare`() = runTest { val expectedSubjectQuery = SubjectQuery() - val expectedRange = 0..10 + val expectedRange = listOf(0..10) val expectedFingerprintIdentities = listOf() coEvery { commCareDataSource.loadFingerprintIdentities( @@ -319,17 +329,22 @@ class EnrolmentRecordRepositoryImplTest { expectedRange, any(), project, + this@runTest, onCandidateLoaded, ) - } returns expectedFingerprintIdentities - - val fingerprintIdentities = repository.loadFingerprintIdentities( - query = expectedSubjectQuery, - range = expectedRange, - dataSource = BiometricDataSource.CommCare(""), - project = project, - onCandidateLoaded = onCandidateLoaded, - ) + } returns createTestChannel(expectedFingerprintIdentities) + val fingerprintIdentities = mutableListOf() + repository + .loadFingerprintIdentities( + query = expectedSubjectQuery, + ranges = expectedRange, + dataSource = BiometricDataSource.CommCare(""), + project = project, + scope = this, + onCandidateLoaded = onCandidateLoaded, + ).consumeEach { + fingerprintIdentities.addAll(it) + } assert(fingerprintIdentities == expectedFingerprintIdentities) coVerify(exactly = 1) { @@ -338,6 +353,7 @@ class EnrolmentRecordRepositoryImplTest { expectedRange, any(), project, + this@runTest, onCandidateLoaded, ) } @@ -346,7 +362,7 @@ class EnrolmentRecordRepositoryImplTest { @Test fun `should forward the call to the local data source when loading face identities and dataSource is Simprints`() = runTest { val expectedSubjectQuery = SubjectQuery() - val expectedRange = 0..10 + val expectedRange = listOf(0..10) val expectedFaceIdentities = listOf() coEvery { localDataSource.loadFaceIdentities( @@ -354,38 +370,66 @@ class EnrolmentRecordRepositoryImplTest { expectedRange, any(), project, + this@runTest, onCandidateLoaded, ) - } returns expectedFaceIdentities - - val faceIdentities = repository.loadFaceIdentities( - query = expectedSubjectQuery, - range = expectedRange, - dataSource = BiometricDataSource.Simprints, - project = project, - onCandidateLoaded = onCandidateLoaded, - ) + } returns createTestChannel(expectedFaceIdentities) + + val faceIdentities = mutableListOf() + repository + .loadFaceIdentities( + query = expectedSubjectQuery, + ranges = expectedRange, + dataSource = BiometricDataSource.Simprints, + project = project, + scope = this, + onCandidateLoaded = onCandidateLoaded, + ).consumeEach { + faceIdentities.addAll(it) + } assert(faceIdentities == expectedFaceIdentities) - coVerify(exactly = 1) { localDataSource.loadFaceIdentities(expectedSubjectQuery, expectedRange, any(), project, onCandidateLoaded) } + coVerify(exactly = 1) { + localDataSource.loadFaceIdentities( + expectedSubjectQuery, + expectedRange, + any(), + project, + this@runTest, + onCandidateLoaded, + ) + } } @Test fun `should forward the call to the commcare data source when loading face identities and dataSource is CommCare`() = runTest { val expectedSubjectQuery = SubjectQuery() - val expectedRange = 0..10 + val expectedRange = listOf(0..10) val expectedFaceIdentities = listOf() coEvery { - commCareDataSource.loadFaceIdentities(expectedSubjectQuery, expectedRange, any(), project, onCandidateLoaded) - } returns expectedFaceIdentities - - val faceIdentities = repository.loadFaceIdentities( - query = expectedSubjectQuery, - range = expectedRange, - dataSource = BiometricDataSource.CommCare(""), - project = project, - onCandidateLoaded = onCandidateLoaded, - ) + commCareDataSource.loadFaceIdentities( + expectedSubjectQuery, + expectedRange, + any(), + project, + this@runTest, + onCandidateLoaded, + ) + } returns createTestChannel(expectedFaceIdentities) + + val faceIdentities = mutableListOf() + + repository + .loadFaceIdentities( + query = expectedSubjectQuery, + ranges = expectedRange, + dataSource = BiometricDataSource.CommCare(""), + project = project, + scope = this, + onCandidateLoaded = onCandidateLoaded, + ).consumeEach { + faceIdentities.addAll(it) + } assert(faceIdentities == expectedFaceIdentities) coVerify(exactly = 1) { @@ -394,8 +438,20 @@ class EnrolmentRecordRepositoryImplTest { expectedRange, any(), project, + this@runTest, onCandidateLoaded, ) } } + + fun createTestChannel(vararg lists: List): ReceiveChannel> { + val channel = Channel>(lists.size) + runBlocking { + for (list in lists) { + channel.send(list) + } + channel.close() + } + return channel + } } diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSourceTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSourceTest.kt index 13662f5b36..db0274fe9a 100644 --- a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSourceTest.kt +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSourceTest.kt @@ -14,6 +14,7 @@ import com.simprints.core.tools.utils.EncodingUtils import com.simprints.infra.config.store.models.Project import com.simprints.infra.enrolment.records.repository.commcare.CommCareIdentityDataSource.Companion.COLUMN_DATUM_ID import com.simprints.infra.enrolment.records.repository.commcare.CommCareIdentityDataSource.Companion.COLUMN_VALUE +import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery @@ -24,6 +25,7 @@ import io.mockk.* import io.mockk.impl.annotations.MockK import junit.framework.TestCase.assertEquals import junit.framework.TestCase.assertTrue +import kotlinx.coroutines.channels.consumeEach import kotlinx.coroutines.test.runTest import org.junit.AfterClass import org.junit.Before @@ -160,6 +162,8 @@ class CommCareIdentityDataSourceTest { @MockK lateinit var project: Project + private val commCareBiometricDataSource = BiometricDataSource.CommCare("") + @Before fun setUp() { MockKAnnotations.init(this) @@ -230,7 +234,19 @@ class CommCareIdentityDataSourceTest { val templateFormat = "ISO_19794_2" val query = SubjectQuery(fingerprintSampleFormat = templateFormat) val range = 0..expectedFingerprintIdentities.size - val actualIdentities = dataSource.loadFingerprintIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + + dataSource + .loadFingerprintIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertEquals(1, actualIdentities.size) val areContentsEqual = @@ -280,7 +296,18 @@ class CommCareIdentityDataSourceTest { subjectId = "b26c91bc-b307-4131-80c3-55090ba5dbf2", ) val range = 0..expectedFaceIdentities.size - val actualIdentities = dataSource.loadFaceIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFaceIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertEquals(1, actualIdentities.size) val areContentsEqual = @@ -322,7 +349,18 @@ class CommCareIdentityDataSourceTest { val templateFormat = "NEC_1_5" val query = SubjectQuery(fingerprintSampleFormat = templateFormat) val range = 0..expectedFingerprintIdentities.size - val actualIdentities = dataSource.loadFingerprintIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFingerprintIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertEquals(1, actualIdentities.size) val areContentsEqual = @@ -369,7 +407,18 @@ class CommCareIdentityDataSourceTest { val templateFormat = "ROC_1_23" val query = SubjectQuery(faceSampleFormat = templateFormat) val range = 0..expectedFaceIdentities.size - val actualIdentities = dataSource.loadFaceIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFaceIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertEquals(1, actualIdentities.size) val areContentsEqual = @@ -408,7 +457,18 @@ class CommCareIdentityDataSourceTest { val templateFormat = "ISO_19794_2" val query = SubjectQuery(fingerprintSampleFormat = templateFormat) val range = 0..expectedFingerprintIdentities.size - val actualIdentities = dataSource.loadFingerprintIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFingerprintIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertEquals(1, actualIdentities.size) val areContentsEqual = @@ -451,7 +511,18 @@ class CommCareIdentityDataSourceTest { val templateFormat = "ROC_1_23" val query = SubjectQuery(faceSampleFormat = templateFormat) val range = 0..expectedFaceIdentities.size - val actualIdentities = dataSource.loadFaceIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFaceIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertEquals(1, actualIdentities.size) val areContentsEqual = @@ -496,7 +567,18 @@ class CommCareIdentityDataSourceTest { val query = SubjectQuery() val range = 0..0 - val actualIdentities = dataSource.loadFingerprintIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFingerprintIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertTrue(actualIdentities.isEmpty()) coVerify { mockContentResolver.query(mockMetadataUri, any(), any(), any(), any()) } @@ -517,7 +599,18 @@ class CommCareIdentityDataSourceTest { val query = SubjectQuery() val range = 2..3 - val actualIdentities = dataSource.loadFingerprintIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFingerprintIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertTrue(actualIdentities.isEmpty()) coVerify { mockContentResolver.query(mockMetadataUri, any(), any(), any(), any()) } @@ -555,7 +648,18 @@ class CommCareIdentityDataSourceTest { val templateFormat = "ISO_19794_2" val query = SubjectQuery(fingerprintSampleFormat = templateFormat) val range = 0..expectedFingerprintIdentities.size - val actualIdentities = dataSource.loadFingerprintIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFingerprintIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertEquals(1, actualIdentities.size) val areContentsEqual = @@ -587,7 +691,18 @@ class CommCareIdentityDataSourceTest { val query = SubjectQuery() val range = 0..2 - val actualIdentities = dataSource.loadFingerprintIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFingerprintIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertEquals(0, actualIdentities.size) coVerify { mockContentResolver.query(mockMetadataUri, any(), any(), any(), any()) } @@ -616,7 +731,18 @@ class CommCareIdentityDataSourceTest { val query = SubjectQuery() val range = 0..2 - val actualIdentities = dataSource.loadFingerprintIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFingerprintIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertEquals(0, actualIdentities.size) coVerify { mockContentResolver.query(mockMetadataUri, any(), any(), any(), any()) } @@ -649,7 +775,18 @@ class CommCareIdentityDataSourceTest { val query = SubjectQuery() val range = 0..2 - val actualIdentities = dataSource.loadFingerprintIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFingerprintIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertEquals(0, actualIdentities.size) coVerify { mockContentResolver.query(mockMetadataUri, any(), any(), any(), any()) } @@ -669,7 +806,18 @@ class CommCareIdentityDataSourceTest { val query = SubjectQuery() val range = 0..2 - val actualIdentities = dataSource.loadFingerprintIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFingerprintIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertEquals(0, actualIdentities.size) coVerify { mockContentResolver.query(mockMetadataUri, any(), any(), any(), any()) } @@ -694,7 +842,18 @@ class CommCareIdentityDataSourceTest { val query = SubjectQuery() val range = 0..2 - val actualIdentities = dataSource.loadFingerprintIdentities(query, range, project = project) {} + val actualIdentities = mutableListOf() + dataSource + .loadFingerprintIdentities( + query = query, + ranges = listOf(range), + project = project, + dataSource = commCareBiometricDataSource, + scope = this, + onCandidateLoaded = {}, + ).consumeEach { + actualIdentities.addAll(it) + } assertEquals(0, actualIdentities.size) coVerify { mockContentResolver.query(mockMetadataUri, any(), any(), any(), any()) } diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImplTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImplTest.kt index e60c866f0d..b855586d02 100644 --- a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImplTest.kt +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImplTest.kt @@ -1,6 +1,6 @@ package com.simprints.infra.enrolment.records.repository.local -import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.* import com.simprints.core.domain.face.FaceSample import com.simprints.core.domain.fingerprint.FingerprintSample import com.simprints.core.domain.fingerprint.IFingerIdentifier @@ -13,6 +13,8 @@ import com.simprints.infra.enrolment.records.realm.store.models.DbFaceSample import com.simprints.infra.enrolment.records.realm.store.models.DbFingerprintSample import com.simprints.infra.enrolment.records.realm.store.models.DbSubject import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource +import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity +import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity import com.simprints.infra.enrolment.records.repository.domain.models.Subject import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAction import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery @@ -21,18 +23,15 @@ import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLoc import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSourceImpl.Companion.FORMAT_FIELD import com.simprints.infra.enrolment.records.repository.local.models.fromDbToDomain import com.simprints.infra.enrolment.records.repository.local.models.fromDomainToDb -import io.mockk.CapturingSlot -import io.mockk.MockKAnnotations -import io.mockk.coEvery -import io.mockk.every +import io.mockk.* import io.mockk.impl.annotations.MockK -import io.mockk.mockk -import io.mockk.slot -import io.mockk.verify import io.realm.kotlin.MutableRealm import io.realm.kotlin.Realm import io.realm.kotlin.query.RealmQuery import io.realm.kotlin.query.RealmSingleQuery +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.channels.consumeEach +import kotlinx.coroutines.test.UnconfinedTestDispatcher import kotlinx.coroutines.test.runTest import org.junit.Before import org.junit.Test @@ -68,6 +67,7 @@ class EnrolmentRecordLocalDataSourceImplTest { private lateinit var enrolmentRecordLocalDataSource: EnrolmentRecordLocalDataSource + @OptIn(ExperimentalCoroutinesApi::class) @Before fun setup() { MockKAnnotations.init(this, relaxed = true) @@ -102,6 +102,7 @@ class EnrolmentRecordLocalDataSourceImplTest { enrolmentRecordLocalDataSource = EnrolmentRecordLocalDataSourceImpl( realmWrapperMock, tokenizationProcessor, + UnconfinedTestDispatcher(), ) } @@ -134,14 +135,16 @@ class EnrolmentRecordLocalDataSourceImplTest { val savedPersons = saveFakePeople(getRandomPeople(20)) val fakePerson = savedPersons[0].fromDomainToDb() - val people = enrolmentRecordLocalDataSource + val people = mutableListOf() + enrolmentRecordLocalDataSource .loadFingerprintIdentities( SubjectQuery(), - IntRange(0, 20), + listOf(IntRange(0, 20)), BiometricDataSource.Simprints, project, + this, onCandidateLoaded, - ).toList() + ).consumeEach { people.addAll(it) } listOf(fakePerson).zip(people).forEach { (subject, identity) -> assertThat(subject.subjectId).isEqualTo(identity.subjectId) @@ -155,11 +158,12 @@ class EnrolmentRecordLocalDataSourceImplTest { enrolmentRecordLocalDataSource .loadFingerprintIdentities( SubjectQuery(fingerprintSampleFormat = format), - IntRange(0, 20), + listOf(IntRange(0, 20)), BiometricDataSource.Simprints, project, + this, onCandidateLoaded, - ).toList() + ).consumeEach { } verify { realmQuery.query( @@ -176,12 +180,12 @@ class EnrolmentRecordLocalDataSourceImplTest { enrolmentRecordLocalDataSource .loadFingerprintIdentities( SubjectQuery(faceSampleFormat = format), - IntRange(0, 20), + listOf(IntRange(0, 20)), BiometricDataSource.Simprints, project, + this, onCandidateLoaded, - ).toList() - + ).consumeEach { } verify { realmQuery.query( "ANY ${FACE_SAMPLES_FIELD}.${FORMAT_FIELD} == $0", @@ -195,15 +199,18 @@ class EnrolmentRecordLocalDataSourceImplTest { val savedPersons = saveFakePeople(getRandomPeople(20)) val fakePerson = savedPersons[0].fromDomainToDb() - val people = enrolmentRecordLocalDataSource + val people = mutableListOf() + enrolmentRecordLocalDataSource .loadFaceIdentities( SubjectQuery(), - IntRange(0, 20), + listOf(IntRange(0, 20)), BiometricDataSource.Simprints, project, + this, onCandidateLoaded, - ).toList() - + ).consumeEach { + people.addAll(it) + } listOf(fakePerson).zip(people).forEach { (subject, identity) -> assertThat(subject.subjectId).isEqualTo(identity.subjectId) }