From f912d39902bf8b6fb699792d86e41907e6f28cca Mon Sep 17 00:00:00 2001 From: Melad Raouf Date: Tue, 27 May 2025 16:29:36 +0300 Subject: [PATCH 01/11] Enables Java 8+ API desugaring for older Android versions by adding the desugar_jdk_libs dependency. --- .../src/main/kotlin/PipelineJacocoConventionPlugin.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-logic/convention/src/main/kotlin/PipelineJacocoConventionPlugin.kt b/build-logic/convention/src/main/kotlin/PipelineJacocoConventionPlugin.kt index 748397d2a4..4264c4a5ed 100644 --- a/build-logic/convention/src/main/kotlin/PipelineJacocoConventionPlugin.kt +++ b/build-logic/convention/src/main/kotlin/PipelineJacocoConventionPlugin.kt @@ -29,20 +29,20 @@ class PipelineJacocoConventionPlugin : Plugin { ) private fun Project.createJacocoTask() { - tasks.create("jacocoTestReport", JacocoReport::class.java) { + tasks.register("jacocoTestReport", JacocoReport::class.java) { dependsOn(tasks.withType().matching { it.name.lowercase().contains("debug") }) reports.xml.required.set(true) reports.html.required.set(false) // Disable html reports to decrease report upload/download time in github pipeline - val javaTree = fileTree("${project.buildDir}/intermediates/javac/debug/classes") { exclude(fileFilter) } - val kotlinTree = fileTree("${project.buildDir}/tmp/kotlin-classes/debug") { exclude(fileFilter) } + val javaTree = fileTree("${project.layout.buildDirectory}/intermediates/javac/debug/classes") { exclude(fileFilter) } + val kotlinTree = fileTree("${project.layout.buildDirectory}/tmp/kotlin-classes/debug") { exclude(fileFilter) } classDirectories.setFrom(files(javaTree, kotlinTree)) sourceDirectories.setFrom(files("${project.projectDir}/src/main/java")) executionData.setFrom( - fileTree("$buildDir") { + fileTree("${layout.buildDirectory}") { include( "jacoco/testDebugUnitTest.exec", "outputs/code-coverage/connected/*coverage.ec", From dbcf05bbf424e942e22c524d3859870fe26cfe31 Mon Sep 17 00:00:00 2001 From: Melad Raouf Date: Thu, 20 Mar 2025 09:55:22 +0200 Subject: [PATCH 02/11] [MS-951]: Introduce Room Database for Enrolment Records --- .github/workflows/pr-checks.yml | 1 + gradle/libs.versions.toml | 2 + .../room-store/build.gradle.kts | 13 +++++ .../records/room/store/SubjectDao.kt | 45 +++++++++++++++ .../records/room/store/SubjectsDatabase.kt | 33 +++++++++++ .../room/store/SubjectsDatabaseFactory.kt | 56 +++++++++++++++++++ .../records/room/store/models/DbFaceSample.kt | 33 +++++++++++ .../room/store/models/DbFingerprintSample.kt | 36 ++++++++++++ .../records/room/store/models/DbSubject.kt | 38 +++++++++++++ .../room/store/models/DbSubjectWithSamples.kt | 19 +++++++ settings.gradle.kts | 1 + 11 files changed, 277 insertions(+) create mode 100644 infra/enrolment-records/room-store/build.gradle.kts create mode 100644 infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectDao.kt create mode 100644 infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt create mode 100644 infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabaseFactory.kt create mode 100644 infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFaceSample.kt create mode 100644 infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFingerprintSample.kt create mode 100644 infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt create mode 100644 infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubjectWithSamples.kt diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 834f703b55..b83003b372 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -32,6 +32,7 @@ jobs: infra:orchestrator-data infra:enrolment-records:repository infra:enrolment-records:realm-store + infra:enrolment-records:room-store infra:recent-user-activity infra:config-store infra:config-sync diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c03b0b807a..7e0d1b9b5a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -24,6 +24,7 @@ androidx_viewpager_version = "1.1.0" androidx_security_version = "1.0.0" androidx_annotation_version = "1.9.1" androidx_arch_core_version = "2.2.0" +androidx_paging_version = "3.3.6" matertial_version = "1.12.0" hilt_version = "2.56.2" @@ -104,6 +105,7 @@ androidX-appcompat-resource = { module = "androidx.appcompat:appcompat-resources androidX-Room-core = { module = "androidx.room:room-runtime", version.ref = "androidx_room_version" } androidX-Room-compiler = { module = "androidx.room:room-compiler", version.ref = "androidx_room_version" } androidX-Room-ktx = { module = "androidx.room:room-ktx", version.ref = "androidx_room_version" } +androidX-paging = { module = "androidx.paging:paging-runtime-ktx", version.ref = "androidx_paging_version" } #Lifecycle androidX-lifecycle = { module = "androidx.lifecycle:lifecycle-viewmodel-ktx", version.ref = "androidx_lifecycle_version" } androidX-lifecycle-scope = { module = "androidx.lifecycle:lifecycle-runtime-ktx", version.ref = "androidx_lifecycle_version" } diff --git a/infra/enrolment-records/room-store/build.gradle.kts b/infra/enrolment-records/room-store/build.gradle.kts new file mode 100644 index 0000000000..7169400be7 --- /dev/null +++ b/infra/enrolment-records/room-store/build.gradle.kts @@ -0,0 +1,13 @@ +plugins { + id("simprints.infra") + id("simprints.library.room") +} + +android { + namespace = "com.simprints.infra.enrolment.records.room.store" +} + +dependencies { + implementation(project(":infra:auth-store")) + implementation(libs.androidX.paging) +} diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectDao.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectDao.kt new file mode 100644 index 0000000000..240a9434e8 --- /dev/null +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectDao.kt @@ -0,0 +1,45 @@ +package com.simprints.infra.enrolment.records.room.store + +import androidx.paging.PagingSource +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy +import androidx.room.Query +import androidx.room.RawQuery +import androidx.room.Transaction +import androidx.sqlite.db.SupportSQLiteQuery +import com.simprints.infra.enrolment.records.room.store.models.DbFaceSample +import com.simprints.infra.enrolment.records.room.store.models.DbFingerprintSample +import com.simprints.infra.enrolment.records.room.store.models.DbSubject +import com.simprints.infra.enrolment.records.room.store.models.DbSubjectWithSamples + +@Dao +interface SubjectDao { + @Insert(onConflict = OnConflictStrategy.REPLACE) + suspend fun insertSubject(subject: DbSubject) + + @Insert(onConflict = OnConflictStrategy.REPLACE) + suspend fun insertFingerprintSamples(fingerprintSamples: List) + + @Insert(onConflict = OnConflictStrategy.REPLACE) + suspend fun insertFaceSamples(faceSamples: List) + + @Query("DELETE FROM DBSUBJECT WHERE subjectId = :subjectId") + suspend fun deleteSubject(subjectId: String) + + @Transaction + @RawQuery + suspend fun loadSubjects(query: SupportSQLiteQuery): List + + @Transaction + @RawQuery + suspend fun count(query: SupportSQLiteQuery): Int + + @Transaction + @RawQuery + fun getFaceSamples(query: SupportSQLiteQuery): PagingSource + + @Transaction + @RawQuery + fun getFingerprintSamples(query: SupportSQLiteQuery): PagingSource +} diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt new file mode 100644 index 0000000000..ba3783fcdc --- /dev/null +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt @@ -0,0 +1,33 @@ +package com.simprints.infra.enrolment.records.room.store + +import android.content.Context +import androidx.annotation.Keep +import androidx.room.Database +import androidx.room.Room +import androidx.room.RoomDatabase +import com.simprints.infra.enrolment.records.room.store.models.DbFaceSample +import com.simprints.infra.enrolment.records.room.store.models.DbFingerprintSample +import com.simprints.infra.enrolment.records.room.store.models.DbSubject +import net.sqlcipher.database.SupportFactory +import javax.inject.Singleton + +@Singleton +@Database(entities = [DbSubject::class, DbFingerprintSample::class, DbFaceSample::class], version = 1, exportSchema = true) +@Keep +abstract class SubjectsDatabase : RoomDatabase() { + abstract val subjectDao: SubjectDao + + companion object { + fun getDatabase( + context: Context, + factory: SupportFactory, + dbName: String, + ): SubjectsDatabase { + val builder = Room.databaseBuilder(context, SubjectsDatabase::class.java, dbName) + if (BuildConfig.DB_ENCRYPTION) { + builder.openHelperFactory(factory) + } + return builder.build() + } + } +} diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabaseFactory.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabaseFactory.kt new file mode 100644 index 0000000000..6933b2cb61 --- /dev/null +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabaseFactory.kt @@ -0,0 +1,56 @@ +package com.simprints.infra.enrolment.records.room.store + +import android.content.Context +import com.simprints.infra.logging.Simber +import com.simprints.infra.security.SecurityManager +import dagger.hilt.android.qualifiers.ApplicationContext +import net.sqlcipher.database.SQLiteDatabase +import net.sqlcipher.database.SupportFactory +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SubjectsDatabaseFactory @Inject constructor( + @ApplicationContext val ctx: Context, + private val secureLocalDbKeyProvider: SecurityManager, +) { + private lateinit var database: SubjectsDatabase + + /** + * Get the database instance. + * If the database is not initialized, it will be built. + */ + fun get(): SubjectsDatabase { + if (!::database.isInitialized) { + database = build() + } + return database + } + + @OptIn(ExperimentalStdlibApi::class) + private fun build(): SubjectsDatabase = try { + val key = getOrCreateKey() + val passphrase: ByteArray = SQLiteDatabase.getBytes(key) + val factory = SupportFactory(passphrase) + SubjectsDatabase.getDatabase( + ctx, + factory, + DB_NAME, + ) + } catch (t: Throwable) { + Simber.e("Error creating subject database", t) + throw t + } + + private fun getOrCreateKey(): CharArray = try { + secureLocalDbKeyProvider.getLocalDbKeyOrThrow(DB_NAME) + } catch (t: Throwable) { + t.message?.let { Simber.d(it) } + secureLocalDbKeyProvider.createLocalDatabaseKeyIfMissing(DB_NAME) + secureLocalDbKeyProvider.getLocalDbKeyOrThrow(DB_NAME) + }.value.decodeToString().toCharArray() + + companion object { + private const val DB_NAME = "db-subjects" + } +} diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFaceSample.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFaceSample.kt new file mode 100644 index 0000000000..7e2c257b50 --- /dev/null +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFaceSample.kt @@ -0,0 +1,33 @@ +package com.simprints.infra.enrolment.records.room.store.models + +import androidx.room.Entity +import androidx.room.ForeignKey +import androidx.room.Index +import androidx.room.PrimaryKey +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.FORMAT_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN + +@Entity( + tableName = "DbFaceSample", + foreignKeys = [ + ForeignKey( + entity = DbSubject::class, + parentColumns = [SUBJECT_ID_COLUMN], + childColumns = [SUBJECT_ID_COLUMN], + onDelete = ForeignKey.CASCADE, + ), + ], + indices = [ + Index(value = [FORMAT_COLUMN]), + Index(value = [SUBJECT_ID_COLUMN]), + ], +) +@Suppress("ArrayInDataClass") +data class DbFaceSample( + @PrimaryKey + val id: String, + val subjectId: String, + val template: ByteArray, + val format: String, + val referenceId: String, +) diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFingerprintSample.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFingerprintSample.kt new file mode 100644 index 0000000000..5c3d0312f2 --- /dev/null +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFingerprintSample.kt @@ -0,0 +1,36 @@ +package com.simprints.infra.enrolment.records.room.store.models + +import androidx.room.Entity +import androidx.room.ForeignKey +import androidx.room.Index +import androidx.room.PrimaryKey +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.FORMAT_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN + +@Entity( + tableName = "DbFingerprintSample", + foreignKeys = [ + ForeignKey( + entity = DbSubject::class, + parentColumns = [SUBJECT_ID_COLUMN], + childColumns = [SUBJECT_ID_COLUMN], + onDelete = ForeignKey.CASCADE, + ), + ], + indices = [ + Index(value = [FORMAT_COLUMN]), + Index(value = [SUBJECT_ID_COLUMN]), + + ], +) +@Suppress("ArrayInDataClass") +data class DbFingerprintSample( + @PrimaryKey + val id: String, + val subjectId: String, + val fingerIdentifier: Int, + val template: ByteArray, + val templateQualityScore: Int, + val format: String, + val referenceId: String, +) diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt new file mode 100644 index 0000000000..0fa1cee291 --- /dev/null +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt @@ -0,0 +1,38 @@ +package com.simprints.infra.enrolment.records.room.store.models + +import androidx.room.Entity +import androidx.room.Index +import androidx.room.PrimaryKey +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.ATTENDANT_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.MODULE_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.PROJECT_ID_COLUMN +import java.util.UUID + +@Entity( + tableName = "DbSubject", + indices = [ + Index(value = [PROJECT_ID_COLUMN]), + Index(value = [ATTENDANT_ID_COLUMN]), + Index(value = [MODULE_ID_COLUMN]), + ], +) +data class DbSubject( + @PrimaryKey + val subjectId: String = UUID.randomUUID().toString(), + val projectId: String? = "", + val attendantId: String? = "", + val moduleId: String? = "", + val createdAt: Long? = 0, + val updatedAt: Long? = 0, + val toSync: Boolean = false, + val isAttendantIdTokenized: Boolean = false, + val isModuleIdTokenized: Boolean = false, +) { + companion object { + const val SUBJECT_ID_COLUMN = "subjectId" + const val PROJECT_ID_COLUMN = "projectId" + const val ATTENDANT_ID_COLUMN = "attendantId" + const val MODULE_ID_COLUMN = "moduleId" + const val FORMAT_COLUMN = "format" + } +} diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubjectWithSamples.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubjectWithSamples.kt new file mode 100644 index 0000000000..a85dbfcaf5 --- /dev/null +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubjectWithSamples.kt @@ -0,0 +1,19 @@ +package com.simprints.infra.enrolment.records.room.store.models + +import androidx.room.Embedded +import androidx.room.Relation +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN + +data class DbSubjectWithSamples( + @Embedded val subject: DbSubject, + @Relation( + parentColumn = SUBJECT_ID_COLUMN, + entityColumn = SUBJECT_ID_COLUMN, + ) + val fingerprintSamples: List, + @Relation( + parentColumn = SUBJECT_ID_COLUMN, + entityColumn = SUBJECT_ID_COLUMN, + ) + val faceSamples: List, +) diff --git a/settings.gradle.kts b/settings.gradle.kts index f193228fe7..2cfa9bdb30 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -134,6 +134,7 @@ include( ":infra:config-sync", ":infra:enrolment-records:repository", ":infra:enrolment-records:realm-store", + ":infra:enrolment-records:room-store", ":infra:images", ":infra:license", ":infra:logging", From d22d470b7a07d4f412e6a57a65fc11f9fbe9ffe1 Mon Sep 17 00:00:00 2001 From: Melad Raouf Date: Sat, 5 Apr 2025 11:23:08 +0200 Subject: [PATCH 03/11] [MS-951] Refactor enrolment records data handling and update database schema for Room integration --- .../feature/dashboard/debug/DebugFragment.kt | 8 +- gradle/libs.versions.toml | 2 - .../simprints/core/domain/face/FaceSample.kt | 4 +- .../repository/build.gradle.kts | 2 + .../EnrolmentRecordRepositoryImpl.kt | 50 ++- .../repository/EnrolmentRecordsStoreModule.kt | 9 +- ...=> RealmEnrolmentRecordLocalDataSource.kt} | 20 +- .../RoomEnrolmentRecordLocalDataSource.kt | 221 ++++++++++++ ...ctEnrolmentRecordLocalDataSourceUseCase.kt | 13 + ...eSample.kt => RealmFaceSampleConverter.kt} | 6 +- ....kt => RealmFingerprintSampleConverter.kt} | 6 +- ...{DbSubject.kt => RealmSubjectConverter.kt} | 14 +- .../local/models/RoomFaceSampleConverter.kt | 19 ++ .../models/RoomFingerprintSampleConverter.kt | 22 ++ .../local/models/RoomSubjectConverter.kt | 40 +++ .../EnrolmentRecordRepositoryImplTest.kt | 13 +- ...ealmEnrolmentRecordLocalDataSourceTest.kt} | 41 +-- .../RoomEnrolmentRecordLocalDataSourceTest.kt | 318 ++++++++++++++++++ .../repository/local/models/DbSubjectTest.kt | 4 +- .../room-store/build.gradle.kts | 1 - .../1.json | 281 ++++++++++++++++ .../records/room/store/SubjectDao.kt | 194 ++++++++++- .../records/room/store/SubjectsDatabase.kt | 6 +- .../records/room/store/models/DbFaceSample.kt | 14 +- .../room/store/models/DbFingerprintSample.kt | 17 +- .../records/room/store/models/DbSubject.kt | 9 +- ...ectWithSamples.kt => SubjectBiometrics.kt} | 2 +- 27 files changed, 1213 insertions(+), 123 deletions(-) rename infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/{EnrolmentRecordLocalDataSourceImpl.kt => RealmEnrolmentRecordLocalDataSource.kt} (96%) create mode 100644 infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt create mode 100644 infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/SelectEnrolmentRecordLocalDataSourceUseCase.kt rename infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/{DbFaceSample.kt => RealmFaceSampleConverter.kt} (67%) rename infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/{DbFingerprintSample.kt => RealmFingerprintSampleConverter.kt} (70%) rename infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/{DbSubject.kt => RealmSubjectConverter.kt} (82%) create mode 100644 infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFaceSampleConverter.kt create mode 100644 infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFingerprintSampleConverter.kt create mode 100644 infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomSubjectConverter.kt rename infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/{EnrolmentRecordLocalDataSourceImplTest.kt => RealmEnrolmentRecordLocalDataSourceTest.kt} (93%) create mode 100644 infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSourceTest.kt create mode 100644 infra/enrolment-records/room-store/schemas/com.simprints.infra.enrolment.records.room.store.SubjectsDatabase/1.json rename infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/{DbSubjectWithSamples.kt => SubjectBiometrics.kt} (94%) diff --git a/feature/dashboard/src/main/java/com/simprints/feature/dashboard/debug/DebugFragment.kt b/feature/dashboard/src/main/java/com/simprints/feature/dashboard/debug/DebugFragment.kt index 06b797e647..3358fe7f4a 100644 --- a/feature/dashboard/src/main/java/com/simprints/feature/dashboard/debug/DebugFragment.kt +++ b/feature/dashboard/src/main/java/com/simprints/feature/dashboard/debug/DebugFragment.kt @@ -1,12 +1,12 @@ package com.simprints.feature.dashboard.debug -import android.graphics.Color import android.os.Bundle import android.text.Spannable import android.text.SpannableString import android.text.SpannableStringBuilder import android.text.style.ForegroundColorSpan import android.view.View +import androidx.core.graphics.toColorInt import androidx.fragment.app.Fragment import androidx.lifecycle.lifecycleScope import androidx.work.WorkManager @@ -14,7 +14,7 @@ import com.simprints.core.DispatcherIO import com.simprints.feature.dashboard.R import com.simprints.feature.dashboard.databinding.FragmentDebugBinding import com.simprints.infra.authstore.AuthStore -import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource +import com.simprints.infra.enrolment.records.repository.EnrolmentRecordRepository import com.simprints.infra.events.EventRepository import com.simprints.infra.eventsync.EventSyncManager import com.simprints.infra.eventsync.status.models.EventSyncWorkerState @@ -43,7 +43,7 @@ internal class DebugFragment : Fragment(R.layout.fragment_debug) { lateinit var eventRepository: EventRepository @Inject - lateinit var enrolmentRecordRepository: EnrolmentRecordLocalDataSource + lateinit var enrolmentRecordRepository: EnrolmentRecordRepository @Inject @DispatcherIO @@ -71,7 +71,7 @@ internal class DebugFragment : Fragment(R.layout.fragment_debug) { val ssb = SpannableStringBuilder( coloredText( "\n$message", - Color.parseColor(getRandomColor()), + getRandomColor().toColorInt(), ), ) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7e0d1b9b5a..c03b0b807a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -24,7 +24,6 @@ androidx_viewpager_version = "1.1.0" androidx_security_version = "1.0.0" androidx_annotation_version = "1.9.1" androidx_arch_core_version = "2.2.0" -androidx_paging_version = "3.3.6" matertial_version = "1.12.0" hilt_version = "2.56.2" @@ -105,7 +104,6 @@ androidX-appcompat-resource = { module = "androidx.appcompat:appcompat-resources androidX-Room-core = { module = "androidx.room:room-runtime", version.ref = "androidx_room_version" } androidX-Room-compiler = { module = "androidx.room:room-compiler", version.ref = "androidx_room_version" } androidX-Room-ktx = { module = "androidx.room:room-ktx", version.ref = "androidx_room_version" } -androidX-paging = { module = "androidx.paging:paging-runtime-ktx", version.ref = "androidx_paging_version" } #Lifecycle androidX-lifecycle = { module = "androidx.lifecycle:lifecycle-viewmodel-ktx", version.ref = "androidx_lifecycle_version" } androidX-lifecycle-scope = { module = "androidx.lifecycle:lifecycle-runtime-ktx", version.ref = "androidx_lifecycle_version" } diff --git a/infra/core/src/main/java/com/simprints/core/domain/face/FaceSample.kt b/infra/core/src/main/java/com/simprints/core/domain/face/FaceSample.kt index fbf1938b6b..57b44fd5f9 100644 --- a/infra/core/src/main/java/com/simprints/core/domain/face/FaceSample.kt +++ b/infra/core/src/main/java/com/simprints/core/domain/face/FaceSample.kt @@ -19,9 +19,7 @@ data class FaceSample( other as FaceSample - if (!template.contentEquals(other.template)) return false - - return true + return template.contentEquals(other.template) } override fun hashCode(): Int = template.contentHashCode() diff --git a/infra/enrolment-records/repository/build.gradle.kts b/infra/enrolment-records/repository/build.gradle.kts index 17bcdc6f6e..064cb2d26e 100644 --- a/infra/enrolment-records/repository/build.gradle.kts +++ b/infra/enrolment-records/repository/build.gradle.kts @@ -1,6 +1,7 @@ plugins { id("simprints.infra") id("kotlin-parcelize") + id("simprints.library.room") } android { @@ -11,6 +12,7 @@ dependencies { implementation(project(":infra:config-store")) implementation(project(":infra:auth-store")) implementation(project(":infra:enrolment-records:realm-store")) + implementation(project(":infra:enrolment-records:room-store")) implementation(project(":infra:events")) implementation(libs.libsimprints) diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt index d5f64e4d66..d69c6c44dd 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt @@ -9,9 +9,13 @@ import com.simprints.infra.config.store.models.TokenKeyType import com.simprints.infra.config.store.tokenization.TokenizationProcessor import com.simprints.infra.enrolment.records.realm.store.exceptions.RealmUninitialisedException import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource +import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity +import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity +import com.simprints.infra.enrolment.records.repository.domain.models.Subject import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAction import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource +import com.simprints.infra.enrolment.records.repository.local.SelectEnrolmentRecordLocalDataSourceUseCase import com.simprints.infra.enrolment.records.repository.remote.EnrolmentRecordRemoteDataSource import com.simprints.infra.logging.Simber import dagger.hilt.android.qualifiers.ApplicationContext @@ -20,42 +24,25 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.withContext import javax.inject.Inject -internal class EnrolmentRecordRepositoryImpl( - context: Context, +internal class EnrolmentRecordRepositoryImpl @Inject constructor( + @ApplicationContext context: Context, private val remoteDataSource: EnrolmentRecordRemoteDataSource, - private val localDataSource: EnrolmentRecordLocalDataSource, - private val commCareDataSource: IdentityDataSource, + @CommCareDataSource private val commCareDataSource: IdentityDataSource, private val tokenizationProcessor: TokenizationProcessor, - private val dispatcher: CoroutineDispatcher, + private val selectEnrolmentRecordLocalDataSource: SelectEnrolmentRecordLocalDataSourceUseCase, + @DispatcherIO private val dispatcher: CoroutineDispatcher, private val batchSize: Int, -) : EnrolmentRecordRepository, - EnrolmentRecordLocalDataSource by localDataSource { - @Inject - constructor( - @ApplicationContext context: Context, - remoteDataSource: EnrolmentRecordRemoteDataSource, - localDataSource: EnrolmentRecordLocalDataSource, - @CommCareDataSource commCareDataSource: IdentityDataSource, - tokenizationProcessor: TokenizationProcessor, - @DispatcherIO dispatcher: CoroutineDispatcher, - ) : this( - context = context, - remoteDataSource = remoteDataSource, - localDataSource = localDataSource, - commCareDataSource = commCareDataSource, - tokenizationProcessor = tokenizationProcessor, - dispatcher = dispatcher, - batchSize = BATCH_SIZE, - ) - +) : EnrolmentRecordRepository { private val prefs = context.getSharedPreferences(PREF_FILE_NAME, Context.MODE_PRIVATE) companion object { - private const val BATCH_SIZE = 80 + const val BATCH_SIZE = 80 private const val PREF_FILE_NAME = "UPLOAD_ENROLMENT_RECORDS_PROGRESS" private const val PROGRESS_KEY = "PROGRESS" } + private val localDataSource: EnrolmentRecordLocalDataSource by lazy { selectEnrolmentRecordLocalDataSource() } + override suspend fun uploadRecords(subjectIds: List) = withContext(dispatcher) { val lastUploadedRecord = prefs.getString(PROGRESS_KEY, null) var query = SubjectQuery(sort = true, afterSubjectId = lastUploadedRecord) @@ -155,4 +142,15 @@ internal class EnrolmentRecordRepositoryImpl( is BiometricDataSource.Simprints -> localDataSource is BiometricDataSource.CommCare -> commCareDataSource } + + override suspend fun load(query: SubjectQuery): List = localDataSource.load(query) + + override suspend fun delete(queries: List) = localDataSource.delete(queries) + + override suspend fun deleteAll() = localDataSource.deleteAll() + + override suspend fun performActions( + actions: List, + project: Project, + ) = localDataSource.performActions(actions, project) } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordsStoreModule.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordsStoreModule.kt index 5aca8df5ad..d6fb683ed4 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordsStoreModule.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordsStoreModule.kt @@ -5,9 +5,8 @@ import com.simprints.core.AvailableProcessors import com.simprints.core.DispatcherIO import com.simprints.core.tools.json.JsonHelper import com.simprints.core.tools.utils.EncodingUtils +import com.simprints.infra.enrolment.records.repository.EnrolmentRecordRepositoryImpl.Companion.BATCH_SIZE import com.simprints.infra.enrolment.records.repository.commcare.CommCareIdentityDataSource -import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource -import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSourceImpl import com.simprints.infra.enrolment.records.repository.remote.EnrolmentRecordRemoteDataSource import com.simprints.infra.enrolment.records.repository.remote.EnrolmentRecordRemoteDataSourceImpl import com.simprints.infra.enrolment.records.repository.usecases.CompareImplicitTokenizedStringsUseCase @@ -30,9 +29,6 @@ abstract class EnrolmentRecordsStoreModule { @Binds internal abstract fun bindEnrolmentRecordRepository(impl: EnrolmentRecordRepositoryImpl): EnrolmentRecordRepository - @Binds - internal abstract fun bindEnrolmentRecordLocalDataSource(impl: EnrolmentRecordLocalDataSourceImpl): EnrolmentRecordLocalDataSource - @Binds internal abstract fun bindEnrolmentRecordRemoteDataSource(impl: EnrolmentRecordRemoteDataSourceImpl): EnrolmentRecordRemoteDataSource } @@ -57,6 +53,9 @@ class IdentityDataSourceModule { context = context, dispatcher = dispatcher, ) + + @Provides + fun provideBatchSize(): Int = BATCH_SIZE } @Qualifier diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImpl.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt similarity index 96% rename from infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImpl.kt rename to infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt index 5f9347936a..85b9441b21 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImpl.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt @@ -15,8 +15,8 @@ import com.simprints.infra.enrolment.records.repository.domain.models.Fingerprin import com.simprints.infra.enrolment.records.repository.domain.models.Subject import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAction import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery -import com.simprints.infra.enrolment.records.repository.local.models.fromDbToDomain -import com.simprints.infra.enrolment.records.repository.local.models.fromDomainToDb +import com.simprints.infra.enrolment.records.repository.local.models.toDomain +import com.simprints.infra.enrolment.records.repository.local.models.toRealmDb import com.simprints.infra.logging.LoggingConstants.CrashReportTag.REALM_DB import com.simprints.infra.logging.Simber import io.realm.kotlin.MutableRealm @@ -30,8 +30,10 @@ import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.channels.ReceiveChannel import javax.inject.Inject +import javax.inject.Singleton -internal class EnrolmentRecordLocalDataSourceImpl @Inject constructor( +@Singleton +internal class RealmEnrolmentRecordLocalDataSource @Inject constructor( private val realmWrapper: RealmWrapper, private val tokenizationProcessor: TokenizationProcessor, @DispatcherIO private val dispatcher: CoroutineDispatcher, @@ -54,7 +56,7 @@ internal class EnrolmentRecordLocalDataSourceImpl @Inject constructor( .query(DbSubject::class) .buildRealmQueryForSubject(query) .find() - .map { dbSubject -> dbSubject.fromDbToDomain() } + .map { dbSubject -> dbSubject.toDomain() } } override fun loadFaceIdentities( @@ -111,7 +113,7 @@ internal class EnrolmentRecordLocalDataSourceImpl @Inject constructor( onCandidateLoaded() FingerprintIdentity( subject.subjectId.toString(), - subject.fingerprintSamples.map(DbFingerprintSample::fromDbToDomain), + subject.fingerprintSamples.map(DbFingerprintSample::toDomain), ) } } @@ -130,7 +132,7 @@ internal class EnrolmentRecordLocalDataSourceImpl @Inject constructor( onCandidateLoaded() FaceIdentity( subject.subjectId.toString(), - subject.faceSamples.map(DbFaceSample::fromDbToDomain), + subject.faceSamples.map(DbFaceSample::toDomain), ) } } @@ -179,7 +181,7 @@ internal class EnrolmentRecordLocalDataSourceImpl @Inject constructor( .copy( moduleId = action.subject.moduleId.tokenizeIfNecessary(TokenKeyType.ModuleId, project), attendantId = action.subject.attendantId.tokenizeIfNecessary(TokenKeyType.AttendantId, project), - ).fromDomainToDb() + ).toRealmDb() val dbSubject: DbSubject? = realm.findSubject(newSubject.subjectId) if (dbSubject != null) { @@ -209,10 +211,10 @@ internal class EnrolmentRecordLocalDataSourceImpl @Inject constructor( // Append new samples to the list of samples that remain after removing dbSubject.faceSamples = ( - faceSamplesMap[false].orEmpty() + action.faceSamplesToAdd.map { it.fromDomainToDb() } + faceSamplesMap[false].orEmpty() + action.faceSamplesToAdd.map { it.toRealmDb() } ).toRealmList() dbSubject.fingerprintSamples = ( - fingerprintSamplesMap[false].orEmpty() + action.fingerprintSamplesToAdd.map { it.fromDomainToDb() } + fingerprintSamplesMap[false].orEmpty() + action.fingerprintSamplesToAdd.map { it.toRealmDb() } ).toRealmList() faceSamplesMap[true]?.forEach { realm.delete(it) } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt new file mode 100644 index 0000000000..c1ae3b0dc9 --- /dev/null +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt @@ -0,0 +1,221 @@ +package com.simprints.infra.enrolment.records.repository.local + +import androidx.room.withTransaction +import androidx.sqlite.db.SimpleSQLiteQuery +import com.simprints.core.DispatcherIO +import com.simprints.infra.config.store.models.Project +import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource +import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity +import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity +import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAction +import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery +import com.simprints.infra.enrolment.records.repository.local.models.toDomain +import com.simprints.infra.enrolment.records.repository.local.models.toRoomDb +import com.simprints.infra.enrolment.records.room.store.SubjectDao +import com.simprints.infra.enrolment.records.room.store.SubjectsDatabase +import com.simprints.infra.enrolment.records.room.store.SubjectsDatabaseFactory +import com.simprints.infra.enrolment.records.room.store.models.DbSubject +import com.simprints.infra.logging.Simber +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.withContext +import javax.inject.Inject +import javax.inject.Singleton +import kotlin.system.measureTimeMillis +import kotlin.time.measureTimedValue +import com.simprints.infra.enrolment.records.repository.domain.models.Subject as SubjectDomain + +@Singleton +internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( + private val subjectsDatabaseFactory: SubjectsDatabaseFactory, + @DispatcherIO private val dispatcherIO: CoroutineDispatcher, +) : EnrolmentRecordLocalDataSource { + val database: SubjectsDatabase by lazy { subjectsDatabaseFactory.get() } + val subjectDao: SubjectDao by lazy { subjectsDatabaseFactory.get().subjectDao } + + override suspend fun load(query: SubjectQuery): List = withContext(dispatcherIO) { + val sqlQuery = buildSubjectQuery(query) + subjectDao.loadSubjects(SimpleSQLiteQuery(sqlQuery.first, sqlQuery.second.toTypedArray())).map { it.toDomain() } + } + + override suspend fun loadFingerprintIdentities( + query: SubjectQuery, + range: IntRange, + dataSource: BiometricDataSource, + project: Project, + onCandidateLoaded: () -> Unit, + ): List = withContext(dispatcherIO) { + subjectDao + .getSubjectsWithFingerprintSamples( + query.projectId, + query.subjectId, + query.subjectIds, + query.attendantId?.value, + query.moduleId?.value, + query.fingerprintSampleFormat, + range.first, + range.last, + ).map { + onCandidateLoaded() + FingerprintIdentity( + subjectId = it.key, + fingerprints = it.value.map { sample -> sample.toDomain() }, + ) + } + } + + override suspend fun loadFaceIdentities( + query: SubjectQuery, + range: IntRange, + dataSource: BiometricDataSource, + project: Project, + onCandidateLoaded: () -> Unit, + ): List = withContext(dispatcherIO) { + val result = measureTimedValue { + subjectDao + .getSubjectsWithFaceSamples( + query.projectId, + query.subjectId, + query.subjectIds, + query.attendantId?.value, + query.moduleId?.value, + query.faceSampleFormat, + range.first, + range.last, + ).map { + onCandidateLoaded() + FaceIdentity( + subjectId = it.key, + faces = it.value.map { sample -> sample.toDomain() }, + ) + } + } + log("loadFaceIdentities ${result.value.size} in ${result.duration.inWholeMilliseconds} ms") + return@withContext result.value + } + + private fun buildSubjectQuery(query: SubjectQuery): Pair> { + val (whereClause, args) = buildWhereClause(query) + val orderBy = if (query.sort) "ORDER BY s.subjectId ASC" else "" + val sql = + """ + SELECT * FROM DbSubject s + LEFT JOIN DbFingerprintSample fingerprint ON s.subjectId = fingerprint.subjectId + LEFT JOIN DbFaceSample face ON s.subjectId = face.subjectId + $whereClause + $orderBy + """.trimIndent() + return Pair(sql, args) + } + + private fun buildWhereClause(query: SubjectQuery): Pair> { + val whereClauses = mutableListOf() + val args = mutableListOf() + + query.projectId?.let { + whereClauses.add("s.projectId = ?") + args.add(it) + } + query.subjectId?.let { + whereClauses.add("s.subjectId = ?") + args.add(it) + } + query.subjectIds?.takeIf { it.isNotEmpty() }?.let { + whereClauses.add("s.subjectId IN (${it.joinToString(",") { "?" }})") + args.addAll(it) + } + query.attendantId?.let { + whereClauses.add("s.attendantId = ?") + args.add(it) + } + query.moduleId?.let { + whereClauses.add("s.moduleId = ?") + args.add(it) + } + if (query.hasUntokenizedFields == true) { + whereClauses.add("(s.isAttendantIdTokenized = 0 OR s.isModuleIdTokenized = 0)") + } + + val whereClause = if (whereClauses.isNotEmpty()) "WHERE ${whereClauses.joinToString(" AND ")}" else "" + return Pair(whereClause, args) + } + + override suspend fun delete(queries: List) { + database.withTransaction { + queries.forEach { + val (whereClause, args) = buildWhereClause(it) + val sql = "DELETE FROM DbSubject $whereClause" + subjectDao.deleteSubjects(SimpleSQLiteQuery(sql, args.toTypedArray())) + } + } + } + + override suspend fun deleteAll() { + subjectDao.deleteSubjects(SimpleSQLiteQuery("DELETE FROM DbSubject")) + } + + override suspend fun count( + query: SubjectQuery, + dataSource: BiometricDataSource, + ): Int = withContext(dispatcherIO) { + var result = 0 + val timeTaken = measureTimeMillis { + result = subjectDao.countSubjects() + } + log("count $result : $timeTaken ms") + result + } + + override suspend fun performActions( + actions: List, + project: Project, + ) { + val timeTaken = measureTimeMillis { + database.withTransaction { + actions.forEach { action -> + when (action) { + is SubjectAction.Creation -> createSubject(action.subject) + is SubjectAction.Update -> Unit + is SubjectAction.Deletion -> deleteSubject(action.subjectId) + } + } + } + } + log("performActions ${actions.size} in : $timeTaken ms") + } + + private suspend fun createSubject(subject: SubjectDomain) { + val subjectId = subject.subjectId + val dbSubject = DbSubject( + subjectId = subject.subjectId, + projectId = subject.projectId, + attendantId = subject.attendantId.value, + moduleId = subject.moduleId.value, + createdAt = subject.createdAt?.time, + updatedAt = subject.updatedAt?.time, + isAttendantIdTokenized = false, + isModuleIdTokenized = false, + ) + + subjectDao.insertSubject(dbSubject) + + // Insert fingerprints + val dbFingerprints = subject.fingerprintSamples.map { it.toRoomDb(subjectId) } + if (dbFingerprints.isNotEmpty()) { + subjectDao.insertFingerprintSamples(dbFingerprints) + } + + // Insert face samples + val dbFaces = subject.faceSamples.map { it.toRoomDb(subjectId) } + if (dbFaces.isNotEmpty()) { + subjectDao.insertFaceSamples(dbFaces) + } + } + + private suspend fun deleteSubject(subjectId: String) { + subjectDao.deleteSubject(subjectId) + } +} + +fun log(message: String) { + Simber.i(message, tag = "roomrecords") +} diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/SelectEnrolmentRecordLocalDataSourceUseCase.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/SelectEnrolmentRecordLocalDataSourceUseCase.kt new file mode 100644 index 0000000000..70db61710b --- /dev/null +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/SelectEnrolmentRecordLocalDataSourceUseCase.kt @@ -0,0 +1,13 @@ +package com.simprints.infra.enrolment.records.repository.local + +import javax.inject.Inject + +internal class SelectEnrolmentRecordLocalDataSourceUseCase @Inject constructor( + private val roomDataSource: RoomEnrolmentRecordLocalDataSource, + private val realmDataSource: RealmEnrolmentRecordLocalDataSource, +) { + operator fun invoke(): EnrolmentRecordLocalDataSource { + // Todo later we will add logic to select the data source + return roomDataSource + } +} diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/DbFaceSample.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmFaceSampleConverter.kt similarity index 67% rename from infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/DbFaceSample.kt rename to infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmFaceSampleConverter.kt index 00ef14df71..1cab7429a8 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/DbFaceSample.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmFaceSampleConverter.kt @@ -1,16 +1,16 @@ package com.simprints.infra.enrolment.records.repository.local.models import com.simprints.core.domain.face.FaceSample -import com.simprints.infra.enrolment.records.realm.store.models.DbFaceSample +import com.simprints.infra.enrolment.records.realm.store.models.DbFaceSample as RealmFaceSample -internal fun DbFaceSample.fromDbToDomain(): FaceSample = FaceSample( +internal fun RealmFaceSample.toDomain(): FaceSample = FaceSample( id = id, template = template, format = format, referenceId = referenceId, ) -internal fun FaceSample.fromDomainToDb(): DbFaceSample = DbFaceSample().also { sample -> +internal fun FaceSample.toRealmDb(): RealmFaceSample = RealmFaceSample().also { sample -> sample.id = id sample.referenceId = referenceId sample.template = template diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/DbFingerprintSample.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmFingerprintSampleConverter.kt similarity index 70% rename from infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/DbFingerprintSample.kt rename to infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmFingerprintSampleConverter.kt index 9b81bf369a..421c598613 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/DbFingerprintSample.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmFingerprintSampleConverter.kt @@ -2,9 +2,9 @@ package com.simprints.infra.enrolment.records.repository.local.models import com.simprints.core.domain.fingerprint.FingerprintSample import com.simprints.core.domain.fingerprint.IFingerIdentifier -import com.simprints.infra.enrolment.records.realm.store.models.DbFingerprintSample +import com.simprints.infra.enrolment.records.realm.store.models.DbFingerprintSample as RealmFingerprintSample -internal fun DbFingerprintSample.fromDbToDomain(): FingerprintSample = FingerprintSample( +internal fun RealmFingerprintSample.toDomain(): FingerprintSample = FingerprintSample( id = id, fingerIdentifier = IFingerIdentifier.entries[fingerIdentifier], template = template, @@ -12,7 +12,7 @@ internal fun DbFingerprintSample.fromDbToDomain(): FingerprintSample = Fingerpri referenceId = referenceId, ) -internal fun FingerprintSample.fromDomainToDb(): DbFingerprintSample = DbFingerprintSample().also { sample -> +internal fun FingerprintSample.toRealmDb(): RealmFingerprintSample = RealmFingerprintSample().also { sample -> sample.id = id sample.referenceId = referenceId sample.fingerIdentifier = fingerIdentifier.ordinal diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/DbSubject.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmSubjectConverter.kt similarity index 82% rename from infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/DbSubject.kt rename to infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmSubjectConverter.kt index 7cace06d8c..cb93aa32b7 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/DbSubject.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmSubjectConverter.kt @@ -7,14 +7,14 @@ import com.simprints.core.domain.tokenization.asTokenizableRaw import com.simprints.core.domain.tokenization.isTokenized import com.simprints.infra.enrolment.records.realm.store.models.DbFaceSample import com.simprints.infra.enrolment.records.realm.store.models.DbFingerprintSample -import com.simprints.infra.enrolment.records.realm.store.models.DbSubject import com.simprints.infra.enrolment.records.realm.store.models.toDate import com.simprints.infra.enrolment.records.realm.store.models.toRealmInstant import com.simprints.infra.enrolment.records.repository.domain.models.Subject import io.realm.kotlin.ext.toRealmList import io.realm.kotlin.types.RealmUUID +import com.simprints.infra.enrolment.records.realm.store.models.DbSubject as RealmSubject -internal fun DbSubject.fromDbToDomain(): Subject { +internal fun RealmSubject.toDomain(): Subject { val attendantId = if (isAttendantIdTokenized) attendantId.asTokenizableEncrypted() else attendantId.asTokenizableRaw() val moduleId = @@ -27,12 +27,12 @@ internal fun DbSubject.fromDbToDomain(): Subject { moduleId = moduleId, createdAt = createdAt?.toDate(), updatedAt = updatedAt?.toDate(), - fingerprintSamples = fingerprintSamples.map(DbFingerprintSample::fromDbToDomain), - faceSamples = faceSamples.map(DbFaceSample::fromDbToDomain), + fingerprintSamples = fingerprintSamples.map(DbFingerprintSample::toDomain), + faceSamples = faceSamples.map(DbFaceSample::toDomain), ) } -internal fun Subject.fromDomainToDb(): DbSubject = DbSubject().also { subject -> +internal fun Subject.toRealmDb(): RealmSubject = RealmSubject().also { subject -> subject.subjectId = RealmUUID.from(subjectId) subject.projectId = projectId subject.attendantId = attendantId.value @@ -40,8 +40,8 @@ internal fun Subject.fromDomainToDb(): DbSubject = DbSubject().also { subject -> subject.createdAt = createdAt?.toRealmInstant() subject.updatedAt = updatedAt?.toRealmInstant() subject.fingerprintSamples = - fingerprintSamples.map(FingerprintSample::fromDomainToDb).toRealmList() - subject.faceSamples = faceSamples.map(FaceSample::fromDomainToDb).toRealmList() + fingerprintSamples.map(FingerprintSample::toRealmDb).toRealmList() + subject.faceSamples = faceSamples.map(FaceSample::toRealmDb).toRealmList() subject.isModuleIdTokenized = moduleId.isTokenized() subject.isAttendantIdTokenized = attendantId.isTokenized() } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFaceSampleConverter.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFaceSampleConverter.kt new file mode 100644 index 0000000000..a1a24949a7 --- /dev/null +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFaceSampleConverter.kt @@ -0,0 +1,19 @@ +package com.simprints.infra.enrolment.records.repository.local.models + +import com.simprints.core.domain.face.FaceSample +import com.simprints.infra.enrolment.records.room.store.models.DbFaceSample as RoomFaceSample + +internal fun RoomFaceSample.toDomain(): FaceSample = FaceSample( + id = uuid, + template = template, + format = format, + referenceId = referenceId, +) + +internal fun FaceSample.toRoomDb(subjectId: String): RoomFaceSample = RoomFaceSample( + uuid = id, + template = template, + format = format, + subjectId = subjectId, + referenceId = referenceId, +) diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFingerprintSampleConverter.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFingerprintSampleConverter.kt new file mode 100644 index 0000000000..3c7b809e3d --- /dev/null +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFingerprintSampleConverter.kt @@ -0,0 +1,22 @@ +package com.simprints.infra.enrolment.records.repository.local.models + +import com.simprints.core.domain.fingerprint.FingerprintSample +import com.simprints.core.domain.fingerprint.IFingerIdentifier +import com.simprints.infra.enrolment.records.room.store.models.DbFingerprintSample as RoomFingerprintSample + +internal fun RoomFingerprintSample.toDomain(): FingerprintSample = FingerprintSample( + id = uuid, + fingerIdentifier = IFingerIdentifier.entries[fingerIdentifier], + template = template, + format = format, + referenceId = referenceId, +) + +internal fun FingerprintSample.toRoomDb(subjectId: String) = RoomFingerprintSample( + uuid = id, + fingerIdentifier = fingerIdentifier.ordinal, + template = template, + format = format, + subjectId = subjectId, + referenceId = referenceId, +) diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomSubjectConverter.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomSubjectConverter.kt new file mode 100644 index 0000000000..7098427754 --- /dev/null +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomSubjectConverter.kt @@ -0,0 +1,40 @@ +package com.simprints.infra.enrolment.records.repository.local.models + +import com.simprints.core.domain.tokenization.asTokenizableEncrypted +import com.simprints.core.domain.tokenization.asTokenizableRaw +import com.simprints.core.domain.tokenization.isTokenized +import com.simprints.infra.enrolment.records.repository.domain.models.Subject +import com.simprints.infra.enrolment.records.room.store.models.SubjectBiometrics +import java.util.Date +import com.simprints.infra.enrolment.records.room.store.models.DbSubject as RoomSubject + +internal fun SubjectBiometrics.toDomain(): Subject { + val attendantId = + if (subject.isAttendantIdTokenized) subject.attendantId.asTokenizableEncrypted() else subject.attendantId.asTokenizableRaw() + val moduleId = + if (subject.isModuleIdTokenized) subject.moduleId.asTokenizableEncrypted() else subject.moduleId.asTokenizableRaw() + + return Subject( + subjectId = subject.subjectId.toString(), + projectId = subject.projectId, + attendantId = attendantId, + moduleId = moduleId, + createdAt = subject.createdAt?.toDate(), + updatedAt = subject.updatedAt?.toDate(), + fingerprintSamples = fingerprintSamples.map { it.toDomain() }, + faceSamples = faceSamples.map { it.toDomain() }, + ) +} + +internal fun Subject.toRoomDb(): RoomSubject = RoomSubject( + subjectId = subjectId, + projectId = projectId, + attendantId = attendantId.value, + moduleId = moduleId.value, + createdAt = createdAt?.time, + updatedAt = updatedAt?.time, + isModuleIdTokenized = moduleId.isTokenized(), + isAttendantIdTokenized = attendantId.isTokenized(), +) + +fun Long.toDate() = Date(this) diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImplTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImplTest.kt index ff0ba31bbc..ae5d863b64 100644 --- a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImplTest.kt +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImplTest.kt @@ -14,6 +14,7 @@ import com.simprints.infra.enrolment.records.repository.domain.models.Subject import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAction import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource +import com.simprints.infra.enrolment.records.repository.local.SelectEnrolmentRecordLocalDataSourceUseCase import com.simprints.infra.enrolment.records.repository.remote.EnrolmentRecordRemoteDataSource import io.mockk.* import kotlinx.coroutines.ExperimentalCoroutinesApi @@ -55,6 +56,7 @@ class EnrolmentRecordRepositoryImplTest { private val onCandidateLoaded: () -> Unit = {} private val tokenizationProcessor = mockk() private val localDataSource = mockk(relaxed = true) + private val selectEnrolmentRecordLocalDataSource = mockk() private val commCareDataSource = mockk(relaxed = true) private val remoteDataSource = mockk(relaxed = true) private val prefsEditor = mockk(relaxed = true) @@ -72,11 +74,11 @@ class EnrolmentRecordRepositoryImplTest { fun setup() { every { prefsEditor.putString(any(), any()) } returns prefsEditor every { prefsEditor.remove(any()) } returns prefsEditor - + every { selectEnrolmentRecordLocalDataSource() } returns localDataSource repository = EnrolmentRecordRepositoryImpl( context = ctx, remoteDataSource = remoteDataSource, - localDataSource = localDataSource, + selectEnrolmentRecordLocalDataSource = selectEnrolmentRecordLocalDataSource, commCareDataSource = commCareDataSource, tokenizationProcessor = tokenizationProcessor, dispatcher = UnconfinedTestDispatcher(), @@ -390,12 +392,7 @@ class EnrolmentRecordRepositoryImplTest { assert(faceIdentities == expectedFaceIdentities) coVerify(exactly = 1) { - localDataSource.loadFaceIdentities( - expectedSubjectQuery, - expectedRange, - any(), - project, - this@runTest, + localDataSource.loadFaceIdentities(expectedSubjectQuery, expectedRange, any(), project,this@runTest, onCandidateLoaded, ) } diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImplTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSourceTest.kt similarity index 93% rename from infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImplTest.kt rename to infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSourceTest.kt index b855586d02..e6e361d1d0 100644 --- a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/EnrolmentRecordLocalDataSourceImplTest.kt +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSourceTest.kt @@ -18,12 +18,15 @@ import com.simprints.infra.enrolment.records.repository.domain.models.Fingerprin import com.simprints.infra.enrolment.records.repository.domain.models.Subject import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAction import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery -import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSourceImpl.Companion.FACE_SAMPLES_FIELD -import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSourceImpl.Companion.FINGERPRINT_SAMPLES_FIELD -import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSourceImpl.Companion.FORMAT_FIELD -import com.simprints.infra.enrolment.records.repository.local.models.fromDbToDomain -import com.simprints.infra.enrolment.records.repository.local.models.fromDomainToDb -import io.mockk.* +import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource.Companion.FACE_SAMPLES_FIELD +import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource.Companion.FINGERPRINT_SAMPLES_FIELD +import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource.Companion.FORMAT_FIELD +import com.simprints.infra.enrolment.records.repository.local.models.toDomain +import com.simprints.infra.enrolment.records.repository.local.models.toRealmDb +import io.mockk.CapturingSlot +import io.mockk.MockKAnnotations +import io.mockk.coEvery +import io.mockk.every import io.mockk.impl.annotations.MockK import io.realm.kotlin.MutableRealm import io.realm.kotlin.Realm @@ -38,7 +41,7 @@ import org.junit.Test import java.util.UUID import kotlin.random.Random -class EnrolmentRecordLocalDataSourceImplTest { +class RealmEnrolmentRecordLocalDataSourceTest { @MockK private lateinit var realm: Realm @@ -77,7 +80,7 @@ class EnrolmentRecordLocalDataSourceImplTest { every { mutableRealm.delete(any()) } answers { localSubjects.clear() } every { mutableRealm.deleteAll() } answers { localSubjects.clear() } every { mutableRealm.copyToRealm(capture(insertedSubject), any()) } answers { - localSubjects.add(insertedSubject.captured.fromDbToDomain()) + localSubjects.add(insertedSubject.captured.toDomain()) insertedSubject.captured } @@ -99,7 +102,7 @@ class EnrolmentRecordLocalDataSourceImplTest { every { realmQuery.query(any(), any()) } returns realmQuery every { realmQuery.first() } returns realmSingleQuery - enrolmentRecordLocalDataSource = EnrolmentRecordLocalDataSourceImpl( + enrolmentRecordLocalDataSource = RealmEnrolmentRecordLocalDataSource( realmWrapperMock, tokenizationProcessor, UnconfinedTestDispatcher(), @@ -133,7 +136,7 @@ class EnrolmentRecordLocalDataSourceImplTest { @Test fun givenValidSerializableQueryForFingerprints_loadIsCalled() = runTest { val savedPersons = saveFakePeople(getRandomPeople(20)) - val fakePerson = savedPersons[0].fromDomainToDb() + val fakePerson = savedPersons[0].toRealmDb() val people = mutableListOf() enrolmentRecordLocalDataSource @@ -197,7 +200,7 @@ class EnrolmentRecordLocalDataSourceImplTest { @Test fun givenValidSerializableQueryForFace_loadIsCalled() = runTest { val savedPersons = saveFakePeople(getRandomPeople(20)) - val fakePerson = savedPersons[0].fromDomainToDb() + val fakePerson = savedPersons[0].toRealmDb() val people = mutableListOf() enrolmentRecordLocalDataSource @@ -224,29 +227,29 @@ class EnrolmentRecordLocalDataSourceImplTest { val people = enrolmentRecordLocalDataSource.load(SubjectQuery()).toList() listOf(fakePerson).zip(people).forEach { (dbSubject, subject) -> - assertThat(dbSubject.deepEquals(subject.fromDomainToDb())).isTrue() + assertThat(dbSubject.deepEquals(subject.toRealmDb())).isTrue() } } @Test fun givenManyPeopleSaved_loadByUserIdShouldReturnTheRightPeople() = runTest { val savedPersons = saveFakePeople(getRandomPeople(20)) - val fakePerson = savedPersons[0].fromDomainToDb() + val fakePerson = savedPersons[0].toRealmDb() val people = enrolmentRecordLocalDataSource.load(SubjectQuery(attendantId = savedPersons[0].attendantId)).toList() listOf(fakePerson).zip(people).forEach { (dbSubject, subject) -> - assertThat(dbSubject.deepEquals(subject.fromDomainToDb())).isTrue() + assertThat(dbSubject.deepEquals(subject.toRealmDb())).isTrue() } } @Test fun givenManyPeopleSaved_loadByModuleIdShouldReturnTheRightPeople() = runTest { val savedPersons = saveFakePeople(getRandomPeople(20)) - val fakePerson = savedPersons[0].fromDomainToDb() + val fakePerson = savedPersons[0].toRealmDb() val people = enrolmentRecordLocalDataSource.load(SubjectQuery(moduleId = fakePerson.moduleId.asTokenizableEncrypted())).toList() listOf(fakePerson).zip(people).forEach { (dbSubject, subject) -> - assertThat(dbSubject.deepEquals(subject.fromDomainToDb())).isTrue() + assertThat(dbSubject.deepEquals(subject.toRealmDb())).isTrue() } } @@ -277,7 +280,7 @@ class EnrolmentRecordLocalDataSourceImplTest { val subject = getFakePerson() enrolmentRecordLocalDataSource.performActions( - listOf(SubjectAction.Creation(subject.fromDbToDomain())), + listOf(SubjectAction.Creation(subject.toDomain())), project, ) @@ -366,9 +369,9 @@ class EnrolmentRecordLocalDataSourceImplTest { assertThat(peopleCount).isEqualTo(0) } - private fun getFakePerson(): DbSubject = getRandomSubject().fromDomainToDb() + private fun getFakePerson(): DbSubject = getRandomSubject().toRealmDb() - private fun saveFakePerson(fakeSubject: DbSubject): DbSubject = fakeSubject.also { localSubjects.add(it.fromDbToDomain()) } + private fun saveFakePerson(fakeSubject: DbSubject): DbSubject = fakeSubject.also { localSubjects.add(it.toDomain()) } private fun saveFakePeople(subjects: List): List = subjects.toMutableList().also { localSubjects.addAll(it) } diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSourceTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSourceTest.kt new file mode 100644 index 0000000000..a7847ca370 --- /dev/null +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSourceTest.kt @@ -0,0 +1,318 @@ +package com.simprints.infra.enrolment.records.repository.local + +import com.google.common.truth.Truth +import com.simprints.core.domain.face.FaceSample +import com.simprints.core.domain.tokenization.asTokenizableEncrypted +import com.simprints.core.domain.tokenization.asTokenizableRaw +import com.simprints.infra.config.store.models.Project +import com.simprints.infra.config.store.tokenization.TokenizationProcessor +import com.simprints.infra.enrolment.records.realm.store.RealmWrapper +import com.simprints.infra.enrolment.records.realm.store.models.DbSubject +import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource +import com.simprints.infra.enrolment.records.repository.domain.models.Subject +import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAction +import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery +import com.simprints.infra.enrolment.records.repository.local.models.toDomain +import com.simprints.infra.enrolment.records.repository.local.models.toRealmDb +import io.mockk.CapturingSlot +import io.mockk.MockKAnnotations +import io.mockk.coEvery +import io.mockk.every +import io.mockk.impl.annotations.MockK +import io.mockk.mockk +import io.mockk.slot +import io.mockk.verify +import io.realm.kotlin.MutableRealm +import io.realm.kotlin.Realm +import io.realm.kotlin.query.RealmQuery +import kotlinx.coroutines.test.runTest +import org.junit.Before +import org.junit.Test +import java.util.UUID +import kotlin.random.Random + +class RealmEnrolmentRecordLocalDataSourceTest { + @MockK + private lateinit var realm: Realm + + @MockK + private lateinit var mutableRealm: MutableRealm + + @MockK + private lateinit var realmWrapperMock: RealmWrapper + + @MockK + private lateinit var realmQuery: RealmQuery + + @MockK + private lateinit var tokenizationProcessor: TokenizationProcessor + + @MockK + private lateinit var project: Project + + private lateinit var blockCapture: CapturingSlot<(Realm) -> Any> + private lateinit var mutableBlockCapture: CapturingSlot<(MutableRealm) -> Any> + private val onCandidateLoaded: () -> Unit = {} + private var localSubjects: MutableList = mutableListOf() + + private lateinit var enrolmentRecordLocalDataSource: EnrolmentRecordLocalDataSource + + @Before + fun setup() { + MockKAnnotations.init(this, relaxed = true) + localSubjects = mutableListOf() + + val insertedSubject = slot() + every { mutableRealm.delete(any()) } answers { localSubjects.clear() } + every { mutableRealm.deleteAll() } answers { localSubjects.clear() } + every { mutableRealm.copyToRealm(capture(insertedSubject), any()) } answers { + localSubjects.add(insertedSubject.captured.toDomain()) + insertedSubject.captured + } + + blockCapture = slot() + coEvery { realmWrapperMock.readRealm(capture(blockCapture)) } answers { + blockCapture.captured.invoke(realm) + } + mutableBlockCapture = slot() + coEvery { realmWrapperMock.writeRealm(capture(mutableBlockCapture)) } answers { + mutableBlockCapture.captured.invoke(mutableRealm) + } + every { realmQuery.count() } answers { + mockk { every { find() } returns localSubjects.size.toLong() } + } + + every { realm.query(DbSubject::class) } returns realmQuery + every { mutableRealm.query(DbSubject::class) } returns realmQuery + + enrolmentRecordLocalDataSource = RealmEnrolmentRecordLocalDataSource(realmWrapperMock, tokenizationProcessor) + } + + @Test + fun givenOneRecordSaved_countShouldReturnOne() = runTest { + saveFakePerson(getFakePerson()) + + val count = enrolmentRecordLocalDataSource.count() + Truth.assertThat(count).isEqualTo(1) + } + + @Test + fun givenManyPeopleSaved_countShouldReturnMany() = runTest { + saveFakePeople(getRandomPeople(20)) + + val count = enrolmentRecordLocalDataSource.count() + Truth.assertThat(count).isEqualTo(20) + } + + @Test + fun givenManyPeopleSaved_countByProjectIdShouldReturnTheRightTotal() = runTest { + saveFakePeople(getRandomPeople(20)) + + val count = enrolmentRecordLocalDataSource.count() + Truth.assertThat(count).isEqualTo(20) + } + + @Test + fun givenValidSerializableQueryForFingerprints_loadIsCalled() = runTest { + val savedPersons = saveFakePeople(getRandomPeople(20)) + val fakePerson = savedPersons[0].toRealmDb() + + val people = enrolmentRecordLocalDataSource + .loadFingerprintIdentities( + SubjectQuery(), + IntRange(0, 20), + BiometricDataSource.Simprints, + project, + onCandidateLoaded, + ).toList() + + listOf(fakePerson).zip(people).forEach { (subject, identity) -> + Truth.assertThat(subject.subjectId).isEqualTo(identity.subjectId) + } + } + + @Test + fun `correctly query supported fingerprint format`() = runTest { + val format = "SupportedFormat" + + enrolmentRecordLocalDataSource + .loadFingerprintIdentities( + SubjectQuery(fingerprintSampleFormat = format), + IntRange(0, 20), + BiometricDataSource.Simprints, + project, + onCandidateLoaded, + ).toList() + + verify { + realmQuery.query( + "ANY ${EnrolmentRecordLocalDataSource.Companion.FINGERPRINT_SAMPLES_FIELD}.${EnrolmentRecordLocalDataSource.Companion.FORMAT_FIELD} == $0", + format, + ) + } + } + + @Test + fun `correctly query supported face format`() = runTest { + val format = "SupportedFormat" + + enrolmentRecordLocalDataSource + .loadFingerprintIdentities( + SubjectQuery(faceSampleFormat = format), + IntRange(0, 20), + BiometricDataSource.Simprints, + project, + onCandidateLoaded, + ).toList() + + verify { + realmQuery.query( + "ANY ${EnrolmentRecordLocalDataSource.Companion.FACE_SAMPLES_FIELD}.${EnrolmentRecordLocalDataSource.Companion.FORMAT_FIELD} == $0", + format, + ) + } + } + + @Test + fun givenValidSerializableQueryForFace_loadIsCalled() = runTest { + val savedPersons = saveFakePeople(getRandomPeople(20)) + val fakePerson = savedPersons[0].toRealmDb() + + val people = enrolmentRecordLocalDataSource + .loadFaceIdentities( + SubjectQuery(), + IntRange(0, 20), + BiometricDataSource.Simprints, + project, + onCandidateLoaded, + ).toList() + + listOf(fakePerson).zip(people).forEach { (subject, identity) -> + Truth.assertThat(subject.subjectId).isEqualTo(identity.subjectId) + } + } + + @Test + fun givenManyPeopleSaved_loadShouldReturnThem() = runTest { + val fakePerson = getFakePerson() + saveFakePerson(fakePerson) + + val people = enrolmentRecordLocalDataSource.load(SubjectQuery()).toList() + + listOf(fakePerson).zip(people).forEach { (dbSubject, subject) -> + Truth.assertThat(dbSubject.deepEquals(subject.toRealmDb())).isTrue() + } + } + + @Test + fun givenManyPeopleSaved_loadByUserIdShouldReturnTheRightPeople() = runTest { + val savedPersons = saveFakePeople(getRandomPeople(20)) + val fakePerson = savedPersons[0].toRealmDb() + + val people = + enrolmentRecordLocalDataSource + .load(SubjectQuery(attendantId = savedPersons[0].attendantId)) + .toList() + listOf(fakePerson).zip(people).forEach { (dbSubject, subject) -> + Truth.assertThat(dbSubject.deepEquals(subject.toRealmDb())).isTrue() + } + } + + @Test + fun givenManyPeopleSaved_loadByModuleIdShouldReturnTheRightPeople() = runTest { + val savedPersons = saveFakePeople(getRandomPeople(20)) + val fakePerson = savedPersons[0].toRealmDb() + + val people = + enrolmentRecordLocalDataSource + .load(SubjectQuery(moduleId = fakePerson.moduleId.asTokenizableEncrypted())) + .toList() + listOf(fakePerson).zip(people).forEach { (dbSubject, subject) -> + Truth.assertThat(dbSubject.deepEquals(subject.toRealmDb())).isTrue() + } + } + + @Test + fun performSubjectCreationAction() = runTest { + val subject = getFakePerson() + enrolmentRecordLocalDataSource.performActions( + listOf(SubjectAction.Creation(subject.toDomain())), + project, + ) + val peopleCount = enrolmentRecordLocalDataSource.count() + Truth.assertThat(peopleCount).isEqualTo(1) + } + + @Test + fun performSubjectDeletionAction() = runTest { + val subject = getFakePerson() + saveFakePerson(subject) + enrolmentRecordLocalDataSource.performActions( + listOf(SubjectAction.Deletion(subject.subjectId.toString())), + project, + ) + val peopleCount = enrolmentRecordLocalDataSource.count() + Truth.assertThat(peopleCount).isEqualTo(0) + } + + @Test + fun performNoAction() = runTest { + val subject = getFakePerson() + saveFakePerson(subject) + enrolmentRecordLocalDataSource.performActions( + listOf(), + project, + ) + val peopleCount = enrolmentRecordLocalDataSource.count() + Truth.assertThat(peopleCount).isEqualTo(1) + } + + @Test + fun shouldDeleteAllSubjects() = runTest { + saveFakePeople(getRandomPeople(5)) + + enrolmentRecordLocalDataSource.deleteAll() + + val peopleCount = enrolmentRecordLocalDataSource.count() + Truth.assertThat(peopleCount).isEqualTo(0) + } + + private fun getFakePerson(): DbSubject = getRandomSubject().toRealmDb() + + private fun saveFakePerson(fakeSubject: DbSubject): DbSubject = fakeSubject.also { localSubjects.add(it.toDomain()) } + + private fun saveFakePeople(subjects: List): List = subjects.toMutableList().also { localSubjects.addAll(it) } + + private fun DbSubject.deepEquals(other: DbSubject): Boolean = when { + this.subjectId != other.subjectId -> false + this.projectId != other.projectId -> false + this.attendantId != other.attendantId -> false + this.moduleId != other.moduleId -> false + this.createdAt != other.createdAt -> false + this.updatedAt != other.updatedAt -> false + else -> true + } + + private fun getRandomPeople(numberOfPeople: Int): ArrayList = arrayListOf().also { list -> + repeat(numberOfPeople) { + list.add(getRandomSubject(UUID.randomUUID().toString())) + } + } + + private fun getRandomSubject( + patientId: String = UUID.randomUUID().toString(), + projectId: String = UUID.randomUUID().toString(), + userId: String = UUID.randomUUID().toString(), + moduleId: String = UUID.randomUUID().toString(), + faceSamples: Array = arrayOf( + FaceSample(Random.Default.nextBytes(64), "faceTemplateFormat"), + FaceSample(Random.Default.nextBytes(64), "faceTemplateFormat"), + ), + ): Subject = Subject( + subjectId = patientId, + projectId = projectId, + attendantId = userId.asTokenizableRaw(), + moduleId = moduleId.asTokenizableRaw(), + faceSamples = faceSamples.toList(), + ) +} diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/models/DbSubjectTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/models/DbSubjectTest.kt index 3986a3cb69..74bd034aee 100644 --- a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/models/DbSubjectTest.kt +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/models/DbSubjectTest.kt @@ -46,7 +46,7 @@ class DbSubjectTest { faceSamples = listOf(faceSample), ) - val dbSubject = domainSubject.fromDomainToDb() + val dbSubject = domainSubject.toRealmDb() with(dbSubject) { assertThat(subjectId).isEqualTo(RealmUUID.from(GUID)) @@ -89,7 +89,7 @@ class DbSubjectTest { isAttendantIdTokenized = true } - val domainSubject = dbSubject.fromDbToDomain() + val domainSubject = dbSubject.toDomain() with(domainSubject) { assertThat(subjectId).isEqualTo(GUID) diff --git a/infra/enrolment-records/room-store/build.gradle.kts b/infra/enrolment-records/room-store/build.gradle.kts index 7169400be7..fdac1ec1fd 100644 --- a/infra/enrolment-records/room-store/build.gradle.kts +++ b/infra/enrolment-records/room-store/build.gradle.kts @@ -9,5 +9,4 @@ android { dependencies { implementation(project(":infra:auth-store")) - implementation(libs.androidX.paging) } diff --git a/infra/enrolment-records/room-store/schemas/com.simprints.infra.enrolment.records.room.store.SubjectsDatabase/1.json b/infra/enrolment-records/room-store/schemas/com.simprints.infra.enrolment.records.room.store.SubjectsDatabase/1.json new file mode 100644 index 0000000000..80c5a6e76a --- /dev/null +++ b/infra/enrolment-records/room-store/schemas/com.simprints.infra.enrolment.records.room.store.SubjectsDatabase/1.json @@ -0,0 +1,281 @@ +{ + "formatVersion": 1, + "database": { + "version": 1, + "identityHash": "f2e5005e3569ee84f0bd388c5485c5bf", + "entities": [ + { + "tableName": "DbSubject", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`subjectId` TEXT NOT NULL, `projectId` TEXT NOT NULL, `attendantId` TEXT NOT NULL, `moduleId` TEXT NOT NULL, `createdAt` INTEGER, `updatedAt` INTEGER, `isAttendantIdTokenized` INTEGER NOT NULL, `isModuleIdTokenized` INTEGER NOT NULL, PRIMARY KEY(`subjectId`))", + "fields": [ + { + "fieldPath": "subjectId", + "columnName": "subjectId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "projectId", + "columnName": "projectId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "attendantId", + "columnName": "attendantId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "moduleId", + "columnName": "moduleId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "updatedAt", + "columnName": "updatedAt", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "isAttendantIdTokenized", + "columnName": "isAttendantIdTokenized", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isModuleIdTokenized", + "columnName": "isModuleIdTokenized", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "subjectId" + ] + }, + "indices": [ + { + "name": "index_DbSubject_subjectId", + "unique": false, + "columnNames": [ + "subjectId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_subjectId` ON `${TABLE_NAME}` (`subjectId`)" + }, + { + "name": "index_DbSubject_projectId", + "unique": false, + "columnNames": [ + "projectId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_projectId` ON `${TABLE_NAME}` (`projectId`)" + }, + { + "name": "index_DbSubject_attendantId", + "unique": false, + "columnNames": [ + "attendantId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_attendantId` ON `${TABLE_NAME}` (`attendantId`)" + }, + { + "name": "index_DbSubject_moduleId", + "unique": false, + "columnNames": [ + "moduleId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_moduleId` ON `${TABLE_NAME}` (`moduleId`)" + } + ], + "foreignKeys": [] + }, + { + "tableName": "DbFingerprintSample", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`rowId` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `uuid` TEXT NOT NULL, `subjectId` TEXT NOT NULL, `fingerIdentifier` INTEGER NOT NULL, `template` BLOB NOT NULL, `format` TEXT NOT NULL, `referenceId` TEXT NOT NULL, FOREIGN KEY(`subjectId`) REFERENCES `DbSubject`(`subjectId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "rowId", + "columnName": "rowId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "uuid", + "columnName": "uuid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "subjectId", + "columnName": "subjectId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fingerIdentifier", + "columnName": "fingerIdentifier", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "template", + "columnName": "template", + "affinity": "BLOB", + "notNull": true + }, + { + "fieldPath": "format", + "columnName": "format", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "referenceId", + "columnName": "referenceId", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "rowId" + ] + }, + "indices": [ + { + "name": "index_DbFingerprintSample_format", + "unique": false, + "columnNames": [ + "format" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbFingerprintSample_format` ON `${TABLE_NAME}` (`format`)" + }, + { + "name": "index_DbFingerprintSample_subjectId", + "unique": false, + "columnNames": [ + "subjectId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbFingerprintSample_subjectId` ON `${TABLE_NAME}` (`subjectId`)" + } + ], + "foreignKeys": [ + { + "table": "DbSubject", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "subjectId" + ], + "referencedColumns": [ + "subjectId" + ] + } + ] + }, + { + "tableName": "DbFaceSample", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`rowId` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `uuid` TEXT NOT NULL, `subjectId` TEXT NOT NULL, `template` BLOB NOT NULL, `format` TEXT NOT NULL, `referenceId` TEXT NOT NULL, FOREIGN KEY(`subjectId`) REFERENCES `DbSubject`(`subjectId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "rowId", + "columnName": "rowId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "uuid", + "columnName": "uuid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "subjectId", + "columnName": "subjectId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "template", + "columnName": "template", + "affinity": "BLOB", + "notNull": true + }, + { + "fieldPath": "format", + "columnName": "format", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "referenceId", + "columnName": "referenceId", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "rowId" + ] + }, + "indices": [ + { + "name": "index_DbFaceSample_format", + "unique": false, + "columnNames": [ + "format" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbFaceSample_format` ON `${TABLE_NAME}` (`format`)" + }, + { + "name": "index_DbFaceSample_subjectId", + "unique": false, + "columnNames": [ + "subjectId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbFaceSample_subjectId` ON `${TABLE_NAME}` (`subjectId`)" + } + ], + "foreignKeys": [ + { + "table": "DbSubject", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "subjectId" + ], + "referencedColumns": [ + "subjectId" + ] + } + ] + } + ], + "views": [], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'f2e5005e3569ee84f0bd388c5485c5bf')" + ] + } +} \ No newline at end of file diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectDao.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectDao.kt index 240a9434e8..0b212477d6 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectDao.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectDao.kt @@ -1,8 +1,8 @@ package com.simprints.infra.enrolment.records.room.store -import androidx.paging.PagingSource import androidx.room.Dao import androidx.room.Insert +import androidx.room.MapColumn import androidx.room.OnConflictStrategy import androidx.room.Query import androidx.room.RawQuery @@ -11,7 +11,8 @@ import androidx.sqlite.db.SupportSQLiteQuery import com.simprints.infra.enrolment.records.room.store.models.DbFaceSample import com.simprints.infra.enrolment.records.room.store.models.DbFingerprintSample import com.simprints.infra.enrolment.records.room.store.models.DbSubject -import com.simprints.infra.enrolment.records.room.store.models.DbSubjectWithSamples +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.SubjectBiometrics @Dao interface SubjectDao { @@ -27,19 +28,190 @@ interface SubjectDao { @Query("DELETE FROM DBSUBJECT WHERE subjectId = :subjectId") suspend fun deleteSubject(subjectId: String) - @Transaction @RawQuery - suspend fun loadSubjects(query: SupportSQLiteQuery): List + suspend fun deleteSubjects(query: SupportSQLiteQuery): Int @Transaction @RawQuery - suspend fun count(query: SupportSQLiteQuery): Int + suspend fun loadSubjects(query: SupportSQLiteQuery): List - @Transaction - @RawQuery - fun getFaceSamples(query: SupportSQLiteQuery): PagingSource + /** + * Counts the number of distinct subjects that match the given filtering criteria. + * + * This method ensures that a subject is counted only once, even if they have multiple + * face or fingerprint samples. It optimizes performance by avoiding expensive `JOIN` + * operations and instead using `EXISTS` subqueries, which efficiently check for the + * presence of related biometric records without generating large intermediate result sets. + * + * ## Query Behavior: + * - Filters subjects based on optional parameters (e.g., `projectId`, `attendantId`, etc.). + * - If `fingerprintSampleFormat` is provided, only counts subjects that have at least one + * fingerprint sample with the specified format. + * - If `faceSampleFormat` is provided, only counts subjects that have at least one face + * sample with the specified format. + * - Uses `EXISTS` instead of `JOIN` to efficiently check for biometric samples. + * - Leverages database indexes for performance optimization. + * + * ## Performance Considerations: + * - **Indexes:** Ensure indexes exist on `DbSubject.subjectId`, `DbFingerprintSample.subjectId`, + * `DbFaceSample.subjectId`, and their respective `format` columns for efficient filtering. + * - **Scalability:** Using `EXISTS` prevents the query from generating excessive row combinations, + * making it efficient even with 50K+ subjects and millions of biometric samples. + * - **Avoids Full Table Scans:** If filtering by biometrics, the query quickly exits once + * a match is found, improving query execution time. + **/ + @Query( + """ + SELECT COUNT(*) FROM DbSubject +""", + ) + fun countSubjects(): Int - @Transaction - @RawQuery - fun getFingerprintSamples(query: SupportSQLiteQuery): PagingSource + /** + * Retrieves a paginated list of distinct subjects along with their fingerprint samples + * that match the given filtering criteria. + * + * This method ensures that: + * - Subjects are retrieved uniquely based on the provided filters. + * - Multiple fingerprint samples per subject are included in the result. + * - Efficient pagination is applied using `LIMIT` and `OFFSET`. + * - Uses `LEFT JOIN` to ensure that multiple fingerprint samples can be selected per subject + * without causing duplicates. + * + * ## Query Behavior: + * - Filters subjects based on optional parameters (e.g., `projectId`, `attendantId`, etc.). + * - If `fingerprintSampleFormat` is provided, only returns subjects with at least one + * fingerprint sample of the specified format. + * - Uses `EXISTS` to efficiently check for fingerprint samples that match the format. + * - Orders by `subjectId` for consistent pagination. + * - Uses `LIMIT` and `OFFSET` for efficient page-wise retrieval. + * + * ## Performance Considerations: + * - **Indexes:** Ensure indexes exist on `DbSubject.subjectId`, `DbFingerprintSample.subjectId`, + * and `DbFingerprintSample.format` to optimize filtering and joins. + * - **Scalability:** Pagination ensures that queries remain performant even with large datasets. + * - **Avoids Duplicate Counting:** Retrieves all fingerprint samples for a subject without + * causing duplicates. + * + * @param projectId (Optional) Filters subjects by project ID. + * @param subjectId (Optional) Filters for a specific subject ID. + * @param subjectIds (Optional) Filters for multiple subject IDs. + * @param attendantId (Optional) Filters subjects assigned to a specific attendant. + * @param moduleId (Optional) Filters subjects assigned to a specific module. + * @param fingerprintSampleFormat (Optional) If provided, only includes subjects with at least + * one fingerprint sample of the specified format. + * @param limit The maximum number of subjects to return per query page. + * @param offset The starting position for pagination. + * @return A map where the keys are subject IDs, and the values are lists of fingerprint samples + * associated with each subject. + */ + @Query( + """ + SELECT s.subjectId, f.* + FROM DbSubject s + LEFT JOIN DbFingerprintSample f ON f.subjectId = s.subjectId + WHERE + (:projectId IS NULL OR s.projectId = :projectId) AND + (:subjectId IS NULL OR s.subjectId = :subjectId) AND + (:subjectIds IS NULL OR s.subjectId IN (:subjectIds)) AND + (:attendantId IS NULL OR s.attendantId = :attendantId) AND + (:moduleId IS NULL OR s.moduleId = :moduleId) AND + ( + (:fingerprintSampleFormat IS NULL OR EXISTS ( + SELECT 1 FROM DbFingerprintSample fs + WHERE fs.subjectId = s.subjectId AND fs.format = :fingerprintSampleFormat + )) + ) + GROUP BY s.subjectId + ORDER BY s.subjectId + LIMIT :limit OFFSET :offset +""", + ) + fun getSubjectsWithFingerprintSamples( + projectId: String?, + subjectId: String?, + subjectIds: List?, + attendantId: String?, + moduleId: String?, + fingerprintSampleFormat: String?, + offset: Int, + limit: Int, + ): Map< + @MapColumn(SUBJECT_ID_COLUMN) String, + List, + > + + /** + * Retrieves a paginated list of distinct subjects along with their face samples + * that match the given filtering criteria. + * + * This method ensures that: + * - Subjects are retrieved uniquely based on the provided filters. + * - Multiple face samples per subject are included in the result. + * - Efficient pagination is applied using `LIMIT` and `OFFSET`. + * - Uses `LEFT JOIN` to ensure that multiple face samples can be selected per subject + * without causing duplicates. + * + * ## Query Behavior: + * - Filters subjects based on optional parameters (e.g., `projectId`, `attendantId`, etc.). + * - If `faceSampleFormat` is provided, only returns subjects with at least one + * face sample of the specified format. + * - Uses `EXISTS` to efficiently check for face samples that match the format. + * - Orders by `subjectId` for consistent pagination. + * - Uses `LIMIT` and `OFFSET` for efficient page-wise retrieval. + * + * ## Performance Considerations: + * - **Indexes:** Ensure indexes exist on `DbSubject.subjectId`, `DbFaceSample.subjectId`, + * and `DbFaceSample.format` to optimize filtering and joins. + * - **Scalability:** Pagination ensures that queries remain performant even with large datasets. + * - **Avoids Duplicate Counting:** Retrieves all face samples for a subject without + * causing duplicates. + * + * @param projectId (Optional) Filters subjects by project ID. + * @param subjectId (Optional) Filters for a specific subject ID. + * @param subjectIds (Optional) Filters for multiple subject IDs. + * @param attendantId (Optional) Filters subjects assigned to a specific attendant. + * @param moduleId (Optional) Filters subjects assigned to a specific module. + * @param faceSampleFormat (Optional) If provided, only includes subjects with at least + * one face sample of the specified format. + * @param limit The maximum number of subjects to return per query page. + * @param offset The starting position for pagination. + * @return A map where the keys are subject IDs, and the values are lists of face samples + * associated with each subject. + */ + @Query( + """ + SELECT s.subjectId, f.* + FROM DbSubject s + LEFT JOIN DbFaceSample f ON f.subjectId = s.subjectId + WHERE + (:projectId IS NULL OR s.projectId = :projectId) AND + (:subjectId IS NULL OR s.subjectId = :subjectId) AND + (:subjectIds IS NULL OR s.subjectId IN (:subjectIds)) AND + (:attendantId IS NULL OR s.attendantId = :attendantId) AND + (:moduleId IS NULL OR s.moduleId = :moduleId) AND + ( + (:faceSampleFormat IS NULL OR EXISTS ( + SELECT 1 FROM DbFaceSample fs + WHERE fs.subjectId = s.subjectId AND fs.format = :faceSampleFormat + )) + ) + GROUP BY s.subjectId + ORDER BY s.subjectId + LIMIT :limit OFFSET :offset +""", + ) + fun getSubjectsWithFaceSamples( + projectId: String?, + subjectId: String?, + subjectIds: List?, + attendantId: String?, + moduleId: String?, + faceSampleFormat: String?, + offset: Int, + limit: Int, + ): Map< + @MapColumn(SUBJECT_ID_COLUMN) String, + List, + > } diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt index ba3783fcdc..1cb809bbbe 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt @@ -12,7 +12,11 @@ import net.sqlcipher.database.SupportFactory import javax.inject.Singleton @Singleton -@Database(entities = [DbSubject::class, DbFingerprintSample::class, DbFaceSample::class], version = 1, exportSchema = true) +@Database( + entities = [DbSubject::class, DbFingerprintSample::class, DbFaceSample::class], + version = 1, + exportSchema = true, +) @Keep abstract class SubjectsDatabase : RoomDatabase() { abstract val subjectDao: SubjectDao diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFaceSample.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFaceSample.kt index 7e2c257b50..9f5335f4d9 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFaceSample.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFaceSample.kt @@ -24,10 +24,12 @@ import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Compani ) @Suppress("ArrayInDataClass") data class DbFaceSample( - @PrimaryKey - val id: String, - val subjectId: String, - val template: ByteArray, - val format: String, - val referenceId: String, + // Auto-incrementing key for pagination + @PrimaryKey(autoGenerate = true) + val rowId: Long = 0, // This field is automatically assigned by Room + val uuid: String = "", + val subjectId: String = "", + val template: ByteArray = byteArrayOf(), + val format: String = "", + val referenceId: String = "", ) diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFingerprintSample.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFingerprintSample.kt index 5c3d0312f2..89e9a5776e 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFingerprintSample.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFingerprintSample.kt @@ -25,12 +25,13 @@ import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Compani ) @Suppress("ArrayInDataClass") data class DbFingerprintSample( - @PrimaryKey - val id: String, - val subjectId: String, - val fingerIdentifier: Int, - val template: ByteArray, - val templateQualityScore: Int, - val format: String, - val referenceId: String, + // Auto-incrementing key for pagination + @PrimaryKey(autoGenerate = true) + val rowId: Long = 0, // This field is automatically assigned by Room + val uuid: String = "", + val subjectId: String = "", + val fingerIdentifier: Int = 0, + val template: ByteArray = byteArrayOf(), + val format: String = "", + val referenceId: String = "", ) diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt index 0fa1cee291..a5608e00bf 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt @@ -6,11 +6,13 @@ import androidx.room.PrimaryKey import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.ATTENDANT_ID_COLUMN import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.MODULE_ID_COLUMN import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.PROJECT_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN import java.util.UUID @Entity( tableName = "DbSubject", indices = [ + Index(value = [SUBJECT_ID_COLUMN]), Index(value = [PROJECT_ID_COLUMN]), Index(value = [ATTENDANT_ID_COLUMN]), Index(value = [MODULE_ID_COLUMN]), @@ -19,12 +21,11 @@ import java.util.UUID data class DbSubject( @PrimaryKey val subjectId: String = UUID.randomUUID().toString(), - val projectId: String? = "", - val attendantId: String? = "", - val moduleId: String? = "", + val projectId: String = "", + val attendantId: String = "", + val moduleId: String = "", val createdAt: Long? = 0, val updatedAt: Long? = 0, - val toSync: Boolean = false, val isAttendantIdTokenized: Boolean = false, val isModuleIdTokenized: Boolean = false, ) { diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubjectWithSamples.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/SubjectBiometrics.kt similarity index 94% rename from infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubjectWithSamples.kt rename to infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/SubjectBiometrics.kt index a85dbfcaf5..d7ca705bee 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubjectWithSamples.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/SubjectBiometrics.kt @@ -4,7 +4,7 @@ import androidx.room.Embedded import androidx.room.Relation import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN -data class DbSubjectWithSamples( +data class SubjectBiometrics( @Embedded val subject: DbSubject, @Relation( parentColumn = SUBJECT_ID_COLUMN, From 734bd23e13866663002996cc41506f2fc6071b97 Mon Sep 17 00:00:00 2001 From: Melad Raouf Date: Thu, 10 Apr 2025 12:55:23 +0200 Subject: [PATCH 04/11] [MS-951] Use Normalized db schema --- .../kotlin/LibraryRoomConventionPlugin.kt | 3 +- .../kotlin/PipelineJacocoConventionPlugin.kt | 8 +- .../kotlin/common/BuildTypeConfigurations.kt | 8 +- gradle/libs.versions.toml | 1 - .../tokenization/TokenizationProcessor.kt | 16 +- .../repository/build.gradle.kts | 1 - .../EnrolmentRecordRepositoryImpl.kt | 3 +- .../repository/EnrolmentRecordsStoreModule.kt | 10 +- .../domain/models/FingerIdentifier.kt | 3 + .../RealmEnrolmentRecordLocalDataSource.kt | 26 +- .../RoomEnrolmentRecordLocalDataSource.kt | 365 ++-- .../local/RoomEnrolmentRecordQueryBuilder.kt | 125 ++ ...ctEnrolmentRecordLocalDataSourceUseCase.kt | 2 +- .../local/models/IFingerIdentifier.kt | 53 + .../models/RealmFingerprintSampleConverter.kt | 5 +- .../models/RoomBiometricTemplateConverter.kt | 40 + .../local/models/RoomFaceSampleConverter.kt | 19 - .../models/RoomFingerprintSampleConverter.kt | 22 - .../local/models/RoomSubjectConverter.kt | 40 +- ...RealmEnrolmentRecordLocalDataSourceTest.kt | 17 +- ...RoomEnrollmentRecordLocalDataSourceTest.kt | 1537 +++++++++++++++++ .../RoomEnrolmentRecordLocalDataSourceTest.kt | 318 ---- .../RoomEnrolmentRecordQueryBuilderTest.kt | 458 +++++ .../1.json | 173 +- .../records/room/store/SubjectDao.kt | 201 +-- .../records/room/store/SubjectsDatabase.kt | 12 +- .../room/store/SubjectsDatabaseFactory.kt | 13 +- ...rprintSample.kt => DbBiometricTemplate.kt} | 26 +- .../records/room/store/models/DbFaceSample.kt | 35 - .../records/room/store/models/DbSubject.kt | 32 +- .../records/room/store/models/Modality.kt | 8 + .../room/store/models/SubjectBiometrics.kt | 7 +- .../infra/logging/LoggingConstants.kt | 1 + infra/test-tools/build.gradle.kts | 1 - 34 files changed, 2620 insertions(+), 969 deletions(-) create mode 100644 infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilder.kt create mode 100644 infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/IFingerIdentifier.kt create mode 100644 infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomBiometricTemplateConverter.kt delete mode 100644 infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFaceSampleConverter.kt delete mode 100644 infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFingerprintSampleConverter.kt create mode 100644 infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrollmentRecordLocalDataSourceTest.kt delete mode 100644 infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSourceTest.kt create mode 100644 infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilderTest.kt rename infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/{DbFingerprintSample.kt => DbBiometricTemplate.kt} (56%) delete mode 100644 infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFaceSample.kt create mode 100644 infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/Modality.kt diff --git a/build-logic/convention/src/main/kotlin/LibraryRoomConventionPlugin.kt b/build-logic/convention/src/main/kotlin/LibraryRoomConventionPlugin.kt index 78437df6df..c9975e1935 100644 --- a/build-logic/convention/src/main/kotlin/LibraryRoomConventionPlugin.kt +++ b/build-logic/convention/src/main/kotlin/LibraryRoomConventionPlugin.kt @@ -1,5 +1,6 @@ import androidx.room.gradle.RoomExtension import com.android.build.api.dsl.LibraryExtension +import common.api import common.configureDbEncryptionBuild import common.getLibs import common.implementation @@ -38,7 +39,7 @@ class LibraryRoomConventionPlugin : Plugin { val libs = getLibs() dependencies { - implementation(libs, "androidX.Room.core") + api(libs, "androidX.Room.core") implementation(libs, "androidX.Room.ktx") ksp(libs, "androidX.Room.compiler") diff --git a/build-logic/convention/src/main/kotlin/PipelineJacocoConventionPlugin.kt b/build-logic/convention/src/main/kotlin/PipelineJacocoConventionPlugin.kt index 4264c4a5ed..748397d2a4 100644 --- a/build-logic/convention/src/main/kotlin/PipelineJacocoConventionPlugin.kt +++ b/build-logic/convention/src/main/kotlin/PipelineJacocoConventionPlugin.kt @@ -29,20 +29,20 @@ class PipelineJacocoConventionPlugin : Plugin { ) private fun Project.createJacocoTask() { - tasks.register("jacocoTestReport", JacocoReport::class.java) { + tasks.create("jacocoTestReport", JacocoReport::class.java) { dependsOn(tasks.withType().matching { it.name.lowercase().contains("debug") }) reports.xml.required.set(true) reports.html.required.set(false) // Disable html reports to decrease report upload/download time in github pipeline - val javaTree = fileTree("${project.layout.buildDirectory}/intermediates/javac/debug/classes") { exclude(fileFilter) } - val kotlinTree = fileTree("${project.layout.buildDirectory}/tmp/kotlin-classes/debug") { exclude(fileFilter) } + val javaTree = fileTree("${project.buildDir}/intermediates/javac/debug/classes") { exclude(fileFilter) } + val kotlinTree = fileTree("${project.buildDir}/tmp/kotlin-classes/debug") { exclude(fileFilter) } classDirectories.setFrom(files(javaTree, kotlinTree)) sourceDirectories.setFrom(files("${project.projectDir}/src/main/java")) executionData.setFrom( - fileTree("${layout.buildDirectory}") { + fileTree("$buildDir") { include( "jacoco/testDebugUnitTest.exec", "outputs/code-coverage/connected/*coverage.ec", diff --git a/build-logic/convention/src/main/kotlin/common/BuildTypeConfigurations.kt b/build-logic/convention/src/main/kotlin/common/BuildTypeConfigurations.kt index e716c7f8e3..fca329d3e4 100644 --- a/build-logic/convention/src/main/kotlin/common/BuildTypeConfigurations.kt +++ b/build-logic/convention/src/main/kotlin/common/BuildTypeConfigurations.kt @@ -37,7 +37,13 @@ fun Project.configureDbEncryptionBuild() { } getByName(BuildTypes.DEBUG) { - buildConfigField("Boolean", "DB_ENCRYPTION", "$propDbEncrypted") + // Set false only if running tests + val isTestTask = gradle.startParameter.taskNames.any { + it.contains("test", ignoreCase = true) || it.contains("connected", ignoreCase = true) + } + + val dbEncryptionValue = if (isTestTask) "false" else "$propDbEncrypted" + buildConfigField("Boolean", "DB_ENCRYPTION", dbEncryptionValue) } } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c03b0b807a..5fd9ec59f4 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -252,7 +252,6 @@ testing-AndroidX-core = { module = "androidx.arch.core:core-testing", version.re testing-AndroidX-orchestrator = { module = "androidx.test:orchestrator", version.ref = "androidx_test_orchestrator_version" } testing-AndroidX-runner = { module = "androidx.test:runner", version.ref = "androidx_version" } testing-AndroidX-room = { module = "androidx.room:room-testing", version.ref = "androidx_room_version" } -testing-AndroidX-navigation = { module = "androidx.navigation:navigation-testing", version.ref = "androidx_navigation_version" } testing-AndroidX-uiAutomator = { module = "androidx.test.uiautomator:uiautomator", version.ref = "uiAutomator_version" } # Dependencies of the included build-logic diff --git a/infra/config-store/src/main/java/com/simprints/infra/config/store/tokenization/TokenizationProcessor.kt b/infra/config-store/src/main/java/com/simprints/infra/config/store/tokenization/TokenizationProcessor.kt index 3d270a923c..d7a8dde943 100644 --- a/infra/config-store/src/main/java/com/simprints/infra/config/store/tokenization/TokenizationProcessor.kt +++ b/infra/config-store/src/main/java/com/simprints/infra/config/store/tokenization/TokenizationProcessor.kt @@ -50,7 +50,7 @@ class TokenizationProcessor @Inject constructor( encrypted: TokenizableString.Tokenized, tokenKeyType: TokenKeyType, project: Project, - logError: Boolean = true + logError: Boolean = true, ): TokenizableString { val moduleKeyset = project.tokenizationKeys[tokenKeyType] ?: return encrypted return try { @@ -62,4 +62,18 @@ class TokenizationProcessor @Inject constructor( encrypted } } + + fun tokenizeIfNecessary( + tokenizableString: TokenizableString, + tokenKeyType: TokenKeyType, + project: Project, + ) = when (tokenizableString) { + is TokenizableString.Raw -> encrypt( + decrypted = tokenizableString, + tokenKeyType = tokenKeyType, + project = project, + ) + + is TokenizableString.Tokenized -> tokenizableString + } } diff --git a/infra/enrolment-records/repository/build.gradle.kts b/infra/enrolment-records/repository/build.gradle.kts index 064cb2d26e..e87c549e8a 100644 --- a/infra/enrolment-records/repository/build.gradle.kts +++ b/infra/enrolment-records/repository/build.gradle.kts @@ -1,7 +1,6 @@ plugins { id("simprints.infra") id("kotlin-parcelize") - id("simprints.library.room") } android { diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt index d69c6c44dd..2259367ac2 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt @@ -31,12 +31,11 @@ internal class EnrolmentRecordRepositoryImpl @Inject constructor( private val tokenizationProcessor: TokenizationProcessor, private val selectEnrolmentRecordLocalDataSource: SelectEnrolmentRecordLocalDataSourceUseCase, @DispatcherIO private val dispatcher: CoroutineDispatcher, - private val batchSize: Int, + @EnrolmentBatchSize private val batchSize: Int, ) : EnrolmentRecordRepository { private val prefs = context.getSharedPreferences(PREF_FILE_NAME, Context.MODE_PRIVATE) companion object { - const val BATCH_SIZE = 80 private const val PREF_FILE_NAME = "UPLOAD_ENROLMENT_RECORDS_PROGRESS" private const val PROGRESS_KEY = "PROGRESS" } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordsStoreModule.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordsStoreModule.kt index d6fb683ed4..60f2199b8b 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordsStoreModule.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordsStoreModule.kt @@ -5,7 +5,6 @@ import com.simprints.core.AvailableProcessors import com.simprints.core.DispatcherIO import com.simprints.core.tools.json.JsonHelper import com.simprints.core.tools.utils.EncodingUtils -import com.simprints.infra.enrolment.records.repository.EnrolmentRecordRepositoryImpl.Companion.BATCH_SIZE import com.simprints.infra.enrolment.records.repository.commcare.CommCareIdentityDataSource import com.simprints.infra.enrolment.records.repository.remote.EnrolmentRecordRemoteDataSource import com.simprints.infra.enrolment.records.repository.remote.EnrolmentRecordRemoteDataSourceImpl @@ -54,10 +53,19 @@ class IdentityDataSourceModule { dispatcher = dispatcher, ) + @EnrolmentBatchSize @Provides fun provideBatchSize(): Int = BATCH_SIZE + + companion object { + const val BATCH_SIZE = 80 + } } @Qualifier @Retention(AnnotationRetention.BINARY) annotation class CommCareDataSource + +@Qualifier +@Retention(AnnotationRetention.BINARY) +annotation class EnrolmentBatchSize diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/domain/models/FingerIdentifier.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/domain/models/FingerIdentifier.kt index f47e72f45d..50ed8caf34 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/domain/models/FingerIdentifier.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/domain/models/FingerIdentifier.kt @@ -3,6 +3,9 @@ package com.simprints.infra.enrolment.records.repository.domain.models import androidx.annotation.Keep import com.simprints.core.domain.fingerprint.IFingerIdentifier +// **IMPORTANT**: Do NOT change the order of this enum as it is used in the database by index. +// Changing the order of the entries in this enum will lead to data corruption or mismatches +// when retrieving or storing data in the database. Always append new entries at the end. @Keep enum class FingerIdentifier { RIGHT_5TH_FINGER, diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt index 85b9441b21..c11bfe02a3 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt @@ -179,8 +179,17 @@ internal class RealmEnrolmentRecordLocalDataSource @Inject constructor( is SubjectAction.Creation -> { val newSubject = action.subject .copy( - moduleId = action.subject.moduleId.tokenizeIfNecessary(TokenKeyType.ModuleId, project), - attendantId = action.subject.attendantId.tokenizeIfNecessary(TokenKeyType.AttendantId, project), + moduleId = + tokenizationProcessor.tokenizeIfNecessary( + action.subject.moduleId, + TokenKeyType.ModuleId, + project, + ), + attendantId = tokenizationProcessor.tokenizeIfNecessary( + action.subject.attendantId, + TokenKeyType.AttendantId, + project, + ), ).toRealmDb() val dbSubject: DbSubject? = realm.findSubject(newSubject.subjectId) @@ -236,19 +245,6 @@ internal class RealmEnrolmentRecordLocalDataSource @Inject constructor( } } - private fun TokenizableString.tokenizeIfNecessary( - tokenKeyType: TokenKeyType, - project: Project, - ) = when (this) { - is TokenizableString.Raw -> tokenizationProcessor.encrypt( - decrypted = this, - tokenKeyType = tokenKeyType, - project = project, - ) - - is TokenizableString.Tokenized -> this - } - private fun MutableRealm.findSubject(subjectId: RealmUUID): DbSubject? = query(DbSubject::class).query("$SUBJECT_ID_FIELD == $0", subjectId).first().find() diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt index c1ae3b0dc9..55874aa711 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt @@ -3,7 +3,12 @@ package com.simprints.infra.enrolment.records.repository.local import androidx.room.withTransaction import androidx.sqlite.db.SimpleSQLiteQuery import com.simprints.core.DispatcherIO +import com.simprints.core.domain.face.FaceSample +import com.simprints.core.domain.fingerprint.FingerprintSample +import com.simprints.core.domain.fingerprint.IFingerIdentifier import com.simprints.infra.config.store.models.Project +import com.simprints.infra.config.store.models.TokenKeyType +import com.simprints.infra.config.store.tokenization.TokenizationProcessor import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity @@ -14,135 +19,192 @@ import com.simprints.infra.enrolment.records.repository.local.models.toRoomDb import com.simprints.infra.enrolment.records.room.store.SubjectDao import com.simprints.infra.enrolment.records.room.store.SubjectsDatabase import com.simprints.infra.enrolment.records.room.store.SubjectsDatabaseFactory +import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate import com.simprints.infra.enrolment.records.room.store.models.DbSubject +import com.simprints.infra.logging.LoggingConstants.CrashReportTag.ROOM_RECORDS_DB import com.simprints.infra.logging.Simber import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.withContext import javax.inject.Inject import javax.inject.Singleton -import kotlin.system.measureTimeMillis -import kotlin.time.measureTimedValue -import com.simprints.infra.enrolment.records.repository.domain.models.Subject as SubjectDomain +import com.simprints.infra.enrolment.records.repository.domain.models.Subject as DomainSubject +/** + * Local data source for enrolment records using Room. + */ @Singleton internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( private val subjectsDatabaseFactory: SubjectsDatabaseFactory, + private val tokenizationProcessor: TokenizationProcessor, + private val queryBuilder: RoomEnrolmentRecordQueryBuilder, @DispatcherIO private val dispatcherIO: CoroutineDispatcher, ) : EnrolmentRecordLocalDataSource { - val database: SubjectsDatabase by lazy { subjectsDatabaseFactory.get() } - val subjectDao: SubjectDao by lazy { subjectsDatabaseFactory.get().subjectDao } + companion object { + // only one concurrent operation is allowed as we use the last seen subject ID to load biometric identities + private const val PARALLELISM = 1 + } + + private val database: SubjectsDatabase by lazy { subjectsDatabaseFactory.get() } + private val subjectDao: SubjectDao by lazy { database.subjectDao } - override suspend fun load(query: SubjectQuery): List = withContext(dispatcherIO) { - val sqlQuery = buildSubjectQuery(query) - subjectDao.loadSubjects(SimpleSQLiteQuery(sqlQuery.first, sqlQuery.second.toTypedArray())).map { it.toDomain() } + /** + * Loads subjects matching the given query. + * Don't use this method if the query contains format fields (faceSampleFormat or fingerprintSampleFormat). + * instead, use loadFaceIdentities or loadFingerprintIdentities methods. + */ + override suspend fun load(query: SubjectQuery): List = withContext(dispatcherIO) { + if (query.hasUntokenizedFields == true) { + Simber.d( + "[load] Query has untokenized fields, returning empty list as all records are tokenized.", + tag = ROOM_RECORDS_DB, + ) + return@withContext emptyList() + } + subjectDao.loadSubjects(queryBuilder.buildSubjectQuery(query)).map { it.toDomain() } } - override suspend fun loadFingerprintIdentities( + /** + * Counts subjects matching the given query. + */ + override suspend fun count( query: SubjectQuery, - range: IntRange, dataSource: BiometricDataSource, - project: Project, - onCandidateLoaded: () -> Unit, - ): List = withContext(dispatcherIO) { - subjectDao - .getSubjectsWithFingerprintSamples( - query.projectId, - query.subjectId, - query.subjectIds, - query.attendantId?.value, - query.moduleId?.value, - query.fingerprintSampleFormat, - range.first, - range.last, - ).map { - onCandidateLoaded() - FingerprintIdentity( - subjectId = it.key, - fingerprints = it.value.map { sample -> sample.toDomain() }, - ) - } + ): Int = withContext(dispatcherIO) { + subjectDao.countSubjects(queryBuilder.buildCountQuery(query)) } - override suspend fun loadFaceIdentities( + /** + * Loads face identities in paged ranges. + */ + override fun loadFaceIdentities( query: SubjectQuery, - range: IntRange, + ranges: List, dataSource: BiometricDataSource, project: Project, + scope: CoroutineScope, onCandidateLoaded: () -> Unit, - ): List = withContext(dispatcherIO) { - val result = measureTimedValue { - subjectDao - .getSubjectsWithFaceSamples( - query.projectId, - query.subjectId, - query.subjectIds, - query.attendantId?.value, - query.moduleId?.value, - query.faceSampleFormat, - range.first, - range.last, - ).map { - onCandidateLoaded() - FaceIdentity( - subjectId = it.key, - faces = it.value.map { sample -> sample.toDomain() }, + ): ReceiveChannel> = loadBiometricIdentitiesPaged( + query = query, + ranges = ranges, + format = requireNotNull(query.faceSampleFormat) { "faceSampleFormat required" }, + createIdentity = { subjectId, templates -> + FaceIdentity( + subjectId = subjectId, + faces = templates.map { sample -> + FaceSample( + template = sample.templateData, + id = sample.uuid, + format = sample.format, + referenceId = sample.referenceId, ) - } - } - log("loadFaceIdentities ${result.value.size} in ${result.duration.inWholeMilliseconds} ms") - return@withContext result.value - } + }, + ) + }, + onCandidateLoaded = onCandidateLoaded, + scope = scope, + ) - private fun buildSubjectQuery(query: SubjectQuery): Pair> { - val (whereClause, args) = buildWhereClause(query) - val orderBy = if (query.sort) "ORDER BY s.subjectId ASC" else "" - val sql = - """ - SELECT * FROM DbSubject s - LEFT JOIN DbFingerprintSample fingerprint ON s.subjectId = fingerprint.subjectId - LEFT JOIN DbFaceSample face ON s.subjectId = face.subjectId - $whereClause - $orderBy - """.trimIndent() - return Pair(sql, args) - } - - private fun buildWhereClause(query: SubjectQuery): Pair> { - val whereClauses = mutableListOf() - val args = mutableListOf() + /** + * Loads fingerprint identities in paged ranges. + */ + override fun loadFingerprintIdentities( + query: SubjectQuery, + ranges: List, + dataSource: BiometricDataSource, + project: Project, + scope: CoroutineScope, + onCandidateLoaded: () -> Unit, + ): ReceiveChannel> = loadBiometricIdentitiesPaged( + query = query, + ranges = ranges, + format = requireNotNull(query.fingerprintSampleFormat) { "fingerprintSampleFormat required" }, + createIdentity = { subjectId, templates -> + FingerprintIdentity( + subjectId = subjectId, + fingerprints = templates.map { sample -> + FingerprintSample( + fingerIdentifier = IFingerIdentifier.entries[sample.identifier!!], + template = sample.templateData, + id = sample.uuid, + format = sample.format, + referenceId = sample.referenceId, + ) + }, + ) + }, + onCandidateLoaded = onCandidateLoaded, + scope = scope, + ) - query.projectId?.let { - whereClauses.add("s.projectId = ?") - args.add(it) - } - query.subjectId?.let { - whereClauses.add("s.subjectId = ?") - args.add(it) - } - query.subjectIds?.takeIf { it.isNotEmpty() }?.let { - whereClauses.add("s.subjectId IN (${it.joinToString(",") { "?" }})") - args.addAll(it) - } - query.attendantId?.let { - whereClauses.add("s.attendantId = ?") - args.add(it) - } - query.moduleId?.let { - whereClauses.add("s.moduleId = ?") - args.add(it) - } - if (query.hasUntokenizedFields == true) { - whereClauses.add("(s.isAttendantIdTokenized = 0 OR s.isModuleIdTokenized = 0)") + private fun loadBiometricIdentitiesPaged( + query: SubjectQuery, + ranges: List, + format: String, + createIdentity: (String, List) -> T, + onCandidateLoaded: () -> Unit, + scope: CoroutineScope, + ): ReceiveChannel> { + var lastSeenSubjectId: String? = null + var lastOffset = 0 + return loadIdentitiesConcurrently( + ranges = ranges, + dispatcher = dispatcherIO, + parallelism = PARALLELISM, + scope = scope, + ) { range -> + require(lastOffset == range.first) { + "[loadBiometricIdentitiesPaged] The range start must match the last seen sample count. " + + "Expected: $lastOffset, Actual: ${range.first}" + } + val identities = loadBiometricIdentities( + query = query, + pageSize = range.last - range.first, + lastSeenSubjectId = lastSeenSubjectId, + format = format, + createIdentity = createIdentity, + onCandidateLoaded = onCandidateLoaded, + ) + lastSeenSubjectId = identities.lastOrNull()?.let { + (it as? FaceIdentity)?.subjectId ?: (it as? FingerprintIdentity)?.subjectId + } + lastOffset = range.last + identities } + } - val whereClause = if (whereClauses.isNotEmpty()) "WHERE ${whereClauses.joinToString(" AND ")}" else "" - return Pair(whereClause, args) + private suspend fun loadBiometricIdentities( + query: SubjectQuery, + pageSize: Int, + format: String?, + lastSeenSubjectId: String?, + createIdentity: (subjectId: String, samples: List) -> T, + onCandidateLoaded: () -> Unit, + ): List = withContext(dispatcherIO) { + requireNotNull(format) { "Appropriate sampleFormat is required for loading biometric identities." } + subjectDao + .loadSamples(queryBuilder.buildBiometricTemplatesQuery(query, pageSize, lastSeenSubjectId)) + .map { (subjectId, templates) -> + onCandidateLoaded() + createIdentity(subjectId, templates) + } } override suspend fun delete(queries: List) { + Simber.i("[delete] Deleting subjects with queries: $queries", tag = ROOM_RECORDS_DB) database.withTransaction { - queries.forEach { - val (whereClause, args) = buildWhereClause(it) + queries.forEach { query -> + require(query.faceSampleFormat == null && query.fingerprintSampleFormat == null) { + val errorMsg = "faceSampleFormat and fingerprintSampleFormat are not supported for deletion" + Simber.i("[delete] $errorMsg", tag = ROOM_RECORDS_DB) + errorMsg + } + val (whereClause, args) = queryBuilder.buildWhereAndOrderByClause( + query, + subjectAlias = "", + templateAlias = "", + ) val sql = "DELETE FROM DbSubject $whereClause" subjectDao.deleteSubjects(SimpleSQLiteQuery(sql, args.toTypedArray())) } @@ -150,72 +212,99 @@ internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( } override suspend fun deleteAll() { + Simber.i("[deleteAll] Deleting all subjects.", tag = ROOM_RECORDS_DB) subjectDao.deleteSubjects(SimpleSQLiteQuery("DELETE FROM DbSubject")) } - override suspend fun count( - query: SubjectQuery, - dataSource: BiometricDataSource, - ): Int = withContext(dispatcherIO) { - var result = 0 - val timeTaken = measureTimeMillis { - result = subjectDao.countSubjects() - } - log("count $result : $timeTaken ms") - result - } - override suspend fun performActions( actions: List, project: Project, ) { - val timeTaken = measureTimeMillis { - database.withTransaction { - actions.forEach { action -> - when (action) { - is SubjectAction.Creation -> createSubject(action.subject) - is SubjectAction.Update -> Unit - is SubjectAction.Deletion -> deleteSubject(action.subjectId) - } + database.withTransaction { + actions.forEach { action -> + Simber.d("[performActions] Performing action: $action", tag = ROOM_RECORDS_DB) + when (action) { + is SubjectAction.Creation -> createSubject(action.subject, project) + is SubjectAction.Update -> updateSubject(action) + is SubjectAction.Deletion -> deleteSubject(action.subjectId) } } } - log("performActions ${actions.size} in : $timeTaken ms") } - private suspend fun createSubject(subject: SubjectDomain) { - val subjectId = subject.subjectId + private suspend fun createSubject( + subject: DomainSubject, + project: Project, + ) { + require(subject.faceSamples.isNotEmpty() || subject.fingerprintSamples.isNotEmpty()) { + val errorMsg = "Subject should include at least one of the face or fingerprint samples" + Simber.i( + "[createSubject] $errorMsg for subjectId: ${subject.subjectId}", + tag = ROOM_RECORDS_DB, + ) + errorMsg + } val dbSubject = DbSubject( subjectId = subject.subjectId, projectId = subject.projectId, - attendantId = subject.attendantId.value, - moduleId = subject.moduleId.value, + attendantId = tokenizationProcessor + .tokenizeIfNecessary( + subject.attendantId, + TokenKeyType.AttendantId, + project, + ).value, + moduleId = tokenizationProcessor + .tokenizeIfNecessary( + subject.moduleId, + TokenKeyType.ModuleId, + project, + ).value, createdAt = subject.createdAt?.time, updatedAt = subject.updatedAt?.time, - isAttendantIdTokenized = false, - isModuleIdTokenized = false, ) - subjectDao.insertSubject(dbSubject) - - // Insert fingerprints - val dbFingerprints = subject.fingerprintSamples.map { it.toRoomDb(subjectId) } - if (dbFingerprints.isNotEmpty()) { - subjectDao.insertFingerprintSamples(dbFingerprints) + subject.fingerprintSamples.takeIf { it.isNotEmpty() }?.let { samples -> + val dbFingerprints = samples.map { it.toRoomDb(subject.subjectId) } + subjectDao.insertBiometricSamples(dbFingerprints) } + subject.faceSamples.takeIf { it.isNotEmpty() }?.let { samples -> + val dbFaces = samples.map { it.toRoomDb(subject.subjectId) } + subjectDao.insertBiometricSamples(dbFaces) + } + } - // Insert face samples - val dbFaces = subject.faceSamples.map { it.toRoomDb(subjectId) } - if (dbFaces.isNotEmpty()) { - subjectDao.insertFaceSamples(dbFaces) + private suspend fun updateSubject(action: SubjectAction.Update) { + val dbSubject = subjectDao.getSubject(action.subjectId) + if (dbSubject != null) { + val referencesToDelete = action.referenceIdsToRemove.toSet() + require( + referencesToDelete.size != dbSubject.biometricTemplates.size || + action.faceSamplesToAdd.isNotEmpty() || + action.fingerprintSamplesToAdd.isNotEmpty(), + ) { + val errorMsg = "Cannot delete all samples for subject ${action.subjectId} without adding new ones" + Simber.i("[updateSubject] $errorMsg", tag = ROOM_RECORDS_DB) + errorMsg + } + dbSubject.biometricTemplates.filter { it.referenceId in referencesToDelete }.forEach { + subjectDao.deleteBiometricSample(it.uuid) + } + val templatesToAdd = + action.faceSamplesToAdd.map { it.toRoomDb(action.subjectId) } + + action.fingerprintSamplesToAdd.map { it.toRoomDb(action.subjectId) } + if (templatesToAdd.isNotEmpty()) { + subjectDao.insertBiometricSamples(templatesToAdd) + } + } else { + Simber.i( + "[updateSubject] Subject ${action.subjectId} not found for update", + tag = ROOM_RECORDS_DB, + ) } } private suspend fun deleteSubject(subjectId: String) { + Simber.d("[deleteSubject] Deleting subject: $subjectId", tag = ROOM_RECORDS_DB) subjectDao.deleteSubject(subjectId) } } - -fun log(message: String) { - Simber.i(message, tag = "roomrecords") -} diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilder.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilder.kt new file mode 100644 index 0000000000..f4131cbeb4 --- /dev/null +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilder.kt @@ -0,0 +1,125 @@ +package com.simprints.infra.enrolment.records.repository.local + +import androidx.sqlite.db.SimpleSQLiteQuery +import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery +import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate.Companion.FORMAT_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate.Companion.TEMPLATE_TABLE_NAME +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.ATTENDANT_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.MODULE_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.PROJECT_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_TABLE_NAME +import jakarta.inject.Inject + +internal class RoomEnrolmentRecordQueryBuilder @Inject constructor() { + fun buildSubjectQuery(query: SubjectQuery): SimpleSQLiteQuery { + // require format not to be set for subject query and guid to use the buildBiometricTemplatesQuery instead + require(query.fingerprintSampleFormat == null && query.faceSampleFormat == null) { + "Cannot set format for subject query, use buildBiometricTemplatesQuery instead" + } + val (whereClause, args) = buildWhereAndOrderByClause(query) + val sql = + """ + SELECT * FROM $SUBJECT_TABLE_NAME S + $whereClause + """.trimIndent() + println(sql) + println("----") + return SimpleSQLiteQuery(sql, args.toTypedArray()) + } + + fun buildCountQuery(query: SubjectQuery): SimpleSQLiteQuery { + val (whereClause, args) = buildWhereAndOrderByClause(query) + val specificFormat = query.fingerprintSampleFormat ?: query.faceSampleFormat + + val sql = if (specificFormat != null) { + "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + + " using(subjectId) $whereClause " + } else { + "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S $whereClause" + } + println(sql) + println("----") + return SimpleSQLiteQuery(sql, args.toTypedArray()) + } + + fun buildBiometricTemplatesQuery( + query: SubjectQuery, + pageSize: Int, + lastSeenSubjectId: String? = null, + ): SimpleSQLiteQuery { + // require format to be set for biometric templates query + val format = query.fingerprintSampleFormat ?: query.faceSampleFormat + require(format != null) { + "Must set format for biometric templates query, use buildSubjectQuery or buildCountQuery instead" + } + val updatedQuery = query.copy(afterSubjectId = lastSeenSubjectId, sort = true) + val (whereClause, args) = buildWhereAndOrderByClause(updatedQuery) + val sql = + """ + SELECT A.* + FROM $TEMPLATE_TABLE_NAME A + INNER JOIN ( + SELECT distinct S.subjectId + FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T + USING(subjectId) + $whereClause + LIMIT $pageSize + ) B USING(subjectId) where A.format ='$format' + """.trimIndent() + println(sql) + println("----") + return SimpleSQLiteQuery(sql, args.toTypedArray()) + } + + fun buildWhereAndOrderByClause( + query: SubjectQuery, + subjectAlias: String = "S.", // Default alias for subject table, dot included. Empty string for no alias. + templateAlias: String = "T.", // Default alias for template table, dot included. Empty string for no alias. + ): Pair> { + val clauses = mutableListOf() + val args = mutableListOf() + if (query.fingerprintSampleFormat != null && query.faceSampleFormat != null) { + throw IllegalArgumentException("Cannot set both fingerprintSampleFormat and faceSampleFormat") + } + // to achieve the highest performance, we should not use OR in the where clause + query.subjectId?.let { + clauses.add("${subjectAlias}$SUBJECT_ID_COLUMN = ?") + args.add(it) + } + query.subjectIds?.takeIf { it.isNotEmpty() }?.let { + clauses.add("${subjectAlias}$SUBJECT_ID_COLUMN IN (${it.joinToString(",") { "?" }})") + args.addAll(it) + } + query.afterSubjectId?.let { + clauses.add("${subjectAlias}$SUBJECT_ID_COLUMN > ?") + args.add(it) + } + query.projectId?.let { + clauses.add("${subjectAlias}$PROJECT_ID_COLUMN = ?") + args.add(it) + } + query.attendantId?.let { + clauses.add("${subjectAlias}$ATTENDANT_ID_COLUMN = ?") + args.add(it.value) + } + query.moduleId?.let { + clauses.add("${subjectAlias}$MODULE_ID_COLUMN = ?") + args.add(it.value) + } + query.faceSampleFormat?.let { + clauses.add("${templateAlias}$FORMAT_COLUMN = ?") + args.add(it) + } + query.fingerprintSampleFormat?.let { + clauses.add("${templateAlias}$FORMAT_COLUMN = ?") + args.add(it) + } + + var whereClauseResult = if (clauses.isNotEmpty()) "WHERE ${clauses.joinToString(" AND ")}" else "" + if (query.sort) { + whereClauseResult += " ORDER BY ${subjectAlias}$SUBJECT_ID_COLUMN ASC" + } + return Pair(whereClauseResult, args) + } +} diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/SelectEnrolmentRecordLocalDataSourceUseCase.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/SelectEnrolmentRecordLocalDataSourceUseCase.kt index 70db61710b..7c4c1a6f22 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/SelectEnrolmentRecordLocalDataSourceUseCase.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/SelectEnrolmentRecordLocalDataSourceUseCase.kt @@ -8,6 +8,6 @@ internal class SelectEnrolmentRecordLocalDataSourceUseCase @Inject constructor( ) { operator fun invoke(): EnrolmentRecordLocalDataSource { // Todo later we will add logic to select the data source - return roomDataSource + return realmDataSource } } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/IFingerIdentifier.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/IFingerIdentifier.kt new file mode 100644 index 0000000000..3fdc0b09ec --- /dev/null +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/IFingerIdentifier.kt @@ -0,0 +1,53 @@ +package com.simprints.infra.enrolment.records.repository.local.models +import com.simprints.core.domain.fingerprint.IFingerIdentifier as CoreFingerIdentifier + +enum class IFingerIdentifier( + val id: Int, +) { + RIGHT_5TH_FINGER(0), + RIGHT_4TH_FINGER(1), + RIGHT_3RD_FINGER(2), + RIGHT_INDEX_FINGER(3), + RIGHT_THUMB(4), + LEFT_THUMB(5), + LEFT_INDEX_FINGER(6), + LEFT_3RD_FINGER(7), + LEFT_4TH_FINGER(8), + LEFT_5TH_FINGER(9), + ; + + // create from id + companion object { + fun fromId(id: Int) = IFingerIdentifier.entries + .firstOrNull { it.id == id } + ?: throw IllegalArgumentException("Invalid id: $id") + } +} + +// convert to CoreFingerIdentifier +internal fun IFingerIdentifier.toDomain() = when (this) { + IFingerIdentifier.RIGHT_5TH_FINGER -> CoreFingerIdentifier.RIGHT_5TH_FINGER + IFingerIdentifier.RIGHT_4TH_FINGER -> CoreFingerIdentifier.RIGHT_4TH_FINGER + IFingerIdentifier.RIGHT_3RD_FINGER -> CoreFingerIdentifier.RIGHT_3RD_FINGER + IFingerIdentifier.RIGHT_INDEX_FINGER -> CoreFingerIdentifier.RIGHT_INDEX_FINGER + IFingerIdentifier.RIGHT_THUMB -> CoreFingerIdentifier.RIGHT_THUMB + IFingerIdentifier.LEFT_THUMB -> CoreFingerIdentifier.LEFT_THUMB + IFingerIdentifier.LEFT_INDEX_FINGER -> CoreFingerIdentifier.LEFT_INDEX_FINGER + IFingerIdentifier.LEFT_3RD_FINGER -> CoreFingerIdentifier.LEFT_3RD_FINGER + IFingerIdentifier.LEFT_4TH_FINGER -> CoreFingerIdentifier.LEFT_4TH_FINGER + IFingerIdentifier.LEFT_5TH_FINGER -> CoreFingerIdentifier.LEFT_5TH_FINGER +} + +// convert from CoreFingerIdentifier +internal fun CoreFingerIdentifier.fromDomain() = when (this) { + CoreFingerIdentifier.RIGHT_5TH_FINGER -> IFingerIdentifier.RIGHT_5TH_FINGER + CoreFingerIdentifier.RIGHT_4TH_FINGER -> IFingerIdentifier.RIGHT_4TH_FINGER + CoreFingerIdentifier.RIGHT_3RD_FINGER -> IFingerIdentifier.RIGHT_3RD_FINGER + CoreFingerIdentifier.RIGHT_INDEX_FINGER -> IFingerIdentifier.RIGHT_INDEX_FINGER + CoreFingerIdentifier.RIGHT_THUMB -> IFingerIdentifier.RIGHT_THUMB + CoreFingerIdentifier.LEFT_THUMB -> IFingerIdentifier.LEFT_THUMB + CoreFingerIdentifier.LEFT_INDEX_FINGER -> IFingerIdentifier.LEFT_INDEX_FINGER + CoreFingerIdentifier.LEFT_3RD_FINGER -> IFingerIdentifier.LEFT_3RD_FINGER + CoreFingerIdentifier.LEFT_4TH_FINGER -> IFingerIdentifier.LEFT_4TH_FINGER + CoreFingerIdentifier.LEFT_5TH_FINGER -> IFingerIdentifier.LEFT_5TH_FINGER +} diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmFingerprintSampleConverter.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmFingerprintSampleConverter.kt index 421c598613..6b2be097ba 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmFingerprintSampleConverter.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RealmFingerprintSampleConverter.kt @@ -1,12 +1,11 @@ package com.simprints.infra.enrolment.records.repository.local.models import com.simprints.core.domain.fingerprint.FingerprintSample -import com.simprints.core.domain.fingerprint.IFingerIdentifier import com.simprints.infra.enrolment.records.realm.store.models.DbFingerprintSample as RealmFingerprintSample internal fun RealmFingerprintSample.toDomain(): FingerprintSample = FingerprintSample( id = id, - fingerIdentifier = IFingerIdentifier.entries[fingerIdentifier], + fingerIdentifier = IFingerIdentifier.fromId(fingerIdentifier).toDomain(), template = template, format = format, referenceId = referenceId, @@ -15,7 +14,7 @@ internal fun RealmFingerprintSample.toDomain(): FingerprintSample = FingerprintS internal fun FingerprintSample.toRealmDb(): RealmFingerprintSample = RealmFingerprintSample().also { sample -> sample.id = id sample.referenceId = referenceId - sample.fingerIdentifier = fingerIdentifier.ordinal + sample.fingerIdentifier = fingerIdentifier.fromDomain().id sample.template = template sample.format = format } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomBiometricTemplateConverter.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomBiometricTemplateConverter.kt new file mode 100644 index 0000000000..f7e7954f34 --- /dev/null +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomBiometricTemplateConverter.kt @@ -0,0 +1,40 @@ +package com.simprints.infra.enrolment.records.repository.local.models + +import com.simprints.core.domain.face.FaceSample +import com.simprints.core.domain.fingerprint.FingerprintSample +import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate +import com.simprints.infra.enrolment.records.room.store.models.Modality + +internal fun DbBiometricTemplate.toFingerprintSample(): FingerprintSample = FingerprintSample( + id = uuid, + fingerIdentifier = IFingerIdentifier.fromId(identifier!!).toDomain(), + template = templateData, + format = format, + referenceId = referenceId, +) + +internal fun DbBiometricTemplate.toFaceSample(): FaceSample = FaceSample( + id = uuid, + template = templateData, + format = format, + referenceId = referenceId, +) + +internal fun FaceSample.toRoomDb(subjectId: String): DbBiometricTemplate = DbBiometricTemplate( + uuid = id, + templateData = template, + format = format, + subjectId = subjectId, + referenceId = referenceId, + modality = Modality.FACE.id, +) + +internal fun FingerprintSample.toRoomDb(subjectId: String) = DbBiometricTemplate( + uuid = id, + identifier = fingerIdentifier.fromDomain().id, + templateData = template, + format = format, + subjectId = subjectId, + referenceId = referenceId, + modality = Modality.FINGERPRINT.id, +) diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFaceSampleConverter.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFaceSampleConverter.kt deleted file mode 100644 index a1a24949a7..0000000000 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFaceSampleConverter.kt +++ /dev/null @@ -1,19 +0,0 @@ -package com.simprints.infra.enrolment.records.repository.local.models - -import com.simprints.core.domain.face.FaceSample -import com.simprints.infra.enrolment.records.room.store.models.DbFaceSample as RoomFaceSample - -internal fun RoomFaceSample.toDomain(): FaceSample = FaceSample( - id = uuid, - template = template, - format = format, - referenceId = referenceId, -) - -internal fun FaceSample.toRoomDb(subjectId: String): RoomFaceSample = RoomFaceSample( - uuid = id, - template = template, - format = format, - subjectId = subjectId, - referenceId = referenceId, -) diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFingerprintSampleConverter.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFingerprintSampleConverter.kt deleted file mode 100644 index 3c7b809e3d..0000000000 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomFingerprintSampleConverter.kt +++ /dev/null @@ -1,22 +0,0 @@ -package com.simprints.infra.enrolment.records.repository.local.models - -import com.simprints.core.domain.fingerprint.FingerprintSample -import com.simprints.core.domain.fingerprint.IFingerIdentifier -import com.simprints.infra.enrolment.records.room.store.models.DbFingerprintSample as RoomFingerprintSample - -internal fun RoomFingerprintSample.toDomain(): FingerprintSample = FingerprintSample( - id = uuid, - fingerIdentifier = IFingerIdentifier.entries[fingerIdentifier], - template = template, - format = format, - referenceId = referenceId, -) - -internal fun FingerprintSample.toRoomDb(subjectId: String) = RoomFingerprintSample( - uuid = id, - fingerIdentifier = fingerIdentifier.ordinal, - template = template, - format = format, - subjectId = subjectId, - referenceId = referenceId, -) diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomSubjectConverter.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomSubjectConverter.kt index 7098427754..385e341830 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomSubjectConverter.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/RoomSubjectConverter.kt @@ -1,40 +1,20 @@ package com.simprints.infra.enrolment.records.repository.local.models import com.simprints.core.domain.tokenization.asTokenizableEncrypted -import com.simprints.core.domain.tokenization.asTokenizableRaw -import com.simprints.core.domain.tokenization.isTokenized import com.simprints.infra.enrolment.records.repository.domain.models.Subject +import com.simprints.infra.enrolment.records.room.store.models.Modality import com.simprints.infra.enrolment.records.room.store.models.SubjectBiometrics import java.util.Date -import com.simprints.infra.enrolment.records.room.store.models.DbSubject as RoomSubject -internal fun SubjectBiometrics.toDomain(): Subject { - val attendantId = - if (subject.isAttendantIdTokenized) subject.attendantId.asTokenizableEncrypted() else subject.attendantId.asTokenizableRaw() - val moduleId = - if (subject.isModuleIdTokenized) subject.moduleId.asTokenizableEncrypted() else subject.moduleId.asTokenizableRaw() - - return Subject( - subjectId = subject.subjectId.toString(), - projectId = subject.projectId, - attendantId = attendantId, - moduleId = moduleId, - createdAt = subject.createdAt?.toDate(), - updatedAt = subject.updatedAt?.toDate(), - fingerprintSamples = fingerprintSamples.map { it.toDomain() }, - faceSamples = faceSamples.map { it.toDomain() }, - ) -} - -internal fun Subject.toRoomDb(): RoomSubject = RoomSubject( - subjectId = subjectId, - projectId = projectId, - attendantId = attendantId.value, - moduleId = moduleId.value, - createdAt = createdAt?.time, - updatedAt = updatedAt?.time, - isModuleIdTokenized = moduleId.isTokenized(), - isAttendantIdTokenized = attendantId.isTokenized(), +internal fun SubjectBiometrics.toDomain() = Subject( + subjectId = subject.subjectId.toString(), + projectId = subject.projectId, + attendantId = subject.attendantId.asTokenizableEncrypted(), + moduleId = subject.moduleId.asTokenizableEncrypted(), + createdAt = subject.createdAt?.toDate(), + updatedAt = subject.updatedAt?.toDate(), + fingerprintSamples = biometricTemplates.filter { it.modality == Modality.FINGERPRINT.id }.map { it.toFingerprintSample() }, + faceSamples = biometricTemplates.filter { it.modality == Modality.FACE.id }.map { it.toFaceSample() }, ) fun Long.toDate() = Date(this) diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSourceTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSourceTest.kt index e6e361d1d0..2b836a20c2 100644 --- a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSourceTest.kt +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSourceTest.kt @@ -18,15 +18,12 @@ import com.simprints.infra.enrolment.records.repository.domain.models.Fingerprin import com.simprints.infra.enrolment.records.repository.domain.models.Subject import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAction import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery -import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource.Companion.FACE_SAMPLES_FIELD -import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource.Companion.FINGERPRINT_SAMPLES_FIELD -import com.simprints.infra.enrolment.records.repository.local.EnrolmentRecordLocalDataSource.Companion.FORMAT_FIELD +import com.simprints.infra.enrolment.records.repository.local.RealmEnrolmentRecordLocalDataSource.Companion.FACE_SAMPLES_FIELD +import com.simprints.infra.enrolment.records.repository.local.RealmEnrolmentRecordLocalDataSource.Companion.FINGERPRINT_SAMPLES_FIELD +import com.simprints.infra.enrolment.records.repository.local.RealmEnrolmentRecordLocalDataSource.Companion.FORMAT_FIELD import com.simprints.infra.enrolment.records.repository.local.models.toDomain import com.simprints.infra.enrolment.records.repository.local.models.toRealmDb -import io.mockk.CapturingSlot -import io.mockk.MockKAnnotations -import io.mockk.coEvery -import io.mockk.every +import io.mockk.* import io.mockk.impl.annotations.MockK import io.realm.kotlin.MutableRealm import io.realm.kotlin.Realm @@ -259,7 +256,7 @@ class RealmEnrolmentRecordLocalDataSourceTest { every { realmSingleQuery.find() } returns null enrolmentRecordLocalDataSource.performActions( - listOf(SubjectAction.Creation(subject.fromDbToDomain())), + listOf(SubjectAction.Creation(subject.toDomain())), project, ) val peopleCount = enrolmentRecordLocalDataSource.count() @@ -276,7 +273,7 @@ class RealmEnrolmentRecordLocalDataSourceTest { fingerprintSamples = listOf( getRandomFingerprintSample("fingerToDelete"), ), - ).fromDomainToDb() + ).toRealmDb() val subject = getFakePerson() enrolmentRecordLocalDataSource.performActions( @@ -304,7 +301,7 @@ class RealmEnrolmentRecordLocalDataSourceTest { getRandomFingerprintSample(referenceId = "fingerToDelete"), getRandomFingerprintSample(), ), - ).fromDomainToDb() + ).toRealmDb() enrolmentRecordLocalDataSource.performActions( listOf( diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrollmentRecordLocalDataSourceTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrollmentRecordLocalDataSourceTest.kt new file mode 100644 index 0000000000..9e72fc9bba --- /dev/null +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrollmentRecordLocalDataSourceTest.kt @@ -0,0 +1,1537 @@ +package com.simprints.infra.enrolment.records.repository.local + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import com.simprints.core.domain.face.FaceSample +import com.simprints.core.domain.fingerprint.FingerprintSample +import com.simprints.core.domain.fingerprint.IFingerIdentifier +import com.simprints.core.domain.tokenization.TokenizableString +import com.simprints.core.domain.tokenization.asTokenizableEncrypted +import com.simprints.infra.config.store.models.Project +import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource.Simprints +import com.simprints.infra.enrolment.records.repository.domain.models.Subject +import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAction +import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery +import com.simprints.infra.enrolment.records.room.store.SubjectsDatabaseFactory +import com.simprints.infra.security.keyprovider.LocalDbKey +import com.simprints.testtools.common.coroutines.TestCoroutineRule +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.channels.toList +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.test.runTest +import org.junit.After +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.util.Date +import java.util.UUID + +@RunWith(RobolectricTestRunner::class) +class RoomEnrollmentRecordLocalDataSourceTest { + companion object { + const val PROJECT_1_ID = "project1" + const val PROJECT_2_ID = "project2" + val ATTENDANT_1_ID = "attendant1".asTokenizableEncrypted() + val ATTENDANT_2_ID = "attendant2".asTokenizableEncrypted() + val MODULE_1_ID = "module1".asTokenizableEncrypted() + val MODULE_2_ID = "module2".asTokenizableEncrypted() + val MODULE_3_ID = "module3".asTokenizableEncrypted() + + const val ROC_1_FORMAT = "roc_123" + const val ROC_3_FORMAT = "roc_3" + const val NEC_FORMAT = "nec" + const val ISO_FORMAT = "iso" + const val UNUSED_FORMAT = "unused" + } + + // Mocks and setup remain the same + private lateinit var dataSource: EnrolmentRecordLocalDataSource + + @get:Rule + val testCoroutineRule = TestCoroutineRule() + + private val subjectDatabaseFactory = SubjectsDatabaseFactory( + ApplicationProvider.getApplicationContext(), + mockk { + every { getLocalDbKeyOrThrow(any()) } returns LocalDbKey("any", byteArrayOf(1, 2, 3)) + }, + ) + + // --- Test Data --- + private val date = Date() // Use a fixed date for consistent timestamps in tests + + // Samples defined first + private val faceSample1 = FaceSample( + template = byteArrayOf(1, 2, 3), + format = ROC_1_FORMAT, + referenceId = "ref-face-1", + id = "face-uuid-1", + ) + private val faceSample2 = FaceSample( + template = byteArrayOf(4, 5, 6), + format = ROC_3_FORMAT, + referenceId = "ref-face-2", + id = "face-uuid-2", + ) + private val faceSample3 = FaceSample( + template = byteArrayOf(7, 8, 9), + format = ROC_1_FORMAT, + referenceId = "ref-face-3-p2", + id = "face-uuid-3-p2", + ) + private val fingerprintSample1 = FingerprintSample( + fingerIdentifier = IFingerIdentifier.LEFT_THUMB, + template = byteArrayOf(10, 11), + format = NEC_FORMAT, + referenceId = "ref-fp-1", + id = "fp-uuid-1", + ) + private val fingerprintSample2 = FingerprintSample( + fingerIdentifier = IFingerIdentifier.RIGHT_THUMB, + template = byteArrayOf(12, 13), + format = ISO_FORMAT, + referenceId = "ref-fp-2", + id = "fp-uuid-2", + ) + private val fingerprintSample3 = FingerprintSample( + fingerIdentifier = IFingerIdentifier.LEFT_INDEX_FINGER, + template = byteArrayOf(14, 15), + format = NEC_FORMAT, + referenceId = "ref-fp-3-p2", + id = "fp-uuid-3-p2", + ) + + // Subjects defined using the samples + private val subject1P1WithFace = Subject( + subjectId = "subj-001", + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_1_ID, + faceSamples = listOf(faceSample1), + fingerprintSamples = emptyList(), + createdAt = date, + updatedAt = date, + ) + private val subject2P1WithFinger = Subject( + subjectId = "subj-002", + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_1_ID, + faceSamples = emptyList(), + fingerprintSamples = listOf(fingerprintSample1), + createdAt = date, + updatedAt = date, + ) + private val subject3P1WithBoth = Subject( + subjectId = "subj-003", + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_2_ID, + faceSamples = listOf(faceSample2), + fingerprintSamples = listOf(fingerprintSample2), + ) + private val subject4P2WithBoth = Subject( + subjectId = "subj-004", + projectId = PROJECT_2_ID, + attendantId = ATTENDANT_2_ID, + moduleId = MODULE_2_ID, + faceSamples = listOf(faceSample3), + fingerprintSamples = listOf(fingerprintSample3), + createdAt = date, + updatedAt = date, + ) + private val subject5P2WithFace = Subject( + // Added subject + subjectId = "subj-005", + projectId = PROJECT_2_ID, + attendantId = ATTENDANT_2_ID, + moduleId = MODULE_3_ID, + faceSamples = listOf(faceSample3.copy(id = UUID.randomUUID().toString())), + fingerprintSamples = emptyList(), + createdAt = Date(date.time + 1000), // Slightly different time + updatedAt = Date(date.time + 1000), + ) + private val subject6P2WithFinger = Subject( + // Added subject + subjectId = "subj-006", + projectId = PROJECT_2_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_3_ID, + faceSamples = emptyList(), + fingerprintSamples = listOf(fingerprintSample3.copy(id = UUID.randomUUID().toString())), + createdAt = Date(date.time + 2000), // Different time + updatedAt = Date(date.time + 2000), + ) + private val subjectInvalidNoSamples = Subject( + subjectId = "subj-invalid", + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_1_ID, + createdAt = date, + updatedAt = date, + ) + + private val project: Project = mockk() + private lateinit var mockCallback: () -> Unit + + @Before + fun setUp() { + dataSource = RoomEnrolmentRecordLocalDataSource( + subjectDatabaseFactory, + mockk { + every { tokenizeIfNecessary(any(), any(), any()) } answers { + val arg1 = it.invocation.args[0] as TokenizableString + arg1 + } + }, + queryBuilder = RoomEnrolmentRecordQueryBuilder(), + dispatcherIO = testCoroutineRule.testCoroutineDispatcher, + ) + mockCallback = mockk(relaxed = true) // Relaxed mock to avoid specifying return values + every { project.id } returns PROJECT_1_ID // Basic mock setup for project ID + } + + @After + fun tearDown() { + runBlocking { + dataSource.deleteAll() + subjectDatabaseFactory.get().close() // Close the database connection + } + } + + val initialData = listOf( + SubjectAction.Creation(subject1P1WithFace), + SubjectAction.Creation(subject2P1WithFinger), + SubjectAction.Creation(subject3P1WithBoth), + SubjectAction.Creation(subject4P2WithBoth), + SubjectAction.Creation(subject5P2WithFace), + SubjectAction.Creation(subject6P2WithFinger), + ) + + private suspend fun setupInitialData() { + dataSource.performActions( + initialData, + project, + ) + } + + // --- Test Cases --- + + @Test + fun `performActions - Creation - should succeed with face sample`() = runTest { + // Given + val action = SubjectAction.Creation(subject1P1WithFace) + + // When + dataSource.performActions(listOf(action), project) + + // Then + val loaded = dataSource.load(SubjectQuery(subjectId = subject1P1WithFace.subjectId)) + assertThat(loaded).hasSize(1) + val createdSubject = loaded[0] + assertThat(createdSubject.subjectId).isEqualTo(subject1P1WithFace.subjectId) + assertThat(createdSubject.faceSamples).hasSize(1) + assertThat(createdSubject.faceSamples).containsExactly(faceSample1) + assertThat(createdSubject.fingerprintSamples).isEmpty() + assertThat(createdSubject.createdAt).isNotNull() + assertThat(createdSubject.updatedAt).isNotNull() + } + + @Test + fun `performActions - Creation - should succeed with fingerprint sample`() = runTest { + // Given + val action = SubjectAction.Creation(subject2P1WithFinger) + + // When + dataSource.performActions(listOf(action), project) + + // Then + val loaded = dataSource.load(SubjectQuery(subjectId = subject2P1WithFinger.subjectId)) + assertThat(loaded).hasSize(1) + val createdSubject = loaded[0] + assertThat(createdSubject.subjectId).isEqualTo(subject2P1WithFinger.subjectId) + assertThat(createdSubject.faceSamples).isEmpty() + assertThat(createdSubject.fingerprintSamples).hasSize(1) + assertThat(createdSubject.fingerprintSamples).containsExactly(fingerprintSample1) + assertThat(createdSubject.createdAt).isNotNull() + assertThat(createdSubject.updatedAt).isNotNull() + } + + @Test + fun `performActions - Creation - should succeed with both samples`() = runTest { + // Given: + val action = SubjectAction.Creation(subject3P1WithBoth) + + // When + dataSource.performActions(listOf(action), project) + + // Then + val loaded = dataSource.load(SubjectQuery(subjectId = subject3P1WithBoth.subjectId)) + assertThat(loaded).hasSize(1) + val createdSubject = loaded[0] + assertThat(createdSubject.subjectId).isEqualTo(subject3P1WithBoth.subjectId) + assertThat(createdSubject.faceSamples).hasSize(1) + assertThat(createdSubject.faceSamples).containsExactly(faceSample2) + assertThat(createdSubject.fingerprintSamples).hasSize(1) + assertThat(createdSubject.fingerprintSamples).containsExactly(fingerprintSample2) + } + + @Test(expected = IllegalArgumentException::class) // Reverted to JUnit exception check + fun `performActions - Creation - should fail without any samples`() = runTest { + // Given + val action = SubjectAction.Creation(subjectInvalidNoSamples) // Subject without samples + + // When + dataSource.performActions(listOf(action), project) // This line will throw + + // Then: Handled by expected exception. + } + + @Test + fun `performActions - Creation - should create multiple valid subjects`() = runTest { + // Given + val actions = listOf( + SubjectAction.Creation(subject1P1WithFace), + SubjectAction.Creation(subject2P1WithFinger), + SubjectAction.Creation(subject4P2WithBoth), // Belongs to Project 2 + ) + + // When + dataSource.performActions(actions, project) + + // Then + val loadedAllP1 = dataSource.load(SubjectQuery(projectId = PROJECT_1_ID, sort = true)) + val loadedAllP2 = dataSource.load(SubjectQuery(projectId = PROJECT_2_ID, sort = true)) + + assertThat(loadedAllP1).hasSize(2) + assertThat(loadedAllP1.map { it.subjectId }) + .containsExactly( + subject1P1WithFace.subjectId, + subject2P1WithFinger.subjectId, + ).inOrder() + + assertThat(loadedAllP2).hasSize(1) + assertThat(loadedAllP2[0].subjectId).isEqualTo(subject4P2WithBoth.subjectId) + } + + @Test + fun `performActions - Update - should add face and fingerprint samples`() = runTest { + // Given: Create initial subject + dataSource.performActions(listOf(SubjectAction.Creation(subject1P1WithFace)), project) + val initialSubject = + dataSource.load(SubjectQuery(subjectId = subject1P1WithFace.subjectId)).first() + assertThat(initialSubject.faceSamples).hasSize(1) + assertThat(initialSubject.fingerprintSamples).isEmpty() + + // Original Update action instantiation style maintained + val updateAction = SubjectAction.Update( + subjectId = subject1P1WithFace.subjectId, + faceSamplesToAdd = listOf(faceSample2), + fingerprintSamplesToAdd = listOf(fingerprintSample1), + referenceIdsToRemove = listOf(), + ) + + // When + dataSource.performActions(listOf(updateAction), project) + + // Then + val loaded = dataSource.load(SubjectQuery(subjectId = subject1P1WithFace.subjectId)) + assertThat(loaded).hasSize(1) + val updatedSubject = loaded[0] + assertThat(updatedSubject.faceSamples).hasSize(2) + assertThat(updatedSubject.faceSamples).containsExactly(faceSample1, faceSample2) + assertThat(updatedSubject.fingerprintSamples).hasSize(1) + assertThat(updatedSubject.fingerprintSamples).containsExactly(fingerprintSample1) + } + + @Test + fun `performActions - Update - should remove face sample when fingerprint sample exists`() = runTest { + // Given + val subjectToUpdate = subject3P1WithBoth + dataSource.performActions(listOf(SubjectAction.Creation(subjectToUpdate)), project) + val initial = + dataSource.load(SubjectQuery(subjectId = subjectToUpdate.subjectId)).first() + assertThat(initial.faceSamples).hasSize(1) + assertThat(initial.fingerprintSamples).hasSize(1) + + // Original Update action instantiation style maintained + val updateAction = SubjectAction.Update( + subjectId = subjectToUpdate.subjectId, + faceSamplesToAdd = listOf(), // Explicitly empty as in original + fingerprintSamplesToAdd = listOf(), // Explicitly empty as in original + referenceIdsToRemove = listOf(faceSample2.referenceId), + ) + + // When + dataSource.performActions(listOf(updateAction), project) + + // Then + val loaded = + dataSource.load(SubjectQuery(subjectId = subjectToUpdate.subjectId)).first() + assertThat(loaded.faceSamples).isEmpty() + assertThat(loaded.fingerprintSamples).hasSize(1) + assertThat(loaded.fingerprintSamples).containsExactly(fingerprintSample2) + } + + @Test + fun `performActions - Update - should remove fingerprint sample when face sample exists`() = runTest { + // Given + val subjectToUpdate = subject3P1WithBoth + dataSource.performActions(listOf(SubjectAction.Creation(subjectToUpdate)), project) + val initial = + dataSource.load(SubjectQuery(subjectId = subjectToUpdate.subjectId)).first() + assertThat(initial.faceSamples).hasSize(1) + assertThat(initial.fingerprintSamples).hasSize(1) + + // Original Update action instantiation style maintained + val updateAction = SubjectAction.Update( + subjectId = subjectToUpdate.subjectId, + faceSamplesToAdd = listOf(), + fingerprintSamplesToAdd = listOf(), + referenceIdsToRemove = listOf(fingerprintSample2.referenceId), + ) + + // When + dataSource.performActions(listOf(updateAction), project) + + // Then + val loaded = + dataSource.load(SubjectQuery(subjectId = subjectToUpdate.subjectId)).first() + assertThat(loaded.faceSamples).hasSize(1) + assertThat(loaded.faceSamples).containsExactly(faceSample2) + assertThat(loaded.fingerprintSamples).isEmpty() + } + + @Test(expected = IllegalArgumentException::class) // Reverted to JUnit exception check + fun `performActions - Update - should fail when removing last face sample`() = runTest { + // Given: Subject with only a face sample + dataSource.performActions(listOf(SubjectAction.Creation(subject1P1WithFace)), project) + val initial = + dataSource.load(SubjectQuery(subjectId = subject1P1WithFace.subjectId)).first() + assertThat(initial.faceSamples).hasSize(1) + assertThat(initial.fingerprintSamples).isEmpty() + + // Original Update action instantiation style maintained + val updateAction = SubjectAction.Update( + subjectId = subject1P1WithFace.subjectId, + faceSamplesToAdd = listOf(), + fingerprintSamplesToAdd = listOf(), + referenceIdsToRemove = listOf(faceSample1.referenceId), + ) + + // When + dataSource.performActions(listOf(updateAction), project) // This line will throw + + // Then: Handled by expected exception + } + + @Test(expected = IllegalArgumentException::class) // Reverted to JUnit exception check + fun `performActions - Update - should fail when removing last fingerprint sample`() = runTest { + // Given: Subject with only a fingerprint sample + dataSource.performActions(listOf(SubjectAction.Creation(subject2P1WithFinger)), project) + val initial = + dataSource.load(SubjectQuery(subjectId = subject2P1WithFinger.subjectId)).first() + assertThat(initial.faceSamples).isEmpty() + assertThat(initial.fingerprintSamples).hasSize(1) + + // Original Update action instantiation style maintained + val updateAction = SubjectAction.Update( + subjectId = subject2P1WithFinger.subjectId, + faceSamplesToAdd = listOf(), + fingerprintSamplesToAdd = listOf(), + referenceIdsToRemove = listOf(fingerprintSample1.referenceId), + ) + + // When + dataSource.performActions(listOf(updateAction), project) // This line will throw + + // Then: Handled by expected exception + } + + @Test + fun `performActions - Update - should not add duplicate samples based on ID`() = runTest { + // Given: Subject already has faceSample1 + dataSource.performActions(listOf(SubjectAction.Creation(subject1P1WithFace)), project) + + // Original Update action instantiation style maintained + val updateAction = SubjectAction.Update( + subjectId = subject1P1WithFace.subjectId, + faceSamplesToAdd = listOf(faceSample1, faceSample2), + fingerprintSamplesToAdd = listOf(fingerprintSample1), + referenceIdsToRemove = listOf(), + ) + + // When + dataSource.performActions(listOf(updateAction), project) + + // Then + val loaded = dataSource.load(SubjectQuery(subjectId = subject1P1WithFace.subjectId)) + assertThat(loaded).hasSize(1) + val finalSubject = loaded[0] + + assertThat(finalSubject.faceSamples).hasSize(2) + assertThat(finalSubject.faceSamples).containsExactly(faceSample1, faceSample2) + + assertThat(finalSubject.fingerprintSamples).hasSize(1) + assertThat(finalSubject.fingerprintSamples).containsExactly(fingerprintSample1) + } + + @Test + fun `performActions - Update - non-existent subjectId - should do nothing`() = runTest { + // Given + setupInitialData() + val initialCount = dataSource.count() + val nonExistentSubjectId = "subj-does-not-exist" + + val updateAction = SubjectAction.Update( + subjectId = nonExistentSubjectId, + faceSamplesToAdd = listOf(faceSample1), // Try to add samples + fingerprintSamplesToAdd = listOf(), + referenceIdsToRemove = listOf(), + ) + + // When + // No exception should be thrown here + dataSource.performActions(listOf(updateAction), project) + + // Then + val finalCount = dataSource.count() + assertThat(finalCount).isEqualTo(initialCount) // Count should not change + + // Verify the subject was not created + val loaded = dataSource.load(SubjectQuery(subjectId = nonExistentSubjectId)) + assertThat(loaded).isEmpty() + } + + @Test + fun `load - by no query - should return all subjects`() = runTest { + // Given + setupInitialData() + + // When + val loaded = dataSource.load(SubjectQuery()) + + // Then + assertThat(loaded).hasSize(initialData.size) + } + + @Test + fun `load - by subjectId - should return correct subject`() = runTest { + // Given + setupInitialData() + + // When + val loaded = dataSource.load(SubjectQuery(subjectId = subject1P1WithFace.subjectId)) + + // Then + assertThat(loaded).hasSize(1) + assertThat(loaded[0].subjectId).isEqualTo(subject1P1WithFace.subjectId) + assertThat(loaded[0].faceSamples).containsExactly(faceSample1) + assertThat(loaded[0].fingerprintSamples).isEmpty() + } + + @Test + fun `load - by hasUntokenizedFields -should return empty list`() = runTest { + // Given + setupInitialData() + + // When + val loaded = dataSource.load(SubjectQuery(hasUntokenizedFields = true)) + + // Then + assertThat(loaded).isEmpty() + } + + @Test + fun `load - by projectId - should return subjects for that project only`() = runTest { + // Given + setupInitialData() + + // When + val loadedP1 = dataSource.load(SubjectQuery(projectId = PROJECT_1_ID)) + val loadedP2 = dataSource.load(SubjectQuery(projectId = PROJECT_2_ID)) + + // Then + assertThat(loadedP1).hasSize(3) + val loadedP1Ids = loadedP1.map { it.subjectId } + assertThat(loadedP1Ids).containsExactly( + subject1P1WithFace.subjectId, + subject2P1WithFinger.subjectId, + subject3P1WithBoth.subjectId, + ) + assertThat(loadedP2).hasSize(3) + } + + @Test + fun `performActions - Deletion - should delete existing subject`() = runTest { + // Given + setupInitialData() + assertThat(dataSource.load(SubjectQuery(subjectId = subject1P1WithFace.subjectId))).isNotEmpty() + + val deleteAction = SubjectAction.Deletion(subjectId = subject1P1WithFace.subjectId) + + // When + dataSource.performActions(listOf(deleteAction), project) + + // Then + val loaded = dataSource.load(SubjectQuery(subjectId = subject1P1WithFace.subjectId)) + assertThat(loaded).isEmpty() + assertThat(dataSource.count()).isEqualTo(initialData.size - 1) + } + + @Test + fun `combined actions - create, update, delete`() = runTest { + // --- Create --- + val createAction = SubjectAction.Creation(subject1P1WithFace) + dataSource.performActions(listOf(createAction), project) + var loadedSubject = + dataSource.load(SubjectQuery(subjectId = subject1P1WithFace.subjectId)).firstOrNull() + assertThat(loadedSubject).isNotNull() + assertThat(loadedSubject!!.faceSamples).hasSize(1) + assertThat(loadedSubject.fingerprintSamples).isEmpty() + + // --- Update (Original style maintained) --- + val updateAction = SubjectAction.Update( + subjectId = subject1P1WithFace.subjectId, + faceSamplesToAdd = listOf(), + fingerprintSamplesToAdd = listOf(fingerprintSample1), + referenceIdsToRemove = listOf(), + ) + dataSource.performActions(listOf(updateAction), project) + loadedSubject = + dataSource.load(SubjectQuery(subjectId = subject1P1WithFace.subjectId)).firstOrNull() + assertThat(loadedSubject).isNotNull() + assertThat(loadedSubject!!.faceSamples).hasSize(1) + assertThat(loadedSubject.fingerprintSamples).hasSize(1) + assertThat(loadedSubject.fingerprintSamples).containsExactly(fingerprintSample1) + + // --- Delete --- + val deleteAction = SubjectAction.Deletion(subjectId = subject1P1WithFace.subjectId) + dataSource.performActions(listOf(deleteAction), project) + val finalLoad = dataSource.load(SubjectQuery(subjectId = subject1P1WithFace.subjectId)) + assertThat(finalLoad).isEmpty() + } + + @Test + fun `count - with no query - should return total number of subjects`() = runTest { + // Given + setupInitialData() + + // When + val count = dataSource.count() + + // Then + assertThat(count).isEqualTo(initialData.size) + } + + @Test + fun `count - by projectId - should return count for that project`() = runTest { + // Given + setupInitialData() + + // When + val countP1 = dataSource.count(SubjectQuery(projectId = PROJECT_1_ID)) + val countP2 = dataSource.count(SubjectQuery(projectId = PROJECT_2_ID)) + + // Then + assertThat(countP1).isEqualTo(3) + assertThat(countP2).isEqualTo(3) + } + + @Test + fun `count - by attendantId - should return count for that attendant`() = runTest { + // Given + setupInitialData() + + // When + val countAtt1 = dataSource.count(SubjectQuery(attendantId = ATTENDANT_1_ID)) + val countAtt2 = dataSource.count(SubjectQuery(attendantId = ATTENDANT_2_ID)) + + // Then + assertThat(countAtt1).isEqualTo(4) + assertThat(countAtt2).isEqualTo(2) + } + + @Test + fun `count - by moduleId - should return count for that module`() = runTest { + // Given + setupInitialData() + + // When + val countModule1 = dataSource.count(SubjectQuery(moduleId = MODULE_1_ID)) + val countModule2 = dataSource.count(SubjectQuery(moduleId = MODULE_2_ID)) + + // Then + assertThat(countModule1).isEqualTo(2) + assertThat(countModule2).isEqualTo(2) + } + + @Test + fun `count - by faceSampleFormat - should return count of subjects with matching format`() = runTest { + // Given + setupInitialData() + + // When + val countRoc1P1 = dataSource.count( + SubjectQuery( + projectId = PROJECT_1_ID, + faceSampleFormat = ROC_1_FORMAT, + ), + ) + val countRoc3P1 = dataSource.count( + SubjectQuery( + projectId = PROJECT_1_ID, + faceSampleFormat = ROC_3_FORMAT, + ), + ) + val countRoc1P2 = dataSource.count( + SubjectQuery( + projectId = PROJECT_2_ID, + faceSampleFormat = ROC_1_FORMAT, + ), + ) + val countAllRoc1 = dataSource.count(SubjectQuery(faceSampleFormat = ROC_1_FORMAT)) + + // Then + assertThat(countRoc1P1).isEqualTo(1) + assertThat(countRoc3P1).isEqualTo(1) + assertThat(countRoc1P2).isEqualTo(2) + assertThat(countAllRoc1).isEqualTo(3) + } + + @Test + fun `count - by fingerprintSampleFormat - should return count of subjects with matching format`() = runTest { + // Given + setupInitialData() + + // When + val countNecP1 = dataSource.count( + SubjectQuery( + projectId = PROJECT_1_ID, + fingerprintSampleFormat = NEC_FORMAT, + ), + ) + val countIsoP1 = dataSource.count( + SubjectQuery( + projectId = PROJECT_1_ID, + fingerprintSampleFormat = ISO_FORMAT, + ), + ) + val countNecP2 = dataSource.count( + SubjectQuery( + projectId = PROJECT_2_ID, + fingerprintSampleFormat = NEC_FORMAT, + ), + ) + val countAllNec = dataSource.count(SubjectQuery(fingerprintSampleFormat = NEC_FORMAT)) + + // Then + assertThat(countNecP1).isEqualTo(1) + assertThat(countIsoP1).isEqualTo(1) + assertThat(countNecP2).isEqualTo(2) + assertThat(countAllNec).isEqualTo(3) + } + + @Test + fun `count - with query matching nothing - should return zero`() = runTest { + // Given + setupInitialData() + + // When + val count = dataSource.count( + SubjectQuery( + projectId = "non-existent-project", + moduleId = "non-existent-module".asTokenizableEncrypted(), + attendantId = "non-existent-attendant".asTokenizableEncrypted(), + ), + ) + + // Then + assertThat(count).isEqualTo(0) + } + + @Test(expected = IllegalArgumentException::class) // Reverted to JUnit exception check + fun `loadFingerprintIdentities - should throw exception if format is missing in query`() = runTest { + // Given + setupInitialData() + val queryWithoutFormat = SubjectQuery(projectId = PROJECT_1_ID) + + // When + dataSource.loadFingerprintIdentities( + // This call will throw + query = queryWithoutFormat, + ranges = listOf(0..10), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ) + + // Then: Handled by expected exception + // Callback should not be called if exception occurs before iteration + verify(exactly = 0) { mockCallback() } + } + + @Test + fun `loadFingerprintIdentities - should load identities matching format for the specified project`() = runTest { + // Given + setupInitialData() + val project2Mock: Project = mockk { every { id } returns PROJECT_2_ID } + + // When - Query P1 for NEC + val loadedP1Nec = dataSource + .loadFingerprintIdentities( + query = SubjectQuery( + projectId = PROJECT_1_ID, + fingerprintSampleFormat = NEC_FORMAT, + ), + ranges = listOf(0..10), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + // When - Query P1 for ISO + val loadedP1Iso = dataSource + .loadFingerprintIdentities( + query = SubjectQuery( + projectId = PROJECT_1_ID, + fingerprintSampleFormat = ISO_FORMAT, + ), + ranges = listOf(0..10), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + // When - Query P2 for NEC + val loadedP2Nec = dataSource + .loadFingerprintIdentities( + query = SubjectQuery( + projectId = PROJECT_2_ID, + fingerprintSampleFormat = NEC_FORMAT, + ), + ranges = listOf(0..10), + project = project2Mock, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + + // Then - P1 NEC + assertThat(loadedP1Nec).hasSize(1) + assertThat(loadedP1Nec[0].subjectId).isEqualTo(subject2P1WithFinger.subjectId) + assertThat(loadedP1Nec[0].fingerprints).hasSize(1) + assertThat(loadedP1Nec[0].fingerprints[0].format).isEqualTo(NEC_FORMAT) + assertThat(loadedP1Nec[0].fingerprints).isEqualTo(subject2P1WithFinger.fingerprintSamples) + + // Then - P1 ISO + assertThat(loadedP1Iso).hasSize(1) + assertThat(loadedP1Iso[0].subjectId).isEqualTo(subject3P1WithBoth.subjectId) + assertThat(loadedP1Iso[0].fingerprints).hasSize(1) + assertThat(loadedP1Iso[0].fingerprints[0].format).isEqualTo(ISO_FORMAT) + assertThat(loadedP1Iso[0].fingerprints).isEqualTo(subject3P1WithBoth.fingerprintSamples) + + // Then - P2 NEC + assertThat(loadedP2Nec).hasSize(2) + assertThat(loadedP2Nec[0].subjectId).isEqualTo(subject4P2WithBoth.subjectId) + assertThat(loadedP2Nec[0].fingerprints).hasSize(1) + assertThat(loadedP2Nec[0].fingerprints[0].format).isEqualTo(NEC_FORMAT) + + verify(exactly = 4) { mockCallback() } + } + + @Test + fun `loadFingerprintIdentities - should respect the range parameter for specific format`() = runTest { + // Given: More data + val subject5P1WithNec = subject2P1WithFinger.copy( + subjectId = "subj-005", + fingerprintSamples = listOf( + fingerprintSample1.copy( + id = "fp-uuid-5", + referenceId = "ref-fp-5", + ), + ), + createdAt = Date(date.time + 1000), + ) + val subject6P1WithNec = subject2P1WithFinger.copy( + subjectId = "subj-006", + fingerprintSamples = listOf( + fingerprintSample1.copy( + id = "fp-uuid-6", + referenceId = "ref-fp-6", + ), + ), + createdAt = Date(date.time + 2000), + ) + setupInitialData() + dataSource.performActions( + listOf( + SubjectAction.Creation(subject5P1WithNec), + SubjectAction.Creation(subject6P1WithNec), + ), + project, + ) + val baseQuery = SubjectQuery( + projectId = PROJECT_1_ID, + fingerprintSampleFormat = NEC_FORMAT, + ) + // When + val loadedRanges = + dataSource + .loadFingerprintIdentities( + query = baseQuery, + ranges = listOf( + 0..1, + 1..2, + ), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + + val loadedFirstTwo = + dataSource + .loadFingerprintIdentities( + query = baseQuery.copy(), // Copy to ensure new instance to avoid using the last subjectId + ranges = listOf( + 0..2, + ), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + val loadedAll = + dataSource + .loadFingerprintIdentities( + query = baseQuery.copy(), + ranges = listOf(0..10), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + + // Then + assertThat(loadedRanges).hasSize(2) + assertThat(loadedRanges[0][0].subjectId).isEqualTo(subject2P1WithFinger.subjectId) + assertThat(loadedRanges[1]).hasSize(1) + assertThat(loadedRanges[1][0].subjectId).isEqualTo(subject5P1WithNec.subjectId) + assertThat(loadedFirstTwo).hasSize(2) + assertThat(loadedFirstTwo.map { it.subjectId }) + .containsExactly(subject2P1WithFinger.subjectId, subject5P1WithNec.subjectId) + .inOrder() + assertThat(loadedAll).hasSize(3) + assertThat(loadedAll.map { it.subjectId }) + .containsExactly( + subject2P1WithFinger.subjectId, + subject5P1WithNec.subjectId, + subject6P1WithNec.subjectId, + ).inOrder() + verify(exactly = 7) { mockCallback() } + } + + @Test + fun `loadFingerprintIdentities - with query format matching nothing - should return empty list`() = runTest { + // Given + setupInitialData() + + // When + val loadedIdentities = dataSource + .loadFingerprintIdentities( + query = SubjectQuery( + projectId = PROJECT_1_ID, + fingerprintSampleFormat = UNUSED_FORMAT, + ), + ranges = listOf(0..10), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + + // Then + assertThat(loadedIdentities).isEmpty() + verify(exactly = 0) { mockCallback() } + } + + @Test(expected = IllegalArgumentException::class) // Reverted to JUnit exception check + fun `loadFaceIdentities - should throw exception if format is missing in query`() = runTest { + // Given + setupInitialData() + val queryWithoutFormat = SubjectQuery(projectId = PROJECT_1_ID) + + // When + dataSource + .loadFaceIdentities( + // This call will throw + query = queryWithoutFormat, + ranges = listOf(0..10), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + + // Then: Handled by expected exception + verify(exactly = 0) { mockCallback() } + } + + @Test + fun `loadFaceIdentities - should load identities matching format for the specified project`() = runTest { + // Given + setupInitialData() + + // When + val loadedP1Roc1 = dataSource + .loadFaceIdentities( + query = SubjectQuery(projectId = PROJECT_1_ID, faceSampleFormat = ROC_1_FORMAT), + ranges = listOf(0..10), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + val loadedP1Roc3 = dataSource + .loadFaceIdentities( + query = SubjectQuery(projectId = PROJECT_1_ID, faceSampleFormat = ROC_3_FORMAT), + ranges = listOf(0..10), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + val loadedP2Roc1 = dataSource + .loadFaceIdentities( + query = SubjectQuery(projectId = PROJECT_2_ID, faceSampleFormat = ROC_1_FORMAT), + ranges = listOf(0..10), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + + // Then - P1 ROC_1 + assertThat(loadedP1Roc1).hasSize(1) + assertThat(loadedP1Roc1[0].subjectId).isEqualTo(subject1P1WithFace.subjectId) + assertThat(loadedP1Roc1[0].faces).hasSize(1) + assertThat(loadedP1Roc1[0].faces[0].format).isEqualTo(ROC_1_FORMAT) + assertThat(loadedP1Roc1[0].faces).isEqualTo(subject1P1WithFace.faceSamples) + + // Then - P1 ROC_3 + assertThat(loadedP1Roc3).hasSize(1) + assertThat(loadedP1Roc3[0].subjectId).isEqualTo(subject3P1WithBoth.subjectId) + assertThat(loadedP1Roc3[0].faces).hasSize(1) + assertThat(loadedP1Roc3[0].faces[0].format).isEqualTo(ROC_3_FORMAT) + assertThat(loadedP1Roc3[0].faces).isEqualTo(subject3P1WithBoth.faceSamples) + + // Then - P2 ROC_1 + assertThat(loadedP2Roc1).hasSize(2) + assertThat(loadedP2Roc1[0].subjectId).isEqualTo(subject4P2WithBoth.subjectId) + + verify(exactly = 4) { mockCallback() } + } + + @Test + fun `loadFaceIdentities - should respect the range parameter for specific format`() = runTest { + // Given: More data + val subject5P1WithRoc1 = subject1P1WithFace.copy( + subjectId = "subj-005", + faceSamples = listOf(faceSample1.copy(id = "face-uuid-5", referenceId = "ref-face-5")), + createdAt = Date(date.time + 1000), + ) + val subject6P1WithRoc1 = subject1P1WithFace.copy( + subjectId = "subj-006", + faceSamples = listOf(faceSample1.copy(id = "face-uuid-6", referenceId = "ref-face-6")), + createdAt = Date(date.time + 2000), + ) + setupInitialData() + dataSource.performActions( + listOf( + SubjectAction.Creation(subject5P1WithRoc1), + SubjectAction.Creation(subject6P1WithRoc1), + ), + project, + ) + val baseQuery = + SubjectQuery(projectId = PROJECT_1_ID, faceSampleFormat = ROC_1_FORMAT, sort = true) + // When + val loadedRanges = + dataSource + .loadFaceIdentities( + query = baseQuery, + ranges = listOf( + 0..1, + 1..2, + ), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + + val loadedFirstTwo = + dataSource + .loadFaceIdentities( + query = baseQuery.copy(), + ranges = listOf( + 0..2, + ), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + val loadedAll = dataSource + .loadFaceIdentities( + query = baseQuery.copy(), + ranges = listOf(0..10), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + + // Then + assertThat(loadedRanges).hasSize(2) // Two ranges loaded + assertThat(loadedRanges[0][0].subjectId).isEqualTo(subject1P1WithFace.subjectId) + assertThat(loadedRanges[1]).hasSize(1) + assertThat(loadedRanges[1][0].subjectId).isEqualTo(subject5P1WithRoc1.subjectId) + assertThat(loadedFirstTwo).hasSize(2) + assertThat(loadedFirstTwo.map { it.subjectId }) + .containsExactly(subject1P1WithFace.subjectId, subject5P1WithRoc1.subjectId) + .inOrder() + assertThat(loadedAll).hasSize(3) + assertThat(loadedAll.map { it.subjectId }) + .containsExactly( + subject1P1WithFace.subjectId, + subject5P1WithRoc1.subjectId, + subject6P1WithRoc1.subjectId, + ).inOrder() + verify(exactly = 7) { mockCallback() } + } + + @Test + fun `loadFaceIdentities - with query format matching nothing - should return empty list`() = runTest { + // Given + setupInitialData() + + // When + val loadedIdentities = dataSource + .loadFaceIdentities( + query = SubjectQuery( + projectId = PROJECT_1_ID, + faceSampleFormat = UNUSED_FORMAT, + ), + ranges = listOf(0..10), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + + // Then + assertThat(loadedIdentities).isEmpty() + verify(exactly = 0) { mockCallback() } + } + + @Test + fun `loadFaceIdentities - by attendantId and moduleId - should return correct identities`() = runTest { + // Given + setupInitialData() + + // Query for Project 1, Attendant 1, Module 1, Format ROC_1 + val queryP1A1M1Roc1 = SubjectQuery( + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_1_ID, + faceSampleFormat = ROC_1_FORMAT, + ) + + // Query for Project 1, Attendant 1, Module 2, Format ROC_3 + val queryP1A1M2Roc3 = SubjectQuery( + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_2_ID, + faceSampleFormat = ROC_3_FORMAT, + ) + + // Query for Project 1, Attendant 1, Module 1, Format ROC_3 (should be empty) + val queryP1A1M1Roc3Empty = SubjectQuery( + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_1_ID, + faceSampleFormat = ROC_3_FORMAT, + ) + + // When + val loadedP1A1M1Roc1 = dataSource + .loadFaceIdentities( + queryP1A1M1Roc1, + listOf(0..10), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + val loadedP1A1M2Roc3 = dataSource + .loadFaceIdentities( + queryP1A1M2Roc3, + listOf( + 0..10, + ), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + val loadedP1A1M1Roc3Empty = + dataSource + .loadFaceIdentities( + queryP1A1M1Roc3Empty, + listOf( + 0..10, + ), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + + // Then + assertThat(loadedP1A1M1Roc1).hasSize(1) + assertThat(loadedP1A1M1Roc1[0].subjectId).isEqualTo(subject1P1WithFace.subjectId) + + assertThat(loadedP1A1M2Roc3).hasSize(1) + assertThat(loadedP1A1M2Roc3[0].subjectId).isEqualTo(subject3P1WithBoth.subjectId) + + assertThat(loadedP1A1M1Roc3Empty).isEmpty() + + verify(exactly = 2) { mockCallback() } // Called for the 3 successful loads + } + + @Test + fun `loadFingerprintIdentities - by attendantId and moduleId - should return correct identities`() = runTest { + // Given + setupInitialData() + + // Query for Project 1, Attendant 1, Module 1, Format NEC + val queryP1A1M1Nec = SubjectQuery( + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_1_ID, + fingerprintSampleFormat = NEC_FORMAT, + ) + + // Query for Project 1, Attendant 1, Module 3, Format NEC + val queryP1A1M3Nec = SubjectQuery( + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_3_ID, + fingerprintSampleFormat = NEC_FORMAT, + ) + + // Query for Project 1, Attendant 1, Module 2, Format ISO + val queryP1A1M2Iso = SubjectQuery( + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_2_ID, + fingerprintSampleFormat = ISO_FORMAT, + ) + + // Query for Project 1, Attendant 2, Module 1, Format NEC (should be empty) + val queryP1A2M1NecEmpty = SubjectQuery( + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_2_ID, + moduleId = MODULE_1_ID, + fingerprintSampleFormat = NEC_FORMAT, + ) + + // When + val loadedP1A1M1Nec = dataSource + .loadFingerprintIdentities( + queryP1A1M1Nec, + listOf( + 0..10, + ), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + val loadedP1A1M3Nec = dataSource + .loadFingerprintIdentities( + queryP1A1M3Nec, + listOf( + 0..10, + ), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + val loadedP1A1M2Iso = dataSource + .loadFingerprintIdentities( + queryP1A1M2Iso, + listOf( + 0..10, + ), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + val loadedP1A2M1NecEmpty = dataSource + .loadFingerprintIdentities( + queryP1A2M1NecEmpty, + listOf( + 0..10, + ), + project = project, + dataSource = Simprints, + scope = this, + onCandidateLoaded = mockCallback, + ).toList() + .first() + + // Then + assertThat(loadedP1A1M1Nec).hasSize(1) + assertThat(loadedP1A1M1Nec[0].subjectId).isEqualTo(subject2P1WithFinger.subjectId) + + assertThat(loadedP1A1M3Nec).hasSize(0) + + assertThat(loadedP1A1M2Iso).hasSize(1) + assertThat(loadedP1A1M2Iso[0].subjectId).isEqualTo(subject3P1WithBoth.subjectId) + + assertThat(loadedP1A2M1NecEmpty).isEmpty() + + verify(exactly = 2) { mockCallback() } + } + + @Test + fun `load - by attendantId and moduleId - should return matching subjects`() = runTest { + // Given + setupInitialData() + + // When + val loadedP1A1M1 = dataSource.load( + SubjectQuery( + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_1_ID, + ), + ) + val loadedP1A1M2 = dataSource.load( + SubjectQuery( + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + moduleId = MODULE_2_ID, + ), + ) + val loadedP1A2M1 = dataSource.load( + SubjectQuery( + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_2_ID, + moduleId = MODULE_1_ID, + ), + ) + val loadedP2A2M2 = dataSource.load( + SubjectQuery( + projectId = PROJECT_2_ID, + attendantId = ATTENDANT_2_ID, + moduleId = MODULE_2_ID, + ), + ) + + // Then + assertThat(loadedP1A1M1).hasSize(2) + assertThat(loadedP1A1M1.map { it.subjectId }).containsExactly( + subject1P1WithFace.subjectId, + subject2P1WithFinger.subjectId, + ) + + assertThat(loadedP1A1M2).hasSize(1) + assertThat(loadedP1A1M2[0].subjectId).isEqualTo(subject3P1WithBoth.subjectId) + + assertThat(loadedP1A2M1).hasSize(0) + + assertThat(loadedP2A2M2).hasSize(1) + assertThat(loadedP2A2M2[0].subjectId).isEqualTo(subject4P2WithBoth.subjectId) + } + + @Test + fun `load - by subjectIds - should return only specified subjects`() = runTest { + // Given + setupInitialData() + val targetIds = listOf( + subject1P1WithFace.subjectId, + subject4P2WithBoth.subjectId, + subject6P2WithFinger.subjectId, + ) + + // When + val loaded = dataSource.load( + SubjectQuery( + subjectIds = targetIds, + sort = true, + ), + ) // Sort for predictable order + + // Then + assertThat(loaded).hasSize(3) + assertThat(loaded.map { it.subjectId }) + .containsExactlyElementsIn(targetIds.sorted()) + .inOrder() // Compare sorted lists + // Check details of one subject + val loadedSubject1 = loaded.find { it.subjectId == subject1P1WithFace.subjectId } + assertThat(loadedSubject1).isNotNull() + assertThat(loadedSubject1?.attendantId).isEqualTo(ATTENDANT_1_ID) + assertThat(loadedSubject1?.moduleId).isEqualTo(MODULE_1_ID) + assertThat(loadedSubject1?.faceSamples).isEqualTo(subject1P1WithFace.faceSamples) + } + + @Test + fun `load - by afterSubjectId - should return subjects after the specified ID`() = runTest { + // Given + setupInitialData() + val allSubjectIdsSorted = listOf( + subject1P1WithFace.subjectId, // subj-001 + subject2P1WithFinger.subjectId, // subj-002 + subject3P1WithBoth.subjectId, // subj-003 + subject4P2WithBoth.subjectId, // subj-004 + subject5P2WithFace.subjectId, // subj-005 + subject6P2WithFinger.subjectId, // subj-006 + ).sorted() + + val afterId = allSubjectIdsSorted[2] // Should be subj-003 + + // When + // Query for all subjects after subj-003, sorted by ID + val loaded = dataSource.load(SubjectQuery(afterSubjectId = afterId, sort = true)) + + // Then + assertThat(loaded).hasSize(3) // subj-004, subj-005, subj-006 remain + assertThat(loaded.map { it.subjectId }) + .containsExactly( + allSubjectIdsSorted[3], // subj-004 + allSubjectIdsSorted[4], // subj-005 + allSubjectIdsSorted[5], // subj-006 + ).inOrder() + } + + @Test + fun `load - combined query - attendantId, moduleId, afterSubjectId, subjectIds - should respect all filters`() = runTest { + // Given + setupInitialData() + // Targets: subj-001, subj-002 (P1, A1, M1), subj-003 (P1, A1, M2) + val targetIds = listOf( + subject1P1WithFace.subjectId, // subj-001 + subject2P1WithFinger.subjectId, // subj-002 + subject3P1WithBoth.subjectId, // subj-003 + "subj-nonexistent", // Include a non-existent ID + ) + val afterId = subject1P1WithFace.subjectId // subj-001 + + // Query: Project 1, Attendant 1, after subj-001, from the targetIds list, sorted + val query = SubjectQuery( + projectId = PROJECT_1_ID, + attendantId = ATTENDANT_1_ID, + afterSubjectId = afterId, + subjectIds = targetIds, + sort = true, + ) + + // When + val loaded = dataSource.load(query) + + // Then + // Expected: subj-002, subj-003 meet all criteria (P1, A1, > subj-001, in targetIds) + assertThat(loaded).hasSize(2) + assertThat(loaded.map { it.subjectId }) + .containsExactly( + subject2P1WithFinger.subjectId, // subj-002 + subject3P1WithBoth.subjectId, // subj-003 + ).inOrder() + } + + // 3. Test delete by module id + + @Test + fun `delete - by moduleId - should delete only subjects with matching moduleId`() = runTest { + // Given + setupInitialData() + val initialCount = dataSource.count() + val countModule1Before = dataSource.count(SubjectQuery(moduleId = MODULE_1_ID)) + val countModule2Before = dataSource.count(SubjectQuery(moduleId = MODULE_2_ID)) + val countModule3Before = dataSource.count(SubjectQuery(moduleId = MODULE_3_ID)) + + assertThat(countModule1Before).isEqualTo(2) + assertThat(countModule2Before).isEqualTo(2) + assertThat(countModule3Before).isEqualTo(2) + + val queryToDeleteModule1 = SubjectQuery(moduleId = MODULE_1_ID) + + // When + dataSource.delete(listOf(queryToDeleteModule1)) + + // Then + val finalCount = dataSource.count() + val countModule1After = dataSource.count(SubjectQuery(moduleId = MODULE_1_ID)) + val countModule2After = dataSource.count(SubjectQuery(moduleId = MODULE_2_ID)) + val countModule3After = dataSource.count(SubjectQuery(moduleId = MODULE_3_ID)) + + assertThat(countModule1After).isEqualTo(0) // All M1 deleted + assertThat(countModule2After).isEqualTo(2) // M2 untouched + assertThat(countModule3After).isEqualTo(2) // M3 untouched + assertThat(finalCount).isEqualTo(initialCount - countModule1Before) // Total count reduced correctly + + // Verify specific subjects are gone/remain + assertThat(dataSource.load(SubjectQuery(subjectId = subject1P1WithFace.subjectId))).isEmpty() + assertThat(dataSource.load(SubjectQuery(subjectId = subject2P1WithFinger.subjectId))).isEmpty() + assertThat(dataSource.load(SubjectQuery(subjectId = subject5P2WithFace.subjectId))).isNotEmpty() + assertThat(dataSource.load(SubjectQuery(subjectId = subject3P1WithBoth.subjectId))).isNotEmpty() + assertThat(dataSource.load(SubjectQuery(subjectId = subject4P2WithBoth.subjectId))).isNotEmpty() + assertThat(dataSource.load(SubjectQuery(subjectId = subject6P2WithFinger.subjectId))).isNotEmpty() + } + + @Test(expected = IllegalArgumentException::class) + fun `delete - by moduleId with format specified - should throw exception`() = runTest { + // Given + setupInitialData() + val queryToDeleteModule1WithFormat = SubjectQuery( + moduleId = MODULE_1_ID, + faceSampleFormat = ROC_1_FORMAT, // Adding format which is not allowed for delete + ) + + // When + dataSource.delete(listOf(queryToDeleteModule1WithFormat)) // This should throw + + // Then: Exception expected + } +} diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSourceTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSourceTest.kt deleted file mode 100644 index a7847ca370..0000000000 --- a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSourceTest.kt +++ /dev/null @@ -1,318 +0,0 @@ -package com.simprints.infra.enrolment.records.repository.local - -import com.google.common.truth.Truth -import com.simprints.core.domain.face.FaceSample -import com.simprints.core.domain.tokenization.asTokenizableEncrypted -import com.simprints.core.domain.tokenization.asTokenizableRaw -import com.simprints.infra.config.store.models.Project -import com.simprints.infra.config.store.tokenization.TokenizationProcessor -import com.simprints.infra.enrolment.records.realm.store.RealmWrapper -import com.simprints.infra.enrolment.records.realm.store.models.DbSubject -import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource -import com.simprints.infra.enrolment.records.repository.domain.models.Subject -import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAction -import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery -import com.simprints.infra.enrolment.records.repository.local.models.toDomain -import com.simprints.infra.enrolment.records.repository.local.models.toRealmDb -import io.mockk.CapturingSlot -import io.mockk.MockKAnnotations -import io.mockk.coEvery -import io.mockk.every -import io.mockk.impl.annotations.MockK -import io.mockk.mockk -import io.mockk.slot -import io.mockk.verify -import io.realm.kotlin.MutableRealm -import io.realm.kotlin.Realm -import io.realm.kotlin.query.RealmQuery -import kotlinx.coroutines.test.runTest -import org.junit.Before -import org.junit.Test -import java.util.UUID -import kotlin.random.Random - -class RealmEnrolmentRecordLocalDataSourceTest { - @MockK - private lateinit var realm: Realm - - @MockK - private lateinit var mutableRealm: MutableRealm - - @MockK - private lateinit var realmWrapperMock: RealmWrapper - - @MockK - private lateinit var realmQuery: RealmQuery - - @MockK - private lateinit var tokenizationProcessor: TokenizationProcessor - - @MockK - private lateinit var project: Project - - private lateinit var blockCapture: CapturingSlot<(Realm) -> Any> - private lateinit var mutableBlockCapture: CapturingSlot<(MutableRealm) -> Any> - private val onCandidateLoaded: () -> Unit = {} - private var localSubjects: MutableList = mutableListOf() - - private lateinit var enrolmentRecordLocalDataSource: EnrolmentRecordLocalDataSource - - @Before - fun setup() { - MockKAnnotations.init(this, relaxed = true) - localSubjects = mutableListOf() - - val insertedSubject = slot() - every { mutableRealm.delete(any()) } answers { localSubjects.clear() } - every { mutableRealm.deleteAll() } answers { localSubjects.clear() } - every { mutableRealm.copyToRealm(capture(insertedSubject), any()) } answers { - localSubjects.add(insertedSubject.captured.toDomain()) - insertedSubject.captured - } - - blockCapture = slot() - coEvery { realmWrapperMock.readRealm(capture(blockCapture)) } answers { - blockCapture.captured.invoke(realm) - } - mutableBlockCapture = slot() - coEvery { realmWrapperMock.writeRealm(capture(mutableBlockCapture)) } answers { - mutableBlockCapture.captured.invoke(mutableRealm) - } - every { realmQuery.count() } answers { - mockk { every { find() } returns localSubjects.size.toLong() } - } - - every { realm.query(DbSubject::class) } returns realmQuery - every { mutableRealm.query(DbSubject::class) } returns realmQuery - - enrolmentRecordLocalDataSource = RealmEnrolmentRecordLocalDataSource(realmWrapperMock, tokenizationProcessor) - } - - @Test - fun givenOneRecordSaved_countShouldReturnOne() = runTest { - saveFakePerson(getFakePerson()) - - val count = enrolmentRecordLocalDataSource.count() - Truth.assertThat(count).isEqualTo(1) - } - - @Test - fun givenManyPeopleSaved_countShouldReturnMany() = runTest { - saveFakePeople(getRandomPeople(20)) - - val count = enrolmentRecordLocalDataSource.count() - Truth.assertThat(count).isEqualTo(20) - } - - @Test - fun givenManyPeopleSaved_countByProjectIdShouldReturnTheRightTotal() = runTest { - saveFakePeople(getRandomPeople(20)) - - val count = enrolmentRecordLocalDataSource.count() - Truth.assertThat(count).isEqualTo(20) - } - - @Test - fun givenValidSerializableQueryForFingerprints_loadIsCalled() = runTest { - val savedPersons = saveFakePeople(getRandomPeople(20)) - val fakePerson = savedPersons[0].toRealmDb() - - val people = enrolmentRecordLocalDataSource - .loadFingerprintIdentities( - SubjectQuery(), - IntRange(0, 20), - BiometricDataSource.Simprints, - project, - onCandidateLoaded, - ).toList() - - listOf(fakePerson).zip(people).forEach { (subject, identity) -> - Truth.assertThat(subject.subjectId).isEqualTo(identity.subjectId) - } - } - - @Test - fun `correctly query supported fingerprint format`() = runTest { - val format = "SupportedFormat" - - enrolmentRecordLocalDataSource - .loadFingerprintIdentities( - SubjectQuery(fingerprintSampleFormat = format), - IntRange(0, 20), - BiometricDataSource.Simprints, - project, - onCandidateLoaded, - ).toList() - - verify { - realmQuery.query( - "ANY ${EnrolmentRecordLocalDataSource.Companion.FINGERPRINT_SAMPLES_FIELD}.${EnrolmentRecordLocalDataSource.Companion.FORMAT_FIELD} == $0", - format, - ) - } - } - - @Test - fun `correctly query supported face format`() = runTest { - val format = "SupportedFormat" - - enrolmentRecordLocalDataSource - .loadFingerprintIdentities( - SubjectQuery(faceSampleFormat = format), - IntRange(0, 20), - BiometricDataSource.Simprints, - project, - onCandidateLoaded, - ).toList() - - verify { - realmQuery.query( - "ANY ${EnrolmentRecordLocalDataSource.Companion.FACE_SAMPLES_FIELD}.${EnrolmentRecordLocalDataSource.Companion.FORMAT_FIELD} == $0", - format, - ) - } - } - - @Test - fun givenValidSerializableQueryForFace_loadIsCalled() = runTest { - val savedPersons = saveFakePeople(getRandomPeople(20)) - val fakePerson = savedPersons[0].toRealmDb() - - val people = enrolmentRecordLocalDataSource - .loadFaceIdentities( - SubjectQuery(), - IntRange(0, 20), - BiometricDataSource.Simprints, - project, - onCandidateLoaded, - ).toList() - - listOf(fakePerson).zip(people).forEach { (subject, identity) -> - Truth.assertThat(subject.subjectId).isEqualTo(identity.subjectId) - } - } - - @Test - fun givenManyPeopleSaved_loadShouldReturnThem() = runTest { - val fakePerson = getFakePerson() - saveFakePerson(fakePerson) - - val people = enrolmentRecordLocalDataSource.load(SubjectQuery()).toList() - - listOf(fakePerson).zip(people).forEach { (dbSubject, subject) -> - Truth.assertThat(dbSubject.deepEquals(subject.toRealmDb())).isTrue() - } - } - - @Test - fun givenManyPeopleSaved_loadByUserIdShouldReturnTheRightPeople() = runTest { - val savedPersons = saveFakePeople(getRandomPeople(20)) - val fakePerson = savedPersons[0].toRealmDb() - - val people = - enrolmentRecordLocalDataSource - .load(SubjectQuery(attendantId = savedPersons[0].attendantId)) - .toList() - listOf(fakePerson).zip(people).forEach { (dbSubject, subject) -> - Truth.assertThat(dbSubject.deepEquals(subject.toRealmDb())).isTrue() - } - } - - @Test - fun givenManyPeopleSaved_loadByModuleIdShouldReturnTheRightPeople() = runTest { - val savedPersons = saveFakePeople(getRandomPeople(20)) - val fakePerson = savedPersons[0].toRealmDb() - - val people = - enrolmentRecordLocalDataSource - .load(SubjectQuery(moduleId = fakePerson.moduleId.asTokenizableEncrypted())) - .toList() - listOf(fakePerson).zip(people).forEach { (dbSubject, subject) -> - Truth.assertThat(dbSubject.deepEquals(subject.toRealmDb())).isTrue() - } - } - - @Test - fun performSubjectCreationAction() = runTest { - val subject = getFakePerson() - enrolmentRecordLocalDataSource.performActions( - listOf(SubjectAction.Creation(subject.toDomain())), - project, - ) - val peopleCount = enrolmentRecordLocalDataSource.count() - Truth.assertThat(peopleCount).isEqualTo(1) - } - - @Test - fun performSubjectDeletionAction() = runTest { - val subject = getFakePerson() - saveFakePerson(subject) - enrolmentRecordLocalDataSource.performActions( - listOf(SubjectAction.Deletion(subject.subjectId.toString())), - project, - ) - val peopleCount = enrolmentRecordLocalDataSource.count() - Truth.assertThat(peopleCount).isEqualTo(0) - } - - @Test - fun performNoAction() = runTest { - val subject = getFakePerson() - saveFakePerson(subject) - enrolmentRecordLocalDataSource.performActions( - listOf(), - project, - ) - val peopleCount = enrolmentRecordLocalDataSource.count() - Truth.assertThat(peopleCount).isEqualTo(1) - } - - @Test - fun shouldDeleteAllSubjects() = runTest { - saveFakePeople(getRandomPeople(5)) - - enrolmentRecordLocalDataSource.deleteAll() - - val peopleCount = enrolmentRecordLocalDataSource.count() - Truth.assertThat(peopleCount).isEqualTo(0) - } - - private fun getFakePerson(): DbSubject = getRandomSubject().toRealmDb() - - private fun saveFakePerson(fakeSubject: DbSubject): DbSubject = fakeSubject.also { localSubjects.add(it.toDomain()) } - - private fun saveFakePeople(subjects: List): List = subjects.toMutableList().also { localSubjects.addAll(it) } - - private fun DbSubject.deepEquals(other: DbSubject): Boolean = when { - this.subjectId != other.subjectId -> false - this.projectId != other.projectId -> false - this.attendantId != other.attendantId -> false - this.moduleId != other.moduleId -> false - this.createdAt != other.createdAt -> false - this.updatedAt != other.updatedAt -> false - else -> true - } - - private fun getRandomPeople(numberOfPeople: Int): ArrayList = arrayListOf().also { list -> - repeat(numberOfPeople) { - list.add(getRandomSubject(UUID.randomUUID().toString())) - } - } - - private fun getRandomSubject( - patientId: String = UUID.randomUUID().toString(), - projectId: String = UUID.randomUUID().toString(), - userId: String = UUID.randomUUID().toString(), - moduleId: String = UUID.randomUUID().toString(), - faceSamples: Array = arrayOf( - FaceSample(Random.Default.nextBytes(64), "faceTemplateFormat"), - FaceSample(Random.Default.nextBytes(64), "faceTemplateFormat"), - ), - ): Subject = Subject( - subjectId = patientId, - projectId = projectId, - attendantId = userId.asTokenizableRaw(), - moduleId = moduleId.asTokenizableRaw(), - faceSamples = faceSamples.toList(), - ) -} diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilderTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilderTest.kt new file mode 100644 index 0000000000..71f47a1dfe --- /dev/null +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilderTest.kt @@ -0,0 +1,458 @@ +package com.simprints.infra.enrolment.records.repository.local + +import com.google.common.truth.Truth.* +import com.simprints.core.domain.tokenization.asTokenizableEncrypted +import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery +import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate +import com.simprints.infra.enrolment.records.room.store.models.DbSubject +import org.junit.Assert.assertThrows +import org.junit.Before +import org.junit.Test + +class RoomEnrolmentRecordQueryBuilderTest { + private lateinit var queryBuilder: RoomEnrolmentRecordQueryBuilder + + companion object { + private const val SUBJECT_ID_1 = "subject_id_1" + private const val SUBJECT_ID_2 = "subject_id_2" + private const val SUBJECT_ID_3 = "subject_id_3" + private const val AFTER_SUBJECT_ID = "after_subject_id" + private const val PROJECT_ID = "project_id_x" + private val ATTENDANT_ID = "attendant_id_y".asTokenizableEncrypted() + private val MODULE_ID = "module_id_z".asTokenizableEncrypted() + private const val FP_FORMAT = "FP_FORMAT_XYZ" + private const val FACE_FORMAT = "FACE_FORMAT_ABC" + private const val PAGE_SIZE = 10 + private const val LAST_SEEN_SUBJECT_ID = "last_seen_subj_001" + private const val SUBJECT_TABLE_NAME = DbSubject.SUBJECT_TABLE_NAME + private const val TEMPLATE_TABLE_NAME = DbBiometricTemplate.TEMPLATE_TABLE_NAME + private const val SUBJECT_ID_COLUMN = DbSubject.SUBJECT_ID_COLUMN + private const val PROJECT_ID_COLUMN = DbSubject.PROJECT_ID_COLUMN + private const val ATTENDANT_ID_COLUMN = DbSubject.ATTENDANT_ID_COLUMN + private const val MODULE_ID_COLUMN = DbSubject.MODULE_ID_COLUMN + private const val FORMAT_COLUMN = DbBiometricTemplate.FORMAT_COLUMN + } + + @Before + fun setUp() { + queryBuilder = RoomEnrolmentRecordQueryBuilder() + } + + // region buildWhereAndOrderByClause Tests + @Test + fun `buildWhereAndOrderByClause with empty query returns empty clause and args`() { + val query = SubjectQuery() + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEmpty() + assertThat(args).isEmpty() + } + + @Test + fun `buildWhereAndOrderByClause with subjectId`() { + val query = SubjectQuery(subjectId = SUBJECT_ID_1) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEqualTo("WHERE S.$SUBJECT_ID_COLUMN = ?") + assertThat(args).containsExactly(SUBJECT_ID_1) + } + + @Test + fun `buildWhereAndOrderByClause with subjectIds not empty`() { + val query = SubjectQuery(subjectIds = listOf(SUBJECT_ID_1, SUBJECT_ID_2)) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEqualTo("WHERE S.$SUBJECT_ID_COLUMN IN (?,?)") + assertThat(args).containsExactly(SUBJECT_ID_1, SUBJECT_ID_2).inOrder() + } + + @Test + fun `buildWhereAndOrderByClause with subjectIds empty`() { + val query = SubjectQuery(subjectIds = emptyList()) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEmpty() + assertThat(args).isEmpty() + } + + @Test + fun `buildWhereAndOrderByClause with afterSubjectId`() { + val query = SubjectQuery(afterSubjectId = AFTER_SUBJECT_ID) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEqualTo("WHERE S.$SUBJECT_ID_COLUMN > ?") + assertThat(args).containsExactly(AFTER_SUBJECT_ID) + } + + @Test + fun `buildWhereAndOrderByClause with projectId`() { + val query = SubjectQuery(projectId = PROJECT_ID) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEqualTo("WHERE S.$PROJECT_ID_COLUMN = ?") + assertThat(args).containsExactly(PROJECT_ID) + } + + @Test + fun `buildWhereAndOrderByClause with attendantId`() { + val query = SubjectQuery(attendantId = ATTENDANT_ID) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEqualTo("WHERE S.$ATTENDANT_ID_COLUMN = ?") + assertThat(args).containsExactly(ATTENDANT_ID.value) + } + + @Test + fun `buildWhereAndOrderByClause with moduleId`() { + val query = SubjectQuery(moduleId = MODULE_ID) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEqualTo("WHERE S.$MODULE_ID_COLUMN = ?") + assertThat(args).containsExactly(MODULE_ID.value) + } + + @Test + fun `buildWhereAndOrderByClause with faceSampleFormat`() { + val query = SubjectQuery(faceSampleFormat = FACE_FORMAT) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEqualTo("WHERE T.$FORMAT_COLUMN = ?") + assertThat(args).containsExactly(FACE_FORMAT) + } + + @Test + fun `buildWhereAndOrderByClause with fingerprintSampleFormat`() { + val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEqualTo("WHERE T.$FORMAT_COLUMN = ?") + assertThat(args).containsExactly(FP_FORMAT) + } + + @Test + fun `buildWhereAndOrderByClause with multiple subject fields`() { + val query = SubjectQuery( + projectId = PROJECT_ID, + attendantId = ATTENDANT_ID, + moduleId = MODULE_ID, + ) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEqualTo( + "WHERE S.$PROJECT_ID_COLUMN = ? AND S.$ATTENDANT_ID_COLUMN = ? AND S.$MODULE_ID_COLUMN = ?", + ) + assertThat(args).containsExactly(PROJECT_ID, ATTENDANT_ID.value, MODULE_ID.value).inOrder() + } + + @Test + fun `buildWhereAndOrderByClause with all parameters and sort`() { + val query = SubjectQuery( + subjectId = SUBJECT_ID_1, + subjectIds = listOf(SUBJECT_ID_2, SUBJECT_ID_3), + afterSubjectId = AFTER_SUBJECT_ID, + projectId = PROJECT_ID, + attendantId = ATTENDANT_ID, + moduleId = MODULE_ID, + fingerprintSampleFormat = FP_FORMAT, + sort = true, + ) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val expectedClause = + "WHERE S.$SUBJECT_ID_COLUMN = ? AND S.$SUBJECT_ID_COLUMN IN (?,?) AND S.$SUBJECT_ID_COLUMN > ? AND S.$PROJECT_ID_COLUMN = ? AND S.$ATTENDANT_ID_COLUMN = ? AND S.$MODULE_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC" + assertThat(clause).isEqualTo(expectedClause) + assertThat(args) + .containsExactly( + SUBJECT_ID_1, + SUBJECT_ID_2, + SUBJECT_ID_3, + AFTER_SUBJECT_ID, + PROJECT_ID, + ATTENDANT_ID.value, + MODULE_ID.value, + FP_FORMAT, + ).inOrder() + } + + @Test + fun `buildWhereAndOrderByClause with sort true and no other clauses`() { + val query = SubjectQuery(sort = true) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEqualTo(" ORDER BY S.$SUBJECT_ID_COLUMN ASC") + assertThat(args).isEmpty() + } + + @Test + fun `buildWhereAndOrderByClause with sort true and other clauses`() { + val query = SubjectQuery(projectId = PROJECT_ID, sort = true) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEqualTo("WHERE S.$PROJECT_ID_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC") + assertThat(args).containsExactly(PROJECT_ID) + } + + @Test + fun `buildWhereAndOrderByClause with sort false and clauses`() { + val query = SubjectQuery(projectId = PROJECT_ID, sort = false) // sort = false is default + val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + assertThat(clause).isEqualTo("WHERE S.$PROJECT_ID_COLUMN = ?") + assertThat(args).containsExactly(PROJECT_ID) + } + + @Test + fun `buildWhereAndOrderByClause with custom aliases`() { + val query = SubjectQuery(subjectId = SUBJECT_ID_1, fingerprintSampleFormat = FP_FORMAT, sort = true) + val (clause, args) = queryBuilder.buildWhereAndOrderByClause( + query, + subjectAlias = "customS.", + templateAlias = "customT.", + ) + val expectedClause = + "WHERE customS.$SUBJECT_ID_COLUMN = ? AND customT.$FORMAT_COLUMN = ? ORDER BY customS.$SUBJECT_ID_COLUMN ASC" + assertThat(clause).isEqualTo(expectedClause) + assertThat(args).containsExactly(SUBJECT_ID_1, FP_FORMAT).inOrder() + } + + @Test + fun `buildWhereAndOrderByClause throws error if both fingerprint and face format set`() { + val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, faceSampleFormat = FACE_FORMAT) + val exception = assertThrows(IllegalArgumentException::class.java) { + queryBuilder.buildWhereAndOrderByClause(query) + } + assertThat(exception.message).isEqualTo("Cannot set both fingerprintSampleFormat and faceSampleFormat") + } + // endregion + + // region buildSubjectQuery Tests + @Test + fun `buildSubjectQuery with empty query`() { + val query = SubjectQuery() + val result = queryBuilder.buildSubjectQuery(query) + val expectedSql = + """ + SELECT * FROM DbSubject S + + """.trimIndent() + assertThat(result.sql).isEqualTo(expectedSql) + assertThat(result.argCount).isEqualTo(0) + } + + @Test + fun `buildSubjectQuery with projectId`() { + val query = SubjectQuery(projectId = PROJECT_ID) + val result = queryBuilder.buildSubjectQuery(query) + val expectedSql = + """ + SELECT * FROM DbSubject S + WHERE S.projectId = ? + """.trimIndent() + assertThat(result.sql).isEqualTo(expectedSql) + assertThat(result.argCount).isEqualTo(1) + } + + @Test + fun `buildSubjectQuery with multiple subject fields`() { + val query = SubjectQuery(projectId = PROJECT_ID, attendantId = ATTENDANT_ID) + val result = queryBuilder.buildSubjectQuery(query) + val expectedSql = + """ + SELECT * FROM $SUBJECT_TABLE_NAME S + WHERE S.$PROJECT_ID_COLUMN = ? AND S.$ATTENDANT_ID_COLUMN = ? + """.trimIndent() + assertThat(result.sql).isEqualTo(expectedSql) + assertThat(result.argCount).isEqualTo(2) + } + + @Test + fun `buildSubjectQuery with sort true`() { + val query = SubjectQuery(sort = true) + val result = queryBuilder.buildSubjectQuery(query) + val expectedSql = + """ + SELECT * FROM $SUBJECT_TABLE_NAME S + ORDER BY S.$SUBJECT_ID_COLUMN ASC + """.trimIndent() + // Note the space before ORDER due to buildWhereAndOrderByClause + assertThat(result.sql).isEqualTo(expectedSql) + assertThat(result.argCount).isEqualTo(0) + } + + @Test + fun `buildSubjectQuery with sort true and projectId`() { + val query = SubjectQuery(projectId = PROJECT_ID, sort = true) + val result = queryBuilder.buildSubjectQuery(query) + val expectedSql = + """ + SELECT * FROM $SUBJECT_TABLE_NAME S + WHERE S.$PROJECT_ID_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC + """.trimIndent() + assertThat(result.sql).isEqualTo(expectedSql) + assertThat(result.argCount).isEqualTo(1) + } + + @Test + fun `buildSubjectQuery throws error if fingerprintSampleFormat is set`() { + val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT) + val exception = assertThrows(IllegalArgumentException::class.java) { + queryBuilder.buildSubjectQuery(query) + } + assertThat(exception.message).isEqualTo("Cannot set format for subject query, use buildBiometricTemplatesQuery instead") + } + + @Test + fun `buildSubjectQuery throws error if faceSampleFormat is set`() { + val query = SubjectQuery(faceSampleFormat = FACE_FORMAT) + val exception = assertThrows(IllegalArgumentException::class.java) { + queryBuilder.buildSubjectQuery(query) + } + assertThat(exception.message).isEqualTo("Cannot set format for subject query, use buildBiometricTemplatesQuery instead") + } + // endregion + + // region buildCountQuery Tests + @Test + fun `buildCountQuery with empty query no format`() { + val query = SubjectQuery() + val result = queryBuilder.buildCountQuery(query) + assertThat(result.sql).isEqualTo("SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S ") + assertThat(result.argCount).isEqualTo(0) + } + + @Test + fun `buildCountQuery with projectId no format`() { + val query = SubjectQuery(projectId = PROJECT_ID) + val result = queryBuilder.buildCountQuery(query) + assertThat( + result.sql, + ).isEqualTo("SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S WHERE S.$PROJECT_ID_COLUMN = ?") + assertThat(result.argCount).isEqualTo(1) + } + + @Test + fun `buildCountQuery with fingerprintSampleFormat no other clauses`() { + val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT) + val result = queryBuilder.buildCountQuery(query) + assertThat(result.sql).isEqualTo( + "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + + " using($SUBJECT_ID_COLUMN) WHERE T.$FORMAT_COLUMN = ? ", + ) + assertThat(result.argCount).isEqualTo(1) + } + + @Test + fun `buildCountQuery with faceSampleFormat and projectId`() { + val query = SubjectQuery(faceSampleFormat = FACE_FORMAT, projectId = PROJECT_ID) + val result = queryBuilder.buildCountQuery(query) + val expectedSql = "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + + " using($SUBJECT_ID_COLUMN) WHERE S.$PROJECT_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ? " + assertThat(result.sql).isEqualTo(expectedSql) + assertThat(result.argCount).isEqualTo(2) + } + + @Test + fun `buildCountQuery with sort true no format`() { + val query = SubjectQuery(sort = true) + val result = queryBuilder.buildCountQuery(query) + // specificFormat == null, whereClause is " ORDER BY S.subjectId ASC" + // "... S ORDER BY S.subjectId ASC" + val expectedSql = "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S ORDER BY S.$SUBJECT_ID_COLUMN ASC" + assertThat(result.sql).isEqualTo(expectedSql) + assertThat(result.argCount).isEqualTo(0) + } + + @Test + fun `buildCountQuery with fingerprintSampleFormat and sort true`() { + val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, sort = true) + val result = queryBuilder.buildCountQuery(query) + val expectedSql = "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + + " using($SUBJECT_ID_COLUMN) WHERE T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC " + assertThat(result.sql).isEqualTo(expectedSql) + assertThat(result.argCount).isEqualTo(1) + } + + @Test + fun `buildCountQuery throws error if both fingerprint and face format set`() { + val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, faceSampleFormat = FACE_FORMAT) + val exception = assertThrows(IllegalArgumentException::class.java) { + queryBuilder.buildCountQuery(query) // This will call buildWhereAndOrderByClause + } + assertThat(exception.message).isEqualTo("Cannot set both fingerprintSampleFormat and faceSampleFormat") + } + // endregion + + // region buildBiometricTemplatesQuery Tests + @Test + fun `buildBiometricTemplatesQuery with fingerprintSampleFormat, no lastSeenId`() { + val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT) + val result = queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE, null) + val expectedSubQueryWhereClause = "WHERE T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC" + val expectedSql = + """ + SELECT A.* + FROM $TEMPLATE_TABLE_NAME A + INNER JOIN ( + SELECT distinct S.$SUBJECT_ID_COLUMN + FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T + USING($SUBJECT_ID_COLUMN) + $expectedSubQueryWhereClause + LIMIT $PAGE_SIZE + ) B USING($SUBJECT_ID_COLUMN) where A.$FORMAT_COLUMN ='$FP_FORMAT' + """.trimIndent() + + assertThat(result.sql).isEqualTo(expectedSql) + assertThat(result.argCount).isEqualTo(1) + } + + @Test + fun `buildBiometricTemplatesQuery with faceSampleFormat and lastSeenId`() { + val query = SubjectQuery(faceSampleFormat = FACE_FORMAT) + val result = queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE, LAST_SEEN_SUBJECT_ID) + val expectedSubQueryWhereClause = "WHERE S.$SUBJECT_ID_COLUMN > ? AND T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC" + val expectedSql = + """ + SELECT A.* + FROM $TEMPLATE_TABLE_NAME A + INNER JOIN ( + SELECT distinct S.$SUBJECT_ID_COLUMN + FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T + USING($SUBJECT_ID_COLUMN) + $expectedSubQueryWhereClause + LIMIT $PAGE_SIZE + ) B USING($SUBJECT_ID_COLUMN) where A.$FORMAT_COLUMN ='$FACE_FORMAT' + """.trimIndent() + + assertThat(result.sql).isEqualTo(expectedSql) + assertThat(result.argCount).isEqualTo(2) + } + + @Test + fun `buildBiometricTemplatesQuery with fingerprintFormat, projectId, lastSeenId`() { + val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, projectId = PROJECT_ID) + val result = queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE, LAST_SEEN_SUBJECT_ID) + val expectedSubQueryWhereClause = + "WHERE S.$SUBJECT_ID_COLUMN > ? AND S.$PROJECT_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC" + val expectedSql = + """ + SELECT A.* + FROM $TEMPLATE_TABLE_NAME A + INNER JOIN ( + SELECT distinct S.$SUBJECT_ID_COLUMN + FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T + USING($SUBJECT_ID_COLUMN) + $expectedSubQueryWhereClause + LIMIT $PAGE_SIZE + ) B USING($SUBJECT_ID_COLUMN) where A.$FORMAT_COLUMN ='$FP_FORMAT' + """.trimIndent() + + assertThat(result.sql).isEqualTo(expectedSql) + assertThat(result.argCount).isEqualTo(3) + } + + @Test + fun `buildBiometricTemplatesQuery throws error if no format is set`() { + val query = SubjectQuery() // No format + val exception = assertThrows(IllegalArgumentException::class.java) { + queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE) + } + assertThat( + exception.message, + ).isEqualTo("Must set format for biometric templates query, use buildSubjectQuery or buildCountQuery instead") + } + + @Test + fun `buildBiometricTemplatesQuery throws error if both fingerprint and face format set`() { + val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, faceSampleFormat = FACE_FORMAT) + val exception = assertThrows(IllegalArgumentException::class.java) { + queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE) + } + assertThat(exception.message).isEqualTo("Cannot set both fingerprintSampleFormat and faceSampleFormat") + } + // endregion +} diff --git a/infra/enrolment-records/room-store/schemas/com.simprints.infra.enrolment.records.room.store.SubjectsDatabase/1.json b/infra/enrolment-records/room-store/schemas/com.simprints.infra.enrolment.records.room.store.SubjectsDatabase/1.json index 80c5a6e76a..bbe29315e1 100644 --- a/infra/enrolment-records/room-store/schemas/com.simprints.infra.enrolment.records.room.store.SubjectsDatabase/1.json +++ b/infra/enrolment-records/room-store/schemas/com.simprints.infra.enrolment.records.room.store.SubjectsDatabase/1.json @@ -2,11 +2,11 @@ "formatVersion": 1, "database": { "version": 1, - "identityHash": "f2e5005e3569ee84f0bd388c5485c5bf", + "identityHash": "94bee827928a2618c6873579bc6bc63a", "entities": [ { "tableName": "DbSubject", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`subjectId` TEXT NOT NULL, `projectId` TEXT NOT NULL, `attendantId` TEXT NOT NULL, `moduleId` TEXT NOT NULL, `createdAt` INTEGER, `updatedAt` INTEGER, `isAttendantIdTokenized` INTEGER NOT NULL, `isModuleIdTokenized` INTEGER NOT NULL, PRIMARY KEY(`subjectId`))", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`subjectId` TEXT NOT NULL, `projectId` TEXT NOT NULL, `attendantId` TEXT NOT NULL, `moduleId` TEXT NOT NULL, `createdAt` INTEGER, `updatedAt` INTEGER, PRIMARY KEY(`subjectId`))", "fields": [ { "fieldPath": "subjectId", @@ -35,26 +35,12 @@ { "fieldPath": "createdAt", "columnName": "createdAt", - "affinity": "INTEGER", - "notNull": false + "affinity": "INTEGER" }, { "fieldPath": "updatedAt", "columnName": "updatedAt", - "affinity": "INTEGER", - "notNull": false - }, - { - "fieldPath": "isAttendantIdTokenized", - "columnName": "isAttendantIdTokenized", - "affinity": "INTEGER", - "notNull": true - }, - { - "fieldPath": "isModuleIdTokenized", - "columnName": "isModuleIdTokenized", - "affinity": "INTEGER", - "notNull": true + "affinity": "INTEGER" } ], "primaryKey": { @@ -65,54 +51,43 @@ }, "indices": [ { - "name": "index_DbSubject_subjectId", + "name": "index_DbSubject_projectId_subjectId", "unique": false, "columnNames": [ + "projectId", "subjectId" ], "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_subjectId` ON `${TABLE_NAME}` (`subjectId`)" + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_projectId_subjectId` ON `${TABLE_NAME}` (`projectId`, `subjectId`)" }, { - "name": "index_DbSubject_projectId", + "name": "index_DbSubject_projectId_moduleId_subjectId", "unique": false, "columnNames": [ - "projectId" - ], - "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_projectId` ON `${TABLE_NAME}` (`projectId`)" - }, - { - "name": "index_DbSubject_attendantId", - "unique": false, - "columnNames": [ - "attendantId" + "projectId", + "moduleId", + "subjectId" ], "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_attendantId` ON `${TABLE_NAME}` (`attendantId`)" + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_projectId_moduleId_subjectId` ON `${TABLE_NAME}` (`projectId`, `moduleId`, `subjectId`)" }, { - "name": "index_DbSubject_moduleId", + "name": "index_DbSubject_projectId_attendantId_subjectId", "unique": false, "columnNames": [ - "moduleId" + "projectId", + "attendantId", + "subjectId" ], "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_moduleId` ON `${TABLE_NAME}` (`moduleId`)" + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbSubject_projectId_attendantId_subjectId` ON `${TABLE_NAME}` (`projectId`, `attendantId`, `subjectId`)" } - ], - "foreignKeys": [] + ] }, { - "tableName": "DbFingerprintSample", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`rowId` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `uuid` TEXT NOT NULL, `subjectId` TEXT NOT NULL, `fingerIdentifier` INTEGER NOT NULL, `template` BLOB NOT NULL, `format` TEXT NOT NULL, `referenceId` TEXT NOT NULL, FOREIGN KEY(`subjectId`) REFERENCES `DbSubject`(`subjectId`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "tableName": "DbBiometricTemplate", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`uuid` TEXT NOT NULL, `subjectId` TEXT NOT NULL, `identifier` INTEGER, `templateData` BLOB NOT NULL, `format` TEXT NOT NULL, `referenceId` TEXT NOT NULL, `modality` INTEGER NOT NULL, PRIMARY KEY(`uuid`), FOREIGN KEY(`subjectId`) REFERENCES `DbSubject`(`subjectId`) ON UPDATE NO ACTION ON DELETE CASCADE )", "fields": [ - { - "fieldPath": "rowId", - "columnName": "rowId", - "affinity": "INTEGER", - "notNull": true - }, { "fieldPath": "uuid", "columnName": "uuid", @@ -126,14 +101,13 @@ "notNull": true }, { - "fieldPath": "fingerIdentifier", - "columnName": "fingerIdentifier", - "affinity": "INTEGER", - "notNull": true + "fieldPath": "identifier", + "columnName": "identifier", + "affinity": "INTEGER" }, { - "fieldPath": "template", - "columnName": "template", + "fieldPath": "templateData", + "columnName": "templateData", "affinity": "BLOB", "notNull": true }, @@ -148,113 +122,39 @@ "columnName": "referenceId", "affinity": "TEXT", "notNull": true - } - ], - "primaryKey": { - "autoGenerate": true, - "columnNames": [ - "rowId" - ] - }, - "indices": [ - { - "name": "index_DbFingerprintSample_format", - "unique": false, - "columnNames": [ - "format" - ], - "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_DbFingerprintSample_format` ON `${TABLE_NAME}` (`format`)" }, { - "name": "index_DbFingerprintSample_subjectId", - "unique": false, - "columnNames": [ - "subjectId" - ], - "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_DbFingerprintSample_subjectId` ON `${TABLE_NAME}` (`subjectId`)" - } - ], - "foreignKeys": [ - { - "table": "DbSubject", - "onDelete": "CASCADE", - "onUpdate": "NO ACTION", - "columns": [ - "subjectId" - ], - "referencedColumns": [ - "subjectId" - ] - } - ] - }, - { - "tableName": "DbFaceSample", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`rowId` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `uuid` TEXT NOT NULL, `subjectId` TEXT NOT NULL, `template` BLOB NOT NULL, `format` TEXT NOT NULL, `referenceId` TEXT NOT NULL, FOREIGN KEY(`subjectId`) REFERENCES `DbSubject`(`subjectId`) ON UPDATE NO ACTION ON DELETE CASCADE )", - "fields": [ - { - "fieldPath": "rowId", - "columnName": "rowId", + "fieldPath": "modality", + "columnName": "modality", "affinity": "INTEGER", "notNull": true - }, - { - "fieldPath": "uuid", - "columnName": "uuid", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "subjectId", - "columnName": "subjectId", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "template", - "columnName": "template", - "affinity": "BLOB", - "notNull": true - }, - { - "fieldPath": "format", - "columnName": "format", - "affinity": "TEXT", - "notNull": true - }, - { - "fieldPath": "referenceId", - "columnName": "referenceId", - "affinity": "TEXT", - "notNull": true } ], "primaryKey": { - "autoGenerate": true, + "autoGenerate": false, "columnNames": [ - "rowId" + "uuid" ] }, "indices": [ { - "name": "index_DbFaceSample_format", + "name": "index_DbBiometricTemplate_format_subjectId", "unique": false, "columnNames": [ - "format" + "format", + "subjectId" ], "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_DbFaceSample_format` ON `${TABLE_NAME}` (`format`)" + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbBiometricTemplate_format_subjectId` ON `${TABLE_NAME}` (`format`, `subjectId`)" }, { - "name": "index_DbFaceSample_subjectId", + "name": "index_DbBiometricTemplate_subjectId", "unique": false, "columnNames": [ "subjectId" ], "orders": [], - "createSql": "CREATE INDEX IF NOT EXISTS `index_DbFaceSample_subjectId` ON `${TABLE_NAME}` (`subjectId`)" + "createSql": "CREATE INDEX IF NOT EXISTS `index_DbBiometricTemplate_subjectId` ON `${TABLE_NAME}` (`subjectId`)" } ], "foreignKeys": [ @@ -272,10 +172,9 @@ ] } ], - "views": [], "setupQueries": [ "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", - "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'f2e5005e3569ee84f0bd388c5485c5bf')" + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '94bee827928a2618c6873579bc6bc63a')" ] } } \ No newline at end of file diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectDao.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectDao.kt index 0b212477d6..d7836eb8cf 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectDao.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectDao.kt @@ -6,12 +6,9 @@ import androidx.room.MapColumn import androidx.room.OnConflictStrategy import androidx.room.Query import androidx.room.RawQuery -import androidx.room.Transaction import androidx.sqlite.db.SupportSQLiteQuery -import com.simprints.infra.enrolment.records.room.store.models.DbFaceSample -import com.simprints.infra.enrolment.records.room.store.models.DbFingerprintSample +import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate import com.simprints.infra.enrolment.records.room.store.models.DbSubject -import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN import com.simprints.infra.enrolment.records.room.store.models.SubjectBiometrics @Dao @@ -20,198 +17,26 @@ interface SubjectDao { suspend fun insertSubject(subject: DbSubject) @Insert(onConflict = OnConflictStrategy.REPLACE) - suspend fun insertFingerprintSamples(fingerprintSamples: List) + suspend fun insertBiometricSamples(samples: List) - @Insert(onConflict = OnConflictStrategy.REPLACE) - suspend fun insertFaceSamples(faceSamples: List) - - @Query("DELETE FROM DBSUBJECT WHERE subjectId = :subjectId") + @Query("DELETE FROM DbSubject WHERE subjectId = :subjectId") suspend fun deleteSubject(subjectId: String) + @Query("DELETE FROM DbBiometricTemplate WHERE uuid = :uuid") + suspend fun deleteBiometricSample(uuid: String) + @RawQuery suspend fun deleteSubjects(query: SupportSQLiteQuery): Int - @Transaction + @Query("SELECT * FROM DbSubject WHERE subjectId = :subjectId") + suspend fun getSubject(subjectId: String): SubjectBiometrics? + @RawQuery suspend fun loadSubjects(query: SupportSQLiteQuery): List - /** - * Counts the number of distinct subjects that match the given filtering criteria. - * - * This method ensures that a subject is counted only once, even if they have multiple - * face or fingerprint samples. It optimizes performance by avoiding expensive `JOIN` - * operations and instead using `EXISTS` subqueries, which efficiently check for the - * presence of related biometric records without generating large intermediate result sets. - * - * ## Query Behavior: - * - Filters subjects based on optional parameters (e.g., `projectId`, `attendantId`, etc.). - * - If `fingerprintSampleFormat` is provided, only counts subjects that have at least one - * fingerprint sample with the specified format. - * - If `faceSampleFormat` is provided, only counts subjects that have at least one face - * sample with the specified format. - * - Uses `EXISTS` instead of `JOIN` to efficiently check for biometric samples. - * - Leverages database indexes for performance optimization. - * - * ## Performance Considerations: - * - **Indexes:** Ensure indexes exist on `DbSubject.subjectId`, `DbFingerprintSample.subjectId`, - * `DbFaceSample.subjectId`, and their respective `format` columns for efficient filtering. - * - **Scalability:** Using `EXISTS` prevents the query from generating excessive row combinations, - * making it efficient even with 50K+ subjects and millions of biometric samples. - * - **Avoids Full Table Scans:** If filtering by biometrics, the query quickly exits once - * a match is found, improving query execution time. - **/ - @Query( - """ - SELECT COUNT(*) FROM DbSubject -""", - ) - fun countSubjects(): Int - - /** - * Retrieves a paginated list of distinct subjects along with their fingerprint samples - * that match the given filtering criteria. - * - * This method ensures that: - * - Subjects are retrieved uniquely based on the provided filters. - * - Multiple fingerprint samples per subject are included in the result. - * - Efficient pagination is applied using `LIMIT` and `OFFSET`. - * - Uses `LEFT JOIN` to ensure that multiple fingerprint samples can be selected per subject - * without causing duplicates. - * - * ## Query Behavior: - * - Filters subjects based on optional parameters (e.g., `projectId`, `attendantId`, etc.). - * - If `fingerprintSampleFormat` is provided, only returns subjects with at least one - * fingerprint sample of the specified format. - * - Uses `EXISTS` to efficiently check for fingerprint samples that match the format. - * - Orders by `subjectId` for consistent pagination. - * - Uses `LIMIT` and `OFFSET` for efficient page-wise retrieval. - * - * ## Performance Considerations: - * - **Indexes:** Ensure indexes exist on `DbSubject.subjectId`, `DbFingerprintSample.subjectId`, - * and `DbFingerprintSample.format` to optimize filtering and joins. - * - **Scalability:** Pagination ensures that queries remain performant even with large datasets. - * - **Avoids Duplicate Counting:** Retrieves all fingerprint samples for a subject without - * causing duplicates. - * - * @param projectId (Optional) Filters subjects by project ID. - * @param subjectId (Optional) Filters for a specific subject ID. - * @param subjectIds (Optional) Filters for multiple subject IDs. - * @param attendantId (Optional) Filters subjects assigned to a specific attendant. - * @param moduleId (Optional) Filters subjects assigned to a specific module. - * @param fingerprintSampleFormat (Optional) If provided, only includes subjects with at least - * one fingerprint sample of the specified format. - * @param limit The maximum number of subjects to return per query page. - * @param offset The starting position for pagination. - * @return A map where the keys are subject IDs, and the values are lists of fingerprint samples - * associated with each subject. - */ - @Query( - """ - SELECT s.subjectId, f.* - FROM DbSubject s - LEFT JOIN DbFingerprintSample f ON f.subjectId = s.subjectId - WHERE - (:projectId IS NULL OR s.projectId = :projectId) AND - (:subjectId IS NULL OR s.subjectId = :subjectId) AND - (:subjectIds IS NULL OR s.subjectId IN (:subjectIds)) AND - (:attendantId IS NULL OR s.attendantId = :attendantId) AND - (:moduleId IS NULL OR s.moduleId = :moduleId) AND - ( - (:fingerprintSampleFormat IS NULL OR EXISTS ( - SELECT 1 FROM DbFingerprintSample fs - WHERE fs.subjectId = s.subjectId AND fs.format = :fingerprintSampleFormat - )) - ) - GROUP BY s.subjectId - ORDER BY s.subjectId - LIMIT :limit OFFSET :offset -""", - ) - fun getSubjectsWithFingerprintSamples( - projectId: String?, - subjectId: String?, - subjectIds: List?, - attendantId: String?, - moduleId: String?, - fingerprintSampleFormat: String?, - offset: Int, - limit: Int, - ): Map< - @MapColumn(SUBJECT_ID_COLUMN) String, - List, - > + @RawQuery + suspend fun countSubjects(query: SupportSQLiteQuery): Int - /** - * Retrieves a paginated list of distinct subjects along with their face samples - * that match the given filtering criteria. - * - * This method ensures that: - * - Subjects are retrieved uniquely based on the provided filters. - * - Multiple face samples per subject are included in the result. - * - Efficient pagination is applied using `LIMIT` and `OFFSET`. - * - Uses `LEFT JOIN` to ensure that multiple face samples can be selected per subject - * without causing duplicates. - * - * ## Query Behavior: - * - Filters subjects based on optional parameters (e.g., `projectId`, `attendantId`, etc.). - * - If `faceSampleFormat` is provided, only returns subjects with at least one - * face sample of the specified format. - * - Uses `EXISTS` to efficiently check for face samples that match the format. - * - Orders by `subjectId` for consistent pagination. - * - Uses `LIMIT` and `OFFSET` for efficient page-wise retrieval. - * - * ## Performance Considerations: - * - **Indexes:** Ensure indexes exist on `DbSubject.subjectId`, `DbFaceSample.subjectId`, - * and `DbFaceSample.format` to optimize filtering and joins. - * - **Scalability:** Pagination ensures that queries remain performant even with large datasets. - * - **Avoids Duplicate Counting:** Retrieves all face samples for a subject without - * causing duplicates. - * - * @param projectId (Optional) Filters subjects by project ID. - * @param subjectId (Optional) Filters for a specific subject ID. - * @param subjectIds (Optional) Filters for multiple subject IDs. - * @param attendantId (Optional) Filters subjects assigned to a specific attendant. - * @param moduleId (Optional) Filters subjects assigned to a specific module. - * @param faceSampleFormat (Optional) If provided, only includes subjects with at least - * one face sample of the specified format. - * @param limit The maximum number of subjects to return per query page. - * @param offset The starting position for pagination. - * @return A map where the keys are subject IDs, and the values are lists of face samples - * associated with each subject. - */ - @Query( - """ - SELECT s.subjectId, f.* - FROM DbSubject s - LEFT JOIN DbFaceSample f ON f.subjectId = s.subjectId - WHERE - (:projectId IS NULL OR s.projectId = :projectId) AND - (:subjectId IS NULL OR s.subjectId = :subjectId) AND - (:subjectIds IS NULL OR s.subjectId IN (:subjectIds)) AND - (:attendantId IS NULL OR s.attendantId = :attendantId) AND - (:moduleId IS NULL OR s.moduleId = :moduleId) AND - ( - (:faceSampleFormat IS NULL OR EXISTS ( - SELECT 1 FROM DbFaceSample fs - WHERE fs.subjectId = s.subjectId AND fs.format = :faceSampleFormat - )) - ) - GROUP BY s.subjectId - ORDER BY s.subjectId - LIMIT :limit OFFSET :offset -""", - ) - fun getSubjectsWithFaceSamples( - projectId: String?, - subjectId: String?, - subjectIds: List?, - attendantId: String?, - moduleId: String?, - faceSampleFormat: String?, - offset: Int, - limit: Int, - ): Map< - @MapColumn(SUBJECT_ID_COLUMN) String, - List, - > + @RawQuery + suspend fun loadSamples(query: SupportSQLiteQuery): Map<@MapColumn(DbSubject.SUBJECT_ID_COLUMN) String, List> } diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt index 1cb809bbbe..16f004f62d 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabase.kt @@ -5,15 +5,17 @@ import androidx.annotation.Keep import androidx.room.Database import androidx.room.Room import androidx.room.RoomDatabase -import com.simprints.infra.enrolment.records.room.store.models.DbFaceSample -import com.simprints.infra.enrolment.records.room.store.models.DbFingerprintSample +import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate import com.simprints.infra.enrolment.records.room.store.models.DbSubject -import net.sqlcipher.database.SupportFactory +import net.zetetic.database.sqlcipher.SupportOpenHelperFactory import javax.inject.Singleton @Singleton @Database( - entities = [DbSubject::class, DbFingerprintSample::class, DbFaceSample::class], + entities = [ + DbSubject::class, + DbBiometricTemplate::class, + ], version = 1, exportSchema = true, ) @@ -24,7 +26,7 @@ abstract class SubjectsDatabase : RoomDatabase() { companion object { fun getDatabase( context: Context, - factory: SupportFactory, + factory: SupportOpenHelperFactory, dbName: String, ): SubjectsDatabase { val builder = Room.databaseBuilder(context, SubjectsDatabase::class.java, dbName) diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabaseFactory.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabaseFactory.kt index 6933b2cb61..90be1fe0c9 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabaseFactory.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/SubjectsDatabaseFactory.kt @@ -4,10 +4,11 @@ import android.content.Context import com.simprints.infra.logging.Simber import com.simprints.infra.security.SecurityManager import dagger.hilt.android.qualifiers.ApplicationContext -import net.sqlcipher.database.SQLiteDatabase -import net.sqlcipher.database.SupportFactory +import net.zetetic.database.sqlcipher.SupportOpenHelperFactory +import java.nio.charset.Charset import javax.inject.Inject import javax.inject.Singleton +import kotlin.text.toByteArray @Singleton class SubjectsDatabaseFactory @Inject constructor( @@ -30,8 +31,8 @@ class SubjectsDatabaseFactory @Inject constructor( @OptIn(ExperimentalStdlibApi::class) private fun build(): SubjectsDatabase = try { val key = getOrCreateKey() - val passphrase: ByteArray = SQLiteDatabase.getBytes(key) - val factory = SupportFactory(passphrase) + val passphrase: ByteArray = key.toByteArray(Charset.forName("UTF-8")) + val factory = SupportOpenHelperFactory(passphrase) SubjectsDatabase.getDatabase( ctx, factory, @@ -42,13 +43,13 @@ class SubjectsDatabaseFactory @Inject constructor( throw t } - private fun getOrCreateKey(): CharArray = try { + private fun getOrCreateKey() = try { secureLocalDbKeyProvider.getLocalDbKeyOrThrow(DB_NAME) } catch (t: Throwable) { t.message?.let { Simber.d(it) } secureLocalDbKeyProvider.createLocalDatabaseKeyIfMissing(DB_NAME) secureLocalDbKeyProvider.getLocalDbKeyOrThrow(DB_NAME) - }.value.decodeToString().toCharArray() + }.value.decodeToString() companion object { private const val DB_NAME = "db-subjects" diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFingerprintSample.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbBiometricTemplate.kt similarity index 56% rename from infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFingerprintSample.kt rename to infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbBiometricTemplate.kt index 89e9a5776e..7590c579bd 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFingerprintSample.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbBiometricTemplate.kt @@ -4,11 +4,12 @@ import androidx.room.Entity import androidx.room.ForeignKey import androidx.room.Index import androidx.room.PrimaryKey -import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.FORMAT_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate.Companion.FORMAT_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate.Companion.TEMPLATE_TABLE_NAME import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN @Entity( - tableName = "DbFingerprintSample", + tableName = TEMPLATE_TABLE_NAME, foreignKeys = [ ForeignKey( entity = DbSubject::class, @@ -18,20 +19,23 @@ import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Compani ), ], indices = [ - Index(value = [FORMAT_COLUMN]), + Index(value = [FORMAT_COLUMN, SUBJECT_ID_COLUMN]), Index(value = [SUBJECT_ID_COLUMN]), - ], ) @Suppress("ArrayInDataClass") -data class DbFingerprintSample( - // Auto-incrementing key for pagination - @PrimaryKey(autoGenerate = true) - val rowId: Long = 0, // This field is automatically assigned by Room +data class DbBiometricTemplate( + @PrimaryKey val uuid: String = "", val subjectId: String = "", - val fingerIdentifier: Int = 0, - val template: ByteArray = byteArrayOf(), + val identifier: Int? = null, // e.g the finger number or other identifier for the biometric + val templateData: ByteArray = byteArrayOf(), val format: String = "", val referenceId: String = "", -) + val modality: Int = Modality.FINGERPRINT.id, +) { + companion object { + const val TEMPLATE_TABLE_NAME = "DbBiometricTemplate" + const val FORMAT_COLUMN = "format" + } +} diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFaceSample.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFaceSample.kt deleted file mode 100644 index 9f5335f4d9..0000000000 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbFaceSample.kt +++ /dev/null @@ -1,35 +0,0 @@ -package com.simprints.infra.enrolment.records.room.store.models - -import androidx.room.Entity -import androidx.room.ForeignKey -import androidx.room.Index -import androidx.room.PrimaryKey -import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.FORMAT_COLUMN -import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN - -@Entity( - tableName = "DbFaceSample", - foreignKeys = [ - ForeignKey( - entity = DbSubject::class, - parentColumns = [SUBJECT_ID_COLUMN], - childColumns = [SUBJECT_ID_COLUMN], - onDelete = ForeignKey.CASCADE, - ), - ], - indices = [ - Index(value = [FORMAT_COLUMN]), - Index(value = [SUBJECT_ID_COLUMN]), - ], -) -@Suppress("ArrayInDataClass") -data class DbFaceSample( - // Auto-incrementing key for pagination - @PrimaryKey(autoGenerate = true) - val rowId: Long = 0, // This field is automatically assigned by Room - val uuid: String = "", - val subjectId: String = "", - val template: ByteArray = byteArrayOf(), - val format: String = "", - val referenceId: String = "", -) diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt index a5608e00bf..042eb0db31 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt @@ -3,19 +3,26 @@ package com.simprints.infra.enrolment.records.room.store.models import androidx.room.Entity import androidx.room.Index import androidx.room.PrimaryKey -import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.ATTENDANT_ID_COLUMN -import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.MODULE_ID_COLUMN -import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.PROJECT_ID_COLUMN -import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_TABLE_NAME import java.util.UUID +/** + * Represents a Subject entry in the local database. + * + * Indexes are crucial for query performance, especially for large datasets. + * The indexes defined here are chosen based on the common query patterns observed + * in SubjectDao, particularly the frequent use of projectId, filtering by either + * moduleId or attendantId, and the ordering requirements of the loadSamples query. + * + * Note: Indexes speed up read operations (SELECT) but can slightly slow down + * write operations (INSERT, UPDATE, DELETE) and consume additional storage space. + */ @Entity( - tableName = "DbSubject", + tableName = SUBJECT_TABLE_NAME, indices = [ - Index(value = [SUBJECT_ID_COLUMN]), - Index(value = [PROJECT_ID_COLUMN]), - Index(value = [ATTENDANT_ID_COLUMN]), - Index(value = [MODULE_ID_COLUMN]), + Index(value = [DbSubject.PROJECT_ID_COLUMN, DbSubject.SUBJECT_ID_COLUMN]), + Index(value = [DbSubject.PROJECT_ID_COLUMN, DbSubject.MODULE_ID_COLUMN, DbSubject.SUBJECT_ID_COLUMN]), + Index(value = [DbSubject.PROJECT_ID_COLUMN, DbSubject.ATTENDANT_ID_COLUMN, DbSubject.SUBJECT_ID_COLUMN]), ], ) data class DbSubject( @@ -26,14 +33,15 @@ data class DbSubject( val moduleId: String = "", val createdAt: Long? = 0, val updatedAt: Long? = 0, - val isAttendantIdTokenized: Boolean = false, - val isModuleIdTokenized: Boolean = false, ) { + /** + * Companion object holding constants for column names. + */ companion object { + const val SUBJECT_TABLE_NAME = "DbSubject" const val SUBJECT_ID_COLUMN = "subjectId" const val PROJECT_ID_COLUMN = "projectId" const val ATTENDANT_ID_COLUMN = "attendantId" const val MODULE_ID_COLUMN = "moduleId" - const val FORMAT_COLUMN = "format" } } diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/Modality.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/Modality.kt new file mode 100644 index 0000000000..4aa4944e19 --- /dev/null +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/Modality.kt @@ -0,0 +1,8 @@ +package com.simprints.infra.enrolment.records.room.store.models + +enum class Modality( + val id: Int, +) { + FINGERPRINT(0), + FACE(1), +} diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/SubjectBiometrics.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/SubjectBiometrics.kt index d7ca705bee..1ea2f722d2 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/SubjectBiometrics.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/SubjectBiometrics.kt @@ -10,10 +10,5 @@ data class SubjectBiometrics( parentColumn = SUBJECT_ID_COLUMN, entityColumn = SUBJECT_ID_COLUMN, ) - val fingerprintSamples: List, - @Relation( - parentColumn = SUBJECT_ID_COLUMN, - entityColumn = SUBJECT_ID_COLUMN, - ) - val faceSamples: List, + val biometricTemplates: List, ) diff --git a/infra/logging/src/main/java/com/simprints/infra/logging/LoggingConstants.kt b/infra/logging/src/main/java/com/simprints/infra/logging/LoggingConstants.kt index 367eb537c3..ef934bcf16 100644 --- a/infra/logging/src/main/java/com/simprints/infra/logging/LoggingConstants.kt +++ b/infra/logging/src/main/java/com/simprints/infra/logging/LoggingConstants.kt @@ -39,6 +39,7 @@ object LoggingConstants { SETTINGS, ALERT, REALM_DB, + ROOM_RECORDS_DB, DB_CORRUPTION, ENROLMENT, APP_SCOPE_ERROR, diff --git a/infra/test-tools/build.gradle.kts b/infra/test-tools/build.gradle.kts index 38c299764e..34b06d0078 100644 --- a/infra/test-tools/build.gradle.kts +++ b/infra/test-tools/build.gradle.kts @@ -16,7 +16,6 @@ dependencies { api(libs.testing.fragment) api(libs.testing.androidX.ext.junit) api(libs.testing.androidX.runner) - api(libs.testing.androidX.navigation) api(libs.testing.navigation) api(libs.testing.hilt) From 70f53c4b6c6a32bddffc15a63ca7b9ec924d1c01 Mon Sep 17 00:00:00 2001 From: Melad Raouf Date: Thu, 29 May 2025 16:04:50 +0300 Subject: [PATCH 05/11] Refactor RoomEnrolmentRecordQueryBuilder to separate where clause and order by clause methods --- .../EnrolmentRecordRepositoryImpl.kt | 2 - .../RealmEnrolmentRecordLocalDataSource.kt | 1 - .../RoomEnrolmentRecordLocalDataSource.kt | 2 +- .../local/RoomEnrolmentRecordQueryBuilder.kt | 47 +++--- .../local/models/IFingerIdentifier.kt | 3 - .../RoomEnrolmentRecordQueryBuilderTest.kt | 141 +++++++----------- .../records/room/store/models/DbSubject.kt | 7 - 7 files changed, 86 insertions(+), 117 deletions(-) diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt index 2259367ac2..6ad240acbc 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt @@ -9,8 +9,6 @@ import com.simprints.infra.config.store.models.TokenKeyType import com.simprints.infra.config.store.tokenization.TokenizationProcessor import com.simprints.infra.enrolment.records.realm.store.exceptions.RealmUninitialisedException import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource -import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity -import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity import com.simprints.infra.enrolment.records.repository.domain.models.Subject import com.simprints.infra.enrolment.records.repository.domain.models.SubjectAction import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt index c11bfe02a3..7306e7a979 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt @@ -1,7 +1,6 @@ package com.simprints.infra.enrolment.records.repository.local import com.simprints.core.DispatcherIO -import com.simprints.core.domain.tokenization.TokenizableString import com.simprints.infra.config.store.models.Project import com.simprints.infra.config.store.models.TokenKeyType import com.simprints.infra.config.store.tokenization.TokenizationProcessor diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt index 55874aa711..9399a95de6 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt @@ -200,7 +200,7 @@ internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( Simber.i("[delete] $errorMsg", tag = ROOM_RECORDS_DB) errorMsg } - val (whereClause, args) = queryBuilder.buildWhereAndOrderByClause( + val (whereClause, args) = queryBuilder.buildWhereClause( query, subjectAlias = "", templateAlias = "", diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilder.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilder.kt index f4131cbeb4..44219cd974 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilder.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilder.kt @@ -9,37 +9,42 @@ import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Compani import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.PROJECT_ID_COLUMN import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_TABLE_NAME -import jakarta.inject.Inject +import javax.inject.Inject internal class RoomEnrolmentRecordQueryBuilder @Inject constructor() { + /** + * Builds a query to select subjects based on the provided [SubjectQuery]. + * The query will be on the `SUBJECT_TABLE_NAME` table and will include filtering criteria + * Don't set the format in the [SubjectQuery] for this method instead use [buildBiometricTemplatesQuery]. + * @param query The [SubjectQuery] containing the filtering criteria. + * @return A [SimpleSQLiteQuery] that can be executed against the database. + */ fun buildSubjectQuery(query: SubjectQuery): SimpleSQLiteQuery { // require format not to be set for subject query and guid to use the buildBiometricTemplatesQuery instead require(query.fingerprintSampleFormat == null && query.faceSampleFormat == null) { "Cannot set format for subject query, use buildBiometricTemplatesQuery instead" } - val (whereClause, args) = buildWhereAndOrderByClause(query) + val (whereClause, args) = buildWhereClause(query) + val orderByClause = buildOrderByClause(query) val sql = """ SELECT * FROM $SUBJECT_TABLE_NAME S $whereClause + $orderByClause """.trimIndent() - println(sql) - println("----") return SimpleSQLiteQuery(sql, args.toTypedArray()) } fun buildCountQuery(query: SubjectQuery): SimpleSQLiteQuery { - val (whereClause, args) = buildWhereAndOrderByClause(query) + val (whereClause, args) = buildWhereClause(query) val specificFormat = query.fingerprintSampleFormat ?: query.faceSampleFormat val sql = if (specificFormat != null) { "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + - " using(subjectId) $whereClause " + " using(subjectId) $whereClause" } else { "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S $whereClause" } - println(sql) - println("----") return SimpleSQLiteQuery(sql, args.toTypedArray()) } @@ -54,7 +59,8 @@ internal class RoomEnrolmentRecordQueryBuilder @Inject constructor() { "Must set format for biometric templates query, use buildSubjectQuery or buildCountQuery instead" } val updatedQuery = query.copy(afterSubjectId = lastSeenSubjectId, sort = true) - val (whereClause, args) = buildWhereAndOrderByClause(updatedQuery) + val (whereClause, args) = buildWhereClause(updatedQuery) + val orderByClause = buildOrderByClause(updatedQuery) val sql = """ SELECT A.* @@ -64,23 +70,22 @@ internal class RoomEnrolmentRecordQueryBuilder @Inject constructor() { FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T USING(subjectId) $whereClause + $orderByClause LIMIT $pageSize ) B USING(subjectId) where A.format ='$format' """.trimIndent() - println(sql) - println("----") return SimpleSQLiteQuery(sql, args.toTypedArray()) } - fun buildWhereAndOrderByClause( + fun buildWhereClause( query: SubjectQuery, subjectAlias: String = "S.", // Default alias for subject table, dot included. Empty string for no alias. templateAlias: String = "T.", // Default alias for template table, dot included. Empty string for no alias. - ): Pair> { + ): Pair> { val clauses = mutableListOf() val args = mutableListOf() - if (query.fingerprintSampleFormat != null && query.faceSampleFormat != null) { - throw IllegalArgumentException("Cannot set both fingerprintSampleFormat and faceSampleFormat") + require(!(query.fingerprintSampleFormat != null && query.faceSampleFormat != null)) { + "Cannot set both fingerprintSampleFormat and faceSampleFormat" } // to achieve the highest performance, we should not use OR in the where clause query.subjectId?.let { @@ -117,9 +122,15 @@ internal class RoomEnrolmentRecordQueryBuilder @Inject constructor() { } var whereClauseResult = if (clauses.isNotEmpty()) "WHERE ${clauses.joinToString(" AND ")}" else "" - if (query.sort) { - whereClauseResult += " ORDER BY ${subjectAlias}$SUBJECT_ID_COLUMN ASC" - } return Pair(whereClauseResult, args) } + + private fun buildOrderByClause( + query: SubjectQuery, + subjectAlias: String = "S.", + ) = if (query.sort) { + "ORDER BY $subjectAlias$SUBJECT_ID_COLUMN ASC" + } else { + "" + } } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/IFingerIdentifier.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/IFingerIdentifier.kt index 3fdc0b09ec..5f7beec9d2 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/IFingerIdentifier.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/models/IFingerIdentifier.kt @@ -16,7 +16,6 @@ enum class IFingerIdentifier( LEFT_5TH_FINGER(9), ; - // create from id companion object { fun fromId(id: Int) = IFingerIdentifier.entries .firstOrNull { it.id == id } @@ -24,7 +23,6 @@ enum class IFingerIdentifier( } } -// convert to CoreFingerIdentifier internal fun IFingerIdentifier.toDomain() = when (this) { IFingerIdentifier.RIGHT_5TH_FINGER -> CoreFingerIdentifier.RIGHT_5TH_FINGER IFingerIdentifier.RIGHT_4TH_FINGER -> CoreFingerIdentifier.RIGHT_4TH_FINGER @@ -38,7 +36,6 @@ internal fun IFingerIdentifier.toDomain() = when (this) { IFingerIdentifier.LEFT_5TH_FINGER -> CoreFingerIdentifier.LEFT_5TH_FINGER } -// convert from CoreFingerIdentifier internal fun CoreFingerIdentifier.fromDomain() = when (this) { CoreFingerIdentifier.RIGHT_5TH_FINGER -> IFingerIdentifier.RIGHT_5TH_FINGER CoreFingerIdentifier.RIGHT_4TH_FINGER -> IFingerIdentifier.RIGHT_4TH_FINGER diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilderTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilderTest.kt index 71f47a1dfe..b24f435aec 100644 --- a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilderTest.kt +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilderTest.kt @@ -38,95 +38,95 @@ class RoomEnrolmentRecordQueryBuilderTest { queryBuilder = RoomEnrolmentRecordQueryBuilder() } - // region buildWhereAndOrderByClause Tests + // region buildWhereClause Tests @Test - fun `buildWhereAndOrderByClause with empty query returns empty clause and args`() { + fun `buildWhereClause with empty query returns empty clause and args`() { val query = SubjectQuery() - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val (clause, args) = queryBuilder.buildWhereClause(query) assertThat(clause).isEmpty() assertThat(args).isEmpty() } @Test - fun `buildWhereAndOrderByClause with subjectId`() { + fun `buildWhereClause with subjectId`() { val query = SubjectQuery(subjectId = SUBJECT_ID_1) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val (clause, args) = queryBuilder.buildWhereClause(query) assertThat(clause).isEqualTo("WHERE S.$SUBJECT_ID_COLUMN = ?") assertThat(args).containsExactly(SUBJECT_ID_1) } @Test - fun `buildWhereAndOrderByClause with subjectIds not empty`() { + fun `buildWhereClause with subjectIds not empty`() { val query = SubjectQuery(subjectIds = listOf(SUBJECT_ID_1, SUBJECT_ID_2)) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val (clause, args) = queryBuilder.buildWhereClause(query) assertThat(clause).isEqualTo("WHERE S.$SUBJECT_ID_COLUMN IN (?,?)") assertThat(args).containsExactly(SUBJECT_ID_1, SUBJECT_ID_2).inOrder() } @Test - fun `buildWhereAndOrderByClause with subjectIds empty`() { + fun `buildWhereClause with subjectIds empty`() { val query = SubjectQuery(subjectIds = emptyList()) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val (clause, args) = queryBuilder.buildWhereClause(query) assertThat(clause).isEmpty() assertThat(args).isEmpty() } @Test - fun `buildWhereAndOrderByClause with afterSubjectId`() { + fun `buildWhereClause with afterSubjectId`() { val query = SubjectQuery(afterSubjectId = AFTER_SUBJECT_ID) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val (clause, args) = queryBuilder.buildWhereClause(query) assertThat(clause).isEqualTo("WHERE S.$SUBJECT_ID_COLUMN > ?") assertThat(args).containsExactly(AFTER_SUBJECT_ID) } @Test - fun `buildWhereAndOrderByClause with projectId`() { + fun `buildWhereClause with projectId`() { val query = SubjectQuery(projectId = PROJECT_ID) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val (clause, args) = queryBuilder.buildWhereClause(query) assertThat(clause).isEqualTo("WHERE S.$PROJECT_ID_COLUMN = ?") assertThat(args).containsExactly(PROJECT_ID) } @Test - fun `buildWhereAndOrderByClause with attendantId`() { + fun `buildWhereClause with attendantId`() { val query = SubjectQuery(attendantId = ATTENDANT_ID) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val (clause, args) = queryBuilder.buildWhereClause(query) assertThat(clause).isEqualTo("WHERE S.$ATTENDANT_ID_COLUMN = ?") assertThat(args).containsExactly(ATTENDANT_ID.value) } @Test - fun `buildWhereAndOrderByClause with moduleId`() { + fun `buildWhereClause with moduleId`() { val query = SubjectQuery(moduleId = MODULE_ID) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val (clause, args) = queryBuilder.buildWhereClause(query) assertThat(clause).isEqualTo("WHERE S.$MODULE_ID_COLUMN = ?") assertThat(args).containsExactly(MODULE_ID.value) } @Test - fun `buildWhereAndOrderByClause with faceSampleFormat`() { + fun `buildWhereClause with faceSampleFormat`() { val query = SubjectQuery(faceSampleFormat = FACE_FORMAT) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val (clause, args) = queryBuilder.buildWhereClause(query) assertThat(clause).isEqualTo("WHERE T.$FORMAT_COLUMN = ?") assertThat(args).containsExactly(FACE_FORMAT) } @Test - fun `buildWhereAndOrderByClause with fingerprintSampleFormat`() { + fun `buildWhereClause with fingerprintSampleFormat`() { val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val (clause, args) = queryBuilder.buildWhereClause(query) assertThat(clause).isEqualTo("WHERE T.$FORMAT_COLUMN = ?") assertThat(args).containsExactly(FP_FORMAT) } @Test - fun `buildWhereAndOrderByClause with multiple subject fields`() { + fun `buildWhereClause with multiple subject fields`() { val query = SubjectQuery( projectId = PROJECT_ID, attendantId = ATTENDANT_ID, moduleId = MODULE_ID, ) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val (clause, args) = queryBuilder.buildWhereClause(query) assertThat(clause).isEqualTo( "WHERE S.$PROJECT_ID_COLUMN = ? AND S.$ATTENDANT_ID_COLUMN = ? AND S.$MODULE_ID_COLUMN = ?", ) @@ -134,7 +134,7 @@ class RoomEnrolmentRecordQueryBuilderTest { } @Test - fun `buildWhereAndOrderByClause with all parameters and sort`() { + fun `buildWhereClause with all parameters`() { val query = SubjectQuery( subjectId = SUBJECT_ID_1, subjectIds = listOf(SUBJECT_ID_2, SUBJECT_ID_3), @@ -143,11 +143,10 @@ class RoomEnrolmentRecordQueryBuilderTest { attendantId = ATTENDANT_ID, moduleId = MODULE_ID, fingerprintSampleFormat = FP_FORMAT, - sort = true, ) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) + val (clause, args) = queryBuilder.buildWhereClause(query) val expectedClause = - "WHERE S.$SUBJECT_ID_COLUMN = ? AND S.$SUBJECT_ID_COLUMN IN (?,?) AND S.$SUBJECT_ID_COLUMN > ? AND S.$PROJECT_ID_COLUMN = ? AND S.$ATTENDANT_ID_COLUMN = ? AND S.$MODULE_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC" + "WHERE S.$SUBJECT_ID_COLUMN = ? AND S.$SUBJECT_ID_COLUMN IN (?,?) AND S.$SUBJECT_ID_COLUMN > ? AND S.$PROJECT_ID_COLUMN = ? AND S.$ATTENDANT_ID_COLUMN = ? AND S.$MODULE_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ?" assertThat(clause).isEqualTo(expectedClause) assertThat(args) .containsExactly( @@ -163,48 +162,24 @@ class RoomEnrolmentRecordQueryBuilderTest { } @Test - fun `buildWhereAndOrderByClause with sort true and no other clauses`() { - val query = SubjectQuery(sort = true) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) - assertThat(clause).isEqualTo(" ORDER BY S.$SUBJECT_ID_COLUMN ASC") - assertThat(args).isEmpty() - } - - @Test - fun `buildWhereAndOrderByClause with sort true and other clauses`() { - val query = SubjectQuery(projectId = PROJECT_ID, sort = true) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) - assertThat(clause).isEqualTo("WHERE S.$PROJECT_ID_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC") - assertThat(args).containsExactly(PROJECT_ID) - } - - @Test - fun `buildWhereAndOrderByClause with sort false and clauses`() { - val query = SubjectQuery(projectId = PROJECT_ID, sort = false) // sort = false is default - val (clause, args) = queryBuilder.buildWhereAndOrderByClause(query) - assertThat(clause).isEqualTo("WHERE S.$PROJECT_ID_COLUMN = ?") - assertThat(args).containsExactly(PROJECT_ID) - } - - @Test - fun `buildWhereAndOrderByClause with custom aliases`() { - val query = SubjectQuery(subjectId = SUBJECT_ID_1, fingerprintSampleFormat = FP_FORMAT, sort = true) - val (clause, args) = queryBuilder.buildWhereAndOrderByClause( + fun `buildWhereClause with custom aliases`() { + val query = SubjectQuery(subjectId = SUBJECT_ID_1, fingerprintSampleFormat = FP_FORMAT) + val (clause, args) = queryBuilder.buildWhereClause( query, subjectAlias = "customS.", templateAlias = "customT.", ) val expectedClause = - "WHERE customS.$SUBJECT_ID_COLUMN = ? AND customT.$FORMAT_COLUMN = ? ORDER BY customS.$SUBJECT_ID_COLUMN ASC" + "WHERE customS.$SUBJECT_ID_COLUMN = ? AND customT.$FORMAT_COLUMN = ?" assertThat(clause).isEqualTo(expectedClause) assertThat(args).containsExactly(SUBJECT_ID_1, FP_FORMAT).inOrder() } @Test - fun `buildWhereAndOrderByClause throws error if both fingerprint and face format set`() { + fun `buildWhereClause throws error if both fingerprint and face format set`() { val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, faceSampleFormat = FACE_FORMAT) val exception = assertThrows(IllegalArgumentException::class.java) { - queryBuilder.buildWhereAndOrderByClause(query) + queryBuilder.buildWhereClause(query) } assertThat(exception.message).isEqualTo("Cannot set both fingerprintSampleFormat and faceSampleFormat") } @@ -218,7 +193,8 @@ class RoomEnrolmentRecordQueryBuilderTest { val expectedSql = """ SELECT * FROM DbSubject S - + + """.trimIndent() assertThat(result.sql).isEqualTo(expectedSql) assertThat(result.argCount).isEqualTo(0) @@ -232,6 +208,7 @@ class RoomEnrolmentRecordQueryBuilderTest { """ SELECT * FROM DbSubject S WHERE S.projectId = ? + """.trimIndent() assertThat(result.sql).isEqualTo(expectedSql) assertThat(result.argCount).isEqualTo(1) @@ -245,6 +222,7 @@ class RoomEnrolmentRecordQueryBuilderTest { """ SELECT * FROM $SUBJECT_TABLE_NAME S WHERE S.$PROJECT_ID_COLUMN = ? AND S.$ATTENDANT_ID_COLUMN = ? + """.trimIndent() assertThat(result.sql).isEqualTo(expectedSql) assertThat(result.argCount).isEqualTo(2) @@ -257,9 +235,10 @@ class RoomEnrolmentRecordQueryBuilderTest { val expectedSql = """ SELECT * FROM $SUBJECT_TABLE_NAME S - ORDER BY S.$SUBJECT_ID_COLUMN ASC + + ORDER BY S.$SUBJECT_ID_COLUMN ASC """.trimIndent() - // Note the space before ORDER due to buildWhereAndOrderByClause + // Note the space before ORDER due to buildWhereClause assertThat(result.sql).isEqualTo(expectedSql) assertThat(result.argCount).isEqualTo(0) } @@ -271,7 +250,8 @@ class RoomEnrolmentRecordQueryBuilderTest { val expectedSql = """ SELECT * FROM $SUBJECT_TABLE_NAME S - WHERE S.$PROJECT_ID_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC + WHERE S.$PROJECT_ID_COLUMN = ? + ORDER BY S.$SUBJECT_ID_COLUMN ASC """.trimIndent() assertThat(result.sql).isEqualTo(expectedSql) assertThat(result.argCount).isEqualTo(1) @@ -321,7 +301,7 @@ class RoomEnrolmentRecordQueryBuilderTest { val result = queryBuilder.buildCountQuery(query) assertThat(result.sql).isEqualTo( "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + - " using($SUBJECT_ID_COLUMN) WHERE T.$FORMAT_COLUMN = ? ", + " using($SUBJECT_ID_COLUMN) WHERE T.$FORMAT_COLUMN = ?", ) assertThat(result.argCount).isEqualTo(1) } @@ -331,28 +311,17 @@ class RoomEnrolmentRecordQueryBuilderTest { val query = SubjectQuery(faceSampleFormat = FACE_FORMAT, projectId = PROJECT_ID) val result = queryBuilder.buildCountQuery(query) val expectedSql = "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + - " using($SUBJECT_ID_COLUMN) WHERE S.$PROJECT_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ? " + " using($SUBJECT_ID_COLUMN) WHERE S.$PROJECT_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ?" assertThat(result.sql).isEqualTo(expectedSql) assertThat(result.argCount).isEqualTo(2) } @Test - fun `buildCountQuery with sort true no format`() { - val query = SubjectQuery(sort = true) - val result = queryBuilder.buildCountQuery(query) - // specificFormat == null, whereClause is " ORDER BY S.subjectId ASC" - // "... S ORDER BY S.subjectId ASC" - val expectedSql = "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S ORDER BY S.$SUBJECT_ID_COLUMN ASC" - assertThat(result.sql).isEqualTo(expectedSql) - assertThat(result.argCount).isEqualTo(0) - } - - @Test - fun `buildCountQuery with fingerprintSampleFormat and sort true`() { - val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, sort = true) + fun `buildCountQuery with fingerprintSampleFormat`() { + val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT) val result = queryBuilder.buildCountQuery(query) val expectedSql = "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + - " using($SUBJECT_ID_COLUMN) WHERE T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC " + " using($SUBJECT_ID_COLUMN) WHERE T.$FORMAT_COLUMN = ?" assertThat(result.sql).isEqualTo(expectedSql) assertThat(result.argCount).isEqualTo(1) } @@ -361,7 +330,7 @@ class RoomEnrolmentRecordQueryBuilderTest { fun `buildCountQuery throws error if both fingerprint and face format set`() { val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, faceSampleFormat = FACE_FORMAT) val exception = assertThrows(IllegalArgumentException::class.java) { - queryBuilder.buildCountQuery(query) // This will call buildWhereAndOrderByClause + queryBuilder.buildCountQuery(query) // This will call buildWhereClause } assertThat(exception.message).isEqualTo("Cannot set both fingerprintSampleFormat and faceSampleFormat") } @@ -372,7 +341,7 @@ class RoomEnrolmentRecordQueryBuilderTest { fun `buildBiometricTemplatesQuery with fingerprintSampleFormat, no lastSeenId`() { val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT) val result = queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE, null) - val expectedSubQueryWhereClause = "WHERE T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC" + val expectedSql = """ SELECT A.* @@ -381,7 +350,8 @@ class RoomEnrolmentRecordQueryBuilderTest { SELECT distinct S.$SUBJECT_ID_COLUMN FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T USING($SUBJECT_ID_COLUMN) - $expectedSubQueryWhereClause + WHERE T.$FORMAT_COLUMN = ? + ORDER BY S.$SUBJECT_ID_COLUMN ASC LIMIT $PAGE_SIZE ) B USING($SUBJECT_ID_COLUMN) where A.$FORMAT_COLUMN ='$FP_FORMAT' """.trimIndent() @@ -394,7 +364,7 @@ class RoomEnrolmentRecordQueryBuilderTest { fun `buildBiometricTemplatesQuery with faceSampleFormat and lastSeenId`() { val query = SubjectQuery(faceSampleFormat = FACE_FORMAT) val result = queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE, LAST_SEEN_SUBJECT_ID) - val expectedSubQueryWhereClause = "WHERE S.$SUBJECT_ID_COLUMN > ? AND T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC" + val expectedSql = """ SELECT A.* @@ -403,7 +373,8 @@ class RoomEnrolmentRecordQueryBuilderTest { SELECT distinct S.$SUBJECT_ID_COLUMN FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T USING($SUBJECT_ID_COLUMN) - $expectedSubQueryWhereClause + WHERE S.$SUBJECT_ID_COLUMN > ? AND T.$FORMAT_COLUMN = ? + ORDER BY S.$SUBJECT_ID_COLUMN ASC LIMIT $PAGE_SIZE ) B USING($SUBJECT_ID_COLUMN) where A.$FORMAT_COLUMN ='$FACE_FORMAT' """.trimIndent() @@ -416,8 +387,7 @@ class RoomEnrolmentRecordQueryBuilderTest { fun `buildBiometricTemplatesQuery with fingerprintFormat, projectId, lastSeenId`() { val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, projectId = PROJECT_ID) val result = queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE, LAST_SEEN_SUBJECT_ID) - val expectedSubQueryWhereClause = - "WHERE S.$SUBJECT_ID_COLUMN > ? AND S.$PROJECT_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC" + val expectedSql = """ SELECT A.* @@ -426,7 +396,8 @@ class RoomEnrolmentRecordQueryBuilderTest { SELECT distinct S.$SUBJECT_ID_COLUMN FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T USING($SUBJECT_ID_COLUMN) - $expectedSubQueryWhereClause + WHERE S.$SUBJECT_ID_COLUMN > ? AND S.$PROJECT_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ? + ORDER BY S.$SUBJECT_ID_COLUMN ASC LIMIT $PAGE_SIZE ) B USING($SUBJECT_ID_COLUMN) where A.$FORMAT_COLUMN ='$FP_FORMAT' """.trimIndent() diff --git a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt index 042eb0db31..76abb42418 100644 --- a/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt +++ b/infra/enrolment-records/room-store/src/main/java/com/simprints/infra/enrolment/records/room/store/models/DbSubject.kt @@ -9,13 +9,6 @@ import java.util.UUID /** * Represents a Subject entry in the local database. * - * Indexes are crucial for query performance, especially for large datasets. - * The indexes defined here are chosen based on the common query patterns observed - * in SubjectDao, particularly the frequent use of projectId, filtering by either - * moduleId or attendantId, and the ordering requirements of the loadSamples query. - * - * Note: Indexes speed up read operations (SELECT) but can slightly slow down - * write operations (INSERT, UPDATE, DELETE) and consume additional storage space. */ @Entity( tableName = SUBJECT_TABLE_NAME, From 8208600a8725f923e236e360aa02a951e3a79a0a Mon Sep 17 00:00:00 2001 From: Melad Raouf Date: Sat, 31 May 2025 22:21:11 +0300 Subject: [PATCH 06/11] Refactor RoomEnrolmentRecordLocalDataSource and QueryBuilder for improved query handling and clarity --- .../RoomEnrolmentRecordLocalDataSource.kt | 72 +-- .../local/RoomEnrolmentRecordQueryBuilder.kt | 52 +- ...RoomEnrolmentRecordLocalDataSourceTest.kt} | 21 +- .../RoomEnrolmentRecordQueryBuilderTest.kt | 606 +++++++++--------- 4 files changed, 393 insertions(+), 358 deletions(-) rename infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/{RoomEnrollmentRecordLocalDataSourceTest.kt => RoomEnrolmentRecordLocalDataSourceTest.kt} (98%) diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt index 9399a95de6..acc9df3295 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt @@ -1,7 +1,6 @@ package com.simprints.infra.enrolment.records.repository.local import androidx.room.withTransaction -import androidx.sqlite.db.SimpleSQLiteQuery import com.simprints.core.DispatcherIO import com.simprints.core.domain.face.FaceSample import com.simprints.core.domain.fingerprint.FingerprintSample @@ -25,7 +24,9 @@ import com.simprints.infra.logging.LoggingConstants.CrashReportTag.ROOM_RECORDS_ import com.simprints.infra.logging.Simber import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.ReceiveChannel +import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import javax.inject.Inject import javax.inject.Singleton @@ -43,7 +44,8 @@ internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( ) : EnrolmentRecordLocalDataSource { companion object { // only one concurrent operation is allowed as we use the last seen subject ID to load biometric identities - private const val PARALLELISM = 1 + // and we need to ensure that the next operation starts after the previous one finishes + private const val CHANNEL_CAPACITY = 3 } private val database: SubjectsDatabase by lazy { subjectsDatabaseFactory.get() } @@ -146,45 +148,44 @@ internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( onCandidateLoaded: () -> Unit, scope: CoroutineScope, ): ReceiveChannel> { - var lastSeenSubjectId: String? = null + var afterSubjectId: String? = null var lastOffset = 0 - return loadIdentitiesConcurrently( - ranges = ranges, - dispatcher = dispatcherIO, - parallelism = PARALLELISM, - scope = scope, - ) { range -> - require(lastOffset == range.first) { - "[loadBiometricIdentitiesPaged] The range start must match the last seen sample count. " + - "Expected: $lastOffset, Actual: ${range.first}" - } - val identities = loadBiometricIdentities( - query = query, - pageSize = range.last - range.first, - lastSeenSubjectId = lastSeenSubjectId, - format = format, - createIdentity = createIdentity, - onCandidateLoaded = onCandidateLoaded, - ) - lastSeenSubjectId = identities.lastOrNull()?.let { - (it as? FaceIdentity)?.subjectId ?: (it as? FingerprintIdentity)?.subjectId - } - lastOffset = range.last - identities + val channel = Channel>(CHANNEL_CAPACITY) + scope.launch(dispatcherIO) { + ranges + .forEach { range -> + require(lastOffset == range.first) { + "[loadBiometricIdentitiesPaged] The range start must match the last seen sample count. " + + "Expected: $lastOffset, Actual: ${range.first}" + } + val identities = loadBiometricIdentities( + query = query.copy(afterSubjectId = afterSubjectId), // update query with the last seen subject ID + pageSize = range.last - range.first, + format = format, + createIdentity = createIdentity, + onCandidateLoaded = onCandidateLoaded, + ) + afterSubjectId = identities.lastOrNull()?.let { + (it as? FaceIdentity)?.subjectId ?: (it as? FingerprintIdentity)?.subjectId + } + lastOffset = range.last + 1 + channel.send(identities) + } + channel.close() } + return channel } private suspend fun loadBiometricIdentities( query: SubjectQuery, pageSize: Int, format: String?, - lastSeenSubjectId: String?, createIdentity: (subjectId: String, samples: List) -> T, onCandidateLoaded: () -> Unit, ): List = withContext(dispatcherIO) { requireNotNull(format) { "Appropriate sampleFormat is required for loading biometric identities." } subjectDao - .loadSamples(queryBuilder.buildBiometricTemplatesQuery(query, pageSize, lastSeenSubjectId)) + .loadSamples(queryBuilder.buildBiometricTemplatesQuery(query, pageSize)) .map { (subjectId, templates) -> onCandidateLoaded() createIdentity(subjectId, templates) @@ -195,25 +196,14 @@ internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( Simber.i("[delete] Deleting subjects with queries: $queries", tag = ROOM_RECORDS_DB) database.withTransaction { queries.forEach { query -> - require(query.faceSampleFormat == null && query.fingerprintSampleFormat == null) { - val errorMsg = "faceSampleFormat and fingerprintSampleFormat are not supported for deletion" - Simber.i("[delete] $errorMsg", tag = ROOM_RECORDS_DB) - errorMsg - } - val (whereClause, args) = queryBuilder.buildWhereClause( - query, - subjectAlias = "", - templateAlias = "", - ) - val sql = "DELETE FROM DbSubject $whereClause" - subjectDao.deleteSubjects(SimpleSQLiteQuery(sql, args.toTypedArray())) + subjectDao.deleteSubjects(queryBuilder.buildDeleteQuery(query)) } } } override suspend fun deleteAll() { Simber.i("[deleteAll] Deleting all subjects.", tag = ROOM_RECORDS_DB) - subjectDao.deleteSubjects(SimpleSQLiteQuery("DELETE FROM DbSubject")) + subjectDao.deleteSubjects(queryBuilder.buildDeleteQuery(SubjectQuery())) } override suspend fun performActions( diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilder.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilder.kt index 44219cd974..61d7a07520 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilder.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilder.kt @@ -9,6 +9,8 @@ import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Compani import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.PROJECT_ID_COLUMN import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_TABLE_NAME +import com.simprints.infra.logging.LoggingConstants.CrashReportTag.ROOM_RECORDS_DB +import com.simprints.infra.logging.Simber import javax.inject.Inject internal class RoomEnrolmentRecordQueryBuilder @Inject constructor() { @@ -20,7 +22,7 @@ internal class RoomEnrolmentRecordQueryBuilder @Inject constructor() { * @return A [SimpleSQLiteQuery] that can be executed against the database. */ fun buildSubjectQuery(query: SubjectQuery): SimpleSQLiteQuery { - // require format not to be set for subject query and guid to use the buildBiometricTemplatesQuery instead + // require format not to be set for subject query and guide to use the buildBiometricTemplatesQuery instead require(query.fingerprintSampleFormat == null && query.faceSampleFormat == null) { "Cannot set format for subject query, use buildBiometricTemplatesQuery instead" } @@ -51,14 +53,13 @@ internal class RoomEnrolmentRecordQueryBuilder @Inject constructor() { fun buildBiometricTemplatesQuery( query: SubjectQuery, pageSize: Int, - lastSeenSubjectId: String? = null, ): SimpleSQLiteQuery { // require format to be set for biometric templates query val format = query.fingerprintSampleFormat ?: query.faceSampleFormat require(format != null) { "Must set format for biometric templates query, use buildSubjectQuery or buildCountQuery instead" } - val updatedQuery = query.copy(afterSubjectId = lastSeenSubjectId, sort = true) + val updatedQuery = query.copy(sort = true) val (whereClause, args) = buildWhereClause(updatedQuery) val orderByClause = buildOrderByClause(updatedQuery) val sql = @@ -77,7 +78,22 @@ internal class RoomEnrolmentRecordQueryBuilder @Inject constructor() { return SimpleSQLiteQuery(sql, args.toTypedArray()) } - fun buildWhereClause( + fun buildDeleteQuery(query: SubjectQuery): SimpleSQLiteQuery { + require(query.faceSampleFormat == null && query.fingerprintSampleFormat == null) { + val errorMsg = "faceSampleFormat and fingerprintSampleFormat are not supported for deletion" + Simber.i("[delete] $errorMsg", tag = ROOM_RECORDS_DB) + errorMsg + } + val (whereClause, args) = buildWhereClause( + query, + subjectAlias = "", + templateAlias = "", + ) + val sql = "DELETE FROM DbSubject $whereClause" + return SimpleSQLiteQuery(sql, args.toTypedArray()) + } + + private fun buildWhereClause( query: SubjectQuery, subjectAlias: String = "S.", // Default alias for subject table, dot included. Empty string for no alias. templateAlias: String = "T.", // Default alias for template table, dot included. Empty string for no alias. @@ -88,18 +104,24 @@ internal class RoomEnrolmentRecordQueryBuilder @Inject constructor() { "Cannot set both fingerprintSampleFormat and faceSampleFormat" } // to achieve the highest performance, we should not use OR in the where clause - query.subjectId?.let { - clauses.add("${subjectAlias}$SUBJECT_ID_COLUMN = ?") - args.add(it) - } - query.subjectIds?.takeIf { it.isNotEmpty() }?.let { - clauses.add("${subjectAlias}$SUBJECT_ID_COLUMN IN (${it.joinToString(",") { "?" }})") - args.addAll(it) - } - query.afterSubjectId?.let { - clauses.add("${subjectAlias}$SUBJECT_ID_COLUMN > ?") - args.add(it) + // subject id params are mutually exclusive, so only one of them will be set at a time + when { + query.subjectId != null -> { + clauses.add("${subjectAlias}$SUBJECT_ID_COLUMN = ?") + args.add(query.subjectId) + } + + query.subjectIds?.isNotEmpty() == true -> { + clauses.add("${subjectAlias}$SUBJECT_ID_COLUMN IN (${query.subjectIds.joinToString(",") { "?" }})") + args.addAll(query.subjectIds) + } + + query.afterSubjectId != null -> { + clauses.add("${subjectAlias}$SUBJECT_ID_COLUMN > ?") + args.add(query.afterSubjectId) + } } + query.projectId?.let { clauses.add("${subjectAlias}$PROJECT_ID_COLUMN = ?") args.add(it) diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrollmentRecordLocalDataSourceTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSourceTest.kt similarity index 98% rename from infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrollmentRecordLocalDataSourceTest.kt rename to infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSourceTest.kt index 9e72fc9bba..4aebfd57e7 100644 --- a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrollmentRecordLocalDataSourceTest.kt +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSourceTest.kt @@ -32,7 +32,7 @@ import java.util.Date import java.util.UUID @RunWith(RobolectricTestRunner::class) -class RoomEnrollmentRecordLocalDataSourceTest { +class RoomEnrolmentRecordLocalDataSourceTest { companion object { const val PROJECT_1_ID = "project1" const val PROJECT_2_ID = "project2" @@ -891,7 +891,7 @@ class RoomEnrollmentRecordLocalDataSourceTest { query = baseQuery, ranges = listOf( 0..1, - 1..2, + 2..3, ), project = project, dataSource = Simprints, @@ -902,7 +902,7 @@ class RoomEnrollmentRecordLocalDataSourceTest { val loadedFirstTwo = dataSource .loadFingerprintIdentities( - query = baseQuery.copy(), // Copy to ensure new instance to avoid using the last subjectId + query = baseQuery, ranges = listOf( 0..2, ), @@ -915,7 +915,7 @@ class RoomEnrollmentRecordLocalDataSourceTest { val loadedAll = dataSource .loadFingerprintIdentities( - query = baseQuery.copy(), + query = baseQuery, ranges = listOf(0..10), project = project, dataSource = Simprints, @@ -1079,7 +1079,7 @@ class RoomEnrollmentRecordLocalDataSourceTest { query = baseQuery, ranges = listOf( 0..1, - 1..2, + 2..3, ), project = project, dataSource = Simprints, @@ -1090,7 +1090,7 @@ class RoomEnrollmentRecordLocalDataSourceTest { val loadedFirstTwo = dataSource .loadFaceIdentities( - query = baseQuery.copy(), + query = baseQuery, ranges = listOf( 0..2, ), @@ -1102,7 +1102,7 @@ class RoomEnrollmentRecordLocalDataSourceTest { .first() val loadedAll = dataSource .loadFaceIdentities( - query = baseQuery.copy(), + query = baseQuery, ranges = listOf(0..10), project = project, dataSource = Simprints, @@ -1446,23 +1446,20 @@ class RoomEnrollmentRecordLocalDataSourceTest { } @Test - fun `load - combined query - attendantId, moduleId, afterSubjectId, subjectIds - should respect all filters`() = runTest { + fun `load - combined query - attendantId, moduleId, subjectIds - should respect all filters`() = runTest { // Given setupInitialData() // Targets: subj-001, subj-002 (P1, A1, M1), subj-003 (P1, A1, M2) val targetIds = listOf( - subject1P1WithFace.subjectId, // subj-001 subject2P1WithFinger.subjectId, // subj-002 subject3P1WithBoth.subjectId, // subj-003 "subj-nonexistent", // Include a non-existent ID ) - val afterId = subject1P1WithFace.subjectId // subj-001 - // Query: Project 1, Attendant 1, after subj-001, from the targetIds list, sorted + // Query: Project 1, Attendant 1, from the targetIds list, sorted val query = SubjectQuery( projectId = PROJECT_1_ID, attendantId = ATTENDANT_1_ID, - afterSubjectId = afterId, subjectIds = targetIds, sort = true, ) diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilderTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilderTest.kt index b24f435aec..d68c21095c 100644 --- a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilderTest.kt +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordQueryBuilderTest.kt @@ -1,429 +1,455 @@ package com.simprints.infra.enrolment.records.repository.local +import androidx.sqlite.db.SimpleSQLiteQuery +import androidx.sqlite.db.SupportSQLiteProgram import com.google.common.truth.Truth.* import com.simprints.core.domain.tokenization.asTokenizableEncrypted import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery -import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate -import com.simprints.infra.enrolment.records.room.store.models.DbSubject -import org.junit.Assert.assertThrows +import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate.Companion.FORMAT_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbBiometricTemplate.Companion.TEMPLATE_TABLE_NAME +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.ATTENDANT_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.MODULE_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.PROJECT_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_ID_COLUMN +import com.simprints.infra.enrolment.records.room.store.models.DbSubject.Companion.SUBJECT_TABLE_NAME +import com.simprints.testtools.common.syntax.assertThrows import org.junit.Before import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE) class RoomEnrolmentRecordQueryBuilderTest { private lateinit var queryBuilder: RoomEnrolmentRecordQueryBuilder - companion object { - private const val SUBJECT_ID_1 = "subject_id_1" - private const val SUBJECT_ID_2 = "subject_id_2" - private const val SUBJECT_ID_3 = "subject_id_3" - private const val AFTER_SUBJECT_ID = "after_subject_id" - private const val PROJECT_ID = "project_id_x" - private val ATTENDANT_ID = "attendant_id_y".asTokenizableEncrypted() - private val MODULE_ID = "module_id_z".asTokenizableEncrypted() - private const val FP_FORMAT = "FP_FORMAT_XYZ" - private const val FACE_FORMAT = "FACE_FORMAT_ABC" - private const val PAGE_SIZE = 10 - private const val LAST_SEEN_SUBJECT_ID = "last_seen_subj_001" - private const val SUBJECT_TABLE_NAME = DbSubject.SUBJECT_TABLE_NAME - private const val TEMPLATE_TABLE_NAME = DbBiometricTemplate.TEMPLATE_TABLE_NAME - private const val SUBJECT_ID_COLUMN = DbSubject.SUBJECT_ID_COLUMN - private const val PROJECT_ID_COLUMN = DbSubject.PROJECT_ID_COLUMN - private const val ATTENDANT_ID_COLUMN = DbSubject.ATTENDANT_ID_COLUMN - private const val MODULE_ID_COLUMN = DbSubject.MODULE_ID_COLUMN - private const val FORMAT_COLUMN = DbBiometricTemplate.FORMAT_COLUMN - } - @Before fun setUp() { queryBuilder = RoomEnrolmentRecordQueryBuilder() } - // region buildWhereClause Tests @Test - fun `buildWhereClause with empty query returns empty clause and args`() { - val query = SubjectQuery() - val (clause, args) = queryBuilder.buildWhereClause(query) - assertThat(clause).isEmpty() - assertThat(args).isEmpty() - } + fun `buildSubjectQuery with empty query returns select all`() { + val subjectQuery = SubjectQuery() + val expectedSql = "SELECT * FROM $SUBJECT_TABLE_NAME S" - @Test - fun `buildWhereClause with subjectId`() { - val query = SubjectQuery(subjectId = SUBJECT_ID_1) - val (clause, args) = queryBuilder.buildWhereClause(query) - assertThat(clause).isEqualTo("WHERE S.$SUBJECT_ID_COLUMN = ?") - assertThat(args).containsExactly(SUBJECT_ID_1) - } + val resultQuery = queryBuilder.buildSubjectQuery(subjectQuery) - @Test - fun `buildWhereClause with subjectIds not empty`() { - val query = SubjectQuery(subjectIds = listOf(SUBJECT_ID_1, SUBJECT_ID_2)) - val (clause, args) = queryBuilder.buildWhereClause(query) - assertThat(clause).isEqualTo("WHERE S.$SUBJECT_ID_COLUMN IN (?,?)") - assertThat(args).containsExactly(SUBJECT_ID_1, SUBJECT_ID_2).inOrder() + assertThat(resultQuery.sql.trim()).isEqualTo(expectedSql) + assertThat(resultQuery.argCount).isEqualTo(0) } @Test - fun `buildWhereClause with subjectIds empty`() { - val query = SubjectQuery(subjectIds = emptyList()) - val (clause, args) = queryBuilder.buildWhereClause(query) - assertThat(clause).isEmpty() - assertThat(args).isEmpty() - } + fun `buildSubjectQuery with subjectId`() { + val subjectId = "test-subject-id" + val subjectQuery = SubjectQuery(subjectId = subjectId) + val expectedSql = "SELECT * FROM $SUBJECT_TABLE_NAME S\n" + + "WHERE S.$SUBJECT_ID_COLUMN = ?" - @Test - fun `buildWhereClause with afterSubjectId`() { - val query = SubjectQuery(afterSubjectId = AFTER_SUBJECT_ID) - val (clause, args) = queryBuilder.buildWhereClause(query) - assertThat(clause).isEqualTo("WHERE S.$SUBJECT_ID_COLUMN > ?") - assertThat(args).containsExactly(AFTER_SUBJECT_ID) - } + val resultQuery = queryBuilder.buildSubjectQuery(subjectQuery) - @Test - fun `buildWhereClause with projectId`() { - val query = SubjectQuery(projectId = PROJECT_ID) - val (clause, args) = queryBuilder.buildWhereClause(query) - assertThat(clause).isEqualTo("WHERE S.$PROJECT_ID_COLUMN = ?") - assertThat(args).containsExactly(PROJECT_ID) + assertThat(resultQuery.sql.trim()).isEqualTo(expectedSql.trim()) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(subjectId)) } @Test - fun `buildWhereClause with attendantId`() { - val query = SubjectQuery(attendantId = ATTENDANT_ID) - val (clause, args) = queryBuilder.buildWhereClause(query) - assertThat(clause).isEqualTo("WHERE S.$ATTENDANT_ID_COLUMN = ?") - assertThat(args).containsExactly(ATTENDANT_ID.value) - } + fun `buildSubjectQuery with subjectIds`() { + val subjectIds = listOf("id1", "id2", "id3") + val subjectQuery = SubjectQuery(subjectIds = subjectIds) + val expectedSql = "SELECT * FROM $SUBJECT_TABLE_NAME S\n" + + "WHERE S.$SUBJECT_ID_COLUMN IN (?,?,?)" - @Test - fun `buildWhereClause with moduleId`() { - val query = SubjectQuery(moduleId = MODULE_ID) - val (clause, args) = queryBuilder.buildWhereClause(query) - assertThat(clause).isEqualTo("WHERE S.$MODULE_ID_COLUMN = ?") - assertThat(args).containsExactly(MODULE_ID.value) - } + val resultQuery = queryBuilder.buildSubjectQuery(subjectQuery) - @Test - fun `buildWhereClause with faceSampleFormat`() { - val query = SubjectQuery(faceSampleFormat = FACE_FORMAT) - val (clause, args) = queryBuilder.buildWhereClause(query) - assertThat(clause).isEqualTo("WHERE T.$FORMAT_COLUMN = ?") - assertThat(args).containsExactly(FACE_FORMAT) + assertThat(resultQuery.sql.trim()).isEqualTo(expectedSql.trim()) + assertThat(getArgs(resultQuery)).isEqualTo(subjectIds.toTypedArray()) } @Test - fun `buildWhereClause with fingerprintSampleFormat`() { - val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT) - val (clause, args) = queryBuilder.buildWhereClause(query) - assertThat(clause).isEqualTo("WHERE T.$FORMAT_COLUMN = ?") - assertThat(args).containsExactly(FP_FORMAT) - } + fun `buildSubjectQuery with afterSubjectId`() { + val afterSubjectId = "last-subject-id" + val subjectQuery = SubjectQuery(afterSubjectId = afterSubjectId) + val expectedSql = "SELECT * FROM $SUBJECT_TABLE_NAME S\n" + + "WHERE S.$SUBJECT_ID_COLUMN > ?" - @Test - fun `buildWhereClause with multiple subject fields`() { - val query = SubjectQuery( - projectId = PROJECT_ID, - attendantId = ATTENDANT_ID, - moduleId = MODULE_ID, - ) - val (clause, args) = queryBuilder.buildWhereClause(query) - assertThat(clause).isEqualTo( - "WHERE S.$PROJECT_ID_COLUMN = ? AND S.$ATTENDANT_ID_COLUMN = ? AND S.$MODULE_ID_COLUMN = ?", - ) - assertThat(args).containsExactly(PROJECT_ID, ATTENDANT_ID.value, MODULE_ID.value).inOrder() - } + val resultQuery = queryBuilder.buildSubjectQuery(subjectQuery) - @Test - fun `buildWhereClause with all parameters`() { - val query = SubjectQuery( - subjectId = SUBJECT_ID_1, - subjectIds = listOf(SUBJECT_ID_2, SUBJECT_ID_3), - afterSubjectId = AFTER_SUBJECT_ID, - projectId = PROJECT_ID, - attendantId = ATTENDANT_ID, - moduleId = MODULE_ID, - fingerprintSampleFormat = FP_FORMAT, - ) - val (clause, args) = queryBuilder.buildWhereClause(query) - val expectedClause = - "WHERE S.$SUBJECT_ID_COLUMN = ? AND S.$SUBJECT_ID_COLUMN IN (?,?) AND S.$SUBJECT_ID_COLUMN > ? AND S.$PROJECT_ID_COLUMN = ? AND S.$ATTENDANT_ID_COLUMN = ? AND S.$MODULE_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ?" - assertThat(clause).isEqualTo(expectedClause) - assertThat(args) - .containsExactly( - SUBJECT_ID_1, - SUBJECT_ID_2, - SUBJECT_ID_3, - AFTER_SUBJECT_ID, - PROJECT_ID, - ATTENDANT_ID.value, - MODULE_ID.value, - FP_FORMAT, - ).inOrder() + assertThat(resultQuery.sql.trim()).isEqualTo(expectedSql.trim()) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(afterSubjectId)) } @Test - fun `buildWhereClause with custom aliases`() { - val query = SubjectQuery(subjectId = SUBJECT_ID_1, fingerprintSampleFormat = FP_FORMAT) - val (clause, args) = queryBuilder.buildWhereClause( - query, - subjectAlias = "customS.", - templateAlias = "customT.", - ) - val expectedClause = - "WHERE customS.$SUBJECT_ID_COLUMN = ? AND customT.$FORMAT_COLUMN = ?" - assertThat(clause).isEqualTo(expectedClause) - assertThat(args).containsExactly(SUBJECT_ID_1, FP_FORMAT).inOrder() - } + fun `buildSubjectQuery with projectId`() { + val projectId = "test-project-id" + val subjectQuery = SubjectQuery(projectId = projectId) + val expectedSql = "SELECT * FROM $SUBJECT_TABLE_NAME S\n" + + "WHERE S.$PROJECT_ID_COLUMN = ?" - @Test - fun `buildWhereClause throws error if both fingerprint and face format set`() { - val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, faceSampleFormat = FACE_FORMAT) - val exception = assertThrows(IllegalArgumentException::class.java) { - queryBuilder.buildWhereClause(query) - } - assertThat(exception.message).isEqualTo("Cannot set both fingerprintSampleFormat and faceSampleFormat") - } - // endregion + val resultQuery = queryBuilder.buildSubjectQuery(subjectQuery) - // region buildSubjectQuery Tests - @Test - fun `buildSubjectQuery with empty query`() { - val query = SubjectQuery() - val result = queryBuilder.buildSubjectQuery(query) - val expectedSql = - """ - SELECT * FROM DbSubject S - - - """.trimIndent() - assertThat(result.sql).isEqualTo(expectedSql) - assertThat(result.argCount).isEqualTo(0) + assertThat(resultQuery.sql.trim()).isEqualTo(expectedSql.trim()) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(projectId)) } @Test - fun `buildSubjectQuery with projectId`() { - val query = SubjectQuery(projectId = PROJECT_ID) - val result = queryBuilder.buildSubjectQuery(query) - val expectedSql = - """ - SELECT * FROM DbSubject S - WHERE S.projectId = ? - - """.trimIndent() - assertThat(result.sql).isEqualTo(expectedSql) - assertThat(result.argCount).isEqualTo(1) + fun `buildSubjectQuery with attendantId`() { + val attendantId = "test-attendant-id".asTokenizableEncrypted() + val subjectQuery = SubjectQuery(attendantId = attendantId) + val expectedSql = "SELECT * FROM $SUBJECT_TABLE_NAME S\n" + + "WHERE S.$ATTENDANT_ID_COLUMN = ?" + + val resultQuery = queryBuilder.buildSubjectQuery(subjectQuery) + + assertThat(resultQuery.sql.trim()).isEqualTo(expectedSql.trim()) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(attendantId.value)) } @Test - fun `buildSubjectQuery with multiple subject fields`() { - val query = SubjectQuery(projectId = PROJECT_ID, attendantId = ATTENDANT_ID) - val result = queryBuilder.buildSubjectQuery(query) - val expectedSql = - """ - SELECT * FROM $SUBJECT_TABLE_NAME S - WHERE S.$PROJECT_ID_COLUMN = ? AND S.$ATTENDANT_ID_COLUMN = ? - - """.trimIndent() - assertThat(result.sql).isEqualTo(expectedSql) - assertThat(result.argCount).isEqualTo(2) + fun `buildSubjectQuery with moduleId`() { + val moduleId = "test-module-id".asTokenizableEncrypted() + val subjectQuery = SubjectQuery(moduleId = moduleId) + val expectedSql = "SELECT * FROM $SUBJECT_TABLE_NAME S\n" + + "WHERE S.$MODULE_ID_COLUMN = ?" + + val resultQuery = queryBuilder.buildSubjectQuery(subjectQuery) + + assertThat(resultQuery.sql.trim()).isEqualTo(expectedSql.trim()) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(moduleId.value)) } @Test fun `buildSubjectQuery with sort true`() { - val query = SubjectQuery(sort = true) - val result = queryBuilder.buildSubjectQuery(query) + val subjectQuery = SubjectQuery(sort = true) val expectedSql = """ SELECT * FROM $SUBJECT_TABLE_NAME S ORDER BY S.$SUBJECT_ID_COLUMN ASC """.trimIndent() - // Note the space before ORDER due to buildWhereClause - assertThat(result.sql).isEqualTo(expectedSql) - assertThat(result.argCount).isEqualTo(0) + + val resultQuery = queryBuilder.buildSubjectQuery(subjectQuery) + assertThat(resultQuery.sql.trim()).isEqualTo(expectedSql.trim()) + assertThat(resultQuery.argCount).isEqualTo(0) } @Test - fun `buildSubjectQuery with sort true and projectId`() { - val query = SubjectQuery(projectId = PROJECT_ID, sort = true) - val result = queryBuilder.buildSubjectQuery(query) - val expectedSql = - """ - SELECT * FROM $SUBJECT_TABLE_NAME S - WHERE S.$PROJECT_ID_COLUMN = ? - ORDER BY S.$SUBJECT_ID_COLUMN ASC - """.trimIndent() - assertThat(result.sql).isEqualTo(expectedSql) - assertThat(result.argCount).isEqualTo(1) + fun `buildSubjectQuery with multiple parameters and sort`() { + val projectId = "proj1" + val attendantId = "att1".asTokenizableEncrypted() + val subjectQuery = SubjectQuery(projectId = projectId, attendantId = attendantId, sort = true) + val expectedSql = "SELECT * FROM $SUBJECT_TABLE_NAME S\n" + + "WHERE S.$PROJECT_ID_COLUMN = ? AND S.$ATTENDANT_ID_COLUMN = ?\n" + + "ORDER BY S.$SUBJECT_ID_COLUMN ASC" + + val resultQuery = queryBuilder.buildSubjectQuery(subjectQuery) + + assertThat(resultQuery.sql.trim()).isEqualTo(expectedSql.trim()) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(projectId, attendantId.value)) } @Test fun `buildSubjectQuery throws error if fingerprintSampleFormat is set`() { - val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT) - val exception = assertThrows(IllegalArgumentException::class.java) { - queryBuilder.buildSubjectQuery(query) + val subjectQuery = SubjectQuery(fingerprintSampleFormat = "ISO_19794_2_2005") + val exception = assertThrows { + queryBuilder.buildSubjectQuery(subjectQuery) } assertThat(exception.message).isEqualTo("Cannot set format for subject query, use buildBiometricTemplatesQuery instead") } @Test fun `buildSubjectQuery throws error if faceSampleFormat is set`() { - val query = SubjectQuery(faceSampleFormat = FACE_FORMAT) - val exception = assertThrows(IllegalArgumentException::class.java) { - queryBuilder.buildSubjectQuery(query) + val subjectQuery = SubjectQuery(faceSampleFormat = "RAW") + val exception = assertThrows { + queryBuilder.buildSubjectQuery(subjectQuery) } assertThat(exception.message).isEqualTo("Cannot set format for subject query, use buildBiometricTemplatesQuery instead") } - // endregion - // region buildCountQuery Tests @Test - fun `buildCountQuery with empty query no format`() { - val query = SubjectQuery() - val result = queryBuilder.buildCountQuery(query) - assertThat(result.sql).isEqualTo("SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S ") - assertThat(result.argCount).isEqualTo(0) + fun `buildCountQuery with empty query`() { + val subjectQuery = SubjectQuery() + val expectedSql = "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S" + + val resultQuery = queryBuilder.buildCountQuery(subjectQuery) + + assertThat(resultQuery.sql.trim()).isEqualTo(expectedSql.trim()) + assertThat(resultQuery.argCount).isEqualTo(0) } @Test - fun `buildCountQuery with projectId no format`() { - val query = SubjectQuery(projectId = PROJECT_ID) - val result = queryBuilder.buildCountQuery(query) - assertThat( - result.sql, - ).isEqualTo("SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S WHERE S.$PROJECT_ID_COLUMN = ?") - assertThat(result.argCount).isEqualTo(1) + fun `buildCountQuery with subjectId`() { + val subjectId = "s1" + val subjectQuery = SubjectQuery(subjectId = subjectId) + val expectedSql = + "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S WHERE S.$SUBJECT_ID_COLUMN = ?" + + val resultQuery = queryBuilder.buildCountQuery(subjectQuery) + + assertThat(resultQuery.sql).isEqualTo(expectedSql) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(subjectId)) + } + + @Test + fun `buildCountQuery with projectId`() { + val projectId = "p1" + val subjectQuery = SubjectQuery(projectId = projectId) + val expectedSql = + "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S WHERE S.$PROJECT_ID_COLUMN = ?" + + val resultQuery = queryBuilder.buildCountQuery(subjectQuery) + + assertThat(resultQuery.sql).isEqualTo(expectedSql) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(projectId)) } @Test - fun `buildCountQuery with fingerprintSampleFormat no other clauses`() { - val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT) - val result = queryBuilder.buildCountQuery(query) - assertThat(result.sql).isEqualTo( + fun `buildCountQuery with fingerprintSampleFormat`() { + val format = "ISO_FP" + val subjectQuery = SubjectQuery(fingerprintSampleFormat = format) + val expectedSql = "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + - " using($SUBJECT_ID_COLUMN) WHERE T.$FORMAT_COLUMN = ?", - ) - assertThat(result.argCount).isEqualTo(1) + " using(subjectId) WHERE T.$FORMAT_COLUMN = ?" + + val resultQuery = queryBuilder.buildCountQuery(subjectQuery) + + assertThat(resultQuery.sql).isEqualTo(expectedSql) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(format)) } @Test - fun `buildCountQuery with faceSampleFormat and projectId`() { - val query = SubjectQuery(faceSampleFormat = FACE_FORMAT, projectId = PROJECT_ID) - val result = queryBuilder.buildCountQuery(query) - val expectedSql = "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + - " using($SUBJECT_ID_COLUMN) WHERE S.$PROJECT_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ?" - assertThat(result.sql).isEqualTo(expectedSql) - assertThat(result.argCount).isEqualTo(2) + fun `buildCountQuery with faceSampleFormat`() { + val format = "RAW_FACE" + val subjectQuery = SubjectQuery(faceSampleFormat = format) + val expectedSql = + "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + + " using(subjectId) WHERE T.$FORMAT_COLUMN = ?" + + val resultQuery = queryBuilder.buildCountQuery(subjectQuery) + + assertThat(resultQuery.sql).isEqualTo(expectedSql) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(format)) } @Test - fun `buildCountQuery with fingerprintSampleFormat`() { - val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT) - val result = queryBuilder.buildCountQuery(query) - val expectedSql = "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + - " using($SUBJECT_ID_COLUMN) WHERE T.$FORMAT_COLUMN = ?" - assertThat(result.sql).isEqualTo(expectedSql) - assertThat(result.argCount).isEqualTo(1) + fun `buildCountQuery with projectId and fingerprintSampleFormat`() { + val projectId = "p1" + val format = "ISO_FP" + val subjectQuery = SubjectQuery(projectId = projectId, fingerprintSampleFormat = format) + val expectedSql = + "SELECT COUNT(DISTINCT S.$SUBJECT_ID_COLUMN) FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T" + + " using(subjectId) WHERE S.$PROJECT_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ?" + + val resultQuery = queryBuilder.buildCountQuery(subjectQuery) + + assertThat(resultQuery.sql).isEqualTo(expectedSql) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(projectId, format)) } @Test - fun `buildCountQuery throws error if both fingerprint and face format set`() { - val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, faceSampleFormat = FACE_FORMAT) - val exception = assertThrows(IllegalArgumentException::class.java) { - queryBuilder.buildCountQuery(query) // This will call buildWhereClause + fun `buildCountQuery throws if both fingerprint and face formats set`() { + val subjectQuery = SubjectQuery(fingerprintSampleFormat = "FP_FORMAT", faceSampleFormat = "FACE_FORMAT") + val exception = assertThrows { + queryBuilder.buildCountQuery(subjectQuery) } assertThat(exception.message).isEqualTo("Cannot set both fingerprintSampleFormat and faceSampleFormat") } - // endregion - // region buildBiometricTemplatesQuery Tests @Test - fun `buildBiometricTemplatesQuery with fingerprintSampleFormat, no lastSeenId`() { - val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT) - val result = queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE, null) + fun `buildBiometricTemplatesQuery throws error if format is not set`() { + val subjectQuery = SubjectQuery() + val pageSize = 10 + val exception = assertThrows { + queryBuilder.buildBiometricTemplatesQuery(subjectQuery, pageSize) + } + assertThat( + exception.message, + ).isEqualTo("Must set format for biometric templates query, use buildSubjectQuery or buildCountQuery instead") + } + @Test + fun `buildBiometricTemplatesQuery with fingerprintSampleFormat`() { + val format = "ISO_FP_TEMPLATE" + val pageSize = 10 + val subjectQuery = SubjectQuery(fingerprintSampleFormat = format) val expectedSql = """ SELECT A.* FROM $TEMPLATE_TABLE_NAME A INNER JOIN ( - SELECT distinct S.$SUBJECT_ID_COLUMN + SELECT distinct S.subjectId FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T - USING($SUBJECT_ID_COLUMN) + USING(subjectId) WHERE T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC - LIMIT $PAGE_SIZE - ) B USING($SUBJECT_ID_COLUMN) where A.$FORMAT_COLUMN ='$FP_FORMAT' + LIMIT $pageSize + ) B USING(subjectId) where A.format ='$format' """.trimIndent() - assertThat(result.sql).isEqualTo(expectedSql) - assertThat(result.argCount).isEqualTo(1) + val resultQuery = queryBuilder.buildBiometricTemplatesQuery(subjectQuery, pageSize) + + assertThat(resultQuery.sql).isEqualTo(expectedSql) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(format)) } @Test - fun `buildBiometricTemplatesQuery with faceSampleFormat and lastSeenId`() { - val query = SubjectQuery(faceSampleFormat = FACE_FORMAT) - val result = queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE, LAST_SEEN_SUBJECT_ID) - + fun `buildBiometricTemplatesQuery with faceSampleFormat and projectId`() { + val format = "RAW_FACE_TEMPLATE" + val projectId = "projX" + val pageSize = 5 + val subjectQuery = SubjectQuery(faceSampleFormat = format, projectId = projectId) val expectedSql = """ SELECT A.* FROM $TEMPLATE_TABLE_NAME A INNER JOIN ( - SELECT distinct S.$SUBJECT_ID_COLUMN + SELECT distinct S.subjectId FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T - USING($SUBJECT_ID_COLUMN) - WHERE S.$SUBJECT_ID_COLUMN > ? AND T.$FORMAT_COLUMN = ? + USING(subjectId) + WHERE S.$PROJECT_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC - LIMIT $PAGE_SIZE - ) B USING($SUBJECT_ID_COLUMN) where A.$FORMAT_COLUMN ='$FACE_FORMAT' + LIMIT $pageSize + ) B USING(subjectId) where A.format ='$format' """.trimIndent() - assertThat(result.sql).isEqualTo(expectedSql) - assertThat(result.argCount).isEqualTo(2) + val resultQuery = queryBuilder.buildBiometricTemplatesQuery(subjectQuery, pageSize) + + assertThat(resultQuery.sql).isEqualTo(expectedSql) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(projectId, format)) } @Test - fun `buildBiometricTemplatesQuery with fingerprintFormat, projectId, lastSeenId`() { - val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, projectId = PROJECT_ID) - val result = queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE, LAST_SEEN_SUBJECT_ID) - + fun `buildBiometricTemplatesQuery uses sort true internally`() { + val format = "ANY_FORMAT" + val pageSize = 15 + val subjectQuery = SubjectQuery(fingerprintSampleFormat = format, sort = false) val expectedSql = """ SELECT A.* FROM $TEMPLATE_TABLE_NAME A INNER JOIN ( - SELECT distinct S.$SUBJECT_ID_COLUMN + SELECT distinct S.subjectId FROM $SUBJECT_TABLE_NAME S INNER JOIN $TEMPLATE_TABLE_NAME T - USING($SUBJECT_ID_COLUMN) - WHERE S.$SUBJECT_ID_COLUMN > ? AND S.$PROJECT_ID_COLUMN = ? AND T.$FORMAT_COLUMN = ? + USING(subjectId) + WHERE T.$FORMAT_COLUMN = ? ORDER BY S.$SUBJECT_ID_COLUMN ASC - LIMIT $PAGE_SIZE - ) B USING($SUBJECT_ID_COLUMN) where A.$FORMAT_COLUMN ='$FP_FORMAT' + LIMIT $pageSize + ) B USING(subjectId) where A.format ='$format' """.trimIndent() - assertThat(result.sql).isEqualTo(expectedSql) - assertThat(result.argCount).isEqualTo(3) + val resultQuery = queryBuilder.buildBiometricTemplatesQuery(subjectQuery, pageSize) + + assertThat(resultQuery.sql).isEqualTo(expectedSql) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(format)) } @Test - fun `buildBiometricTemplatesQuery throws error if no format is set`() { - val query = SubjectQuery() // No format - val exception = assertThrows(IllegalArgumentException::class.java) { - queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE) + fun `buildBiometricTemplatesQuery throws if both fingerprint and face formats set`() { + val subjectQuery = SubjectQuery(fingerprintSampleFormat = "FP_FORMAT", faceSampleFormat = "FACE_FORMAT") + val pageSize = 10 + val exception = assertThrows { + queryBuilder.buildBiometricTemplatesQuery(subjectQuery, pageSize) } - assertThat( - exception.message, - ).isEqualTo("Must set format for biometric templates query, use buildSubjectQuery or buildCountQuery instead") + assertThat(exception.message).isEqualTo("Cannot set both fingerprintSampleFormat and faceSampleFormat") } @Test - fun `buildBiometricTemplatesQuery throws error if both fingerprint and face format set`() { - val query = SubjectQuery(fingerprintSampleFormat = FP_FORMAT, faceSampleFormat = FACE_FORMAT) - val exception = assertThrows(IllegalArgumentException::class.java) { - queryBuilder.buildBiometricTemplatesQuery(query, PAGE_SIZE) + fun `buildDeleteQuery throws error if fingerprintSampleFormat is set`() { + val subjectQuery = SubjectQuery(fingerprintSampleFormat = "ISO_19794_2_2005") + val exception = assertThrows { + queryBuilder.buildDeleteQuery(subjectQuery) } - assertThat(exception.message).isEqualTo("Cannot set both fingerprintSampleFormat and faceSampleFormat") + assertThat(exception.message).isEqualTo("faceSampleFormat and fingerprintSampleFormat are not supported for deletion") + } + + @Test + fun `buildDeleteQuery throws error if faceSampleFormat is set`() { + val subjectQuery = SubjectQuery(faceSampleFormat = "RAW") + val exception = assertThrows { + queryBuilder.buildDeleteQuery(subjectQuery) + } + assertThat(exception.message).isEqualTo("faceSampleFormat and fingerprintSampleFormat are not supported for deletion") + } + + @Test + fun `buildDeleteQuery with empty query`() { + val subjectQuery = SubjectQuery() + val expectedSql = "DELETE FROM DbSubject" + + val resultQuery = queryBuilder.buildDeleteQuery(subjectQuery) + + assertThat(resultQuery.sql.trim()).isEqualTo(expectedSql.trim()) + assertThat(resultQuery.argCount).isEqualTo(0) + } + + @Test + fun `buildDeleteQuery with subjectId`() { + val subjectId = "id-to-delete" + val subjectQuery = SubjectQuery(subjectId = subjectId) + val expectedSql = "DELETE FROM DbSubject WHERE $SUBJECT_ID_COLUMN = ?" + + val resultQuery = queryBuilder.buildDeleteQuery(subjectQuery) + + assertThat(resultQuery.sql).isEqualTo(expectedSql) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(subjectId)) + } + + @Test + fun `buildDeleteQuery with projectId and moduleId`() { + val projectId = "proj-del" + val moduleId = "mod-del".asTokenizableEncrypted() + val subjectQuery = SubjectQuery(projectId = projectId, moduleId = moduleId) + val expectedSql = "DELETE FROM DbSubject WHERE $PROJECT_ID_COLUMN = ? AND $MODULE_ID_COLUMN = ?" + val resultQuery = queryBuilder.buildDeleteQuery(subjectQuery) + assertThat(resultQuery.sql).isEqualTo(expectedSql) + assertThat(getArgs(resultQuery)).isEqualTo(arrayOf(projectId, moduleId.value)) + } + + private fun getArgs(query: SimpleSQLiteQuery): Array { + val argsMap = mutableMapOf() + val program = object : SupportSQLiteProgram { + override fun bindNull(index: Int) { + argsMap[index] = null + } + + override fun bindLong( + index: Int, + value: Long, + ) { + argsMap[index] = value + } + + override fun bindDouble( + index: Int, + value: Double, + ) { + argsMap[index] = value + } + + override fun bindString( + index: Int, + value: String, + ) { + argsMap[index] = value + } + + override fun bindBlob( + index: Int, + value: ByteArray, + ) { + argsMap[index] = value + } + + override fun clearBindings() { + argsMap.clear() + } + + override fun close() {} + } + + query.bindTo(program) + + val maxIndex = argsMap.keys.maxOrNull() ?: 0 + return Array(maxIndex) { i -> argsMap[i + 1] } } - // endregion } From 786340e9931e3a9f00cb102a4ce40d5f2d118b8d Mon Sep 17 00:00:00 2001 From: Melad Raouf Date: Wed, 4 Jun 2025 15:20:19 +0300 Subject: [PATCH 07/11] Prevent Reader Blocking by Increasing Channel Capacity for Matching Pipeline And ensure all DB reads happen in the IO dispatcher --- .../matcher/usecases/FaceMatcherUseCase.kt | 5 +- .../usecases/FingerprintMatcherUseCase.kt | 5 +- .../usecases/FaceMatcherUseCaseTest.kt | 1 - .../usecases/FingerprintMatcherUseCaseTest.kt | 1 - .../biosdkimpl/matching/SimAfisMatcher.kt | 2 - .../records/repository/IdentityDataSource.kt | 28 ----- .../commcare/CommCareIdentityDataSource.kt | 79 +++++++++----- .../RealmEnrolmentRecordLocalDataSource.kt | 101 +++++++++--------- .../RoomEnrolmentRecordLocalDataSource.kt | 8 +- 9 files changed, 107 insertions(+), 123 deletions(-) diff --git a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt index 293cf9ed11..ba2056c4ac 100644 --- a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt +++ b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt @@ -2,7 +2,6 @@ package com.simprints.matcher.usecases import com.simprints.core.AvailableProcessors import com.simprints.core.DispatcherBG -import com.simprints.core.DispatcherIO import com.simprints.face.infra.basebiosdk.matching.FaceIdentity import com.simprints.face.infra.basebiosdk.matching.FaceMatcher import com.simprints.face.infra.basebiosdk.matching.FaceSample @@ -19,7 +18,6 @@ import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.channelFlow -import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.launch import java.util.concurrent.atomic.AtomicInteger import javax.inject.Inject @@ -32,7 +30,6 @@ internal class FaceMatcherUseCase @Inject constructor( private val createRanges: CreateRangesUseCase, @AvailableProcessors private val availableProcessors: Int, @DispatcherBG private val dispatcherBG: CoroutineDispatcher, - @DispatcherIO private val dispatcherIO: CoroutineDispatcher, ) : MatcherUseCase { override val crashReportTag = LoggingConstants.CrashReportTag.FACE_MATCHING @@ -99,7 +96,7 @@ internal class FaceMatcherUseCase @Inject constructor( // Wait for all to complete consumerJobs.forEach { it.join() } send(MatcherState.Success(resultSet.toList(), loadedCandidates.get(), bioSdk.matcherName)) - }.flowOn(dispatcherIO) + } suspend fun consumeAndMatch( candidatesChannel: ReceiveChannel>, diff --git a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt index 1d6f0d6557..313c346786 100644 --- a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt +++ b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt @@ -2,7 +2,6 @@ package com.simprints.matcher.usecases import com.simprints.core.AvailableProcessors import com.simprints.core.DispatcherBG -import com.simprints.core.DispatcherIO import com.simprints.core.domain.common.FlowType import com.simprints.core.domain.fingerprint.IFingerIdentifier import com.simprints.fingerprint.infra.basebiosdk.matching.domain.FingerIdentifier @@ -24,7 +23,6 @@ import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.channelFlow -import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.launch import java.util.concurrent.atomic.AtomicInteger import javax.inject.Inject @@ -38,7 +36,6 @@ internal class FingerprintMatcherUseCase @Inject constructor( private val createRanges: CreateRangesUseCase, @AvailableProcessors private val availableProcessors: Int, @DispatcherBG private val dispatcherBG: CoroutineDispatcher, - @DispatcherIO private val dispatcherIO: CoroutineDispatcher, ) : MatcherUseCase { override val crashReportTag = LoggingConstants.CrashReportTag.FINGER_MATCHING @@ -104,7 +101,7 @@ internal class FingerprintMatcherUseCase @Inject constructor( Simber.i("Matched $loadedCandidates candidates", tag = crashReportTag) send(MatcherState.Success(resultSet.toList(), loadedCandidates.get(), bioSdkWrapper.matcherName)) - }.flowOn(dispatcherIO) + } private suspend fun consumeAndMatch( channel: ReceiveChannel>, diff --git a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt index 9aa941e8c4..6cfe1d355b 100644 --- a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt +++ b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt @@ -56,7 +56,6 @@ internal class FaceMatcherUseCaseTest { createRangesUseCase, 4, testCoroutineRule.testCoroutineDispatcher, - testCoroutineRule.testCoroutineDispatcher, ) } diff --git a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FingerprintMatcherUseCaseTest.kt b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FingerprintMatcherUseCaseTest.kt index 465241b33f..eb99bc32d7 100644 --- a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FingerprintMatcherUseCaseTest.kt +++ b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FingerprintMatcherUseCaseTest.kt @@ -68,7 +68,6 @@ internal class FingerprintMatcherUseCaseTest { createRangesUseCase, 4, testCoroutineRule.testCoroutineDispatcher, - testCoroutineRule.testCoroutineDispatcher, ) } diff --git a/fingerprint/infra/simprints-bio-sdk/src/main/java/com/simprints/fingerprint/infra/biosdkimpl/matching/SimAfisMatcher.kt b/fingerprint/infra/simprints-bio-sdk/src/main/java/com/simprints/fingerprint/infra/biosdkimpl/matching/SimAfisMatcher.kt index bd067642ed..b4ceb2530f 100644 --- a/fingerprint/infra/simprints-bio-sdk/src/main/java/com/simprints/fingerprint/infra/biosdkimpl/matching/SimAfisMatcher.kt +++ b/fingerprint/infra/simprints-bio-sdk/src/main/java/com/simprints/fingerprint/infra/biosdkimpl/matching/SimAfisMatcher.kt @@ -56,8 +56,6 @@ internal class SimAfisMatcher @Inject constructor( ): List { val simAfisCandidates = candidates.map { it.toSimAfisPerson() } - println("Matching ${simAfisCandidates.size} candidates using all ${jniLibAfis.getNbCores()} cores") - val results = jniLibAfis.identify( probe.toSimAfisPerson(), simAfisCandidates, diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/IdentityDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/IdentityDataSource.kt index 7aceac3deb..59c007ce8d 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/IdentityDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/IdentityDataSource.kt @@ -5,14 +5,8 @@ import com.simprints.infra.enrolment.records.repository.domain.models.BiometricD import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity import com.simprints.infra.enrolment.records.repository.domain.models.SubjectQuery -import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.ReceiveChannel -import kotlinx.coroutines.joinAll -import kotlinx.coroutines.launch -import kotlinx.coroutines.sync.Semaphore -import kotlinx.coroutines.sync.withPermit interface IdentityDataSource { suspend fun count( @@ -42,26 +36,4 @@ interface IdentityDataSource { * Loads identities concurrently using the provided dispatcher and parallelism level. * */ - fun loadIdentitiesConcurrently( - ranges: List, - dispatcher: CoroutineDispatcher, - parallelism: Int, - scope: CoroutineScope, - load: suspend (IntRange) -> List, - ): ReceiveChannel> { - val channel = Channel>(parallelism) - val semaphore = Semaphore(parallelism) - scope.launch { - ranges - .map { range -> - launch { - semaphore.withPermit { - channel.send(load(range)) - } - } - }.joinAll() - channel.close() - } - return channel - } } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt index 7a3f58a4a3..263b544d79 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt @@ -6,8 +6,8 @@ import android.net.Uri import androidx.core.net.toUri import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.module.SimpleModule -import com.simprints.core.DispatcherBG import com.simprints.core.AvailableProcessors +import com.simprints.core.DispatcherBG import com.simprints.core.domain.face.FaceSample import com.simprints.core.domain.fingerprint.FingerprintSample import com.simprints.core.domain.tokenization.TokenizableString @@ -33,7 +33,13 @@ import com.simprints.libsimprints.Constants.SIMPRINTS_COSYNC_SUBJECT_ACTIONS import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.async +import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.ReceiveChannel +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.withContext import org.json.JSONException import javax.inject.Inject @@ -56,24 +62,25 @@ internal class CommCareIdentityDataSource @Inject constructor( dataSource: BiometricDataSource, project: Project, onCandidateLoaded: () -> Unit, - ): List = loadEnrolmentRecordCreationEvents(range, dataSource.callerPackageName(), query, project, onCandidateLoaded) - .filter { erce -> - erce.payload.biometricReferences.any { it is FingerprintReference && it.format == query.fingerprintSampleFormat } - }.map { - FingerprintIdentity( - it.payload.subjectId, - it.payload.biometricReferences.filterIsInstance().flatMap { fingerprintReference -> - fingerprintReference.templates.map { fingerprintTemplate -> - FingerprintSample( - fingerIdentifier = fingerprintTemplate.finger, - template = encoder.base64ToBytes(fingerprintTemplate.template), - format = fingerprintReference.format, - referenceId = fingerprintReference.id, - ) - } - }, - ) - } + ): List = + loadEnrolmentRecordCreationEvents(range, dataSource.callerPackageName(), query, project, onCandidateLoaded) + .filter { erce -> + erce.payload.biometricReferences.any { it is FingerprintReference && it.format == query.fingerprintSampleFormat } + }.map { + FingerprintIdentity( + it.payload.subjectId, + it.payload.biometricReferences.filterIsInstance().flatMap { fingerprintReference -> + fingerprintReference.templates.map { fingerprintTemplate -> + FingerprintSample( + fingerIdentifier = fingerprintTemplate.finger, + template = encoder.base64ToBytes(fingerprintTemplate.template), + format = fingerprintReference.format, + referenceId = fingerprintReference.id, + ) + } + }, + ) + } private fun loadEnrolmentRecordCreationEvents( range: IntRange, @@ -100,9 +107,10 @@ internal class CommCareIdentityDataSource @Inject constructor( if (caseMetadataCursor.moveToPosition(range.first)) { do { caseMetadataCursor.getString(caseMetadataCursor.getColumnIndexOrThrow(COLUMN_CASE_ID))?.let { caseId -> - enrolmentRecordCreationEvents.addAll( - loadEnrolmentRecordCreationEvents(caseId, callerPackageName, query, project), - ).also { onCandidateLoaded() } + enrolmentRecordCreationEvents + .addAll( + loadEnrolmentRecordCreationEvents(caseId, callerPackageName, query, project), + ).also { onCandidateLoaded() } } } while (caseMetadataCursor.moveToNext() && caseMetadataCursor.position <= range.last) } @@ -159,7 +167,6 @@ internal class CommCareIdentityDataSource @Inject constructor( .query(caseDataUri, null, null, null, null) ?.use { caseDataCursor -> val subjectActions = getSubjectActionsValue(caseDataCursor) - Simber.d(subjectActions) val coSyncEnrolmentRecordEvents = parseRecordEvents(subjectActions) coSyncEnrolmentRecordEvents?.events?.filterIsInstance()?.filter { event -> @@ -268,8 +275,6 @@ internal class CommCareIdentityDataSource @Inject constructor( onCandidateLoaded: () -> Unit, ): ReceiveChannel> = loadIdentitiesConcurrently( ranges = ranges, - dispatcher = dispatcher, - parallelism = availableProcessors, scope = scope, ) { range -> loadFaceIdentities( @@ -290,8 +295,6 @@ internal class CommCareIdentityDataSource @Inject constructor( onCandidateLoaded: () -> Unit, ): ReceiveChannel> = loadIdentitiesConcurrently( ranges = ranges, - dispatcher = dispatcher, - parallelism = availableProcessors, scope = scope, ) { range -> loadFingerprintIdentities( @@ -303,11 +306,31 @@ internal class CommCareIdentityDataSource @Inject constructor( ) } + fun loadIdentitiesConcurrently( + ranges: List, + scope: CoroutineScope, + load: suspend (IntRange) -> List, + ): ReceiveChannel> { + val channel = Channel>(availableProcessors) + val semaphore = Semaphore(availableProcessors) + scope.launch(dispatcher) { + ranges + .map { range -> + async(dispatcher) { + semaphore.withPermit { + channel.send(load(range)) + } + } + }.joinAll() + channel.close() + } + return channel + } + companion object { const val COLUMN_CASE_ID = "case_id" const val COLUMN_DATUM_ID = "datum_id" const val COLUMN_VALUE = "value" - const val ARG_CASE_ID = "caseId" } } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt index 7306e7a979..ad0e515f71 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt @@ -5,8 +5,6 @@ import com.simprints.infra.config.store.models.Project import com.simprints.infra.config.store.models.TokenKeyType import com.simprints.infra.config.store.tokenization.TokenizationProcessor import com.simprints.infra.enrolment.records.realm.store.RealmWrapper -import com.simprints.infra.enrolment.records.realm.store.models.DbFaceSample -import com.simprints.infra.enrolment.records.realm.store.models.DbFingerprintSample import com.simprints.infra.enrolment.records.realm.store.models.DbSubject import com.simprints.infra.enrolment.records.repository.domain.models.BiometricDataSource import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity @@ -27,7 +25,9 @@ import io.realm.kotlin.query.find import io.realm.kotlin.types.RealmUUID import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.ReceiveChannel +import kotlinx.coroutines.launch import javax.inject.Inject import javax.inject.Singleton @@ -35,7 +35,7 @@ import javax.inject.Singleton internal class RealmEnrolmentRecordLocalDataSource @Inject constructor( private val realmWrapper: RealmWrapper, private val tokenizationProcessor: TokenizationProcessor, - @DispatcherIO private val dispatcher: CoroutineDispatcher, + @DispatcherIO private val dispatcherIO: CoroutineDispatcher, ) : EnrolmentRecordLocalDataSource { companion object { const val PROJECT_ID_FIELD = "projectId" @@ -47,7 +47,9 @@ internal class RealmEnrolmentRecordLocalDataSource @Inject constructor( const val FINGERPRINT_SAMPLES_FIELD = "fingerprintSamples" const val FACE_SAMPLES_FIELD = "faceSamples" const val FORMAT_FIELD = "format" - const val PARALLELISM = 1 + + // Although batches are processed sequentially, we use a small channel capacity to prevent blocking and reduce the risk of out-of-memory errors. + const val CHANNEL_CAPACITY = 4 } override suspend fun load(query: SubjectQuery): List = realmWrapper.readRealm { @@ -65,17 +67,26 @@ internal class RealmEnrolmentRecordLocalDataSource @Inject constructor( project: Project, scope: CoroutineScope, onCandidateLoaded: () -> Unit, - ): ReceiveChannel> = loadIdentitiesConcurrently( - ranges = ranges, - dispatcher = dispatcher, - parallelism = PARALLELISM, - scope = scope, - ) { range -> - loadFaceIdentities( - query = query, - range = range, - onCandidateLoaded = onCandidateLoaded, - ) + ): ReceiveChannel> { + val channel = Channel>(CHANNEL_CAPACITY) + scope.launch(dispatcherIO) { + ranges.forEach { range -> + val identities = loadIdentitiesRange( + query = query, + range = range, + mapper = { dbSubject -> + FaceIdentity( + subjectId = dbSubject.subjectId.toString(), + faces = dbSubject.faceSamples.map { it.toDomain() }, + ) + }, + onCandidateLoaded = onCandidateLoaded, + ) + channel.send(identities) + } + channel.close() + } + return channel } override fun loadFingerprintIdentities( @@ -85,54 +96,42 @@ internal class RealmEnrolmentRecordLocalDataSource @Inject constructor( project: Project, scope: CoroutineScope, onCandidateLoaded: () -> Unit, - ): ReceiveChannel> = loadIdentitiesConcurrently( - ranges = ranges, - dispatcher = dispatcher, - parallelism = PARALLELISM, - scope = scope, - ) { range -> - loadFingerprintIdentities( - query = query, - range = range, - onCandidateLoaded = onCandidateLoaded, - ) - } - - private suspend fun loadFingerprintIdentities( - query: SubjectQuery, - range: IntRange, - onCandidateLoaded: () -> Unit, - ): List = realmWrapper.readRealm { realm -> - realm - .query(DbSubject::class) - .buildRealmQueryForSubject(query) - // subList's second parameter is exclusive, so we need to add 1 to the last index - .find { it.subList(range.first, range.last+1) } - .map { subject -> - onCandidateLoaded() - FingerprintIdentity( - subject.subjectId.toString(), - subject.fingerprintSamples.map(DbFingerprintSample::toDomain), + ): ReceiveChannel> { + val channel = Channel>(CHANNEL_CAPACITY) + scope.launch(dispatcherIO) { + ranges.forEach { range -> + val identities = loadIdentitiesRange( + query = query, + range = range, + mapper = { dbSubject -> + FingerprintIdentity( + subjectId = dbSubject.subjectId.toString(), + fingerprints = dbSubject.fingerprintSamples.map { it.toDomain() }, + ) + }, + onCandidateLoaded = onCandidateLoaded, ) + channel.send(identities) } + channel.close() + } + return channel } - private suspend fun loadFaceIdentities( + private suspend fun loadIdentitiesRange( query: SubjectQuery, range: IntRange, + mapper: (DbSubject) -> T, onCandidateLoaded: () -> Unit, - ): List = realmWrapper.readRealm { realm -> + ): List = realmWrapper.readRealm { realm -> realm .query(DbSubject::class) .buildRealmQueryForSubject(query) // subList's second parameter is exclusive, so we need to add 1 to the last index - .find { it.subList(range.first, range.last+1) } - .map { subject -> + .find { it.subList(range.first, range.last + 1) } + .map { dbSubject -> onCandidateLoaded() - FaceIdentity( - subject.subjectId.toString(), - subject.faceSamples.map(DbFaceSample::toDomain), - ) + mapper(dbSubject) } } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt index acc9df3295..76adb64cc7 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt @@ -43,9 +43,8 @@ internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( @DispatcherIO private val dispatcherIO: CoroutineDispatcher, ) : EnrolmentRecordLocalDataSource { companion object { - // only one concurrent operation is allowed as we use the last seen subject ID to load biometric identities - // and we need to ensure that the next operation starts after the previous one finishes - private const val CHANNEL_CAPACITY = 3 + // Although batches are processed sequentially, we use a small channel capacity to prevent blocking and reduce the risk of out-of-memory errors. + private const val CHANNEL_CAPACITY = 4 } private val database: SubjectsDatabase by lazy { subjectsDatabaseFactory.get() } @@ -272,7 +271,8 @@ internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( action.faceSamplesToAdd.isNotEmpty() || action.fingerprintSamplesToAdd.isNotEmpty(), ) { - val errorMsg = "Cannot delete all samples for subject ${action.subjectId} without adding new ones" + val errorMsg = + "Cannot delete all samples for subject ${action.subjectId} without adding new ones" Simber.i("[updateSubject] $errorMsg", tag = ROOM_RECORDS_DB) errorMsg } From 59d01e622aa60908d20afe55edd107f1bad8ffbc Mon Sep 17 00:00:00 2001 From: Melad Raouf Date: Wed, 4 Jun 2025 20:13:04 +0300 Subject: [PATCH 08/11] Refactor FaceMatcherUseCase and FingerprintMatcherUseCase to remove use single consumer --- .../matcher/usecases/FaceMatcherUseCase.kt | 24 ++++++------------- .../usecases/FingerprintMatcherUseCase.kt | 22 ++++++----------- .../usecases/FaceMatcherUseCaseTest.kt | 1 - .../usecases/FingerprintMatcherUseCaseTest.kt | 1 - 4 files changed, 14 insertions(+), 34 deletions(-) diff --git a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt index ba2056c4ac..6fabe19a21 100644 --- a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt +++ b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt @@ -1,6 +1,5 @@ package com.simprints.matcher.usecases -import com.simprints.core.AvailableProcessors import com.simprints.core.DispatcherBG import com.simprints.face.infra.basebiosdk.matching.FaceIdentity import com.simprints.face.infra.basebiosdk.matching.FaceMatcher @@ -18,17 +17,16 @@ import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.channelFlow -import kotlinx.coroutines.launch +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.runBlocking import java.util.concurrent.atomic.AtomicInteger import javax.inject.Inject -import kotlin.math.min import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity as DomainFaceIdentity internal class FaceMatcherUseCase @Inject constructor( private val enrolmentRecordRepository: EnrolmentRecordRepository, private val resolveFaceBioSdk: ResolveFaceBioSdkUseCase, private val createRanges: CreateRangesUseCase, - @AvailableProcessors private val availableProcessors: Int, @DispatcherBG private val dispatcherBG: CoroutineDispatcher, ) : MatcherUseCase { override val crashReportTag = LoggingConstants.CrashReportTag.FACE_MATCHING @@ -70,9 +68,6 @@ internal class FaceMatcherUseCase @Inject constructor( // as it's count function does not take into account filtering criteria val loadedCandidates = AtomicInteger(0) val ranges = createRanges(expectedCandidates) - // if number of ranges less than the number of cores then use the number of ranges - val numConsumers = min(availableProcessors, ranges.size) - val resultSet = MatchResultSet() val candidatesChannel = enrolmentRecordRepository .loadFaceIdentities( @@ -83,20 +78,15 @@ internal class FaceMatcherUseCase @Inject constructor( scope = this, onCandidateLoaded = { loadedCandidates.incrementAndGet() - this.trySend(MatcherState.CandidateLoaded) + runBlocking { + this@channelFlow.send(MatcherState.CandidateLoaded) + } }, ) + consumeAndMatch(candidatesChannel, samples, resultSet, bioSdk) - // Start Consumers in BG thread - val consumerJobs = List(numConsumers) { - launch(dispatcherBG) { - consumeAndMatch(candidatesChannel, samples, resultSet, bioSdk) - } - } - // Wait for all to complete - consumerJobs.forEach { it.join() } send(MatcherState.Success(resultSet.toList(), loadedCandidates.get(), bioSdk.matcherName)) - } + }.flowOn(dispatcherBG) suspend fun consumeAndMatch( candidatesChannel: ReceiveChannel>, diff --git a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt index 313c346786..cb48059d7b 100644 --- a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt +++ b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt @@ -1,6 +1,5 @@ package com.simprints.matcher.usecases -import com.simprints.core.AvailableProcessors import com.simprints.core.DispatcherBG import com.simprints.core.domain.common.FlowType import com.simprints.core.domain.fingerprint.IFingerIdentifier @@ -23,10 +22,10 @@ import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.channelFlow -import kotlinx.coroutines.launch +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.runBlocking import java.util.concurrent.atomic.AtomicInteger import javax.inject.Inject -import kotlin.math.min import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity as DomainFingerprintIdentity internal class FingerprintMatcherUseCase @Inject constructor( @@ -34,7 +33,6 @@ internal class FingerprintMatcherUseCase @Inject constructor( private val resolveBioSdkWrapper: ResolveBioSdkWrapperUseCase, private val configManager: ConfigManager, private val createRanges: CreateRangesUseCase, - @AvailableProcessors private val availableProcessors: Int, @DispatcherBG private val dispatcherBG: CoroutineDispatcher, ) : MatcherUseCase { override val crashReportTag = LoggingConstants.CrashReportTag.FINGER_MATCHING @@ -76,7 +74,6 @@ internal class FingerprintMatcherUseCase @Inject constructor( val loadedCandidates = AtomicInteger(0) val ranges = createRanges(expectedCandidates) // if number of ranges less than the number of cores then use the number of ranges - val numConsumers = min(availableProcessors, ranges.size) val channel = enrolmentRecordRepository.loadFingerprintIdentities( query = queryWithSupportedFormat, ranges = ranges, @@ -85,23 +82,18 @@ internal class FingerprintMatcherUseCase @Inject constructor( project = project, ) { loadedCandidates.incrementAndGet() - trySend(MatcherState.CandidateLoaded) + runBlocking { + this@channelFlow.send(MatcherState.CandidateLoaded) + } } val resultSet = MatchResultSet() - // Start Consumers in BG thread - val consumerJobs = List(numConsumers) { - launch(dispatcherBG) { - consumeAndMatch(channel, samples, resultSet, bioSdkWrapper, matchParams) - } - } - // Wait for all to complete - consumerJobs.forEach { it.join() } + consumeAndMatch(channel, samples, resultSet, bioSdkWrapper, matchParams) Simber.i("Matched $loadedCandidates candidates", tag = crashReportTag) send(MatcherState.Success(resultSet.toList(), loadedCandidates.get(), bioSdkWrapper.matcherName)) - } + }.flowOn(dispatcherBG) private suspend fun consumeAndMatch( channel: ReceiveChannel>, diff --git a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt index 6cfe1d355b..e48281fede 100644 --- a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt +++ b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt @@ -54,7 +54,6 @@ internal class FaceMatcherUseCaseTest { enrolmentRecordRepository, resolveFaceBioSdk, createRangesUseCase, - 4, testCoroutineRule.testCoroutineDispatcher, ) } diff --git a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FingerprintMatcherUseCaseTest.kt b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FingerprintMatcherUseCaseTest.kt index eb99bc32d7..8fb25993a1 100644 --- a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FingerprintMatcherUseCaseTest.kt +++ b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FingerprintMatcherUseCaseTest.kt @@ -66,7 +66,6 @@ internal class FingerprintMatcherUseCaseTest { resolveBioSdkWrapperUseCase, configManager, createRangesUseCase, - 4, testCoroutineRule.testCoroutineDispatcher, ) } From 2e17db4992e99d06e311c7922d0c9feba0c36b72 Mon Sep 17 00:00:00 2001 From: Melad Raouf Date: Wed, 4 Jun 2025 20:13:21 +0300 Subject: [PATCH 09/11] Fix logging level for subject update failure in RoomEnrolmentRecordLocalDataSource --- .../repository/local/RoomEnrolmentRecordLocalDataSource.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt index 76adb64cc7..15e01e26ba 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt @@ -286,8 +286,11 @@ internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( subjectDao.insertBiometricSamples(templatesToAdd) } } else { - Simber.i( + Simber.e( "[updateSubject] Subject ${action.subjectId} not found for update", + IllegalStateException( + "Subject ${action.subjectId} not found for update", + ), tag = ROOM_RECORDS_DB, ) } From 74d5a3cb0cce39101912622f975358d346a1ba9c Mon Sep 17 00:00:00 2001 From: Melad Raouf Date: Thu, 5 Jun 2025 21:06:19 +0300 Subject: [PATCH 10/11] Update onCandidateLoaded callbacks to be suspend functions --- .../matcher/usecases/FaceMatcherUseCase.kt | 6 +----- .../usecases/FingerprintMatcherUseCase.kt | 5 +---- .../biosdkimpl/matching/SimAfisMatcher.kt | 2 ++ .../records/realm/store/RealmWrapper.kt | 2 +- .../records/realm/store/RealmWrapperImpl.kt | 2 +- .../EnrolmentRecordRepositoryImpl.kt | 4 ++-- .../records/repository/IdentityDataSource.kt | 4 ++-- .../commcare/CommCareIdentityDataSource.kt | 19 ++++++++++--------- .../RealmEnrolmentRecordLocalDataSource.kt | 6 +++--- .../RoomEnrolmentRecordLocalDataSource.kt | 8 ++++---- ...RealmEnrolmentRecordLocalDataSourceTest.kt | 7 ++++--- 11 files changed, 31 insertions(+), 34 deletions(-) diff --git a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt index 6fabe19a21..4c0bd34f02 100644 --- a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt +++ b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt @@ -18,7 +18,6 @@ import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.flowOn -import kotlinx.coroutines.runBlocking import java.util.concurrent.atomic.AtomicInteger import javax.inject.Inject import com.simprints.infra.enrolment.records.repository.domain.models.FaceIdentity as DomainFaceIdentity @@ -78,13 +77,10 @@ internal class FaceMatcherUseCase @Inject constructor( scope = this, onCandidateLoaded = { loadedCandidates.incrementAndGet() - runBlocking { - this@channelFlow.send(MatcherState.CandidateLoaded) - } + this@channelFlow.send(MatcherState.CandidateLoaded) }, ) consumeAndMatch(candidatesChannel, samples, resultSet, bioSdk) - send(MatcherState.Success(resultSet.toList(), loadedCandidates.get(), bioSdk.matcherName)) }.flowOn(dispatcherBG) diff --git a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt index cb48059d7b..1c1a91e2b0 100644 --- a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt +++ b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FingerprintMatcherUseCase.kt @@ -23,7 +23,6 @@ import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.flowOn -import kotlinx.coroutines.runBlocking import java.util.concurrent.atomic.AtomicInteger import javax.inject.Inject import com.simprints.infra.enrolment.records.repository.domain.models.FingerprintIdentity as DomainFingerprintIdentity @@ -82,9 +81,7 @@ internal class FingerprintMatcherUseCase @Inject constructor( project = project, ) { loadedCandidates.incrementAndGet() - runBlocking { - this@channelFlow.send(MatcherState.CandidateLoaded) - } + this@channelFlow.send(MatcherState.CandidateLoaded) } val resultSet = MatchResultSet() diff --git a/fingerprint/infra/simprints-bio-sdk/src/main/java/com/simprints/fingerprint/infra/biosdkimpl/matching/SimAfisMatcher.kt b/fingerprint/infra/simprints-bio-sdk/src/main/java/com/simprints/fingerprint/infra/biosdkimpl/matching/SimAfisMatcher.kt index b4ceb2530f..bd067642ed 100644 --- a/fingerprint/infra/simprints-bio-sdk/src/main/java/com/simprints/fingerprint/infra/biosdkimpl/matching/SimAfisMatcher.kt +++ b/fingerprint/infra/simprints-bio-sdk/src/main/java/com/simprints/fingerprint/infra/biosdkimpl/matching/SimAfisMatcher.kt @@ -56,6 +56,8 @@ internal class SimAfisMatcher @Inject constructor( ): List { val simAfisCandidates = candidates.map { it.toSimAfisPerson() } + println("Matching ${simAfisCandidates.size} candidates using all ${jniLibAfis.getNbCores()} cores") + val results = jniLibAfis.identify( probe.toSimAfisPerson(), simAfisCandidates, diff --git a/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/RealmWrapper.kt b/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/RealmWrapper.kt index c3d71bbb3b..4790990ddc 100644 --- a/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/RealmWrapper.kt +++ b/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/RealmWrapper.kt @@ -7,7 +7,7 @@ interface RealmWrapper { /** * Returns read-only Realm instance for data fetching. */ - suspend fun readRealm(block: (Realm) -> R): R + suspend fun readRealm(block: suspend (Realm) -> R): R /** * Executes provided block with a writable Realm instance ensuring diff --git a/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/RealmWrapperImpl.kt b/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/RealmWrapperImpl.kt index 0514544a8e..35307646e7 100644 --- a/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/RealmWrapperImpl.kt +++ b/infra/enrolment-records/realm-store/src/main/java/com/simprints/infra/enrolment/records/realm/store/RealmWrapperImpl.kt @@ -86,7 +86,7 @@ class RealmWrapperImpl @Inject constructor( /** * Executes provided block ensuring a valid Realm instance is used and closed. */ - override suspend fun readRealm(block: (Realm) -> R): R = withContext(dispatcher) { block(getRealm()) } + override suspend fun readRealm(block: suspend (Realm) -> R): R = withContext(dispatcher) { block(getRealm()) } /** * Executes provided block in a transaction ensuring a valid Realm instance is used and closed. diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt index 6ad240acbc..38e286c804 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/EnrolmentRecordRepositoryImpl.kt @@ -109,7 +109,7 @@ internal class EnrolmentRecordRepositoryImpl @Inject constructor( dataSource: BiometricDataSource, project: Project, scope: CoroutineScope, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ) = fromIdentityDataSource(dataSource).loadFingerprintIdentities( query = query, ranges = ranges, @@ -125,7 +125,7 @@ internal class EnrolmentRecordRepositoryImpl @Inject constructor( dataSource: BiometricDataSource, project: Project, scope: CoroutineScope, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ) = fromIdentityDataSource(dataSource).loadFaceIdentities( query = query, ranges = ranges, diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/IdentityDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/IdentityDataSource.kt index 59c007ce8d..6918a7e759 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/IdentityDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/IdentityDataSource.kt @@ -20,7 +20,7 @@ interface IdentityDataSource { dataSource: BiometricDataSource, project: Project, scope: CoroutineScope, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): ReceiveChannel> fun loadFaceIdentities( @@ -29,7 +29,7 @@ interface IdentityDataSource { dataSource: BiometricDataSource, project: Project, scope: CoroutineScope, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): ReceiveChannel> /** diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt index 263b544d79..1b9692a784 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/commcare/CommCareIdentityDataSource.kt @@ -56,12 +56,12 @@ internal class CommCareIdentityDataSource @Inject constructor( private fun getCaseDataUri(packageName: String): Uri = "content://$packageName.case/casedb/data".toUri() - private fun loadFingerprintIdentities( + private suspend fun loadFingerprintIdentities( query: SubjectQuery, range: IntRange, dataSource: BiometricDataSource, project: Project, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): List = loadEnrolmentRecordCreationEvents(range, dataSource.callerPackageName(), query, project, onCandidateLoaded) .filter { erce -> @@ -82,12 +82,12 @@ internal class CommCareIdentityDataSource @Inject constructor( ) } - private fun loadEnrolmentRecordCreationEvents( + private suspend fun loadEnrolmentRecordCreationEvents( range: IntRange, callerPackageName: String, query: SubjectQuery, project: Project, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): List { val enrolmentRecordCreationEvents: MutableList = mutableListOf() try { @@ -130,12 +130,12 @@ internal class CommCareIdentityDataSource @Inject constructor( } } - private fun loadFaceIdentities( + private suspend fun loadFaceIdentities( query: SubjectQuery, range: IntRange, dataSource: BiometricDataSource, project: Project, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): List = loadEnrolmentRecordCreationEvents(range, dataSource.callerPackageName(), query, project, onCandidateLoaded) .filter { erce -> erce.payload.biometricReferences.any { it is FaceReference && it.format == query.faceSampleFormat } @@ -167,6 +167,7 @@ internal class CommCareIdentityDataSource @Inject constructor( .query(caseDataUri, null, null, null, null) ?.use { caseDataCursor -> val subjectActions = getSubjectActionsValue(caseDataCursor) + Simber.d(subjectActions) val coSyncEnrolmentRecordEvents = parseRecordEvents(subjectActions) coSyncEnrolmentRecordEvents?.events?.filterIsInstance()?.filter { event -> @@ -272,7 +273,7 @@ internal class CommCareIdentityDataSource @Inject constructor( dataSource: BiometricDataSource, project: Project, scope: CoroutineScope, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): ReceiveChannel> = loadIdentitiesConcurrently( ranges = ranges, scope = scope, @@ -292,7 +293,7 @@ internal class CommCareIdentityDataSource @Inject constructor( dataSource: BiometricDataSource, project: Project, scope: CoroutineScope, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): ReceiveChannel> = loadIdentitiesConcurrently( ranges = ranges, scope = scope, @@ -316,7 +317,7 @@ internal class CommCareIdentityDataSource @Inject constructor( scope.launch(dispatcher) { ranges .map { range -> - async(dispatcher) { + async { semaphore.withPermit { channel.send(load(range)) } diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt index ad0e515f71..8866a9809c 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSource.kt @@ -66,7 +66,7 @@ internal class RealmEnrolmentRecordLocalDataSource @Inject constructor( dataSource: BiometricDataSource, project: Project, scope: CoroutineScope, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): ReceiveChannel> { val channel = Channel>(CHANNEL_CAPACITY) scope.launch(dispatcherIO) { @@ -95,7 +95,7 @@ internal class RealmEnrolmentRecordLocalDataSource @Inject constructor( dataSource: BiometricDataSource, project: Project, scope: CoroutineScope, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): ReceiveChannel> { val channel = Channel>(CHANNEL_CAPACITY) scope.launch(dispatcherIO) { @@ -122,7 +122,7 @@ internal class RealmEnrolmentRecordLocalDataSource @Inject constructor( query: SubjectQuery, range: IntRange, mapper: (DbSubject) -> T, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): List = realmWrapper.readRealm { realm -> realm .query(DbSubject::class) diff --git a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt index 15e01e26ba..daef9e8a5d 100644 --- a/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt +++ b/infra/enrolment-records/repository/src/main/java/com/simprints/infra/enrolment/records/repository/local/RoomEnrolmentRecordLocalDataSource.kt @@ -85,7 +85,7 @@ internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( dataSource: BiometricDataSource, project: Project, scope: CoroutineScope, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): ReceiveChannel> = loadBiometricIdentitiesPaged( query = query, ranges = ranges, @@ -116,7 +116,7 @@ internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( dataSource: BiometricDataSource, project: Project, scope: CoroutineScope, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): ReceiveChannel> = loadBiometricIdentitiesPaged( query = query, ranges = ranges, @@ -144,7 +144,7 @@ internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( ranges: List, format: String, createIdentity: (String, List) -> T, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, scope: CoroutineScope, ): ReceiveChannel> { var afterSubjectId: String? = null @@ -180,7 +180,7 @@ internal class RoomEnrolmentRecordLocalDataSource @Inject constructor( pageSize: Int, format: String?, createIdentity: (subjectId: String, samples: List) -> T, - onCandidateLoaded: () -> Unit, + onCandidateLoaded: suspend () -> Unit, ): List = withContext(dispatcherIO) { requireNotNull(format) { "Appropriate sampleFormat is required for loading biometric identities." } subjectDao diff --git a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSourceTest.kt b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSourceTest.kt index 2b836a20c2..20df337b8e 100644 --- a/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSourceTest.kt +++ b/infra/enrolment-records/repository/src/test/java/com/simprints/infra/enrolment/records/repository/local/RealmEnrolmentRecordLocalDataSourceTest.kt @@ -31,6 +31,7 @@ import io.realm.kotlin.query.RealmQuery import io.realm.kotlin.query.RealmSingleQuery import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.consumeEach +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.UnconfinedTestDispatcher import kotlinx.coroutines.test.runTest import org.junit.Before @@ -60,9 +61,9 @@ class RealmEnrolmentRecordLocalDataSourceTest { @MockK private lateinit var project: Project - private lateinit var blockCapture: CapturingSlot<(Realm) -> Any> + private lateinit var blockCapture: CapturingSlot Any> private lateinit var mutableBlockCapture: CapturingSlot<(MutableRealm) -> Any> - private val onCandidateLoaded: () -> Unit = {} + private val onCandidateLoaded: suspend () -> Unit = {} private var localSubjects: MutableList = mutableListOf() private lateinit var enrolmentRecordLocalDataSource: EnrolmentRecordLocalDataSource @@ -83,7 +84,7 @@ class RealmEnrolmentRecordLocalDataSourceTest { blockCapture = slot() coEvery { realmWrapperMock.readRealm(capture(blockCapture)) } answers { - blockCapture.captured.invoke(realm) + runBlocking { blockCapture.captured.invoke(realm) } } mutableBlockCapture = slot() coEvery { realmWrapperMock.writeRealm(capture(mutableBlockCapture)) } answers { From a3a162c3c6fbfa9ce2c5cbd4c856de69433e3eaf Mon Sep 17 00:00:00 2001 From: Melad Raouf Date: Thu, 5 Jun 2025 23:17:12 +0300 Subject: [PATCH 11/11] Refactor FaceMatcherUseCase to use suspend functions for onCandidateLoaded callbacks --- .../simprints/matcher/usecases/FaceMatcherUseCase.kt | 10 +++++----- .../matcher/usecases/FaceMatcherUseCaseTest.kt | 11 ++++++++--- .../local/migrations/ProjectRealmMigrationTest.kt | 5 +++-- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt index 4c0bd34f02..70157fc190 100644 --- a/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt +++ b/feature/matcher/src/main/java/com/simprints/matcher/usecases/FaceMatcherUseCase.kt @@ -75,11 +75,11 @@ internal class FaceMatcherUseCase @Inject constructor( dataSource = matchParams.biometricDataSource, project = project, scope = this, - onCandidateLoaded = { - loadedCandidates.incrementAndGet() - this@channelFlow.send(MatcherState.CandidateLoaded) - }, - ) + ) { + loadedCandidates.incrementAndGet() + this@channelFlow.send(MatcherState.CandidateLoaded) + } + consumeAndMatch(candidatesChannel, samples, resultSet, bioSdk) send(MatcherState.Success(resultSet.toList(), loadedCandidates.get(), bioSdk.matcherName)) }.flowOn(dispatcherBG) diff --git a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt index e48281fede..04015195a6 100644 --- a/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt +++ b/feature/matcher/src/test/java/com/simprints/matcher/usecases/FaceMatcherUseCaseTest.kt @@ -18,6 +18,7 @@ import com.simprints.testtools.common.coroutines.TestCoroutineRule import io.mockk.* import io.mockk.impl.annotations.MockK import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import org.junit.Before import org.junit.Rule @@ -125,10 +126,14 @@ internal class FaceMatcherUseCaseTest { ) coEvery { enrolmentRecordRepository.count(any(), any()) } returns 1 coEvery { createRangesUseCase(any()) } returns listOf(0..99) - coEvery { enrolmentRecordRepository.loadFaceIdentities(any(), any(), any(), any(), any(), any()) } coAnswers { + every { + enrolmentRecordRepository.loadFaceIdentities(any(), any(), any(), any(), any(), any()) + } answers { // Call the onCandidateLoaded callback (5th parameter) - val onCandidateLoaded = arg<() -> Unit>(5) - onCandidateLoaded() + val onCandidateLoaded: suspend () -> Unit = arg(5) + runBlocking { + onCandidateLoaded() + } // Return the face identities createTestChannel(faceIdentities) diff --git a/infra/config-store/src/test/java/com/simprints/infra/config/store/local/migrations/ProjectRealmMigrationTest.kt b/infra/config-store/src/test/java/com/simprints/infra/config/store/local/migrations/ProjectRealmMigrationTest.kt index 64c0397a0c..e8c6881026 100644 --- a/infra/config-store/src/test/java/com/simprints/infra/config/store/local/migrations/ProjectRealmMigrationTest.kt +++ b/infra/config-store/src/test/java/com/simprints/infra/config/store/local/migrations/ProjectRealmMigrationTest.kt @@ -18,6 +18,7 @@ import io.realm.kotlin.MutableRealm import io.realm.kotlin.Realm import io.realm.kotlin.query.RealmQuery import io.realm.kotlin.query.RealmSingleQuery +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import org.junit.Before import org.junit.Test @@ -43,7 +44,7 @@ class ProjectRealmMigrationTest { private lateinit var realmQuery: RealmQuery private lateinit var blockCapture: CapturingSlot<(MutableRealm) -> Any> - private lateinit var readBlockCapture: CapturingSlot<(Realm) -> Any> + private lateinit var readBlockCapture: CapturingSlot Any> private lateinit var projectRealmMigration: ProjectRealmMigration @@ -56,7 +57,7 @@ class ProjectRealmMigrationTest { every { realm.query(any(), any(), any()) } returns realmQuery coEvery { realmWrapper.readRealm(capture(readBlockCapture)) } answers { - readBlockCapture.captured.invoke(realm) + runBlocking { readBlockCapture.captured.invoke(realm) } } coEvery { realmWrapper.writeRealm(capture(blockCapture)) } answers { blockCapture.captured.invoke(mutableRealm)