From b26913c15833722db7844bcddbb00953b24b4311 Mon Sep 17 00:00:00 2001 From: Sergejs Luhmirins Date: Thu, 2 Oct 2025 11:26:26 +0300 Subject: [PATCH 1/2] Report sample upload event for firebase uploader --- .../ApiEventSampleUpSyncRequestPayload.kt | 2 +- .../samples/SampleUpSyncRequestEvent.kt | 4 +- .../firestore/FirestoreSampleUploader.kt | 37 +++++++++-- .../firestore/FirestoreSampleUploaderTest.kt | 61 ++++++++++++++++++- 4 files changed, 94 insertions(+), 10 deletions(-) diff --git a/infra/event-sync/src/main/java/com/simprints/infra/eventsync/event/remote/models/samples/ApiEventSampleUpSyncRequestPayload.kt b/infra/event-sync/src/main/java/com/simprints/infra/eventsync/event/remote/models/samples/ApiEventSampleUpSyncRequestPayload.kt index 2feaf75161..4d3c950dbc 100644 --- a/infra/event-sync/src/main/java/com/simprints/infra/eventsync/event/remote/models/samples/ApiEventSampleUpSyncRequestPayload.kt +++ b/infra/event-sync/src/main/java/com/simprints/infra/eventsync/event/remote/models/samples/ApiEventSampleUpSyncRequestPayload.kt @@ -11,7 +11,7 @@ import com.simprints.infra.eventsync.event.remote.models.fromDomainToApi internal data class ApiEventSampleUpSyncRequestPayload( override val startTime: ApiTimestamp, val endTime: ApiTimestamp?, - val requestId: String, + val requestId: String?, val sampleId: String, val size: Long, val errorType: String?, diff --git a/infra/events/src/main/java/com/simprints/infra/events/event/domain/models/samples/SampleUpSyncRequestEvent.kt b/infra/events/src/main/java/com/simprints/infra/events/event/domain/models/samples/SampleUpSyncRequestEvent.kt index 89389d57de..9f9ecbccde 100644 --- a/infra/events/src/main/java/com/simprints/infra/events/event/domain/models/samples/SampleUpSyncRequestEvent.kt +++ b/infra/events/src/main/java/com/simprints/infra/events/event/domain/models/samples/SampleUpSyncRequestEvent.kt @@ -20,7 +20,7 @@ class SampleUpSyncRequestEvent( constructor( createdAt: Timestamp, endedAt: Timestamp, - requestId: String, + requestId: String?, sampleId: String, size: Long, errorType: String? = null, @@ -42,7 +42,7 @@ class SampleUpSyncRequestEvent( data class SampleUpSyncRequestPayload( override val createdAt: Timestamp, override val endedAt: Timestamp?, - val requestId: String, + val requestId: String?, val sampleId: String, val size: Long, val errorType: String?, diff --git a/infra/images/src/main/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploader.kt b/infra/images/src/main/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploader.kt index 5f100c3ab8..6aae146b6e 100644 --- a/infra/images/src/main/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploader.kt +++ b/infra/images/src/main/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploader.kt @@ -3,12 +3,19 @@ package com.simprints.infra.images.remote.firestore import com.google.firebase.storage.FirebaseStorage import com.google.firebase.storage.StorageMetadata import com.google.firebase.storage.StorageReference +import com.google.firebase.storage.UploadTask +import com.simprints.core.tools.time.TimeHelper import com.simprints.infra.authstore.AuthStore import com.simprints.infra.config.sync.ConfigManager +import com.simprints.infra.events.EventRepository +import com.simprints.infra.events.event.domain.models.samples.SampleUpSyncRequestEvent +import com.simprints.infra.events.event.domain.models.scope.EventScopeEndCause +import com.simprints.infra.events.event.domain.models.scope.EventScopeType import com.simprints.infra.images.local.ImageLocalDataSource import com.simprints.infra.images.metadata.ImageMetadataStore import com.simprints.infra.images.model.SecuredImageRef import com.simprints.infra.images.remote.SampleUploader +import com.simprints.infra.images.usecase.SamplePathConverter import com.simprints.infra.logging.LoggingConstants.CrashReportTag.SAMPLE_UPLOAD import com.simprints.infra.logging.LoggingConstants.CrashReportTag.SYNC import com.simprints.infra.logging.Simber @@ -17,10 +24,13 @@ import java.io.FileInputStream import javax.inject.Inject internal class FirestoreSampleUploader @Inject constructor( + private val timeHelper: TimeHelper, private val configManager: ConfigManager, private val authStore: AuthStore, private val localDataSource: ImageLocalDataSource, private val metadataStore: ImageMetadataStore, + private val samplePathUtil: SamplePathConverter, + private val eventRepository: EventRepository, ) : SampleUploader { override suspend fun uploadAllSamples( projectId: String, @@ -39,27 +49,45 @@ internal class FirestoreSampleUploader @Inject constructor( .getInstance(firebaseApp, bucketUrl) .reference + val urlRequestScope = eventRepository.createEventScope(type = EventScopeType.SAMPLE_UP_SYNC) + val sampleReferences = localDataSource.listImages(projectId) sampleReferences.forEachIndexed { index, imageRef -> Simber.i("Reading sample file: ${imageRef.relativePath.parts.last()}", tag = SAMPLE_UPLOAD) + progressCallback?.invoke(index, sampleReferences.size) try { + val requestStartTime = timeHelper.now() localDataSource.decryptImage(imageRef)?.let { stream -> val metadata = metadataStore.getMetadata(imageRef.relativePath) - val uploadSuccessful = uploadSample(rootRef, stream, imageRef, metadata) - if (uploadSuccessful) { + + val task = uploadSample(rootRef, stream, imageRef, metadata) + if (task.task.isSuccessful) { localDataSource.deleteImage(imageRef) metadataStore.deleteMetadata(imageRef.relativePath) } else { allImagesUploaded = false Simber.i("Failed to upload image without exception", tag = SAMPLE_UPLOAD) } + + eventRepository.addOrUpdateEvent( + scope = urlRequestScope, + event = SampleUpSyncRequestEvent( + createdAt = requestStartTime, + endedAt = timeHelper.now(), + requestId = null, + sampleId = samplePathUtil.extract(imageRef.relativePath)?.sampleId.orEmpty(), + size = task.bytesTransferred, + errorType = task.error?.javaClass?.simpleName, + ), + ) } } catch (t: Throwable) { allImagesUploaded = false Simber.e("Failed to upload images", t, tag = SYNC) } } + eventRepository.closeEventScope(urlRequestScope, EventScopeEndCause.WORKFLOW_ENDED) return allImagesUploaded } @@ -69,13 +97,13 @@ internal class FirestoreSampleUploader @Inject constructor( imageStream: FileInputStream, imageRef: SecuredImageRef, metadata: Map, - ): Boolean { + ): UploadTask.TaskSnapshot { val fileRef = imageRef.relativePath.parts .fold(rootRef) { ref, pathPart -> ref.child(pathPart) } Simber.i("Uploading ${fileRef.path.last()}", tag = SAMPLE_UPLOAD) - val uploadTask = if (metadata.isEmpty()) { + return if (metadata.isEmpty()) { fileRef.putStream(imageStream).await() } else { val storeMetadata = StorageMetadata @@ -84,6 +112,5 @@ internal class FirestoreSampleUploader @Inject constructor( .build() fileRef.putStream(imageStream, storeMetadata).await() } - return uploadTask.task.isSuccessful } } diff --git a/infra/images/src/test/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploaderTest.kt b/infra/images/src/test/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploaderTest.kt index d1a8f1f144..49ca106e95 100644 --- a/infra/images/src/test/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploaderTest.kt +++ b/infra/images/src/test/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploaderTest.kt @@ -3,12 +3,19 @@ package com.simprints.infra.images.remote.firestore import androidx.test.ext.junit.runners.* import com.google.common.truth.Truth.* import com.google.firebase.storage.FirebaseStorage +import com.simprints.core.tools.time.TimeHelper +import com.simprints.core.tools.time.Timestamp import com.simprints.infra.authstore.AuthStore +import com.simprints.infra.config.store.models.GeneralConfiguration import com.simprints.infra.config.sync.ConfigManager +import com.simprints.infra.events.EventRepository +import com.simprints.infra.events.event.domain.models.scope.EventScope +import com.simprints.infra.events.event.domain.models.scope.EventScopeType import com.simprints.infra.images.local.ImageLocalDataSource import com.simprints.infra.images.metadata.ImageMetadataStore import com.simprints.infra.images.model.Path import com.simprints.infra.images.model.SecuredImageRef +import com.simprints.infra.images.usecase.SamplePathConverter import io.mockk.* import io.mockk.impl.annotations.MockK import kotlinx.coroutines.tasks.await @@ -22,6 +29,9 @@ import java.io.FileInputStream @Suppress("DEPRECATION") @RunWith(AndroidJUnit4::class) class FirestoreSampleUploaderTest { + @MockK + private lateinit var timeHelper: TimeHelper + @MockK private lateinit var configManager: ConfigManager @@ -34,6 +44,12 @@ class FirestoreSampleUploaderTest { @MockK private lateinit var metadataStore: ImageMetadataStore + @MockK + private lateinit var samplePathUtil: SamplePathConverter + + @MockK + private lateinit var eventRepository: EventRepository + @MockK private lateinit var localDataSource: ImageLocalDataSource @@ -46,12 +62,21 @@ class FirestoreSampleUploaderTest { every { mockSecuredImageRef.relativePath.parts } returns arrayOf("Test1") remoteDataSource = FirestoreSampleUploader( + timeHelper = timeHelper, configManager = configManager, authStore = authStore, localDataSource = localDataSource, metadataStore = metadataStore, + samplePathUtil = samplePathUtil, + eventRepository = eventRepository, ) + every { timeHelper.now() } returns Timestamp(0L) + every { samplePathUtil.extract(any()) } returns + SamplePathConverter.PathData("sessionID", GeneralConfiguration.Modality.FACE, "sampleId") + coEvery { eventRepository.createEventScope(any(), any()) } returns mockk() + coJustRun { eventRepository.closeEventScope(any(), any()) } + // We need to mock statics and global extensions mockkStatic(FirebaseStorage::class) mockkStatic("kotlinx.coroutines.tasks.TasksKt") @@ -118,6 +143,30 @@ class FirestoreSampleUploaderTest { coVerify(exactly = 3) { metadataStore.deleteMetadata(any()) } } + @Test + fun `test upload and report all upload events`() = runTest { + setupProjectConfig() + setupStorageMock() + configureLocalImageFiles(numberOfValidFiles = 3) + + assertThat(remoteDataSource.uploadAllSamples(PROJECT_ID)).isTrue() + coVerify(exactly = 1) { eventRepository.createEventScope(EventScopeType.SAMPLE_UP_SYNC, any()) } + coVerify(exactly = 3) { eventRepository.addOrUpdateEvent(any(), any()) } + coVerify(exactly = 1) { eventRepository.closeEventScope(any(), any()) } + } + + @Test + fun `test upload failed and report all upload events`() = runTest { + setupProjectConfig() + setupStorageMock(success = false) + configureLocalImageFiles(numberOfValidFiles = 3) + + assertThat(remoteDataSource.uploadAllSamples(PROJECT_ID)).isFalse() + coVerify(exactly = 1) { eventRepository.createEventScope(EventScopeType.SAMPLE_UP_SYNC, any()) } + coVerify(exactly = 3) { eventRepository.addOrUpdateEvent(any(), any()) } + coVerify(exactly = 1) { eventRepository.closeEventScope(any(), any()) } + } + @Test fun `test failed decryption should not return success`() = runTest { setupProjectConfig() @@ -167,10 +216,18 @@ class FirestoreSampleUploaderTest { every { reference.child(any()) } returns mockk { every { path } returns "testPath" every { putStream(any()) } returns mockk { - coEvery { await().task.isSuccessful } returns success + coEvery { await() } returns mockk { + coEvery { bytesTransferred } returns 1L + coEvery { task.isSuccessful } returns success + coEvery { error } returns null + } } every { putStream(any(), any()) } returns mockk { - coEvery { await().task.isSuccessful } returns success + coEvery { await() } returns mockk { + coEvery { bytesTransferred } returns 1L + coEvery { task.isSuccessful } returns success + coEvery { error } returns null + } } } } From c20ee2361e9e2c6864a4a7c648abc3647f821e26 Mon Sep 17 00:00:00 2001 From: Sergejs Luhmirins Date: Thu, 2 Oct 2025 12:00:45 +0300 Subject: [PATCH 2/2] Rename firestore to firebase classes to refer to the correct storage service --- .../FirebaseSampleUploader.kt} | 6 +++--- .../infra/images/usecase/GetUploaderUseCase.kt | 4 ++-- .../FirebaseSampleUploaderTest.kt} | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) rename infra/images/src/main/java/com/simprints/infra/images/remote/{firestore/FirestoreSampleUploader.kt => firebase/FirebaseSampleUploader.kt} (96%) rename infra/images/src/test/java/com/simprints/infra/images/remote/{firestore/FirestoreSampleUploaderTest.kt => firebase/FirebaseSampleUploaderTest.kt} (97%) diff --git a/infra/images/src/main/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploader.kt b/infra/images/src/main/java/com/simprints/infra/images/remote/firebase/FirebaseSampleUploader.kt similarity index 96% rename from infra/images/src/main/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploader.kt rename to infra/images/src/main/java/com/simprints/infra/images/remote/firebase/FirebaseSampleUploader.kt index 6aae146b6e..82276db1e3 100644 --- a/infra/images/src/main/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploader.kt +++ b/infra/images/src/main/java/com/simprints/infra/images/remote/firebase/FirebaseSampleUploader.kt @@ -1,4 +1,4 @@ -package com.simprints.infra.images.remote.firestore +package com.simprints.infra.images.remote.firebase import com.google.firebase.storage.FirebaseStorage import com.google.firebase.storage.StorageMetadata @@ -23,7 +23,7 @@ import kotlinx.coroutines.tasks.await import java.io.FileInputStream import javax.inject.Inject -internal class FirestoreSampleUploader @Inject constructor( +internal class FirebaseSampleUploader @Inject constructor( private val timeHelper: TimeHelper, private val configManager: ConfigManager, private val authStore: AuthStore, @@ -43,7 +43,7 @@ internal class FirestoreSampleUploader @Inject constructor( } var allImagesUploaded = true - Simber.i("Starting sample upload to Firestore") + Simber.i("Starting sample upload to Firebase storage") val bucketUrl = configManager.getProject(projectId).imageBucket val rootRef = FirebaseStorage .getInstance(firebaseApp, bucketUrl) diff --git a/infra/images/src/main/java/com/simprints/infra/images/usecase/GetUploaderUseCase.kt b/infra/images/src/main/java/com/simprints/infra/images/usecase/GetUploaderUseCase.kt index 61986f9637..e6cfe2cab1 100644 --- a/infra/images/src/main/java/com/simprints/infra/images/usecase/GetUploaderUseCase.kt +++ b/infra/images/src/main/java/com/simprints/infra/images/usecase/GetUploaderUseCase.kt @@ -3,13 +3,13 @@ package com.simprints.infra.images.usecase import com.simprints.infra.config.store.ConfigRepository import com.simprints.infra.config.store.models.experimental import com.simprints.infra.images.remote.SampleUploader -import com.simprints.infra.images.remote.firestore.FirestoreSampleUploader +import com.simprints.infra.images.remote.firebase.FirebaseSampleUploader import com.simprints.infra.images.remote.signedurl.SignedUrlSampleUploader import javax.inject.Inject internal class GetUploaderUseCase @Inject constructor( private val configRepository: ConfigRepository, - private val firestoreUploader: FirestoreSampleUploader, + private val firestoreUploader: FirebaseSampleUploader, private val signedUrlUploader: SignedUrlSampleUploader, ) { suspend operator fun invoke(): SampleUploader = configRepository diff --git a/infra/images/src/test/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploaderTest.kt b/infra/images/src/test/java/com/simprints/infra/images/remote/firebase/FirebaseSampleUploaderTest.kt similarity index 97% rename from infra/images/src/test/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploaderTest.kt rename to infra/images/src/test/java/com/simprints/infra/images/remote/firebase/FirebaseSampleUploaderTest.kt index 49ca106e95..6d89faac9e 100644 --- a/infra/images/src/test/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploaderTest.kt +++ b/infra/images/src/test/java/com/simprints/infra/images/remote/firebase/FirebaseSampleUploaderTest.kt @@ -1,4 +1,4 @@ -package com.simprints.infra.images.remote.firestore +package com.simprints.infra.images.remote.firebase import androidx.test.ext.junit.runners.* import com.google.common.truth.Truth.* @@ -28,7 +28,7 @@ import java.io.FileInputStream @Suppress("DEPRECATION") @RunWith(AndroidJUnit4::class) -class FirestoreSampleUploaderTest { +class FirebaseSampleUploaderTest { @MockK private lateinit var timeHelper: TimeHelper @@ -53,7 +53,7 @@ class FirestoreSampleUploaderTest { @MockK private lateinit var localDataSource: ImageLocalDataSource - private lateinit var remoteDataSource: FirestoreSampleUploader + private lateinit var remoteDataSource: FirebaseSampleUploader @Before fun setup() { @@ -61,7 +61,7 @@ class FirestoreSampleUploaderTest { every { mockSecuredImageRef.relativePath.parts } returns arrayOf("Test1") - remoteDataSource = FirestoreSampleUploader( + remoteDataSource = FirebaseSampleUploader( timeHelper = timeHelper, configManager = configManager, authStore = authStore, @@ -261,7 +261,7 @@ class FirestoreSampleUploaderTest { private fun mockImage() = SecuredImageRef(Path(VALID_PATH)) - companion object { + companion object Companion { private const val VALID_PATH = "valid.txt" private const val PROJECT_ID = "projectId" }