diff --git a/infra/event-sync/src/main/java/com/simprints/infra/eventsync/event/remote/models/samples/ApiEventSampleUpSyncRequestPayload.kt b/infra/event-sync/src/main/java/com/simprints/infra/eventsync/event/remote/models/samples/ApiEventSampleUpSyncRequestPayload.kt index 2feaf75161..4d3c950dbc 100644 --- a/infra/event-sync/src/main/java/com/simprints/infra/eventsync/event/remote/models/samples/ApiEventSampleUpSyncRequestPayload.kt +++ b/infra/event-sync/src/main/java/com/simprints/infra/eventsync/event/remote/models/samples/ApiEventSampleUpSyncRequestPayload.kt @@ -11,7 +11,7 @@ import com.simprints.infra.eventsync.event.remote.models.fromDomainToApi internal data class ApiEventSampleUpSyncRequestPayload( override val startTime: ApiTimestamp, val endTime: ApiTimestamp?, - val requestId: String, + val requestId: String?, val sampleId: String, val size: Long, val errorType: String?, diff --git a/infra/events/src/main/java/com/simprints/infra/events/event/domain/models/samples/SampleUpSyncRequestEvent.kt b/infra/events/src/main/java/com/simprints/infra/events/event/domain/models/samples/SampleUpSyncRequestEvent.kt index 89389d57de..9f9ecbccde 100644 --- a/infra/events/src/main/java/com/simprints/infra/events/event/domain/models/samples/SampleUpSyncRequestEvent.kt +++ b/infra/events/src/main/java/com/simprints/infra/events/event/domain/models/samples/SampleUpSyncRequestEvent.kt @@ -20,7 +20,7 @@ class SampleUpSyncRequestEvent( constructor( createdAt: Timestamp, endedAt: Timestamp, - requestId: String, + requestId: String?, sampleId: String, size: Long, errorType: String? = null, @@ -42,7 +42,7 @@ class SampleUpSyncRequestEvent( data class SampleUpSyncRequestPayload( override val createdAt: Timestamp, override val endedAt: Timestamp?, - val requestId: String, + val requestId: String?, val sampleId: String, val size: Long, val errorType: String?, diff --git a/infra/images/src/main/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploader.kt b/infra/images/src/main/java/com/simprints/infra/images/remote/firebase/FirebaseSampleUploader.kt similarity index 64% rename from infra/images/src/main/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploader.kt rename to infra/images/src/main/java/com/simprints/infra/images/remote/firebase/FirebaseSampleUploader.kt index 5f100c3ab8..82276db1e3 100644 --- a/infra/images/src/main/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploader.kt +++ b/infra/images/src/main/java/com/simprints/infra/images/remote/firebase/FirebaseSampleUploader.kt @@ -1,14 +1,21 @@ -package com.simprints.infra.images.remote.firestore +package com.simprints.infra.images.remote.firebase import com.google.firebase.storage.FirebaseStorage import com.google.firebase.storage.StorageMetadata import com.google.firebase.storage.StorageReference +import com.google.firebase.storage.UploadTask +import com.simprints.core.tools.time.TimeHelper import com.simprints.infra.authstore.AuthStore import com.simprints.infra.config.sync.ConfigManager +import com.simprints.infra.events.EventRepository +import com.simprints.infra.events.event.domain.models.samples.SampleUpSyncRequestEvent +import com.simprints.infra.events.event.domain.models.scope.EventScopeEndCause +import com.simprints.infra.events.event.domain.models.scope.EventScopeType import com.simprints.infra.images.local.ImageLocalDataSource import com.simprints.infra.images.metadata.ImageMetadataStore import com.simprints.infra.images.model.SecuredImageRef import com.simprints.infra.images.remote.SampleUploader +import com.simprints.infra.images.usecase.SamplePathConverter import com.simprints.infra.logging.LoggingConstants.CrashReportTag.SAMPLE_UPLOAD import com.simprints.infra.logging.LoggingConstants.CrashReportTag.SYNC import com.simprints.infra.logging.Simber @@ -16,11 +23,14 @@ import kotlinx.coroutines.tasks.await import java.io.FileInputStream import javax.inject.Inject -internal class FirestoreSampleUploader @Inject constructor( +internal class FirebaseSampleUploader @Inject constructor( + private val timeHelper: TimeHelper, private val configManager: ConfigManager, private val authStore: AuthStore, private val localDataSource: ImageLocalDataSource, private val metadataStore: ImageMetadataStore, + private val samplePathUtil: SamplePathConverter, + private val eventRepository: EventRepository, ) : SampleUploader { override suspend fun uploadAllSamples( projectId: String, @@ -33,33 +43,51 @@ internal class FirestoreSampleUploader @Inject constructor( } var allImagesUploaded = true - Simber.i("Starting sample upload to Firestore") + Simber.i("Starting sample upload to Firebase storage") val bucketUrl = configManager.getProject(projectId).imageBucket val rootRef = FirebaseStorage .getInstance(firebaseApp, bucketUrl) .reference + val urlRequestScope = eventRepository.createEventScope(type = EventScopeType.SAMPLE_UP_SYNC) + val sampleReferences = localDataSource.listImages(projectId) sampleReferences.forEachIndexed { index, imageRef -> Simber.i("Reading sample file: ${imageRef.relativePath.parts.last()}", tag = SAMPLE_UPLOAD) + progressCallback?.invoke(index, sampleReferences.size) try { + val requestStartTime = timeHelper.now() localDataSource.decryptImage(imageRef)?.let { stream -> val metadata = metadataStore.getMetadata(imageRef.relativePath) - val uploadSuccessful = uploadSample(rootRef, stream, imageRef, metadata) - if (uploadSuccessful) { + + val task = uploadSample(rootRef, stream, imageRef, metadata) + if (task.task.isSuccessful) { localDataSource.deleteImage(imageRef) metadataStore.deleteMetadata(imageRef.relativePath) } else { allImagesUploaded = false Simber.i("Failed to upload image without exception", tag = SAMPLE_UPLOAD) } + + eventRepository.addOrUpdateEvent( + scope = urlRequestScope, + event = SampleUpSyncRequestEvent( + createdAt = requestStartTime, + endedAt = timeHelper.now(), + requestId = null, + sampleId = samplePathUtil.extract(imageRef.relativePath)?.sampleId.orEmpty(), + size = task.bytesTransferred, + errorType = task.error?.javaClass?.simpleName, + ), + ) } } catch (t: Throwable) { allImagesUploaded = false Simber.e("Failed to upload images", t, tag = SYNC) } } + eventRepository.closeEventScope(urlRequestScope, EventScopeEndCause.WORKFLOW_ENDED) return allImagesUploaded } @@ -69,13 +97,13 @@ internal class FirestoreSampleUploader @Inject constructor( imageStream: FileInputStream, imageRef: SecuredImageRef, metadata: Map, - ): Boolean { + ): UploadTask.TaskSnapshot { val fileRef = imageRef.relativePath.parts .fold(rootRef) { ref, pathPart -> ref.child(pathPart) } Simber.i("Uploading ${fileRef.path.last()}", tag = SAMPLE_UPLOAD) - val uploadTask = if (metadata.isEmpty()) { + return if (metadata.isEmpty()) { fileRef.putStream(imageStream).await() } else { val storeMetadata = StorageMetadata @@ -84,6 +112,5 @@ internal class FirestoreSampleUploader @Inject constructor( .build() fileRef.putStream(imageStream, storeMetadata).await() } - return uploadTask.task.isSuccessful } } diff --git a/infra/images/src/main/java/com/simprints/infra/images/usecase/GetUploaderUseCase.kt b/infra/images/src/main/java/com/simprints/infra/images/usecase/GetUploaderUseCase.kt index 61986f9637..e6cfe2cab1 100644 --- a/infra/images/src/main/java/com/simprints/infra/images/usecase/GetUploaderUseCase.kt +++ b/infra/images/src/main/java/com/simprints/infra/images/usecase/GetUploaderUseCase.kt @@ -3,13 +3,13 @@ package com.simprints.infra.images.usecase import com.simprints.infra.config.store.ConfigRepository import com.simprints.infra.config.store.models.experimental import com.simprints.infra.images.remote.SampleUploader -import com.simprints.infra.images.remote.firestore.FirestoreSampleUploader +import com.simprints.infra.images.remote.firebase.FirebaseSampleUploader import com.simprints.infra.images.remote.signedurl.SignedUrlSampleUploader import javax.inject.Inject internal class GetUploaderUseCase @Inject constructor( private val configRepository: ConfigRepository, - private val firestoreUploader: FirestoreSampleUploader, + private val firestoreUploader: FirebaseSampleUploader, private val signedUrlUploader: SignedUrlSampleUploader, ) { suspend operator fun invoke(): SampleUploader = configRepository diff --git a/infra/images/src/test/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploaderTest.kt b/infra/images/src/test/java/com/simprints/infra/images/remote/firebase/FirebaseSampleUploaderTest.kt similarity index 69% rename from infra/images/src/test/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploaderTest.kt rename to infra/images/src/test/java/com/simprints/infra/images/remote/firebase/FirebaseSampleUploaderTest.kt index d1a8f1f144..6d89faac9e 100644 --- a/infra/images/src/test/java/com/simprints/infra/images/remote/firestore/FirestoreSampleUploaderTest.kt +++ b/infra/images/src/test/java/com/simprints/infra/images/remote/firebase/FirebaseSampleUploaderTest.kt @@ -1,14 +1,21 @@ -package com.simprints.infra.images.remote.firestore +package com.simprints.infra.images.remote.firebase import androidx.test.ext.junit.runners.* import com.google.common.truth.Truth.* import com.google.firebase.storage.FirebaseStorage +import com.simprints.core.tools.time.TimeHelper +import com.simprints.core.tools.time.Timestamp import com.simprints.infra.authstore.AuthStore +import com.simprints.infra.config.store.models.GeneralConfiguration import com.simprints.infra.config.sync.ConfigManager +import com.simprints.infra.events.EventRepository +import com.simprints.infra.events.event.domain.models.scope.EventScope +import com.simprints.infra.events.event.domain.models.scope.EventScopeType import com.simprints.infra.images.local.ImageLocalDataSource import com.simprints.infra.images.metadata.ImageMetadataStore import com.simprints.infra.images.model.Path import com.simprints.infra.images.model.SecuredImageRef +import com.simprints.infra.images.usecase.SamplePathConverter import io.mockk.* import io.mockk.impl.annotations.MockK import kotlinx.coroutines.tasks.await @@ -21,7 +28,10 @@ import java.io.FileInputStream @Suppress("DEPRECATION") @RunWith(AndroidJUnit4::class) -class FirestoreSampleUploaderTest { +class FirebaseSampleUploaderTest { + @MockK + private lateinit var timeHelper: TimeHelper + @MockK private lateinit var configManager: ConfigManager @@ -34,10 +44,16 @@ class FirestoreSampleUploaderTest { @MockK private lateinit var metadataStore: ImageMetadataStore + @MockK + private lateinit var samplePathUtil: SamplePathConverter + + @MockK + private lateinit var eventRepository: EventRepository + @MockK private lateinit var localDataSource: ImageLocalDataSource - private lateinit var remoteDataSource: FirestoreSampleUploader + private lateinit var remoteDataSource: FirebaseSampleUploader @Before fun setup() { @@ -45,13 +61,22 @@ class FirestoreSampleUploaderTest { every { mockSecuredImageRef.relativePath.parts } returns arrayOf("Test1") - remoteDataSource = FirestoreSampleUploader( + remoteDataSource = FirebaseSampleUploader( + timeHelper = timeHelper, configManager = configManager, authStore = authStore, localDataSource = localDataSource, metadataStore = metadataStore, + samplePathUtil = samplePathUtil, + eventRepository = eventRepository, ) + every { timeHelper.now() } returns Timestamp(0L) + every { samplePathUtil.extract(any()) } returns + SamplePathConverter.PathData("sessionID", GeneralConfiguration.Modality.FACE, "sampleId") + coEvery { eventRepository.createEventScope(any(), any()) } returns mockk() + coJustRun { eventRepository.closeEventScope(any(), any()) } + // We need to mock statics and global extensions mockkStatic(FirebaseStorage::class) mockkStatic("kotlinx.coroutines.tasks.TasksKt") @@ -118,6 +143,30 @@ class FirestoreSampleUploaderTest { coVerify(exactly = 3) { metadataStore.deleteMetadata(any()) } } + @Test + fun `test upload and report all upload events`() = runTest { + setupProjectConfig() + setupStorageMock() + configureLocalImageFiles(numberOfValidFiles = 3) + + assertThat(remoteDataSource.uploadAllSamples(PROJECT_ID)).isTrue() + coVerify(exactly = 1) { eventRepository.createEventScope(EventScopeType.SAMPLE_UP_SYNC, any()) } + coVerify(exactly = 3) { eventRepository.addOrUpdateEvent(any(), any()) } + coVerify(exactly = 1) { eventRepository.closeEventScope(any(), any()) } + } + + @Test + fun `test upload failed and report all upload events`() = runTest { + setupProjectConfig() + setupStorageMock(success = false) + configureLocalImageFiles(numberOfValidFiles = 3) + + assertThat(remoteDataSource.uploadAllSamples(PROJECT_ID)).isFalse() + coVerify(exactly = 1) { eventRepository.createEventScope(EventScopeType.SAMPLE_UP_SYNC, any()) } + coVerify(exactly = 3) { eventRepository.addOrUpdateEvent(any(), any()) } + coVerify(exactly = 1) { eventRepository.closeEventScope(any(), any()) } + } + @Test fun `test failed decryption should not return success`() = runTest { setupProjectConfig() @@ -167,10 +216,18 @@ class FirestoreSampleUploaderTest { every { reference.child(any()) } returns mockk { every { path } returns "testPath" every { putStream(any()) } returns mockk { - coEvery { await().task.isSuccessful } returns success + coEvery { await() } returns mockk { + coEvery { bytesTransferred } returns 1L + coEvery { task.isSuccessful } returns success + coEvery { error } returns null + } } every { putStream(any(), any()) } returns mockk { - coEvery { await().task.isSuccessful } returns success + coEvery { await() } returns mockk { + coEvery { bytesTransferred } returns 1L + coEvery { task.isSuccessful } returns success + coEvery { error } returns null + } } } } @@ -204,7 +261,7 @@ class FirestoreSampleUploaderTest { private fun mockImage() = SecuredImageRef(Path(VALID_PATH)) - companion object { + companion object Companion { private const val VALID_PATH = "valid.txt" private const val PROJECT_ID = "projectId" }