diff --git a/src/main/kotlin/simplerag/ragback/domain/document/controller/DataFileController.kt b/src/main/kotlin/simplerag/ragback/domain/document/controller/DataFileController.kt index 64693ba..fa7914b 100644 --- a/src/main/kotlin/simplerag/ragback/domain/document/controller/DataFileController.kt +++ b/src/main/kotlin/simplerag/ragback/domain/document/controller/DataFileController.kt @@ -3,6 +3,8 @@ package simplerag.ragback.domain.document.controller import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.media.Content import jakarta.validation.Valid +import jakarta.validation.constraints.Max +import jakarta.validation.constraints.Min import jakarta.validation.constraints.Size import org.springframework.http.HttpStatus import org.springframework.http.MediaType @@ -10,6 +12,7 @@ import org.springframework.validation.annotation.Validated import org.springframework.web.bind.annotation.* import org.springframework.web.multipart.MultipartFile import simplerag.ragback.domain.document.dto.DataFileBulkCreateRequest +import simplerag.ragback.domain.document.dto.DataFileDetailResponseList import simplerag.ragback.domain.document.dto.DataFileResponseList import simplerag.ragback.domain.document.service.DataFileService import simplerag.ragback.global.response.ApiResponse @@ -41,4 +44,13 @@ class DataFileController( return ApiResponse.ok(saved, "업로드 완료") } + @GetMapping + fun getDataFiles( + @RequestParam(name = "cursor") cursor: Long, + @RequestParam(name = "take") @Min(1) @Max(100) take: Int, + ): ApiResponse { + val data = dataFileService.getDataFiles(cursor, take) + return ApiResponse.ok(data) + } + } diff --git a/src/main/kotlin/simplerag/ragback/domain/document/dto/DataFileResponseDTO.kt b/src/main/kotlin/simplerag/ragback/domain/document/dto/DataFileResponseDTO.kt index 04244f3..d91e725 100644 --- a/src/main/kotlin/simplerag/ragback/domain/document/dto/DataFileResponseDTO.kt +++ b/src/main/kotlin/simplerag/ragback/domain/document/dto/DataFileResponseDTO.kt @@ -1,5 +1,8 @@ package simplerag.ragback.domain.document.dto +import simplerag.ragback.domain.document.entity.DataFile +import java.time.LocalDateTime + data class DataFileResponseList( val dataFilePreviewResponseList: List, ) @@ -7,4 +10,39 @@ data class DataFileResponseList( data class DataFilePreviewResponse( val id: Long, val sha256: String, -) \ No newline at end of file +) + +data class DataFileDetailResponseList( + val dataFileDetailResponseList: List, + val cursor: Long?, + val hasNext: Boolean, +) + +data class DataFileDetailResponse( + var id: Long?, + val title: String, + val type: String, + val lastModified: LocalDateTime, + val sizeMB: Double, + val tags: List, + val sha256: String, +) { + companion object { + fun of(dataFile: DataFile, tags: List): DataFileDetailResponse { + return DataFileDetailResponse( + dataFile.id, + dataFile.title, + dataFile.type, + dataFile.updatedAt, + dataFile.sizeBytes / (1024.0 * 1024.0), + tags, + dataFile.sha256, + ) + } + } +} + +data class TagDTO( + val id: Long?, + val name: String, +) diff --git a/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileRepository.kt b/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileRepository.kt index 11590fc..539ff04 100644 --- a/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileRepository.kt +++ b/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileRepository.kt @@ -1,8 +1,12 @@ package simplerag.ragback.domain.document.repository +import org.springframework.data.domain.Pageable +import org.springframework.data.domain.Slice import org.springframework.data.jpa.repository.JpaRepository import simplerag.ragback.domain.document.entity.DataFile interface DataFileRepository : JpaRepository { fun existsBySha256(sha256: String): Boolean + + fun findByIdGreaterThanOrderById(cursorId: Long, pageable: Pageable): Slice } \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileTagRepository.kt b/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileTagRepository.kt index 69509a4..37691b3 100644 --- a/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileTagRepository.kt +++ b/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileTagRepository.kt @@ -1,8 +1,19 @@ package simplerag.ragback.domain.document.repository import org.springframework.data.jpa.repository.JpaRepository +import org.springframework.data.jpa.repository.Query +import org.springframework.data.repository.query.Param +import simplerag.ragback.domain.document.entity.DataFile import simplerag.ragback.domain.document.entity.DataFileTag interface DataFileTagRepository : JpaRepository { fun existsByDataFileIdAndTagId(dataFileId: Long, tagId: Long): Boolean + + @Query(""" + SELECT dft + FROM DataFileTag dft + JOIN FETCH dft.tag t + WHERE dft.dataFile = :dataFile + """) + fun findTagsByDataFile(@Param("dataFile") dataFile: DataFile): List } \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/domain/document/service/DataFileService.kt b/src/main/kotlin/simplerag/ragback/domain/document/service/DataFileService.kt index 8678574..8331da5 100644 --- a/src/main/kotlin/simplerag/ragback/domain/document/service/DataFileService.kt +++ b/src/main/kotlin/simplerag/ragback/domain/document/service/DataFileService.kt @@ -1,14 +1,14 @@ package simplerag.ragback.domain.document.service import org.springframework.dao.DataIntegrityViolationException +import org.springframework.data.domain.PageRequest +import org.springframework.data.domain.Pageable import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import org.springframework.transaction.support.TransactionSynchronization import org.springframework.transaction.support.TransactionSynchronizationManager import org.springframework.web.multipart.MultipartFile -import simplerag.ragback.domain.document.dto.DataFileBulkCreateRequest -import simplerag.ragback.domain.document.dto.DataFilePreviewResponse -import simplerag.ragback.domain.document.dto.DataFileResponseList +import simplerag.ragback.domain.document.dto.* import simplerag.ragback.domain.document.entity.DataFile import simplerag.ragback.domain.document.entity.DataFileTag import simplerag.ragback.domain.document.entity.Tag @@ -24,6 +24,7 @@ import simplerag.ragback.global.util.computeMetricsStreaming import simplerag.ragback.global.util.resolveContentType import java.time.LocalDateTime import java.util.* +import kotlin.collections.ArrayList @Service class DataFileService( @@ -76,6 +77,28 @@ class DataFileService( return DataFileResponseList(responses) } + @Transactional(readOnly = true) + fun getDataFiles(cursor: Long, take: Int): DataFileDetailResponseList { + + val dataSlice = dataFileRepository.findByIdGreaterThanOrderById(cursor, PageRequest.of(0, take)) + + val dataFileList: MutableList = ArrayList() + dataSlice.forEach{ dataFile -> + val dataFileTags: List = dataFileTagRepository.findTagsByDataFile(dataFile) + + val tagDtos: List = dataFileTags.map{ + dataFileTag -> + val tag = dataFileTag.tag + TagDTO(tag.id, tag.name) + } + + dataFileList.add(DataFileDetailResponse.of(dataFile, tagDtos)) + } + + val nextCursor: Long? = dataFileList.lastOrNull()?.id + return DataFileDetailResponseList(dataFileList, nextCursor, dataSlice.hasNext()) + } + private fun registerRollbackCleanup(uploadedUrls: MutableList) { if (TransactionSynchronizationManager.isSynchronizationActive()) { TransactionSynchronizationManager.registerSynchronization(object : TransactionSynchronization { diff --git a/src/main/kotlin/simplerag/ragback/global/storage/FakeS3Util.kt b/src/main/kotlin/simplerag/ragback/global/storage/FakeS3Util.kt index c701405..7256546 100644 --- a/src/main/kotlin/simplerag/ragback/global/storage/FakeS3Util.kt +++ b/src/main/kotlin/simplerag/ragback/global/storage/FakeS3Util.kt @@ -50,4 +50,5 @@ class FakeS3Util : S3Util { // 테스트 용 함수 fun exists(url: String): Boolean = keyFromUrl(url)?.let { store.containsKey(it) } == true fun count(): Int = store.size + fun clear() = store.clear() } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index a403076..bc96fb8 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -16,3 +16,4 @@ spring: hibernate: dialect: org.hibernate.dialect.PostgreSQLDialect format_sql: true + default_batch_fetch_size: 500 diff --git a/src/test/kotlin/simplerag/ragback/domain/document/service/DataFileServiceTest.kt b/src/test/kotlin/simplerag/ragback/domain/document/service/DataFileServiceTest.kt index 31264d1..3ac7b70 100644 --- a/src/test/kotlin/simplerag/ragback/domain/document/service/DataFileServiceTest.kt +++ b/src/test/kotlin/simplerag/ragback/domain/document/service/DataFileServiceTest.kt @@ -1,6 +1,8 @@ package simplerag.ragback.domain.document.service +import jakarta.annotation.PostConstruct import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -23,7 +25,6 @@ import simplerag.ragback.global.storage.FakeS3Util import simplerag.ragback.global.util.S3Type import simplerag.ragback.global.util.sha256Hex import java.security.MessageDigest -import java.time.LocalDateTime @SpringBootTest @ActiveProfiles("test") @@ -40,6 +41,14 @@ class DataFileServiceTest( private fun txTemplate() = TransactionTemplate(txManager) + @BeforeEach + fun clean() { + dataFileTagRepository.deleteAll() + tagRepository.deleteAll() + dataFileRepository.deleteAll() + s3Util.clear() + } + @Test @Transactional @DisplayName("업로드 시 잘 저장이 된다.") @@ -207,6 +216,57 @@ class DataFileServiceTest( assertFalse(s3Util.exists(expectedUrl), "롤백 시 S3도 보상 삭제되어야 합니다") } + @Test + @DisplayName("데이터 조회가 잘 된다") + @Transactional + fun getDataFilesOK() { + // given + val bytes1 = "test1".toByteArray() + val sha1 = sha256Hex(bytes1) + val bytes2 = "test2".toByteArray() + val sha2 = sha256Hex(bytes2) + dataFileRepository.saveAll( + listOf( + DataFile( + title = "exists", + type = "text/plain", + sizeBytes = 0, + sha256 = sha1, + fileUrl = "fake://original/exists.txt", + ), + DataFile( + title = "exists2", + type = "text/pdf", + sizeBytes = 0, + sha256 = sha2, + fileUrl = "fake://original/exists.txt", + ) + ) + ) + + val cursor = 0L + val take = 2 + + // when + val dataFiles = dataFileService.getDataFiles(cursor, take) + + // then + val dataFileDetailResponse = dataFiles.dataFileDetailResponseList[0] + assertEquals(dataFileDetailResponse.title, "exists") + assertEquals(dataFileDetailResponse.type, "text/plain") + assertEquals(dataFileDetailResponse.sizeMB, 0.0) + assertEquals(dataFileDetailResponse.sha256, sha1) + + val dataFileDetailResponse2 = dataFiles.dataFileDetailResponseList[1] + assertEquals(dataFileDetailResponse2.title, "exists2") + assertEquals(dataFileDetailResponse2.type, "text/pdf") + assertEquals(dataFileDetailResponse2.sizeMB, 0.0) + assertEquals(dataFileDetailResponse2.sha256, sha2) + + assertEquals(dataFiles.cursor, dataFileDetailResponse2.id) + assertEquals(dataFiles.hasNext, false) + } + // ----------------------- // helpers // -----------------------