diff --git a/build.gradle b/build.gradle index e8b033f..fe060a4 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,7 @@ ext { } dependencies { + implementation("com.fasterxml.jackson.module:jackson-module-kotlin") implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.boot:spring-boot-starter-validation' implementation 'org.springframework.boot:spring-boot-starter-web' @@ -41,8 +42,17 @@ dependencies { annotationProcessor 'org.projectlombok:lombok' testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.jetbrains.kotlin:kotlin-test-junit5' + testImplementation("org.mockito.kotlin:mockito-kotlin:5.3.1") testRuntimeOnly 'org.junit.platform:junit-platform-launcher' runtimeOnly 'org.postgresql:postgresql' + testRuntimeOnly("com.h2database:h2") + + // swagger + implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.7.0' + + // s3 + implementation(platform("software.amazon.awssdk:bom:2.25.70")) + implementation("software.amazon.awssdk:s3") } dependencyManagement { diff --git a/src/main/kotlin/simplerag/ragback/domain/document/controller/DataFileController.kt b/src/main/kotlin/simplerag/ragback/domain/document/controller/DataFileController.kt new file mode 100644 index 0000000..64693ba --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/domain/document/controller/DataFileController.kt @@ -0,0 +1,44 @@ +package simplerag.ragback.domain.document.controller + +import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.media.Content +import jakarta.validation.Valid +import jakarta.validation.constraints.Size +import org.springframework.http.HttpStatus +import org.springframework.http.MediaType +import org.springframework.validation.annotation.Validated +import org.springframework.web.bind.annotation.* +import org.springframework.web.multipart.MultipartFile +import simplerag.ragback.domain.document.dto.DataFileBulkCreateRequest +import simplerag.ragback.domain.document.dto.DataFileResponseList +import simplerag.ragback.domain.document.service.DataFileService +import simplerag.ragback.global.response.ApiResponse + +@RestController +@RequestMapping("/api/v1/data-files") +@Validated +class DataFileController( + private val dataFileService: DataFileService +) { + + @PostMapping( + consumes = [ + MediaType.MULTIPART_FORM_DATA_VALUE, + ] + ) + @ResponseStatus(HttpStatus.CREATED) + fun upload( + @RequestPart("files") + @Size(min = 1, message = "최소 하나 이상 업로드해야 합니다") + files: List, + + @Parameter(content = [Content(mediaType = MediaType.APPLICATION_JSON_VALUE)]) + @RequestPart("request") + @Valid + req: DataFileBulkCreateRequest + ): ApiResponse { + val saved = dataFileService.upload(files, req) + return ApiResponse.ok(saved, "업로드 완료") + } + +} diff --git a/src/main/kotlin/simplerag/ragback/domain/document/dto/DataFileRequestDTO.kt b/src/main/kotlin/simplerag/ragback/domain/document/dto/DataFileRequestDTO.kt new file mode 100644 index 0000000..985c912 --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/domain/document/dto/DataFileRequestDTO.kt @@ -0,0 +1,20 @@ +package simplerag.ragback.domain.document.dto + +import jakarta.validation.Valid +import jakarta.validation.constraints.NotBlank +import jakarta.validation.constraints.Size + +data class DataFileBulkCreateRequest( + @field:Size(min = 1, message = "최소 하나 이상 업로드해야 합니다") + @Valid + val items: List +) + +data class DataFileCreateItem( + @field:NotBlank(message = "title은 비어있을 수 없습니다") + @field:Size(max = 100) + val title: String, + + @field:Size(max = 10, message = "태그는 최대 10개까지 가능합니다") + val tags: List = emptyList() +) diff --git a/src/main/kotlin/simplerag/ragback/domain/document/dto/DataFileResponseDTO.kt b/src/main/kotlin/simplerag/ragback/domain/document/dto/DataFileResponseDTO.kt new file mode 100644 index 0000000..04244f3 --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/domain/document/dto/DataFileResponseDTO.kt @@ -0,0 +1,10 @@ +package simplerag.ragback.domain.document.dto + +data class DataFileResponseList( + val dataFilePreviewResponseList: List, +) + +data class DataFilePreviewResponse( + val id: Long, + val sha256: String, +) \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/domain/document/entity/DataFile.kt b/src/main/kotlin/simplerag/ragback/domain/document/entity/DataFile.kt new file mode 100644 index 0000000..6a2cc6b --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/domain/document/entity/DataFile.kt @@ -0,0 +1,36 @@ +package simplerag.ragback.domain.document.entity + +import jakarta.persistence.* +import java.time.LocalDateTime + +@Entity +@Table( + name = "data_file", + uniqueConstraints = [UniqueConstraint(columnNames = ["sha256"])] +) +class DataFile( + + @Column(nullable = false, length = 255) + val title: String, + + @Column(name = "file_type", nullable = false, length = 120) + val type: String, + + @Column(name = "size_bytes", nullable = false) + val sizeBytes: Long, + + @Column(nullable = false, length = 64) + val sha256: String, + + @Column(nullable = false, length = 2048) + val fileUrl: String, + + @Column(nullable = false) + val updatedAt: LocalDateTime, + + @Column(nullable = false) + val createdAt: LocalDateTime, + + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) + val id: Long? = null, +) diff --git a/src/main/kotlin/simplerag/ragback/domain/document/entity/DataFileTag.kt b/src/main/kotlin/simplerag/ragback/domain/document/entity/DataFileTag.kt new file mode 100644 index 0000000..a9771db --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/domain/document/entity/DataFileTag.kt @@ -0,0 +1,22 @@ +package simplerag.ragback.domain.document.entity + +import jakarta.persistence.* + +@Entity +@Table( + name = "data_file_tags", + uniqueConstraints = [UniqueConstraint(columnNames = ["data_file_id", "tag_id"])] +) +class DataFileTag( + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "tag_id", nullable = false) + var tag: Tag, + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "data_file_id", nullable = false) + var dataFile: DataFile, + + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) + val id: Long? = null, +) \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/domain/document/entity/Tag.kt b/src/main/kotlin/simplerag/ragback/domain/document/entity/Tag.kt new file mode 100644 index 0000000..7669439 --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/domain/document/entity/Tag.kt @@ -0,0 +1,17 @@ +package simplerag.ragback.domain.document.entity + +import jakarta.persistence.* + +@Entity +@Table( + name = "tags", + uniqueConstraints = [UniqueConstraint(columnNames = ["name"])] +) +class Tag( + + @Column(nullable = false, length = 60) + val name: String, + + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) + val id: Long? = null, +) \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileRepository.kt b/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileRepository.kt new file mode 100644 index 0000000..11590fc --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileRepository.kt @@ -0,0 +1,8 @@ +package simplerag.ragback.domain.document.repository + +import org.springframework.data.jpa.repository.JpaRepository +import simplerag.ragback.domain.document.entity.DataFile + +interface DataFileRepository : JpaRepository { + fun existsBySha256(sha256: String): Boolean +} \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileTagRepository.kt b/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileTagRepository.kt new file mode 100644 index 0000000..69509a4 --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/domain/document/repository/DataFileTagRepository.kt @@ -0,0 +1,8 @@ +package simplerag.ragback.domain.document.repository + +import org.springframework.data.jpa.repository.JpaRepository +import simplerag.ragback.domain.document.entity.DataFileTag + +interface DataFileTagRepository : JpaRepository { + fun existsByDataFileIdAndTagId(dataFileId: Long, tagId: Long): Boolean +} \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/domain/document/repository/TagRepository.kt b/src/main/kotlin/simplerag/ragback/domain/document/repository/TagRepository.kt new file mode 100644 index 0000000..fd15dc6 --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/domain/document/repository/TagRepository.kt @@ -0,0 +1,11 @@ +package simplerag.ragback.domain.document.repository + +import org.springframework.data.jpa.repository.JpaRepository +import simplerag.ragback.domain.document.entity.Tag + +interface TagRepository : JpaRepository { + + fun findByName(name: String): Tag? + + fun findByNameIn(names: Collection): List +} diff --git a/src/main/kotlin/simplerag/ragback/domain/document/service/DataFileService.kt b/src/main/kotlin/simplerag/ragback/domain/document/service/DataFileService.kt new file mode 100644 index 0000000..bcde47d --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/domain/document/service/DataFileService.kt @@ -0,0 +1,133 @@ +package simplerag.ragback.domain.document.service + +import org.springframework.dao.DataIntegrityViolationException +import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional +import org.springframework.transaction.support.TransactionSynchronization +import org.springframework.transaction.support.TransactionSynchronizationManager +import org.springframework.web.multipart.MultipartFile +import simplerag.ragback.domain.document.dto.DataFileBulkCreateRequest +import simplerag.ragback.domain.document.dto.DataFilePreviewResponse +import simplerag.ragback.domain.document.dto.DataFileResponseList +import simplerag.ragback.domain.document.entity.DataFile +import simplerag.ragback.domain.document.entity.DataFileTag +import simplerag.ragback.domain.document.entity.Tag +import simplerag.ragback.domain.document.repository.DataFileRepository +import simplerag.ragback.domain.document.repository.DataFileTagRepository +import simplerag.ragback.domain.document.repository.TagRepository +import simplerag.ragback.global.error.CustomException +import simplerag.ragback.global.error.ErrorCode +import simplerag.ragback.global.error.FileException +import simplerag.ragback.global.util.S3Type +import simplerag.ragback.global.util.S3Util +import simplerag.ragback.global.util.computeMetricsStreaming +import simplerag.ragback.global.util.resolveContentType +import java.time.LocalDateTime +import java.util.* + +@Service +class DataFileService( + private val dataFileRepository: DataFileRepository, + private val tagRepository: TagRepository, + private val dataFileTagRepository: DataFileTagRepository, + private val s3Util: S3Util, +) { + + @Transactional + fun upload( + files: List, + req: DataFileBulkCreateRequest + ): DataFileResponseList { + if (files.isEmpty() || files.size != req.items.size) { + throw CustomException(ErrorCode.INVALID_INPUT) + } + + val now = LocalDateTime.now() + val uploadedUrls = mutableListOf() + + registerRollbackCleanup(uploadedUrls) + + val responses = files.mapIndexed { idx, file -> + val meta = req.items[idx] + + val metrics = file.computeMetricsStreaming() + val sha256 = metrics.sha256 + val sizeBytes = metrics.sizeByte + val type = file.resolveContentType() + + if (dataFileRepository.existsBySha256(sha256)) { + throw FileException(ErrorCode.ALREADY_FILE, sha256) + } + + val fileUrl = s3Util.upload(file, S3Type.ORIGINAL_FILE) + uploadedUrls += fileUrl + + val dataFile = try { + dataFileRepository.save(DataFile(meta.title, type, sizeBytes, sha256, fileUrl, now, now)) + } catch (ex: DataIntegrityViolationException) { + throw FileException(ErrorCode.ALREADY_FILE, sha256) + } + + val tags = getOrCreateTags(meta.tags) + attachTagsIfMissing(dataFile, tags) + + DataFilePreviewResponse(requireNotNull(dataFile.id), dataFile.sha256) + } + + return DataFileResponseList(responses) + } + + private fun registerRollbackCleanup(uploadedUrls: MutableList) { + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(object : TransactionSynchronization { + override fun afterCompletion(status: Int) { + if (status == TransactionSynchronization.STATUS_ROLLED_BACK) { + uploadedUrls.forEach { runCatching { s3Util.deleteByUrl(it) } } + } + } + }) + } + } + + + private fun getOrCreateTags(names: List): List { + val normalized = names + .map { it.trim().uppercase(Locale.ROOT) } + .filter { it.isNotEmpty() } + .distinct() + + if (normalized.isEmpty()) return emptyList() + + val existing = tagRepository.findByNameIn(normalized) + val existingByName = existing.associateBy { it.name } + + val toCreate = normalized + .asSequence() + .filter { it !in existingByName } + .map { Tag(name = it) } + .toList() + + val created = if (toCreate.isNotEmpty()) { + try { + tagRepository.saveAllAndFlush(toCreate) + } catch (ex: DataIntegrityViolationException) { + tagRepository.findByNameIn(toCreate.map { it.name }) + } + } else emptyList() + + return existing + created + } + + + private fun attachTagsIfMissing(dataFile: DataFile, tags: List) { + val fileId = dataFile.id ?: return + tags.forEach { tag -> + val tagId = tag.id ?: return@forEach + val exists = dataFileTagRepository.existsByDataFileIdAndTagId(fileId, tagId) + if (!exists) { + dataFileTagRepository.save(DataFileTag(tag = tag, dataFile = dataFile)) + } + } + } + +} diff --git a/src/main/kotlin/simplerag/ragback/global/config/S3Config.kt b/src/main/kotlin/simplerag/ragback/global/config/S3Config.kt new file mode 100644 index 0000000..c71b40c --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/global/config/S3Config.kt @@ -0,0 +1,71 @@ +package simplerag.ragback.global.config + +import org.springframework.beans.factory.annotation.Value +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Profile +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.S3Configuration +import software.amazon.awssdk.services.s3.presigner.S3Presigner + +@Configuration +@Profile("!test") +class S3Config( + + @Value("\${cloud.aws.region.static}") + private val region: String, + + @Value("\${cloud.aws.s3.bucket}") + val bucket: String, + + @Value("\${cloud.aws.credentials.access-key:}") + private val accessKey: String, + + @Value("\${cloud.aws.credentials.secret-key:}") + private val secretKey: String, +) { + + @Bean + fun s3Client(): S3Client { + val regionObj = Region.of(region) + + val creds: AwsCredentialsProvider = + if (accessKey.isNotBlank() && secretKey.isNotBlank()) + StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey)) + else + DefaultCredentialsProvider.create() + + val builder = S3Client.builder() + .region(regionObj) + .credentialsProvider(creds) + .serviceConfiguration( + S3Configuration.builder() + .checksumValidationEnabled(true) + .build() + ) + + return builder.build() + } + + @Bean + fun s3Presigner(): S3Presigner { + val regionObj = Region.of(region) + + val creds: AwsCredentialsProvider = + if (accessKey.isNotBlank() && secretKey.isNotBlank()) + StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey)) + else + DefaultCredentialsProvider.create() + + val builder = S3Presigner.builder() + .region(regionObj) + .credentialsProvider(creds) + + return builder.build() + } +} \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/global/config/SwaggerConfig.kt b/src/main/kotlin/simplerag/ragback/global/config/SwaggerConfig.kt new file mode 100644 index 0000000..2effd47 --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/global/config/SwaggerConfig.kt @@ -0,0 +1,38 @@ +package simplerag.ragback.global.config + +import io.swagger.v3.oas.models.Components +import io.swagger.v3.oas.models.OpenAPI +import io.swagger.v3.oas.models.info.Info +import io.swagger.v3.oas.models.security.SecurityRequirement +import io.swagger.v3.oas.models.security.SecurityScheme +import io.swagger.v3.oas.models.servers.Server +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +@Configuration +class SwaggerConfig { + @Bean + fun openAPI(): OpenAPI { + val info = Info().title("SimpleRAG API").description("SimpleRAG API 명세").version("0.0.1") + + val jwtSchemeName = "JWT TOKEN" + val securityRequirement = SecurityRequirement().addList(jwtSchemeName) + + val components = + Components() + .addSecuritySchemes( + jwtSchemeName, + SecurityScheme() + .name(jwtSchemeName) + .type(SecurityScheme.Type.HTTP) + .scheme("bearer") + .bearerFormat("JWT") + ) + + return OpenAPI() + .addServersItem(Server().url("/")) + .info(info) + .addSecurityItem(securityRequirement) + .components(components) + } +} \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/global/error/CustomException.kt b/src/main/kotlin/simplerag/ragback/global/error/CustomException.kt index 1fc0b21..723a62c 100644 --- a/src/main/kotlin/simplerag/ragback/global/error/CustomException.kt +++ b/src/main/kotlin/simplerag/ragback/global/error/CustomException.kt @@ -1,5 +1,18 @@ package simplerag.ragback.global.error -class CustomException( - val errorCode: ErrorCode -) : RuntimeException(errorCode.message) +open class CustomException( + open val errorCode: ErrorCode, + override val message: String = errorCode.message, + override val cause: Throwable? = null, +) : RuntimeException(message, cause) + +class S3Exception( + override val errorCode: ErrorCode, + override val cause: Throwable? = null, +) : CustomException(errorCode, errorCode.message, cause) + +class FileException( + override val errorCode: ErrorCode, + override val message: String, + override val cause: Throwable? = null, +) : CustomException(errorCode, message, cause) \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/global/error/ErrorCode.kt b/src/main/kotlin/simplerag/ragback/global/error/ErrorCode.kt index 7d329f4..f1a2810 100644 --- a/src/main/kotlin/simplerag/ragback/global/error/ErrorCode.kt +++ b/src/main/kotlin/simplerag/ragback/global/error/ErrorCode.kt @@ -9,5 +9,17 @@ enum class ErrorCode( ) { INVALID_INPUT(HttpStatus.BAD_REQUEST, "INVALID_INPUT", "잘못된 요청입니다."), NOT_FOUND(HttpStatus.NOT_FOUND, "NOT_FOUND", "리소스를 찾을 수 없습니다."), - INTERNAL_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, "INTERNAL_ERROR", "서버 오류가 발생했습니다.") + INTERNAL_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, "INTERNAL_ERROR", "서버 오류가 발생했습니다."), + ALREADY_FILE(HttpStatus.BAD_REQUEST, "ALREADY_FILE", "같은 내용의 파일이 이미 존재합니다."), + FILE_PART_MISSING(HttpStatus.BAD_REQUEST, "FILE_PART_MISSING", "필수 파트가 존재하지 않습니다."), + INVALID_JSON(HttpStatus.BAD_REQUEST, "INVALID_JSON", "JSON이 유효하지 않습니다."), + + // S3 + S3_OBJECT_NOT_FOUND(HttpStatus.NOT_FOUND, "S3_001", "S3 오브젝트를 찾을 수 없습니다."), + S3_UPLOAD_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, "S3_002", "S3 업로드 실패"), + S3_DELETE_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, "S3_003", "S3 삭제 실패"), + S3_INVALID_URL(HttpStatus.BAD_REQUEST, "S3_004", "유효하지 않은 S3 URL 입니다."), + S3_EMPTY_FILE(HttpStatus.BAD_REQUEST, "S3_005", "빈 파일은 업로드할 수 없습니다."), + S3_PRESIGN_FAIL(HttpStatus.INTERNAL_SERVER_ERROR, "S3_006", "프리사인 URL 발급 실패"), + S3_UNSUPPORTED_CONTENT_TYPE(HttpStatus.BAD_REQUEST, "S3_007", "지원하지 않는 Content-Type 입니다.") } diff --git a/src/main/kotlin/simplerag/ragback/global/error/GlobalExceptionHandler.kt b/src/main/kotlin/simplerag/ragback/global/error/GlobalExceptionHandler.kt index 5ef0541..22cc0b4 100644 --- a/src/main/kotlin/simplerag/ragback/global/error/GlobalExceptionHandler.kt +++ b/src/main/kotlin/simplerag/ragback/global/error/GlobalExceptionHandler.kt @@ -1,20 +1,28 @@ package simplerag.ragback.global.error +import com.fasterxml.jackson.databind.exc.InvalidFormatException +import com.fasterxml.jackson.databind.exc.InvalidNullException +import com.fasterxml.jackson.databind.exc.MismatchedInputException import jakarta.validation.ConstraintViolationException +import org.slf4j.LoggerFactory import org.springframework.http.ResponseEntity +import org.springframework.http.converter.HttpMessageNotReadableException import org.springframework.web.bind.MethodArgumentNotValidException import org.springframework.web.bind.annotation.ExceptionHandler import org.springframework.web.bind.annotation.RestControllerAdvice +import org.springframework.web.multipart.support.MissingServletRequestPartException import simplerag.ragback.global.response.ApiResponse @RestControllerAdvice class GlobalExceptionHandler { + private val log = LoggerFactory.getLogger(GlobalExceptionHandler::class.java) + @ExceptionHandler(MethodArgumentNotValidException::class) fun handleValidationException(ex: MethodArgumentNotValidException): ResponseEntity> { val message = ex.bindingResult.allErrors.first().defaultMessage ?: "잘못된 요청" return ResponseEntity - .badRequest() + .status(ErrorCode.INVALID_INPUT.status) .body(ApiResponse.fail(ErrorCode.INVALID_INPUT.code, message)) } @@ -22,10 +30,53 @@ class GlobalExceptionHandler { fun handleConstraintViolation(ex: ConstraintViolationException): ResponseEntity> { val message = ex.constraintViolations.firstOrNull()?.message ?: "잘못된 요청" return ResponseEntity - .badRequest() + .status(ErrorCode.INVALID_INPUT.status) .body(ApiResponse.fail(ErrorCode.INVALID_INPUT.code, message)) } + @ExceptionHandler(MissingServletRequestPartException::class) + fun handleMissingPart(e: MissingServletRequestPartException): ResponseEntity> { + val msg = "필수 '${e.requestPartName}' 가 없습니다." + return ResponseEntity + .status(ErrorCode.FILE_PART_MISSING.status) + .body(ApiResponse.fail(ErrorCode.FILE_PART_MISSING.code, message = msg)) + } + + + @ExceptionHandler(HttpMessageNotReadableException::class) + fun handleUnreadable(e: HttpMessageNotReadableException): ResponseEntity> { + val cause = e.cause + val msg = when (cause) { + is InvalidNullException -> { + val field = cause.path.lastOrNull()?.fieldName ?: "unknown" + "'$field' 값이 비어있습니다." + } + + is InvalidFormatException -> { + val field = cause.path.lastOrNull()?.fieldName ?: "unknown" + "'$field' 값 형식이 올바르지 않습니다." + } + + is MismatchedInputException -> { + val field = cause.path.lastOrNull()?.fieldName ?: "unknown" + "'$field' 값 타입이 올바르지 않습니다." + } + + else -> "유효하지 않은 요청입니다." + } + return ResponseEntity + .status(ErrorCode.INVALID_JSON.status) + .body(ApiResponse.fail(ErrorCode.INVALID_JSON.code, message = msg)) + } + + @ExceptionHandler(FileException::class) + fun handleFileException(ex: FileException): ResponseEntity> { + val errorCode = ex.errorCode + return ResponseEntity + .status(errorCode.status) + .body(ApiResponse.fail(errorCode.code, "${errorCode.message} sha256: ${ex.message}")) + } + @ExceptionHandler(CustomException::class) fun handleCustomException(ex: CustomException): ResponseEntity> { val errorCode = ex.errorCode @@ -36,6 +87,7 @@ class GlobalExceptionHandler { @ExceptionHandler(Exception::class) fun handleGeneralException(ex: Exception): ResponseEntity> { + log.error("Unhandled exception", ex) return ResponseEntity .status(ErrorCode.INTERNAL_ERROR.status) .body(ApiResponse.fail(ErrorCode.INTERNAL_ERROR.code, ErrorCode.INTERNAL_ERROR.message)) diff --git a/src/main/kotlin/simplerag/ragback/global/storage/FakeS3Util.kt b/src/main/kotlin/simplerag/ragback/global/storage/FakeS3Util.kt new file mode 100644 index 0000000..c701405 --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/global/storage/FakeS3Util.kt @@ -0,0 +1,53 @@ +package simplerag.ragback.global.storage + +import org.springframework.context.annotation.Primary +import org.springframework.context.annotation.Profile +import org.springframework.stereotype.Component +import org.springframework.web.multipart.MultipartFile +import simplerag.ragback.global.util.S3Type +import simplerag.ragback.global.util.S3Util +import simplerag.ragback.global.util.sha256Hex +import java.util.concurrent.ConcurrentHashMap + +@Component +@Primary +@Profile("test") +class FakeS3Util : S3Util { + + private val store = ConcurrentHashMap() + + override fun upload(file: MultipartFile, dir: S3Type): String { + val clean = (file.originalFilename ?: "file") + .substringAfterLast('/') + .substringAfterLast('\\') + .ifBlank { "file" } + + val hash = sha256Hex(file.bytes).take(12) + val prefix = dir.label.trim('/') + val key = "$prefix/${hash}_$clean" + + store[key] = file.bytes + return urlFromKey(key) + } + + override fun urlFromKey(key: String): String = "fake://$key" + + override fun deleteByUrl(url: String) { + val key = keyFromUrl(url) ?: throw simplerag.ragback.global.error.S3Exception( + simplerag.ragback.global.error.ErrorCode.S3_INVALID_URL + ) + store.remove(key) + } + + override fun delete(key: String) { + store.remove(key) + } + + override fun keyFromUrl(url: String): String? = url.removePrefix("fake://") + .removePrefix("/") + .ifBlank { null } + + // 테스트 용 함수 + fun exists(url: String): Boolean = keyFromUrl(url)?.let { store.containsKey(it) } == true + fun count(): Int = store.size +} diff --git a/src/main/kotlin/simplerag/ragback/global/util/FileConvertUtil.kt b/src/main/kotlin/simplerag/ragback/global/util/FileConvertUtil.kt new file mode 100644 index 0000000..ca50c62 --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/global/util/FileConvertUtil.kt @@ -0,0 +1,57 @@ +package simplerag.ragback.global.util + +import org.springframework.web.multipart.MultipartFile +import java.io.BufferedInputStream +import java.security.DigestInputStream +import java.security.MessageDigest + +data class FileMetrics( + val sha256: String, + val sizeByte: Long +) + +fun sha256Hex(bytes: ByteArray): String = + MessageDigest.getInstance("SHA-256") + .digest(bytes) + .joinToString("") { "%02x".format(it) } + +fun MultipartFile.resolveContentType(): String { + if (!this.contentType.isNullOrBlank()) return this.contentType!! + val ext = this.originalFilename?.substringAfterLast('.', "")?.lowercase() + return when (ext) { + "png" -> "image/png" + "jpg", "jpeg" -> "image/jpeg" + "pdf" -> "application/pdf" + "txt" -> "text/plain" + "csv" -> "text/csv" + "md" -> "text/markdown" + "json" -> "application/json" + "zip" -> "application/zip" + "doc" -> "application/msword" + "docx" -> "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + "xls" -> "application/vnd.ms-excel" + "xlsx" -> "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + "ppt" -> "application/vnd.ms-powerpoint" + "pptx" -> "application/vnd.openxmlformats-officedocument.presentationml.presentation" + else -> "application/octet-stream" + } +} + +fun MultipartFile.computeMetricsStreaming(): FileMetrics { + val digest = MessageDigest.getInstance("SHA-256") + var totalBytes = 0L + + inputStream.use { input -> + DigestInputStream(BufferedInputStream(input), digest).use { digestStream -> + val buffer = ByteArray(8192) // 8KB buffer + var bytesRead: Int + + while (digestStream.read(buffer).also { bytesRead = it } != -1) { + totalBytes += bytesRead + } + } + } + + val sha256 = digest.digest().joinToString("") { "%02x".format(it) } + return FileMetrics(sha256, totalBytes) +} \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/global/util/MultipartJackson2HttpMessageConverter.kt b/src/main/kotlin/simplerag/ragback/global/util/MultipartJackson2HttpMessageConverter.kt new file mode 100644 index 0000000..9e276dd --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/global/util/MultipartJackson2HttpMessageConverter.kt @@ -0,0 +1,25 @@ +package simplerag.ragback.global.util + +import com.fasterxml.jackson.databind.ObjectMapper +import org.springframework.http.MediaType +import org.springframework.http.converter.json.AbstractJackson2HttpMessageConverter +import org.springframework.stereotype.Component +import java.lang.reflect.Type + +@Component +class MultipartJackson2HttpMessageConverter + (objectMapper: ObjectMapper) : + AbstractJackson2HttpMessageConverter(objectMapper, MediaType.APPLICATION_OCTET_STREAM) { + + override fun canWrite(clazz: Class<*>, mediaType: MediaType?): Boolean { + return false + } + + override fun canWrite(type: Type?, clazz: Class<*>, mediaType: MediaType?): Boolean { + return false + } + + override fun canWrite(mediaType: MediaType?): Boolean { + return false + } +} \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/global/util/S3Type.kt b/src/main/kotlin/simplerag/ragback/global/util/S3Type.kt new file mode 100644 index 0000000..f7136a3 --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/global/util/S3Type.kt @@ -0,0 +1,8 @@ +package simplerag.ragback.global.util + + +enum class S3Type( + val label: String, +) { + ORIGINAL_FILE("/ORIGINAL/"), +} \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/global/util/S3Util.kt b/src/main/kotlin/simplerag/ragback/global/util/S3Util.kt new file mode 100644 index 0000000..e1aba27 --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/global/util/S3Util.kt @@ -0,0 +1,11 @@ +package simplerag.ragback.global.util + +import org.springframework.web.multipart.MultipartFile + +interface S3Util { + fun upload(file: MultipartFile, dir: S3Type): String + fun urlFromKey(key: String): String + fun deleteByUrl(url: String) + fun delete(key: String) + fun keyFromUrl(url: String): String? +} \ No newline at end of file diff --git a/src/main/kotlin/simplerag/ragback/global/util/S3UtilImpl.kt b/src/main/kotlin/simplerag/ragback/global/util/S3UtilImpl.kt new file mode 100644 index 0000000..c657152 --- /dev/null +++ b/src/main/kotlin/simplerag/ragback/global/util/S3UtilImpl.kt @@ -0,0 +1,110 @@ +package simplerag.ragback.global.util + +import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Profile +import org.springframework.stereotype.Component +import org.springframework.web.multipart.MultipartFile +import simplerag.ragback.global.config.S3Config +import simplerag.ragback.global.error.ErrorCode +import simplerag.ragback.global.error.GlobalExceptionHandler +import simplerag.ragback.global.error.S3Exception +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest +import software.amazon.awssdk.services.s3.model.PutObjectRequest +import java.net.URI +import java.util.* + +@Component +@Profile("!test") +class S3UtilImpl( + private val s3: S3Client, + private val s3Config: S3Config, +) : S3Util { + + private val bucket get() = s3Config.bucket + private val log = LoggerFactory.getLogger(GlobalExceptionHandler::class.java) + + override fun upload(file: MultipartFile, dir: S3Type): String { + if (file.isEmpty) throw S3Exception(ErrorCode.S3_EMPTY_FILE) + + val key = buildKey(dir.label, file.originalFilename) + val contentType = file.contentType ?: "application/octet-stream" + + try { + file.inputStream.use { input -> + val putReq = PutObjectRequest.builder() + .bucket(bucket) + .key(key) + .contentType(contentType) + .build() + + val body = RequestBody.fromInputStream(input, file.size) + s3.putObject(putReq, body) + } + + return urlFromKey(key) + } catch (e: software.amazon.awssdk.services.s3.model.S3Exception) { + log.error( + "S3 putObject fail bucket={}, key={}, status={}, awsCode={}, reqId={}, msg={}", + bucket, key, e.statusCode(), e.awsErrorDetails()?.errorCode(), e.requestId(), + e.awsErrorDetails()?.errorMessage(), e + ) + throw S3Exception(ErrorCode.S3_UPLOAD_FAIL) + } catch (e: Exception) { + log.error(e.message, e) + throw S3Exception(ErrorCode.S3_UPLOAD_FAIL) + } + } + + override fun urlFromKey(key: String): String = + s3.utilities() + .getUrl { it.bucket(bucket).key(key) } + .toExternalForm() + + override fun deleteByUrl(url: String) { + val key = keyFromUrl(url) ?: throw S3Exception(ErrorCode.S3_INVALID_URL) + delete(key) + } + + override fun delete(key: String) { + try { + val req = DeleteObjectRequest.builder() + .bucket(bucket) + .key(key) + .build() + s3.deleteObject(req) + } catch (e: software.amazon.awssdk.services.s3.model.S3Exception) { + // NoSuchKey 등 + throw S3Exception(ErrorCode.S3_OBJECT_NOT_FOUND) + } catch (e: Exception) { + throw S3Exception(ErrorCode.S3_DELETE_FAIL) + } + } + + private fun buildKey(dir: String, originalFilename: String?): String { + val cleanName = (originalFilename ?: "file") + .substringAfterLast('/') + .substringAfterLast('\\') + .ifBlank { "file" } + + val prefix = dir.trim('/') + + val key = if (prefix.isBlank()) { + "${UUID.randomUUID()}_$cleanName" + } else { + "$prefix/${UUID.randomUUID()}_$cleanName" + } + + return key + } + + override fun keyFromUrl(url: String): String? { + val path = try { + URI(url).path // e.g. "/market/menu/uuid_name.jpg" + } catch (_: Exception) { + return null + } + return path.removePrefix("/").ifBlank { null } + } +} diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml new file mode 100644 index 0000000..5019101 --- /dev/null +++ b/src/main/resources/application-local.yml @@ -0,0 +1,26 @@ +spring: + datasource: + url: ${DB_URL_LOCAL} + username: ${DB_USERNAME_LOCAL} + password: ${DB_PASSWORD_LOCAL} + driver-class-name: org.postgresql.Driver + jpa: + hibernate: + ddl-auto: update + show-sql: true + +logging: + level: + root: INFO + org.hibernate.SQL: DEBUG + org.hibernate.type.descriptor.sql.BasicBinder: TRACE + +cloud: + aws: + region: + static: ${REGION_LOCAL} + s3: + bucket: ${BUCKET_LOCAL} + credentials: + access-key: ${AWS_ACCESS_KEY_ID_LOCAL} + secret-key: ${AWS_SECRET_ACCESS_KEY_LOCAL} \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 081211d..a403076 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -4,24 +4,15 @@ server: spring: application: name: rag-backend - datasource: - url: ${DB_URL} - username: ${DB_USERNAME} - password: - driver-class-name: org.postgresql.Driver + servlet: + multipart: + max-file-size: 50MB + max-request-size: 200MB + jackson: + time-zone: Asia/Seoul + jpa: - hibernate: - ddl-auto: update properties: hibernate: dialect: org.hibernate.dialect.PostgreSQLDialect format_sql: true - show_sql: true - jackson: - time-zone: Asia/Seoul - -logging: - level: - root: INFO - org.hibernate.SQL: DEBUG - org.hibernate.type.descriptor.sql.BasicBinder: TRACE \ No newline at end of file diff --git a/src/test/kotlin/simplerag/ragback/domain/document/service/DataFileServiceTest.kt b/src/test/kotlin/simplerag/ragback/domain/document/service/DataFileServiceTest.kt new file mode 100644 index 0000000..2d58831 --- /dev/null +++ b/src/test/kotlin/simplerag/ragback/domain/document/service/DataFileServiceTest.kt @@ -0,0 +1,224 @@ +package simplerag.ragback.domain.document.service + +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.mock.web.MockMultipartFile +import org.springframework.test.context.ActiveProfiles +import org.springframework.transaction.annotation.Transactional +import org.springframework.transaction.support.TransactionTemplate +import org.springframework.web.multipart.MultipartFile +import simplerag.ragback.domain.document.dto.DataFileBulkCreateRequest +import simplerag.ragback.domain.document.dto.DataFileCreateItem +import simplerag.ragback.domain.document.entity.DataFile +import simplerag.ragback.domain.document.repository.DataFileRepository +import simplerag.ragback.domain.document.repository.DataFileTagRepository +import simplerag.ragback.domain.document.repository.TagRepository +import simplerag.ragback.global.error.CustomException +import simplerag.ragback.global.error.ErrorCode +import simplerag.ragback.global.error.FileException +import simplerag.ragback.global.storage.FakeS3Util +import simplerag.ragback.global.util.S3Type +import simplerag.ragback.global.util.sha256Hex +import java.security.MessageDigest +import java.time.LocalDateTime + +@SpringBootTest +@ActiveProfiles("test") +class DataFileServiceTest( + @Autowired private val dataFileService: DataFileService, + @Autowired private val dataFileRepository: DataFileRepository, + @Autowired private val tagRepository: TagRepository, + @Autowired private val dataFileTagRepository: DataFileTagRepository, + @Autowired private val s3Util: FakeS3Util +) { + + @Autowired + lateinit var txManager: org.springframework.transaction.PlatformTransactionManager + + private fun txTemplate() = TransactionTemplate(txManager) + + @Test + @Transactional + @DisplayName("업로드 시 잘 저장이 된다.") + fun uploadOk() { + // given + val bytes = "hello world".toByteArray() + val req = DataFileBulkCreateRequest( + listOf(DataFileCreateItem(title = "greeting", tags = listOf(" ai ", "RAG", "ai"))) + ) + val f = file("greet.txt", bytes, contentType = "text/plain") + + // when + val res = dataFileService.upload(listOf(f), req) + + // then + assertEquals(1, res.dataFilePreviewResponseList.size) + val r0 = res.dataFilePreviewResponseList.first() + assertTrue(r0.id > 0) + assertEquals(sha256Hex(bytes), r0.sha256) + + val saved = dataFileRepository.findById(r0.id).orElseThrow() + assertEquals("greeting", saved.title) + assertEquals("text/plain", saved.type) + assertEquals(sha256Hex(bytes), saved.sha256) + assertFalse(saved.fileUrl.isNullOrBlank()) + + val ai = tagRepository.findByName("AI") + val rag = tagRepository.findByName("RAG") + assertNotNull(ai); assertNotNull(rag) + assertTrue(dataFileTagRepository.existsByDataFileIdAndTagId(saved.id!!, ai!!.id!!)) + assertTrue(dataFileTagRepository.existsByDataFileIdAndTagId(saved.id!!, rag!!.id!!)) + } + + @Test + @DisplayName("files와 items 개수 불일치 시 예외를 던진다.") + fun invalidInputCountMismatch() { + // given + val req = DataFileBulkCreateRequest(items = emptyList()) + val f = file("a.txt", "a".toByteArray()) + + // when + val ex = assertThrows(CustomException::class.java) { dataFileService.upload(listOf(f), req) } + + // then + assertEquals(ErrorCode.INVALID_INPUT, ex.errorCode) + } + + @Test + @Transactional + @DisplayName("동일 sha256이 이미 있으면 업로드 거부한다.") + fun rejectDuplicateSha() { + // given + val bytes = "same".toByteArray() + val sha = sha256Hex(bytes) + val now = LocalDateTime.now() + dataFileRepository.save( + DataFile( + title = "exists", + type = "text/plain", + sizeBytes = 0, + sha256 = sha, + fileUrl = "fake://original/exists.txt", + updatedAt = now, + createdAt = now + ) + ) + val req = DataFileBulkCreateRequest(listOf(DataFileCreateItem("dup", listOf("tag")))) + val f = file("dup.txt", bytes) + + // when + val ex = assertThrows(FileException::class.java) { dataFileService.upload(listOf(f), req) } + + // then + assertEquals(ErrorCode.ALREADY_FILE, ex.errorCode) + } + + @Test + @Transactional + @DisplayName("컨텐츠 타입 null이거나 확장자 없을 시 application/octet-stream 저장이 된다") + fun unknownTypeOctetStream() { + // given + val bytes = "x".toByteArray() + val req = DataFileBulkCreateRequest(listOf(DataFileCreateItem("noext", emptyList()))) + val f = file(name = "noext", content = bytes, contentType = null) // no extension + + // when + val res = dataFileService.upload(listOf(f), req) + + // then + val saved = dataFileRepository.findById(res.dataFilePreviewResponseList.first().id).orElseThrow() + assertEquals("application/octet-stream", saved.type) + } + + @Test + @Transactional + @DisplayName("같은 파일 두 번 업로드 시 에러가 난다.") + fun secondUploadAlreadyFile() { + // given + val bytes = "dupcontent".toByteArray() + val req = DataFileBulkCreateRequest(listOf(DataFileCreateItem("t", emptyList()))) + val f1 = file("x.txt", bytes) + val f2 = file("y.txt", bytes) + + // when + dataFileService.upload(listOf(f1), req) + val ex = assertThrows(FileException::class.java) { dataFileService.upload(listOf(f2), req) } + + // then + assertEquals(ErrorCode.ALREADY_FILE, ex.errorCode) + } + + @Test + @DisplayName("트랜잭션이 커밋되면 DB와 S3에 정상 저장된다") + fun uploadCommitPersist() { + // given + val bytes = "commit-case".toByteArray() + val req = DataFileBulkCreateRequest( + listOf(DataFileCreateItem(title = "commit-title", tags = listOf("t1"))) + ) + val f = file("c.txt", bytes) + + // when + val resultIds = txTemplate().execute { + val res = dataFileService.upload(listOf(f), req) + res.dataFilePreviewResponseList.map { it.id } + }!! + + // then (DB) + assertEquals(1, resultIds.size) + val saved = dataFileRepository.findById(resultIds.first()).orElseThrow() + assertEquals("commit-title", saved.title) + assertEquals(sha256Hex(bytes), saved.sha256) + assertFalse(saved.fileUrl.isNullOrBlank()) + + // then (S3 - FakeS3Util 기준) + assertTrue(s3Util.exists(saved.fileUrl!!), "커밋 시 S3에 파일이 존재해야 합니다") + } + + + @Test + @DisplayName("파일 업로드 중 트랜잭션이 롤백되면 DB와 S3에서 모두 정리된다") + fun uploadRollbackCleansDBandS3() { + // given + val bytes = "rollback-case".toByteArray() + val filename = "r.txt" + val req = DataFileBulkCreateRequest( + listOf(DataFileCreateItem(title = "rollback-title", tags = listOf("t2"))) + ) + val f = file(filename, bytes) + + val hash12 = MessageDigest.getInstance("SHA-256") + .digest(bytes).joinToString("") { "%02x".format(it) } + .take(12) + val expectedKey = "${S3Type.ORIGINAL_FILE.label}/${hash12}_$filename" + val expectedUrl = s3Util.urlFromKey(expectedKey) + + // when: 트랜잭션 내에서 업로드 후 강제 롤백 + txTemplate().execute { status -> + dataFileService.upload(listOf(f), req) + status!!.setRollbackOnly() + } + + // then + val sha = sha256Hex(bytes) + val existsInDb = dataFileRepository.findAll().any { it.sha256 == sha } + assertFalse(existsInDb, "롤백되었으므로 DB에 남으면 안 됩니다") + + assertFalse(s3Util.exists(expectedUrl), "롤백 시 S3도 보상 삭제되어야 합니다") + } + + // ----------------------- + // helpers + // ----------------------- + + private fun file( + name: String, + content: ByteArray, + contentType: String? = "text/plain", + paramName: String = "files" + ): MultipartFile = + MockMultipartFile(paramName, name, contentType, content) +} diff --git a/src/test/resources/application-test.yml b/src/test/resources/application-test.yml new file mode 100644 index 0000000..5885995 --- /dev/null +++ b/src/test/resources/application-test.yml @@ -0,0 +1,10 @@ +spring: + datasource: + url: jdbc:h2:mem:simplerag;MODE=PostgreSQL;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE + driver-class-name: org.h2.Driver + username: sa + password: + jpa: + database-platform: org.hibernate.dialect.H2Dialect + hibernate: + ddl-auto: create-drop