From d9877ee2688d63ab7452ee10bf86b4c6d8f141e4 Mon Sep 17 00:00:00 2001 From: longjuan <769022681@qq.com> Date: Sat, 25 May 2024 21:46:33 +0800 Subject: [PATCH 1/3] faet: Add file prefix search function in s3link --- ...sS3OsHaloRunV1Alpha1ObjectsByPolicyName.ts | 2 ++ console/src/utils/request.ts | 4 ++- console/src/views/S3Link.vue | 18 ++++++++++++- .../java/run/halo/s3os/S3LinkController.java | 8 +++--- .../java/run/halo/s3os/S3LinkService.java | 4 +-- .../java/run/halo/s3os/S3LinkServiceImpl.java | 27 +++++++++++++++---- 6 files changed, 50 insertions(+), 13 deletions(-) diff --git a/console/src/controller/s-3-link-controller/getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName.ts b/console/src/controller/s-3-link-controller/getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName.ts index 5c346f4..fe3890c 100644 --- a/console/src/controller/s-3-link-controller/getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName.ts +++ b/console/src/controller/s-3-link-controller/getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName.ts @@ -10,6 +10,7 @@ export function getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName(params: GetApisS3O continuationObject: params.continuationObject, pageSize: params.pageSize, unlinked: params.unlinked, + filePrefix: params.filePrefix, }; return request.get>(`/apis/s3os.halo.run/v1alpha1/objects/${params.policyName}`, { params: paramsInput, @@ -22,4 +23,5 @@ interface GetApisS3OsHaloRunV1Alpha1ObjectsByPolicyNameParams { continuationObject?: any; pageSize: any; unlinked?: any; + filePrefix?: any; } diff --git a/console/src/utils/request.ts b/console/src/utils/request.ts index 1a6877a..f765aac 100644 --- a/console/src/utils/request.ts +++ b/console/src/utils/request.ts @@ -18,7 +18,9 @@ request.interceptors.response.use( return Promise.reject(error); } const { status } = errorResponse; - if (status !== 200) { + if (status === 400) { + Toast.error(errorResponse.data.detail); + } else if (status !== 200) { Toast.error("status: " + status); } return Promise.reject(error); diff --git a/console/src/views/S3Link.vue b/console/src/views/S3Link.vue index c788ccd..3de06ee 100644 --- a/console/src/views/S3Link.vue +++ b/console/src/views/S3Link.vue @@ -14,6 +14,7 @@ import { VTag, } from "@halo-dev/components"; import CarbonFolderDetailsReference from "~icons/carbon/folder-details-reference"; +import IconErrorWarning from "~icons/ri/error-warning-line"; import {computed, onMounted, ref, watch} from "vue"; import { getApisS3OsHaloRunV1Alpha1ObjectsByPolicyName, @@ -31,6 +32,10 @@ const policyOptions = ref<{ label: string; value: string; attrs: any }[]>([{ value: "", attrs: {disabled: true} }]); +// update when fetch first page +const filePrefix = ref(""); +// update when user input +const filePrefixBind = ref(""); const s3Objects = ref({ objects: [], hasMore: false, @@ -139,6 +144,8 @@ const clearTokenAndObject = () => { s3Objects.value.nextContinuationObject = ""; }; +// filePrefix will not be updated from user input +// if you want to update filePrefix, please call `handleFirstPage` const fetchObjects = async () => { if (!policyName.value) { return; @@ -152,6 +159,7 @@ const fetchObjects = async () => { continuationToken: s3Objects.value.currentToken, continuationObject: s3Objects.value.currentContinuationObject, unlinked: selectedLinkedStatusItem.value, + filePrefix: filePrefix.value }); if (objectsData.status == 200) { s3Objects.value = objectsData.data; @@ -222,6 +230,7 @@ const handleFirstPage = () => { isFetching.value = true; page.value = 1; clearTokenAndObject(); + filePrefix.value = filePrefixBind.value; fetchObjects(); }; @@ -266,8 +275,15 @@ const handleModalClose = () => { name="policyName" type="select" :options="policyOptions" - @change="fetchObjects()" + @change="handleFirstPage" > + + diff --git a/src/main/java/run/halo/s3os/S3LinkController.java b/src/main/java/run/halo/s3os/S3LinkController.java index 2600a54..0baa5ee 100644 --- a/src/main/java/run/halo/s3os/S3LinkController.java +++ b/src/main/java/run/halo/s3os/S3LinkController.java @@ -28,13 +28,13 @@ public Mono listObjects(@PathVariable(value = "policyName") String @RequestParam(name = "continuationToken", required = false) String continuationToken, @RequestParam(name = "continuationObject", required = false) String continuationObject, @RequestParam(name = "pageSize") Integer pageSize, - @RequestParam(name = "unlinked", required = false, defaultValue = "false") - Boolean unlinked) { + @RequestParam(name = "unlinked", required = false, defaultValue = "false") Boolean unlinked, + @RequestParam(name = "filePrefix", required = false) String filePrefix) { if (unlinked) { return s3LinkService.listObjectsUnlinked(policyName, continuationToken, - continuationObject, pageSize); + continuationObject, pageSize, filePrefix); } else { - return s3LinkService.listObjects(policyName, continuationToken, pageSize); + return s3LinkService.listObjects(policyName, continuationToken, pageSize, filePrefix); } } diff --git a/src/main/java/run/halo/s3os/S3LinkService.java b/src/main/java/run/halo/s3os/S3LinkService.java index c6b94eb..2a1bb45 100644 --- a/src/main/java/run/halo/s3os/S3LinkService.java +++ b/src/main/java/run/halo/s3os/S3LinkService.java @@ -10,10 +10,10 @@ public interface S3LinkService { Flux listS3Policies(); Mono listObjects(String policyName, String continuationToken, - Integer pageSize); + Integer pageSize, String filePrefix); Mono addAttachmentRecords(String policyName, Set objectKeys); Mono listObjectsUnlinked(String policyName, String continuationToken, - String continuationObject, Integer pageSize); + String continuationObject, Integer pageSize, String filePrefix); } diff --git a/src/main/java/run/halo/s3os/S3LinkServiceImpl.java b/src/main/java/run/halo/s3os/S3LinkServiceImpl.java index 6b59ed6..f415d6c 100644 --- a/src/main/java/run/halo/s3os/S3LinkServiceImpl.java +++ b/src/main/java/run/halo/s3os/S3LinkServiceImpl.java @@ -58,7 +58,7 @@ public Flux listS3Policies() { @Override public Mono listObjects(String policyName, String continuationToken, - Integer pageSize) { + Integer pageSize, String filePrefix) { return client.fetch(Policy.class, policyName) .flatMap((policy) -> { var configMapName = policy.getSpec().getConfigMapName(); @@ -68,11 +68,11 @@ public Mono listObjects(String policyName, String continuationToke var properties = handler.getProperties(configMap); var finalLocation = FilePathUtils.getFilePathByPlaceholder(properties.getLocation()); return Mono.using(() -> handler.buildS3Client(properties), + // 执行 listObjects (s3Client) -> Mono.fromCallable( () -> s3Client.listObjectsV2(ListObjectsV2Request.builder() .bucket(properties.getBucket()) - .prefix(StringUtils.isNotEmpty(finalLocation) - ? finalLocation + "/" : null) + .prefix(buildPrefix(finalLocation, filePrefix)) .delimiter("/") .maxKeys(pageSize) .continuationToken(StringUtils.isNotEmpty(continuationToken) @@ -81,10 +81,12 @@ public Mono listObjects(String policyName, String continuationToke S3Client::close) .flatMap(listObjectsV2Response -> { List contents = listObjectsV2Response.contents(); + // 过滤掉目录并转换为ObjectVo var objectVos = contents .stream().map(S3ListResult.ObjectVo::fromS3Object) .filter(objectVo -> !objectVo.getKey().endsWith("/")) .collect(Collectors.toMap(S3ListResult.ObjectVo::getKey, o -> o)); + // 获取已经关联的附件并标记 ListOptions listOptions = new ListOptions(); listOptions.setFieldSelector( FieldSelector.of(QueryFactory.equal("spec.policyName", policyName))); @@ -163,7 +165,7 @@ private Flux getLinkResultItems(Set objectKey @Override public Mono listObjectsUnlinked(String policyName, String continuationToken, - String continuationObject, Integer pageSize) { + String continuationObject, Integer pageSize, String filePrefix) { // TODO 优化成查一次数据库 return Mono.defer(() -> { List s3Objects = new ArrayList<>(); @@ -172,7 +174,8 @@ public Mono listObjectsUnlinked(String policyName, String continua return Flux.defer(() -> Flux.just( new TokenState(null, currToken.get() == null ? "" : currToken.get()))) - .flatMap(tokenState -> listObjects(policyName, tokenState.nextToken, pageSize)) + .flatMap(tokenState -> listObjects(policyName, tokenState.nextToken, + pageSize, filePrefix)) .flatMap(s3ListResult -> { var filteredObjects = s3ListResult.getObjects(); if (!continuationObjectMatched.get()) { @@ -267,4 +270,18 @@ private Mono authenticationConsumer(Function> fun .flatMap(func); } + public String buildPrefix(String finalLocation, String filePrefix) { + if (StringUtils.isBlank(finalLocation) && StringUtils.isBlank(filePrefix)) { + return null; + } + StringBuilder sb = new StringBuilder(); + if (StringUtils.isNotBlank(finalLocation)) { + sb.append(finalLocation).append("/"); + } + if (StringUtils.isNotBlank(filePrefix)) { + sb.append(filePrefix); + } + return sb.toString(); + } + } From 20ffb6ee437b3a3ba5158d6a0c148b68f6e5801c Mon Sep 17 00:00:00 2001 From: longjuan <769022681@qq.com> Date: Sat, 25 May 2024 21:55:54 +0800 Subject: [PATCH 2/3] Set buildPrefix from public to default --- src/main/java/run/halo/s3os/S3LinkServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/run/halo/s3os/S3LinkServiceImpl.java b/src/main/java/run/halo/s3os/S3LinkServiceImpl.java index f415d6c..552a3ae 100644 --- a/src/main/java/run/halo/s3os/S3LinkServiceImpl.java +++ b/src/main/java/run/halo/s3os/S3LinkServiceImpl.java @@ -270,7 +270,7 @@ private Mono authenticationConsumer(Function> fun .flatMap(func); } - public String buildPrefix(String finalLocation, String filePrefix) { + String buildPrefix(String finalLocation, String filePrefix) { if (StringUtils.isBlank(finalLocation) && StringUtils.isBlank(filePrefix)) { return null; } From 8662cdbdaf549d6291c53526d9b7bb0e345503ce Mon Sep 17 00:00:00 2001 From: longjuan <769022681@qq.com> Date: Sat, 25 May 2024 23:22:13 +0800 Subject: [PATCH 3/3] wrap style --- console/src/views/S3Link.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/console/src/views/S3Link.vue b/console/src/views/S3Link.vue index 3de06ee..ebb5689 100644 --- a/console/src/views/S3Link.vue +++ b/console/src/views/S3Link.vue @@ -264,9 +264,9 @@ const handleModalClose = () => {
- 存储策略: + 存储策略: