From 5b01f4dccb8b933ec5ef9e2a80bde52cbdb0d6df Mon Sep 17 00:00:00 2001 From: yantian Date: Thu, 26 Dec 2024 17:48:34 +0800 Subject: [PATCH 01/17] [core] Support partition API: createPartition --- .../org/apache/paimon/rest/RESTCatalog.java | 16 ++++- .../org/apache/paimon/rest/ResourcePaths.java | 11 ++++ .../rest/requests/CreatePartitionRequest.java | 61 +++++++++++++++++++ .../responses/ListPartitionsResponse.java | 27 ++++++++ 4 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index e0b3dccdd56b..dd176185d221 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -41,6 +41,7 @@ import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.requests.CreatePartitionRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; @@ -49,6 +50,7 @@ import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetTableResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; +import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; @@ -360,7 +362,19 @@ public void dropTable(Identifier identifier, boolean ignoreIfNotExists) @Override public void createPartition(Identifier identifier, Map partitionSpec) throws TableNotExistException { - throw new UnsupportedOperationException(); + try { + CreatePartitionRequest request = new CreatePartitionRequest(identifier, partitionSpec); + client.post( + resourcePaths.partitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + ListPartitionsResponse.class, + headers()); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } catch (ForbiddenException e) { + throw new TableNoPermissionException(identifier, e); + } } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java index f006713fe2b6..ebfdd2db1eec 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -82,4 +82,15 @@ public String renameTable(String databaseName, String tableName) { .add("rename") .toString(); } + + public String partitions(String databaseName, String tableName) { + return SLASH.add("v1") + .add(prefix) + .add("databases") + .add(databaseName) + .add("tables") + .add(tableName) + .add("partitions") + .toString(); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java new file mode 100644 index 000000000000..094274514029 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; + +/** Request for creating partition. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class CreatePartitionRequest implements RESTRequest { + + private static final String FIELD_IDENTIFIER = "identifier"; + private static final String FIELD_PARTITION_SPEC = "partitionSpec"; + + @JsonProperty(FIELD_IDENTIFIER) + private final Identifier identifier; + + @JsonProperty(FIELD_PARTITION_SPEC) + private final Map partitionSpec; + + @JsonCreator + public CreatePartitionRequest( + @JsonProperty(FIELD_IDENTIFIER) Identifier identifier, + @JsonProperty(FIELD_PARTITION_SPEC) Map partitionSpec) { + this.identifier = identifier; + this.partitionSpec = partitionSpec; + } + + @JsonGetter(FIELD_IDENTIFIER) + public Identifier getIdentifier() { + return identifier; + } + + @JsonGetter(FIELD_PARTITION_SPEC) + public Map getPartitionSpec() { + return partitionSpec; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java new file mode 100644 index 000000000000..b0943a3fcc30 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** Response for listing partitions. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ListPartitionsResponse implements RESTResponse {} From 541d1184226988a119cc345e7a20ebac83bc203a Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 27 Dec 2024 09:53:31 +0800 Subject: [PATCH 02/17] support drop partition API --- .../org/apache/paimon/rest/HttpClient.java | 17 +++++++ .../org/apache/paimon/rest/RESTCatalog.java | 50 +++++++++++++++++-- .../org/apache/paimon/rest/RESTClient.java | 2 + .../rest/requests/DropPartitionRequest.java | 48 ++++++++++++++++++ .../rest/responses/SuccessResponse.java | 27 ++++++++++ 5 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/responses/SuccessResponse.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java index 87f3fad9b2fd..d92cab510201 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -94,6 +94,23 @@ public T delete(String path, Map header return exec(request, null); } + @Override + public T delete( + String path, RESTRequest body, Map headers) { + try { + RequestBody requestBody = buildRequestBody(body); + Request request = + new Request.Builder() + .url(uri + path) + .delete(requestBody) + .headers(Headers.of(headers)) + .build(); + return exec(request, null); + } catch (JsonProcessingException e) { + throw new RESTException(e, "build request failed."); + } + } + @Override public void close() throws IOException { okHttpClient.dispatcher().cancelAll(); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index dd176185d221..87b752748461 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -29,6 +29,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.operation.FileStoreCommit; import org.apache.paimon.operation.Lock; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; @@ -43,6 +44,7 @@ import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreatePartitionRequest; import org.apache.paimon.rest.requests.CreateTableRequest; +import org.apache.paimon.rest.requests.DropPartitionRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.ConfigResponse; @@ -50,8 +52,8 @@ import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetTableResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; -import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; +import org.apache.paimon.rest.responses.SuccessResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.TableSchema; @@ -60,6 +62,7 @@ import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.Table; import org.apache.paimon.table.object.ObjectTable; +import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Preconditions; @@ -73,13 +76,16 @@ import java.io.IOException; import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; +import static org.apache.paimon.CoreOptions.createCommitUser; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase; +import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable; import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE; import static org.apache.paimon.utils.Preconditions.checkNotNull; @@ -368,7 +374,7 @@ public void createPartition(Identifier identifier, Map partition resourcePaths.partitions( identifier.getDatabaseName(), identifier.getTableName()), request, - ListPartitionsResponse.class, + SuccessResponse.class, headers()); } catch (NoSuchResourceException e) { throw new TableNotExistException(identifier); @@ -379,7 +385,29 @@ public void createPartition(Identifier identifier, Map partition @Override public void dropPartition(Identifier identifier, Map partitions) - throws TableNotExistException, PartitionNotExistException {} + throws TableNotExistException, PartitionNotExistException { + checkNotSystemTable(identifier, "dropPartition"); + Table table = null; + try { + dropPartitionMetadata(identifier, partitions); + table = getTable(identifier); + } catch (Exception e) { + if (table != null) { + FileStoreTable fileStoreTable = (FileStoreTable) table; + try (FileStoreCommit commit = + fileStoreTable + .store() + .newCommit( + createCommitUser( + fileStoreTable.coreOptions().toConfiguration()))) { + commit.dropPartitions( + Collections.singletonList(partitions), + BatchWriteBuilder.COMMIT_IDENTIFIER); + } + } + throw e; + } + } @Override public List listPartitions(Identifier identifier) @@ -448,6 +476,22 @@ protected GetTableResponse getTableResponse(Identifier identifier) } } + protected SuccessResponse dropPartitionMetadata( + Identifier identifier, Map partitions) throws TableNotExistException { + try { + DropPartitionRequest request = new DropPartitionRequest(partitions); + return client.delete( + resourcePaths.partitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + headers()); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } catch (ForbiddenException e) { + throw new TableNoPermissionException(identifier, e); + } + } + private static Map configHeaders(Map properties) { return RESTUtil.extractPrefixMap(properties, "header."); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java index a255d688bc52..d90cb5fa4ad9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java @@ -30,4 +30,6 @@ T post( String path, RESTRequest body, Class responseType, Map headers); T delete(String path, Map headers); + + T delete(String path, RESTRequest body, Map headers); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java new file mode 100644 index 000000000000..98ea175ce1b4 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; + +/** Request for deleting partition. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DropPartitionRequest implements RESTRequest { + private static final String FIELD_PARTITION_SPEC = "partitionSpec"; + + @JsonProperty(FIELD_PARTITION_SPEC) + private final Map partitionSpec; + + @JsonCreator + public DropPartitionRequest( + @JsonProperty(FIELD_PARTITION_SPEC) Map partitionSpec) { + this.partitionSpec = partitionSpec; + } + + @JsonGetter(FIELD_PARTITION_SPEC) + public Map getPartitionSpec() { + return partitionSpec; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/SuccessResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/SuccessResponse.java new file mode 100644 index 000000000000..0169731779c6 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/SuccessResponse.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** Response for do action success. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class SuccessResponse implements RESTResponse {} From e28a51555de9e45efdba0323c23e96934c3be6bb Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 27 Dec 2024 11:35:08 +0800 Subject: [PATCH 03/17] support list partitions API --- .../src/main/java/org/apache/paimon/rest/RESTCatalog.java | 8 +++++++- .../java/org/apache/paimon/rest/RESTCatalogOptions.java | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 87b752748461..256cca59ab4d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -88,6 +88,7 @@ import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable; import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE; +import static org.apache.paimon.rest.RESTCatalogOptions.METASTORE_PARTITIONED; import static org.apache.paimon.utils.Preconditions.checkNotNull; import static org.apache.paimon.utils.ThreadPoolUtils.createScheduledThreadPool; @@ -412,7 +413,12 @@ public void dropPartition(Identifier identifier, Map partitions) @Override public List listPartitions(Identifier identifier) throws TableNotExistException { - throw new UnsupportedOperationException(); + boolean whetherSupportListPartitions = context.options().get(METASTORE_PARTITIONED); + if (whetherSupportListPartitions) { + return null; + } else { + return getTable(identifier).newReadBuilder().newScan().listPartitionEntries(); + } } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index 1af64def4f71..82c06ea546cb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -71,4 +71,10 @@ public class RESTCatalogOptions { .stringType() .noDefaultValue() .withDescription("REST Catalog auth token provider path."); + + public static final ConfigOption METASTORE_PARTITIONED = + ConfigOptions.key("metastore-partitioned") + .booleanType() + .defaultValue(false) + .withDescription("REST Catalog whether support list partitions."); } From d0cae621d2f0c109749dfab1ac705117d7e20154 Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 27 Dec 2024 15:16:13 +0800 Subject: [PATCH 04/17] define list partitions rest api --- .../utils/InternalRowPartitionComputer.java | 12 + .../org/apache/paimon/rest/RESTCatalog.java | 70 +++-- .../responses/ListPartitionsResponse.java | 100 ++++++- .../paimon/rest/RESTObjectMapperTest.java | 16 ++ paimon-open-api/rest-catalog-open-api.yaml | 249 ++++++++++++++---- .../open/api/RESTCatalogController.java | 91 +++++++ 6 files changed, 472 insertions(+), 66 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java index 6bb26d76138e..f42176061c5c 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java @@ -120,6 +120,18 @@ public static GenericRow convertSpecToInternalRow( return partRow; } + public static GenericRow convertSpecToInternalRow(Map spec, RowType partType) { + checkArgument(spec.size() == partType.getFieldCount()); + GenericRow partRow = new GenericRow(spec.size()); + List fieldNames = partType.getFieldNames(); + for (Map.Entry entry : spec.entrySet()) { + Object value = + castFromString(entry.getValue(), partType.getField(entry.getKey()).type()); + partRow.setField(fieldNames.indexOf(entry.getKey()), value); + } + return partRow; + } + public static String partToSimpleString( RowType partitionType, BinaryRow partition, String delimiter, int maxLength) { FieldGetter[] getters = partitionType.fieldGetters(); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 256cca59ab4d..920e66c93c2f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -26,6 +26,8 @@ import org.apache.paimon.catalog.Database; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.catalog.PropertyChange; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.PartitionEntry; @@ -52,6 +54,7 @@ import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetTableResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; +import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; import org.apache.paimon.rest.responses.SuccessResponse; import org.apache.paimon.schema.Schema; @@ -89,6 +92,7 @@ import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE; import static org.apache.paimon.rest.RESTCatalogOptions.METASTORE_PARTITIONED; +import static org.apache.paimon.utils.InternalRowPartitionComputer.convertSpecToInternalRow; import static org.apache.paimon.utils.Preconditions.checkNotNull; import static org.apache.paimon.utils.ThreadPoolUtils.createScheduledThreadPool; @@ -385,28 +389,23 @@ public void createPartition(Identifier identifier, Map partition } @Override + // todo: if table not exist how to drop partition public void dropPartition(Identifier identifier, Map partitions) throws TableNotExistException, PartitionNotExistException { checkNotSystemTable(identifier, "dropPartition"); - Table table = null; - try { - dropPartitionMetadata(identifier, partitions); - table = getTable(identifier); - } catch (Exception e) { - if (table != null) { - FileStoreTable fileStoreTable = (FileStoreTable) table; - try (FileStoreCommit commit = - fileStoreTable - .store() - .newCommit( - createCommitUser( - fileStoreTable.coreOptions().toConfiguration()))) { - commit.dropPartitions( - Collections.singletonList(partitions), - BatchWriteBuilder.COMMIT_IDENTIFIER); - } + dropPartitionMetadata(identifier, partitions); + Table table = getTable(identifier); + if (table != null) { + FileStoreTable fileStoreTable = (FileStoreTable) table; + try (FileStoreCommit commit = + fileStoreTable + .store() + .newCommit( + createCommitUser( + fileStoreTable.coreOptions().toConfiguration()))) { + commit.dropPartitions( + Collections.singletonList(partitions), BatchWriteBuilder.COMMIT_IDENTIFIER); } - throw e; } } @@ -415,7 +414,7 @@ public List listPartitions(Identifier identifier) throws TableNotExistException { boolean whetherSupportListPartitions = context.options().get(METASTORE_PARTITIONED); if (whetherSupportListPartitions) { - return null; + return listPartitionsFromServer(identifier); } else { return getTable(identifier).newReadBuilder().newScan().listPartitionEntries(); } @@ -468,6 +467,39 @@ Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException return table; } + @VisibleForTesting + public List listPartitionsFromServer(Identifier identifier) + throws TableNotExistException { + try { + ListPartitionsResponse response = + client.get( + resourcePaths.partitions( + identifier.getDatabaseName(), identifier.getTableName()), + ListPartitionsResponse.class, + headers()); + List partitionEntries = new ArrayList<>(); + for (ListPartitionsResponse.Partition partition : response.getPartitions()) { + InternalRowSerializer serializer = + new InternalRowSerializer(partition.getPartitionType()); + GenericRow row = + convertSpecToInternalRow(partition.getSpec(), partition.getPartitionType()); + PartitionEntry partitionEntry = + new PartitionEntry( + serializer.toBinaryRow(row).copy(), + partition.getRecordCount(), + partition.getFileSizeInBytes(), + partition.getFileCount(), + partition.getLastFileCreationTime()); + partitionEntries.add(partitionEntry); + } + return partitionEntries; + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } catch (ForbiddenException e) { + throw new TableNoPermissionException(identifier, e); + } + } + protected GetTableResponse getTableResponse(Identifier identifier) throws TableNotExistException { try { diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java index b0943a3fcc30..a76187ca4c2c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java @@ -19,9 +19,107 @@ package org.apache.paimon.rest.responses; import org.apache.paimon.rest.RESTResponse; +import org.apache.paimon.types.RowType; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Map; /** Response for listing partitions. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class ListPartitionsResponse implements RESTResponse {} +public class ListPartitionsResponse implements RESTResponse { + public static final String FIELD_PARTITIONS = "partitions"; + + @JsonProperty(FIELD_PARTITIONS) + private final List partitions; + + @JsonCreator + public ListPartitionsResponse(@JsonProperty(FIELD_PARTITIONS) List partitions) { + this.partitions = partitions; + } + + @JsonGetter(FIELD_PARTITIONS) + public List getPartitions() { + return partitions; + } + + /** Partition for rest api. */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Partition implements RESTResponse { + + private static final String FIELD_SPEC = "spec"; + public static final String FIELD_PARTITION_TYPE = "partitionType"; + public static final String FIELD_RECORD_COUNT = "recordCount"; + public static final String FIELD_FILE_SIZE_IN_BYTES = "fileSizeInBytes"; + public static final String FIELD_FILE_COUNT = "fileCount"; + public static final String FIELD_LAST_FILE_CREATION_TIME = "lastFileCreationTime"; + + @JsonProperty(FIELD_SPEC) + private final Map spec; + + @JsonProperty(FIELD_PARTITION_TYPE) + private final RowType partitionType; + + @JsonProperty(FIELD_RECORD_COUNT) + private final long recordCount; + + @JsonProperty(FIELD_FILE_SIZE_IN_BYTES) + private final long fileSizeInBytes; + + @JsonProperty(FIELD_FILE_COUNT) + private final long fileCount; + + @JsonProperty(FIELD_LAST_FILE_CREATION_TIME) + private final long lastFileCreationTime; + + @JsonCreator + public Partition( + @JsonProperty(FIELD_SPEC) Map spec, + @JsonProperty(FIELD_PARTITION_TYPE) RowType partitionType, + @JsonProperty(FIELD_RECORD_COUNT) long recordCount, + @JsonProperty(FIELD_FILE_SIZE_IN_BYTES) long fileSizeInBytes, + @JsonProperty(FIELD_FILE_COUNT) long fileCount, + @JsonProperty(FIELD_LAST_FILE_CREATION_TIME) long lastFileCreationTime) { + this.spec = spec; + this.partitionType = partitionType; + this.recordCount = recordCount; + this.fileSizeInBytes = fileSizeInBytes; + this.fileCount = fileCount; + this.lastFileCreationTime = lastFileCreationTime; + } + + @JsonGetter(FIELD_SPEC) + public Map getSpec() { + return spec; + } + + @JsonGetter(FIELD_PARTITION_TYPE) + public RowType getPartitionType() { + return partitionType; + } + + @JsonGetter(FIELD_RECORD_COUNT) + public long getRecordCount() { + return recordCount; + } + + @JsonGetter(FIELD_FILE_SIZE_IN_BYTES) + public long getFileSizeInBytes() { + return fileSizeInBytes; + } + + @JsonGetter(FIELD_FILE_COUNT) + public long getFileCount() { + return fileCount; + } + + @JsonGetter(FIELD_LAST_FILE_CREATION_TIME) + public long getLastFileCreationTime() { + return lastFileCreationTime; + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java index 26b3db615d11..1313f6cc19e2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java @@ -18,6 +18,7 @@ package org.apache.paimon.rest; +import org.apache.paimon.data.GenericRow; import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; @@ -34,6 +35,8 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.InternalRowPartitionComputer; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; @@ -41,6 +44,7 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -193,4 +197,16 @@ public void alterTableRequestParseTest() throws Exception { AlterTableRequest parseData = mapper.readValue(requestStr, AlterTableRequest.class); assertEquals(parseData.getChanges().size(), parseData.getChanges().size()); } + + @Test + public void testPartitionSpecToInternalRow() { + Map spec = new HashMap<>(); + spec.put("a", "1"); + spec.put("b", "2"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "a", DataTypes.INT())); + fields.add(new DataField(1, "b", DataTypes.STRING())); + RowType partitionType = new RowType(false, fields); + GenericRow row = InternalRowPartitionComputer.convertSpecToInternalRow(spec, partitionType); + } } diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml index 7fefd0254b1b..e344399dccda 100644 --- a/paimon-open-api/rest-catalog-open-api.yaml +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -28,6 +28,21 @@ servers: - url: http://localhost:8080 description: Server URL in Development environment paths: + /v1/config: + get: + tags: + - config + summary: Get Config + operationId: getConfig + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/ConfigResponse' + "500": + description: Internal Server Error /v1/{prefix}/databases: get: tags: @@ -80,6 +95,100 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": description: Internal Server Error + /v1/{prefix}/databases/{database}: + get: + tags: + - database + summary: Get Database + operationId: getDatabases + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetDatabaseResponse' + "404": + description: Resource not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + description: Internal Server Error + delete: + tags: + - database + summary: Drop Database + operationId: dropDatabase + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + responses: + "404": + description: Resource not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + description: Internal Server Error + /v1/{prefix}/databases/{database}/properties: + post: + tags: + - database + summary: Alter Database + operationId: alterDatabase + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/AlterDatabaseRequest' + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/AlterDatabaseResponse' + "404": + description: Resource not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + description: Internal Server Error /v1/{prefix}/databases/{database}/tables: get: tags: @@ -293,12 +402,12 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": description: Internal Server Error - /v1/{prefix}/databases/{database}/properties: - post: + /v1/{prefix}/databases/{database}/tables/{table}/partitions: + get: tags: - - database - summary: Alter Database - operationId: alterDatabase + - partition + summary: List partitions + operationId: listPartitions parameters: - name: prefix in: path @@ -310,18 +419,18 @@ paths: required: true schema: type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/AlterDatabaseRequest' + - name: table + in: path + required: true + schema: + type: string responses: "200": description: OK content: application/json: schema: - $ref: '#/components/schemas/AlterDatabaseResponse' + $ref: '#/components/schemas/ListPartitionsResponse' "404": description: Resource not found content: @@ -330,12 +439,11 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": description: Internal Server Error - /v1/{prefix}/databases/{database}: - get: + post: tags: - - database - summary: Get Database - operationId: getDatabases + - partition + summary: Create partition + operationId: createPartition parameters: - name: prefix in: path @@ -347,13 +455,23 @@ paths: required: true schema: type: string + - name: table + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CreatePartitionRequest' responses: "200": description: OK content: application/json: schema: - $ref: '#/components/schemas/GetDatabaseResponse' + $ref: '#/components/schemas/SuccessResponse' "404": description: Resource not found content: @@ -364,9 +482,9 @@ paths: description: Internal Server Error delete: tags: - - database - summary: Drop Database - operationId: dropDatabase + - partition + summary: Drop partition + operationId: dropPartition parameters: - name: prefix in: path @@ -378,28 +496,29 @@ paths: required: true schema: type: string + - name: table + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DropPartitionRequest' responses: - "404": - description: Resource not found + "200": + description: OK content: application/json: schema: - $ref: '#/components/schemas/ErrorResponse' - "500": - description: Internal Server Error - /v1/config: - get: - tags: - - config - summary: Get Config - operationId: getConfig - responses: - "200": - description: OK + $ref: '#/components/schemas/SuccessResponse' + "404": + description: Resource not found content: application/json: schema: - $ref: '#/components/schemas/ConfigResponse' + $ref: '#/components/schemas/ErrorResponse' "500": description: Internal Server Error components: @@ -413,6 +532,18 @@ components: type: object additionalProperties: type: string + CreatePartitionRequest: + type: object + properties: + identifier: + $ref: '#/components/schemas/Identifier' + partitionSpec: + type: object + DropPartitionRequest: + type: object + properties: + partitionSpec: + type: object CreateDatabaseResponse: type: object properties: @@ -468,39 +599,38 @@ components: type: type: string pattern: ^ARRAY.* + example: ARRAY element: - type: - $ref: '#/components/schemas/DataType' + $ref: '#/components/schemas/DataType' MultisetType: type: object properties: type: type: string pattern: ^MULTISET.* + example: MULTISET element: - type: - $ref: '#/components/schemas/DataType' + $ref: '#/components/schemas/DataType' MapType: type: object properties: type: type: string pattern: ^MAP.* + example: MAP key: - type: - $ref: '#/components/schemas/DataType' + $ref: '#/components/schemas/DataType' value: - type: - $ref: '#/components/schemas/DataType' + $ref: '#/components/schemas/DataType' RowType: type: object properties: type: type: string pattern: ^ROW.* + example: ROW fields: - type: - $ref: '#/components/schemas/DataField' + $ref: '#/components/schemas/DataField' Identifier: type: object properties: @@ -744,7 +874,34 @@ components: type: object additionalProperties: type: string - + ListPartitionsResponse: + type: object + properties: + partitions: + type: array + items: + $ref: '#/components/schemas/Partition' + Partition: + type: object + properties: + spec: + type: object + partitionType: + $ref: '#/components/schemas/RowType' + recordCount: + type: integer + format: int64 + fileSizeInBytes: + type: integer + format: int64 + fileCount: + type: integer + format: int64 + lastFileCreationTime: + type: integer + format: int64 + SuccessResponse: + type: object securitySchemes: BearerAuth: type: http diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java index 179c23ce46f8..324f93740e4c 100644 --- a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java @@ -22,7 +22,9 @@ import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.requests.CreatePartitionRequest; import org.apache.paimon.rest.requests.CreateTableRequest; +import org.apache.paimon.rest.requests.DropPartitionRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.ConfigResponse; @@ -31,7 +33,12 @@ import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetTableResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; +import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; +import org.apache.paimon.rest.responses.SuccessResponse; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; @@ -49,7 +56,9 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RestController; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; /** RESTCatalog management APIs. */ @@ -346,4 +355,86 @@ public GetTableResponse renameTable( new HashMap<>(), "comment")); } + + @Operation( + summary = "List partitions", + tags = {"partition"}) + @ApiResponses({ + @ApiResponse( + responseCode = "200", + content = { + @Content(schema = @Schema(implementation = ListPartitionsResponse.class)) + }), + @ApiResponse( + responseCode = "404", + description = "Resource not found", + content = {@Content(schema = @Schema(implementation = ErrorResponse.class))}), + @ApiResponse( + responseCode = "500", + content = {@Content(schema = @Schema())}) + }) + @GetMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions") + public ListPartitionsResponse listPartitions( + @PathVariable String prefix, + @PathVariable String database, + @PathVariable String table) { + Map spec = new HashMap<>(); + spec.put("a", "1"); + spec.put("b", "2"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "a", DataTypes.INT())); + fields.add(new DataField(1, "b", DataTypes.STRING())); + RowType partitionType = new RowType(false, fields); + ListPartitionsResponse.Partition partition = + new ListPartitionsResponse.Partition(spec, partitionType, 1, 2, 3, 4); + return new ListPartitionsResponse(ImmutableList.of(partition)); + } + + @Operation( + summary = "Create partition", + tags = {"partition"}) + @ApiResponses({ + @ApiResponse( + responseCode = "200", + content = {@Content(schema = @Schema(implementation = SuccessResponse.class))}), + @ApiResponse( + responseCode = "404", + description = "Resource not found", + content = {@Content(schema = @Schema(implementation = ErrorResponse.class))}), + @ApiResponse( + responseCode = "500", + content = {@Content(schema = @Schema())}) + }) + @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions") + public SuccessResponse createPartition( + @PathVariable String prefix, + @PathVariable String database, + @PathVariable String table, + @RequestBody CreatePartitionRequest request) { + return new SuccessResponse(); + } + + @Operation( + summary = "Drop partition", + tags = {"partition"}) + @ApiResponses({ + @ApiResponse( + responseCode = "200", + content = {@Content(schema = @Schema(implementation = SuccessResponse.class))}), + @ApiResponse( + responseCode = "404", + description = "Resource not found", + content = {@Content(schema = @Schema(implementation = ErrorResponse.class))}), + @ApiResponse( + responseCode = "500", + content = {@Content(schema = @Schema())}) + }) + @DeleteMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions") + public SuccessResponse dropPartition( + @PathVariable String prefix, + @PathVariable String database, + @PathVariable String table, + @RequestBody DropPartitionRequest request) { + return new SuccessResponse(); + } } From 35aa8ec271e1eadf1c70bbe33ad7c0742176a853 Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 27 Dec 2024 16:17:08 +0800 Subject: [PATCH 05/17] add ut --- .../java/org/apache/paimon/types/RowType.java | 11 +- .../utils/InternalRowPartitionComputer.java | 14 +-- .../org/apache/paimon/rest/RESTCatalog.java | 6 +- .../apache/paimon/rest/MockRESTMessage.java | 26 +++++ .../apache/paimon/rest/RESTCatalogTest.java | 105 +++++++++++++++--- .../paimon/rest/RESTObjectMapperTest.java | 53 ++++++--- 6 files changed, 172 insertions(+), 43 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java index fecb5bed9ebd..537c362e9c00 100644 --- a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java +++ b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java @@ -24,6 +24,9 @@ import org.apache.paimon.utils.Preconditions; import org.apache.paimon.utils.StringUtils; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonGenerator; import java.io.IOException; @@ -52,9 +55,13 @@ public final class RowType extends DataType { private static final long serialVersionUID = 1L; + public static final String FILED_FIELDS = "fields"; + public static final String FORMAT = "ROW<%s>"; + @JsonProperty(FILED_FIELDS) private final List fields; + private InternalRow.FieldGetter[] fieldGetters; public RowType(boolean isNullable, List fields) { @@ -67,7 +74,8 @@ public RowType(boolean isNullable, List fields) { validateFields(fields); } - public RowType(List fields) { + @JsonCreator + public RowType(@JsonProperty(FILED_FIELDS) List fields) { this(true, fields); } @@ -75,6 +83,7 @@ public RowType copy(List newFields) { return new RowType(isNullable(), newFields); } + @JsonGetter(FILED_FIELDS) public List getFields() { return fields; } diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java index f42176061c5c..f4aad8f03f4b 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java @@ -111,7 +111,7 @@ public static GenericRow convertSpecToInternalRow( List fieldNames = partType.getFieldNames(); for (Map.Entry entry : spec.entrySet()) { Object value = - defaultPartValue.equals(entry.getValue()) + defaultPartValue != null && defaultPartValue.equals(entry.getValue()) ? null : castFromString( entry.getValue(), partType.getField(entry.getKey()).type()); @@ -120,18 +120,6 @@ public static GenericRow convertSpecToInternalRow( return partRow; } - public static GenericRow convertSpecToInternalRow(Map spec, RowType partType) { - checkArgument(spec.size() == partType.getFieldCount()); - GenericRow partRow = new GenericRow(spec.size()); - List fieldNames = partType.getFieldNames(); - for (Map.Entry entry : spec.entrySet()) { - Object value = - castFromString(entry.getValue(), partType.getField(entry.getKey()).type()); - partRow.setField(fieldNames.indexOf(entry.getKey()), value); - } - return partRow; - } - public static String partToSimpleString( RowType partitionType, BinaryRow partition, String delimiter, int maxLength) { FieldGetter[] getters = partitionType.fieldGetters(); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 920e66c93c2f..d5808a852b05 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -145,7 +145,8 @@ public RESTCatalog(CatalogContext catalogContext) { Map initHeaders = RESTUtil.merge( configHeaders(catalogOptions.toMap()), this.catalogAuth.getHeaders()); - Options options = new Options(fetchOptionsFromServer(initHeaders, initHeaders)); + Options options = + new Options(fetchOptionsFromServer(initHeaders, catalogContext.options().toMap())); this.context = CatalogContext.create( options, catalogContext.preferIO(), catalogContext.fallbackIO()); @@ -482,7 +483,8 @@ public List listPartitionsFromServer(Identifier identifier) InternalRowSerializer serializer = new InternalRowSerializer(partition.getPartitionType()); GenericRow row = - convertSpecToInternalRow(partition.getSpec(), partition.getPartitionType()); + convertSpecToInternalRow( + partition.getSpec(), partition.getPartitionType(), null); PartitionEntry partitionEntry = new PartitionEntry( serializer.toBinaryRow(row).copy(), diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index 96d0c9d7c724..ee52dc0b6f23 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -22,7 +22,9 @@ import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.requests.CreatePartitionRequest; import org.apache.paimon.rest.requests.CreateTableRequest; +import org.apache.paimon.rest.requests.DropPartitionRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; @@ -30,6 +32,7 @@ import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetTableResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; +import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; @@ -39,6 +42,7 @@ import org.apache.paimon.types.IntType; import org.apache.paimon.types.RowType; +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; import java.util.ArrayList; @@ -131,6 +135,28 @@ public static AlterTableRequest alterTableRequest() { return new AlterTableRequest(getChanges()); } + public static CreatePartitionRequest createPartitionRequest(String tableName) { + Identifier identifier = Identifier.create(databaseName(), tableName); + return new CreatePartitionRequest(identifier, Collections.singletonMap("pt", "1")); + } + + public static DropPartitionRequest dropPartitionRequest() { + return new DropPartitionRequest(Collections.singletonMap("pt", "1")); + } + + public static ListPartitionsResponse listPartitionsResponse() { + Map spec = new HashMap<>(); + spec.put("a", "1"); + spec.put("b", "2"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "a", DataTypes.INT())); + fields.add(new DataField(1, "b", DataTypes.STRING())); + RowType partitionType = new RowType(false, fields); + ListPartitionsResponse.Partition partition = + new ListPartitionsResponse.Partition(spec, partitionType, 1, 1, 1, 1); + return new ListPartitionsResponse(ImmutableList.of(partition)); + } + public static List getChanges() { // add option SchemaChange addOption = SchemaChange.setOption("snapshot.time-retained", "2h"); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 01555adc3df4..ef4534763b26 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -22,6 +22,7 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.Database; import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.rest.requests.CreateTableRequest; @@ -31,7 +32,9 @@ import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetTableResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; +import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; +import org.apache.paimon.rest.responses.SuccessResponse; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.Table; @@ -71,27 +74,17 @@ public class RESTCatalogTest { private RESTCatalog restCatalog; private RESTCatalog mockRestCatalog; private String warehouseStr; + private String serverUrl; @Rule public TemporaryFolder folder = new TemporaryFolder(); @Before public void setUp() throws IOException { mockWebServer = new MockWebServer(); mockWebServer.start(); - String baseUrl = mockWebServer.url("").toString(); - Options options = new Options(); - options.set(RESTCatalogOptions.URI, baseUrl); - String initToken = "init_token"; - options.set(RESTCatalogOptions.TOKEN, initToken); - options.set(RESTCatalogOptions.THREAD_POOL_SIZE, 1); + serverUrl = mockWebServer.url("").toString(); + Options options = mockInitOptions(); warehouseStr = folder.getRoot().getPath(); - String mockResponse = - String.format( - "{\"defaults\": {\"%s\": \"%s\", \"%s\": \"%s\"}}", - RESTCatalogInternalOptions.PREFIX.key(), - "prefix", - CatalogOptions.WAREHOUSE.key(), - warehouseStr); - mockResponse(mockResponse, 200); + mockConfig(warehouseStr); restCatalog = new RESTCatalog(CatalogContext.create(options)); mockRestCatalog = spy(restCatalog); } @@ -359,6 +352,70 @@ public void testDropTableWhenTableNotExistAndIgnoreIfNotExistsIsFalse() throws E () -> restCatalog.dropTable(Identifier.create(databaseName, tableName), false)); } + @Test + public void testCreatePartition() throws Exception { + String databaseName = MockRESTMessage.databaseName(); + Map partitionSpec = new HashMap<>(); + partitionSpec.put("p1", "v1"); + mockResponse(mapper.writeValueAsString(new SuccessResponse()), 200); + assertDoesNotThrow( + () -> + mockRestCatalog.createPartition( + Identifier.create(databaseName, "table"), partitionSpec)); + } + + @Test + public void testCreatePartitionWhenTableNotExist() throws Exception { + String databaseName = MockRESTMessage.databaseName(); + Map partitionSpec = new HashMap<>(); + partitionSpec.put("p1", "v1"); + mockResponse("", 404); + assertThrows( + Catalog.TableNotExistException.class, + () -> + mockRestCatalog.createPartition( + Identifier.create(databaseName, "table"), partitionSpec)); + } + + @Test + public void testCreatePartitionWhenTableNoPermissionException() throws Exception { + String databaseName = MockRESTMessage.databaseName(); + Map partitionSpec = new HashMap<>(); + partitionSpec.put("p1", "v1"); + mockResponse("", 403); + assertThrows( + Catalog.TableNoPermissionException.class, + () -> + mockRestCatalog.createPartition( + Identifier.create(databaseName, "table"), partitionSpec)); + } + + @Test + public void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception { + Options options = mockInitOptions(); + options = options.set(RESTCatalogOptions.METASTORE_PARTITIONED, true); + mockConfig(warehouseStr); + RESTCatalog restCatalog = new RESTCatalog(CatalogContext.create(options)); + RESTCatalog mockRestCatalog = spy(restCatalog); + String databaseName = MockRESTMessage.databaseName(); + ListPartitionsResponse response = MockRESTMessage.listPartitionsResponse(); + mockResponse(mapper.writeValueAsString(response), 200); + List result = + mockRestCatalog.listPartitions(Identifier.create(databaseName, "table")); + verify(mockRestCatalog, times(1)).listPartitionsFromServer(any()); + assertEquals(response.getPartitions().size(), result.size()); + } + + @Test + public void testListPartitionsFromFile() throws Exception { + String databaseName = MockRESTMessage.databaseName(); + GetTableResponse response = MockRESTMessage.getTableResponse(); + mockResponse(mapper.writeValueAsString(response), 200); + mockRestCatalog.listPartitions(Identifier.create(databaseName, "table")); + verify(mockRestCatalog, times(1)).getTable(any()); + verify(mockRestCatalog, times(0)).listPartitionsFromServer(any()); + } + private void mockResponse(String mockResponse, int httpCode) { MockResponse mockResponseObj = new MockResponse() @@ -367,4 +424,24 @@ private void mockResponse(String mockResponse, int httpCode) { .addHeader("Content-Type", "application/json"); mockWebServer.enqueue(mockResponseObj); } + + private void mockConfig(String warehouseStr) { + String mockResponse = + String.format( + "{\"defaults\": {\"%s\": \"%s\", \"%s\": \"%s\"}}", + RESTCatalogInternalOptions.PREFIX.key(), + "prefix", + CatalogOptions.WAREHOUSE.key(), + warehouseStr); + mockResponse(mockResponse, 200); + } + + public Options mockInitOptions() { + Options options = new Options(); + options.set(RESTCatalogOptions.URI, serverUrl); + String initToken = "init_token"; + options.set(RESTCatalogOptions.TOKEN, initToken); + options.set(RESTCatalogOptions.THREAD_POOL_SIZE, 1); + return options; + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java index 1313f6cc19e2..b3a5c378d812 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java @@ -18,11 +18,12 @@ package org.apache.paimon.rest; -import org.apache.paimon.data.GenericRow; import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.requests.CreatePartitionRequest; import org.apache.paimon.rest.requests.CreateTableRequest; +import org.apache.paimon.rest.requests.DropPartitionRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.ConfigResponse; @@ -31,23 +32,24 @@ import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetTableResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; +import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; +import org.apache.paimon.rest.responses.SuccessResponse; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; -import org.apache.paimon.types.RowType; -import org.apache.paimon.utils.InternalRowPartitionComputer; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; /** Test for {@link RESTObjectMapper}. */ public class RESTObjectMapperTest { @@ -199,14 +201,39 @@ public void alterTableRequestParseTest() throws Exception { } @Test - public void testPartitionSpecToInternalRow() { - Map spec = new HashMap<>(); - spec.put("a", "1"); - spec.put("b", "2"); - List fields = new ArrayList<>(); - fields.add(new DataField(0, "a", DataTypes.INT())); - fields.add(new DataField(1, "b", DataTypes.STRING())); - RowType partitionType = new RowType(false, fields); - GenericRow row = InternalRowPartitionComputer.convertSpecToInternalRow(spec, partitionType); + public void createPartitionRequestParseTest() throws JsonProcessingException { + CreatePartitionRequest request = MockRESTMessage.createPartitionRequest("t1"); + String requestStr = mapper.writeValueAsString(request); + CreatePartitionRequest parseData = + mapper.readValue(requestStr, CreatePartitionRequest.class); + assertEquals(parseData.getIdentifier(), parseData.getIdentifier()); + assertEquals(parseData.getPartitionSpec().size(), parseData.getPartitionSpec().size()); + } + + @Test + public void dropPartitionRequestParseTest() throws JsonProcessingException { + DropPartitionRequest request = MockRESTMessage.dropPartitionRequest(); + String requestStr = mapper.writeValueAsString(request); + DropPartitionRequest parseData = mapper.readValue(requestStr, DropPartitionRequest.class); + assertEquals(parseData.getPartitionSpec().size(), parseData.getPartitionSpec().size()); + } + + @Test + public void listPartitionsResponseParseTest() throws Exception { + ListPartitionsResponse response = MockRESTMessage.listPartitionsResponse(); + String responseStr = mapper.writeValueAsString(response); + ListPartitionsResponse parseData = + mapper.readValue(responseStr, ListPartitionsResponse.class); + assertEquals( + response.getPartitions().get(0).getFileCount(), + parseData.getPartitions().get(0).getFileCount()); + } + + @Test + public void successResponseParseTest() throws Exception { + SuccessResponse response = new SuccessResponse(); + assertDoesNotThrow(() -> mapper.writeValueAsString(response)); + assertDoesNotThrow( + () -> mapper.readValue(mapper.writeValueAsString(response), SuccessResponse.class)); } } From f0bddf0e55b9c8931d4bcca76a7c959a26c3a95b Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 27 Dec 2024 17:06:25 +0800 Subject: [PATCH 06/17] add ut for drop partition --- .../org/apache/paimon/rest/RESTCatalog.java | 32 ++++---- .../apache/paimon/rest/RESTCatalogTest.java | 74 +++++++++++++++++++ 2 files changed, 92 insertions(+), 14 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index d5808a852b05..dc142e77c54e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -390,23 +390,26 @@ public void createPartition(Identifier identifier, Map partition } @Override - // todo: if table not exist how to drop partition public void dropPartition(Identifier identifier, Map partitions) throws TableNotExistException, PartitionNotExistException { checkNotSystemTable(identifier, "dropPartition"); dropPartitionMetadata(identifier, partitions); Table table = getTable(identifier); if (table != null) { - FileStoreTable fileStoreTable = (FileStoreTable) table; - try (FileStoreCommit commit = - fileStoreTable - .store() - .newCommit( - createCommitUser( - fileStoreTable.coreOptions().toConfiguration()))) { - commit.dropPartitions( - Collections.singletonList(partitions), BatchWriteBuilder.COMMIT_IDENTIFIER); - } + cleanPartitionsInFileSystem(table, partitions); + } + } + + @VisibleForTesting + void cleanPartitionsInFileSystem(Table table, Map partitions) { + FileStoreTable fileStoreTable = (FileStoreTable) table; + try (FileStoreCommit commit = + fileStoreTable + .store() + .newCommit( + createCommitUser(fileStoreTable.coreOptions().toConfiguration()))) { + commit.dropPartitions( + Collections.singletonList(partitions), BatchWriteBuilder.COMMIT_IDENTIFIER); } } @@ -517,7 +520,8 @@ protected GetTableResponse getTableResponse(Identifier identifier) } protected SuccessResponse dropPartitionMetadata( - Identifier identifier, Map partitions) throws TableNotExistException { + Identifier identifier, Map partitions) + throws TableNoPermissionException { try { DropPartitionRequest request = new DropPartitionRequest(partitions); return client.delete( @@ -525,8 +529,8 @@ protected SuccessResponse dropPartitionMetadata( identifier.getDatabaseName(), identifier.getTableName()), request, headers()); - } catch (NoSuchResourceException e) { - throw new TableNotExistException(identifier); + } catch (NoSuchResourceException ignore) { + return new SuccessResponse(); } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index ef4534763b26..8a72f749d86b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -62,6 +62,7 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -390,6 +391,79 @@ public void testCreatePartitionWhenTableNoPermissionException() throws Exception Identifier.create(databaseName, "table"), partitionSpec)); } + @Test + public void testDropPartition() throws Exception { + String databaseName = MockRESTMessage.databaseName(); + Map partitionSpec = new HashMap<>(); + GetTableResponse response = MockRESTMessage.getTableResponse(); + partitionSpec.put(response.getSchema().primaryKeys().get(0), "1"); + mockResponse(mapper.writeValueAsString(new SuccessResponse()), 200); + mockResponse(mapper.writeValueAsString(response), 200); + doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any()); + assertDoesNotThrow( + () -> + mockRestCatalog.dropPartition( + Identifier.create(databaseName, "table"), partitionSpec)); + verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any()); + verify(mockRestCatalog, times(1)).getTable(any()); + verify(mockRestCatalog, times(1)).cleanPartitionsInFileSystem(any(), any()); + } + + @Test + public void testDropPartitionWhenPartitionNoExist() throws Exception { + String databaseName = MockRESTMessage.databaseName(); + Map partitionSpec = new HashMap<>(); + GetTableResponse response = MockRESTMessage.getTableResponse(); + partitionSpec.put(response.getSchema().primaryKeys().get(0), "1"); + mockResponse(mapper.writeValueAsString(new SuccessResponse()), 404); + mockResponse(mapper.writeValueAsString(response), 200); + doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any()); + assertDoesNotThrow( + () -> + mockRestCatalog.dropPartition( + Identifier.create(databaseName, "table"), partitionSpec)); + verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any()); + verify(mockRestCatalog, times(1)).getTable(any()); + verify(mockRestCatalog, times(1)).cleanPartitionsInFileSystem(any(), any()); + } + + @Test + public void testDropPartitionWhenTableNoPermission() throws Exception { + String databaseName = MockRESTMessage.databaseName(); + Map partitionSpec = new HashMap<>(); + GetTableResponse response = MockRESTMessage.getTableResponse(); + partitionSpec.put(response.getSchema().primaryKeys().get(0), "1"); + mockResponse(mapper.writeValueAsString(new SuccessResponse()), 403); + doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any()); + assertThrows( + Catalog.TableNoPermissionException.class, + () -> + mockRestCatalog.dropPartition( + Identifier.create(databaseName, "table"), partitionSpec)); + verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any()); + verify(mockRestCatalog, times(0)).getTable(any()); + verify(mockRestCatalog, times(0)).cleanPartitionsInFileSystem(any(), any()); + } + + @Test + public void testDropPartitionWhenTableNoExist() throws Exception { + String databaseName = MockRESTMessage.databaseName(); + Map partitionSpec = new HashMap<>(); + GetTableResponse response = MockRESTMessage.getTableResponse(); + partitionSpec.put(response.getSchema().primaryKeys().get(0), "1"); + mockResponse(mapper.writeValueAsString(new SuccessResponse()), 200); + mockResponse("", 404); + doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any()); + assertThrows( + Catalog.TableNotExistException.class, + () -> + mockRestCatalog.dropPartition( + Identifier.create(databaseName, "table"), partitionSpec)); + verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any()); + verify(mockRestCatalog, times(1)).getTable(any()); + verify(mockRestCatalog, times(0)).cleanPartitionsInFileSystem(any(), any()); + } + @Test public void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception { Options options = mockInitOptions(); From ed42f4366673a26b29778bc891727f83b66b58d8 Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 27 Dec 2024 17:46:14 +0800 Subject: [PATCH 07/17] add ut for partition convert --- .../java/org/apache/paimon/types/RowType.java | 2 + .../org/apache/paimon/rest/RESTCatalog.java | 64 ++++++++++--------- .../apache/paimon/rest/RESTCatalogTest.java | 30 +++++++++ 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java index 537c362e9c00..55f0be9fd614 100644 --- a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java +++ b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java @@ -26,6 +26,7 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonGenerator; @@ -51,6 +52,7 @@ * @since 0.4.0 */ @Public +@JsonIgnoreProperties(ignoreUnknown = true) public final class RowType extends DataType { private static final long serialVersionUID = 1L; diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index dc142e77c54e..8cfe6fe9ecfc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -20,6 +20,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.TableType; +import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogUtils; @@ -69,7 +70,6 @@ import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Preconditions; -import org.apache.paimon.shade.guava30.com.google.common.annotations.VisibleForTesting; import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; @@ -85,6 +85,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Collectors; import static org.apache.paimon.CoreOptions.createCommitUser; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase; @@ -400,19 +401,6 @@ public void dropPartition(Identifier identifier, Map partitions) } } - @VisibleForTesting - void cleanPartitionsInFileSystem(Table table, Map partitions) { - FileStoreTable fileStoreTable = (FileStoreTable) table; - try (FileStoreCommit commit = - fileStoreTable - .store() - .newCommit( - createCommitUser(fileStoreTable.coreOptions().toConfiguration()))) { - commit.dropPartitions( - Collections.singletonList(partitions), BatchWriteBuilder.COMMIT_IDENTIFIER); - } - } - @Override public List listPartitions(Identifier identifier) throws TableNotExistException { @@ -481,23 +469,13 @@ public List listPartitionsFromServer(Identifier identifier) identifier.getDatabaseName(), identifier.getTableName()), ListPartitionsResponse.class, headers()); - List partitionEntries = new ArrayList<>(); - for (ListPartitionsResponse.Partition partition : response.getPartitions()) { - InternalRowSerializer serializer = - new InternalRowSerializer(partition.getPartitionType()); - GenericRow row = - convertSpecToInternalRow( - partition.getSpec(), partition.getPartitionType(), null); - PartitionEntry partitionEntry = - new PartitionEntry( - serializer.toBinaryRow(row).copy(), - partition.getRecordCount(), - partition.getFileSizeInBytes(), - partition.getFileCount(), - partition.getLastFileCreationTime()); - partitionEntries.add(partitionEntry); + if (response != null && response.getPartitions() != null) { + return response.getPartitions().stream() + .map(this::convertToPartitionEntry) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); } - return partitionEntries; } catch (NoSuchResourceException e) { throw new TableNotExistException(identifier); } catch (ForbiddenException e) { @@ -505,6 +483,32 @@ public List listPartitionsFromServer(Identifier identifier) } } + @VisibleForTesting + PartitionEntry convertToPartitionEntry(ListPartitionsResponse.Partition partition) { + InternalRowSerializer serializer = new InternalRowSerializer(partition.getPartitionType()); + GenericRow row = + convertSpecToInternalRow(partition.getSpec(), partition.getPartitionType(), null); + return new PartitionEntry( + serializer.toBinaryRow(row).copy(), + partition.getRecordCount(), + partition.getFileSizeInBytes(), + partition.getFileCount(), + partition.getLastFileCreationTime()); + } + + @VisibleForTesting + void cleanPartitionsInFileSystem(Table table, Map partitions) { + FileStoreTable fileStoreTable = (FileStoreTable) table; + try (FileStoreCommit commit = + fileStoreTable + .store() + .newCommit( + createCommitUser(fileStoreTable.coreOptions().toConfiguration()))) { + commit.dropPartitions( + Collections.singletonList(partitions), BatchWriteBuilder.COMMIT_IDENTIFIER); + } + } + protected GetTableResponse getTableResponse(Identifier identifier) throws TableNotExistException { try { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 8a72f749d86b..211813d89889 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -37,6 +37,11 @@ import org.apache.paimon.rest.responses.SuccessResponse; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.Table; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.FileStorePathFactory; +import org.apache.paimon.utils.InternalRowPartitionComputer; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; @@ -490,6 +495,31 @@ public void testListPartitionsFromFile() throws Exception { verify(mockRestCatalog, times(0)).listPartitionsFromServer(any()); } + @Test + public void convertToPartitionEntryTest() { + Map spec = new HashMap<>(); + spec.put("a", "1"); + spec.put("b", "2"); + List fields = new ArrayList<>(); + fields.add(new DataField(0, "a", DataTypes.INT())); + fields.add(new DataField(1, "b", DataTypes.STRING())); + RowType partitionRowType = new RowType(false, fields); + ListPartitionsResponse.Partition partition = + new ListPartitionsResponse.Partition(spec, partitionRowType, 1, 1, 1, 1); + PartitionEntry partitionEntry = mockRestCatalog.convertToPartitionEntry(partition); + InternalRowPartitionComputer partitionComputer = + FileStorePathFactory.getPartitionComputer(partitionRowType, null, false); + Map partValues = + partitionComputer.generatePartValues(partitionEntry.partition()); + for (Map.Entry entry : spec.entrySet()) { + assertEquals(entry.getValue(), partValues.get(entry.getKey())); + } + assertEquals(partitionEntry.recordCount(), partition.getRecordCount()); + assertEquals(partitionEntry.fileSizeInBytes(), partition.getFileSizeInBytes()); + assertEquals(partitionEntry.fileCount(), partition.getFileCount()); + assertEquals(partitionEntry.lastFileCreationTime(), partition.getLastFileCreationTime()); + } + private void mockResponse(String mockResponse, int httpCode) { MockResponse mockResponseObj = new MockResponse() From 1dfe8eb637c2d7634cbd83432458025aa9158fbf Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 27 Dec 2024 17:58:43 +0800 Subject: [PATCH 08/17] format --- .../org/apache/paimon/rest/requests/DropPartitionRequest.java | 1 + .../org/apache/paimon/rest/responses/ListPartitionsResponse.java | 1 + 2 files changed, 2 insertions(+) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java index 98ea175ce1b4..47fbf5651881 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java @@ -30,6 +30,7 @@ /** Request for deleting partition. */ @JsonIgnoreProperties(ignoreUnknown = true) public class DropPartitionRequest implements RESTRequest { + private static final String FIELD_PARTITION_SPEC = "partitionSpec"; @JsonProperty(FIELD_PARTITION_SPEC) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java index a76187ca4c2c..bdf1aa5345ca 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java @@ -32,6 +32,7 @@ /** Response for listing partitions. */ @JsonIgnoreProperties(ignoreUnknown = true) public class ListPartitionsResponse implements RESTResponse { + public static final String FIELD_PARTITIONS = "partitions"; @JsonProperty(FIELD_PARTITIONS) From 427f5a73d8019bb0d755e3b1a11166d7e4506c2c Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 30 Dec 2024 10:10:01 +0800 Subject: [PATCH 09/17] update desc of METASTORE_PARTITIONED and update ut dropPartition throw exception when table is nul --- .../src/main/java/org/apache/paimon/rest/RESTCatalog.java | 2 ++ .../java/org/apache/paimon/rest/RESTCatalogOptions.java | 2 +- .../test/java/org/apache/paimon/rest/RESTCatalogTest.java | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 8cfe6fe9ecfc..4190f29168a0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -398,6 +398,8 @@ public void dropPartition(Identifier identifier, Map partitions) Table table = getTable(identifier); if (table != null) { cleanPartitionsInFileSystem(table, partitions); + } else { + throw new TableNotExistException(identifier); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index 82c06ea546cb..d750654f94af 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -76,5 +76,5 @@ public class RESTCatalogOptions { ConfigOptions.key("metastore-partitioned") .booleanType() .defaultValue(false) - .withDescription("REST Catalog whether support list partitions."); + .withDescription("REST Catalog Server whether support list partitions."); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 211813d89889..4f1e33255c86 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -402,7 +402,7 @@ public void testDropPartition() throws Exception { Map partitionSpec = new HashMap<>(); GetTableResponse response = MockRESTMessage.getTableResponse(); partitionSpec.put(response.getSchema().primaryKeys().get(0), "1"); - mockResponse(mapper.writeValueAsString(new SuccessResponse()), 200); + mockResponse(mapper.writeValueAsString(""), 200); mockResponse(mapper.writeValueAsString(response), 200); doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any()); assertDoesNotThrow( @@ -420,7 +420,7 @@ public void testDropPartitionWhenPartitionNoExist() throws Exception { Map partitionSpec = new HashMap<>(); GetTableResponse response = MockRESTMessage.getTableResponse(); partitionSpec.put(response.getSchema().primaryKeys().get(0), "1"); - mockResponse(mapper.writeValueAsString(new SuccessResponse()), 404); + mockResponse(mapper.writeValueAsString(""), 404); mockResponse(mapper.writeValueAsString(response), 200); doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any()); assertDoesNotThrow( @@ -438,7 +438,7 @@ public void testDropPartitionWhenTableNoPermission() throws Exception { Map partitionSpec = new HashMap<>(); GetTableResponse response = MockRESTMessage.getTableResponse(); partitionSpec.put(response.getSchema().primaryKeys().get(0), "1"); - mockResponse(mapper.writeValueAsString(new SuccessResponse()), 403); + mockResponse(mapper.writeValueAsString(""), 403); doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any()); assertThrows( Catalog.TableNoPermissionException.class, From 4ec183ca489fe4acf7592c339abfc52ca93a91a9 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 30 Dec 2024 11:47:10 +0800 Subject: [PATCH 10/17] update list partition api: get partition type from table --- .../java/org/apache/paimon/types/RowType.java | 13 +------------ .../org/apache/paimon/rest/RESTCatalog.java | 17 ++++++++++------- .../rest/responses/ListPartitionsResponse.java | 12 ------------ .../org/apache/paimon/rest/MockRESTMessage.java | 9 ++------- .../org/apache/paimon/rest/RESTCatalogTest.java | 11 +++++++---- paimon-open-api/rest-catalog-open-api.yaml | 2 -- .../paimon/open/api/RESTCatalogController.java | 14 ++------------ 7 files changed, 22 insertions(+), 56 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java index 55f0be9fd614..fecb5bed9ebd 100644 --- a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java +++ b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java @@ -24,10 +24,6 @@ import org.apache.paimon.utils.Preconditions; import org.apache.paimon.utils.StringUtils; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonGenerator; import java.io.IOException; @@ -52,18 +48,13 @@ * @since 0.4.0 */ @Public -@JsonIgnoreProperties(ignoreUnknown = true) public final class RowType extends DataType { private static final long serialVersionUID = 1L; - public static final String FILED_FIELDS = "fields"; - public static final String FORMAT = "ROW<%s>"; - @JsonProperty(FILED_FIELDS) private final List fields; - private InternalRow.FieldGetter[] fieldGetters; public RowType(boolean isNullable, List fields) { @@ -76,8 +67,7 @@ public RowType(boolean isNullable, List fields) { validateFields(fields); } - @JsonCreator - public RowType(@JsonProperty(FILED_FIELDS) List fields) { + public RowType(List fields) { this(true, fields); } @@ -85,7 +75,6 @@ public RowType copy(List newFields) { return new RowType(isNullable(), newFields); } - @JsonGetter(FILED_FIELDS) public List getFields() { return fields; } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 4190f29168a0..bd0447c40a69 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -67,6 +67,7 @@ import org.apache.paimon.table.Table; import org.apache.paimon.table.object.ObjectTable; import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.types.RowType; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Preconditions; @@ -408,7 +409,9 @@ public List listPartitions(Identifier identifier) throws TableNotExistException { boolean whetherSupportListPartitions = context.options().get(METASTORE_PARTITIONED); if (whetherSupportListPartitions) { - return listPartitionsFromServer(identifier); + FileStoreTable table = (FileStoreTable) getTable(identifier); + RowType rowType = table.schema().logicalPartitionType(); + return listPartitionsFromServer(identifier, rowType); } else { return getTable(identifier).newReadBuilder().newScan().listPartitionEntries(); } @@ -462,7 +465,7 @@ Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException } @VisibleForTesting - public List listPartitionsFromServer(Identifier identifier) + public List listPartitionsFromServer(Identifier identifier, RowType rowType) throws TableNotExistException { try { ListPartitionsResponse response = @@ -473,7 +476,7 @@ public List listPartitionsFromServer(Identifier identifier) headers()); if (response != null && response.getPartitions() != null) { return response.getPartitions().stream() - .map(this::convertToPartitionEntry) + .map(p -> convertToPartitionEntry(p, rowType)) .collect(Collectors.toList()); } else { return Collections.emptyList(); @@ -486,10 +489,10 @@ public List listPartitionsFromServer(Identifier identifier) } @VisibleForTesting - PartitionEntry convertToPartitionEntry(ListPartitionsResponse.Partition partition) { - InternalRowSerializer serializer = new InternalRowSerializer(partition.getPartitionType()); - GenericRow row = - convertSpecToInternalRow(partition.getSpec(), partition.getPartitionType(), null); + PartitionEntry convertToPartitionEntry( + ListPartitionsResponse.Partition partition, RowType rowType) { + InternalRowSerializer serializer = new InternalRowSerializer(rowType); + GenericRow row = convertSpecToInternalRow(partition.getSpec(), rowType, null); return new PartitionEntry( serializer.toBinaryRow(row).copy(), partition.getRecordCount(), diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java index bdf1aa5345ca..e2ce322fd7e3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java @@ -19,7 +19,6 @@ package org.apache.paimon.rest.responses; import org.apache.paimon.rest.RESTResponse; -import org.apache.paimon.types.RowType; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; @@ -53,7 +52,6 @@ public List getPartitions() { public static class Partition implements RESTResponse { private static final String FIELD_SPEC = "spec"; - public static final String FIELD_PARTITION_TYPE = "partitionType"; public static final String FIELD_RECORD_COUNT = "recordCount"; public static final String FIELD_FILE_SIZE_IN_BYTES = "fileSizeInBytes"; public static final String FIELD_FILE_COUNT = "fileCount"; @@ -62,9 +60,6 @@ public static class Partition implements RESTResponse { @JsonProperty(FIELD_SPEC) private final Map spec; - @JsonProperty(FIELD_PARTITION_TYPE) - private final RowType partitionType; - @JsonProperty(FIELD_RECORD_COUNT) private final long recordCount; @@ -80,13 +75,11 @@ public static class Partition implements RESTResponse { @JsonCreator public Partition( @JsonProperty(FIELD_SPEC) Map spec, - @JsonProperty(FIELD_PARTITION_TYPE) RowType partitionType, @JsonProperty(FIELD_RECORD_COUNT) long recordCount, @JsonProperty(FIELD_FILE_SIZE_IN_BYTES) long fileSizeInBytes, @JsonProperty(FIELD_FILE_COUNT) long fileCount, @JsonProperty(FIELD_LAST_FILE_CREATION_TIME) long lastFileCreationTime) { this.spec = spec; - this.partitionType = partitionType; this.recordCount = recordCount; this.fileSizeInBytes = fileSizeInBytes; this.fileCount = fileCount; @@ -98,11 +91,6 @@ public Map getSpec() { return spec; } - @JsonGetter(FIELD_PARTITION_TYPE) - public RowType getPartitionType() { - return partitionType; - } - @JsonGetter(FIELD_RECORD_COUNT) public long getRecordCount() { return recordCount; diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index ee52dc0b6f23..33f8451f3f6b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -146,14 +146,9 @@ public static DropPartitionRequest dropPartitionRequest() { public static ListPartitionsResponse listPartitionsResponse() { Map spec = new HashMap<>(); - spec.put("a", "1"); - spec.put("b", "2"); - List fields = new ArrayList<>(); - fields.add(new DataField(0, "a", DataTypes.INT())); - fields.add(new DataField(1, "b", DataTypes.STRING())); - RowType partitionType = new RowType(false, fields); + spec.put("f0", "1"); ListPartitionsResponse.Partition partition = - new ListPartitionsResponse.Partition(spec, partitionType, 1, 1, 1, 1); + new ListPartitionsResponse.Partition(spec, 1, 1, 1, 1); return new ListPartitionsResponse(ImmutableList.of(partition)); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 4f1e33255c86..b52d81e8e7cd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -477,11 +477,13 @@ public void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception RESTCatalog restCatalog = new RESTCatalog(CatalogContext.create(options)); RESTCatalog mockRestCatalog = spy(restCatalog); String databaseName = MockRESTMessage.databaseName(); + GetTableResponse getTableResponse = MockRESTMessage.getTableResponse(); + mockResponse(mapper.writeValueAsString(getTableResponse), 200); ListPartitionsResponse response = MockRESTMessage.listPartitionsResponse(); mockResponse(mapper.writeValueAsString(response), 200); List result = mockRestCatalog.listPartitions(Identifier.create(databaseName, "table")); - verify(mockRestCatalog, times(1)).listPartitionsFromServer(any()); + verify(mockRestCatalog, times(1)).listPartitionsFromServer(any(), any()); assertEquals(response.getPartitions().size(), result.size()); } @@ -492,7 +494,7 @@ public void testListPartitionsFromFile() throws Exception { mockResponse(mapper.writeValueAsString(response), 200); mockRestCatalog.listPartitions(Identifier.create(databaseName, "table")); verify(mockRestCatalog, times(1)).getTable(any()); - verify(mockRestCatalog, times(0)).listPartitionsFromServer(any()); + verify(mockRestCatalog, times(0)).listPartitionsFromServer(any(), any()); } @Test @@ -505,8 +507,9 @@ public void convertToPartitionEntryTest() { fields.add(new DataField(1, "b", DataTypes.STRING())); RowType partitionRowType = new RowType(false, fields); ListPartitionsResponse.Partition partition = - new ListPartitionsResponse.Partition(spec, partitionRowType, 1, 1, 1, 1); - PartitionEntry partitionEntry = mockRestCatalog.convertToPartitionEntry(partition); + new ListPartitionsResponse.Partition(spec, 1, 1, 1, 1); + PartitionEntry partitionEntry = + mockRestCatalog.convertToPartitionEntry(partition, partitionRowType); InternalRowPartitionComputer partitionComputer = FileStorePathFactory.getPartitionComputer(partitionRowType, null, false); Map partValues = diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml index e344399dccda..9664dcb41f9a 100644 --- a/paimon-open-api/rest-catalog-open-api.yaml +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -886,8 +886,6 @@ components: properties: spec: type: object - partitionType: - $ref: '#/components/schemas/RowType' recordCount: type: integer format: int64 diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java index 324f93740e4c..b09c352099ba 100644 --- a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java @@ -36,9 +36,6 @@ import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; import org.apache.paimon.rest.responses.SuccessResponse; -import org.apache.paimon.types.DataField; -import org.apache.paimon.types.DataTypes; -import org.apache.paimon.types.RowType; import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; @@ -56,9 +53,7 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RestController; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; /** RESTCatalog management APIs. */ @@ -379,14 +374,9 @@ public ListPartitionsResponse listPartitions( @PathVariable String database, @PathVariable String table) { Map spec = new HashMap<>(); - spec.put("a", "1"); - spec.put("b", "2"); - List fields = new ArrayList<>(); - fields.add(new DataField(0, "a", DataTypes.INT())); - fields.add(new DataField(1, "b", DataTypes.STRING())); - RowType partitionType = new RowType(false, fields); + spec.put("f1", "1"); ListPartitionsResponse.Partition partition = - new ListPartitionsResponse.Partition(spec, partitionType, 1, 2, 3, 4); + new ListPartitionsResponse.Partition(spec, 1, 2, 3, 4); return new ListPartitionsResponse(ImmutableList.of(partition)); } From 1d8faffef1d94dcd185bef1761529c6a71ffebdb Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 30 Dec 2024 14:36:17 +0800 Subject: [PATCH 11/17] create partition return PartitionResponse and drop partition return boolean --- .../org/apache/paimon/rest/RESTCatalog.java | 25 ++--- .../paimon/rest/RESTCatalogOptions.java | 6 -- .../responses/ListPartitionsResponse.java | 73 +-------------- .../rest/responses/PartitionResponse.java | 92 +++++++++++++++++++ .../apache/paimon/rest/MockRESTMessage.java | 4 +- .../apache/paimon/rest/RESTCatalogTest.java | 7 +- paimon-open-api/rest-catalog-open-api.yaml | 12 +-- .../open/api/RESTCatalogController.java | 26 +++--- 8 files changed, 127 insertions(+), 118 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index bd0447c40a69..4cb5087e8340 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -57,7 +57,7 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.SuccessResponse; +import org.apache.paimon.rest.responses.PartitionResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.TableSchema; @@ -93,7 +93,6 @@ import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable; import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE; -import static org.apache.paimon.rest.RESTCatalogOptions.METASTORE_PARTITIONED; import static org.apache.paimon.utils.InternalRowPartitionComputer.convertSpecToInternalRow; import static org.apache.paimon.utils.Preconditions.checkNotNull; import static org.apache.paimon.utils.ThreadPoolUtils.createScheduledThreadPool; @@ -382,7 +381,7 @@ public void createPartition(Identifier identifier, Map partition resourcePaths.partitions( identifier.getDatabaseName(), identifier.getTableName()), request, - SuccessResponse.class, + PartitionResponse.class, headers()); } catch (NoSuchResourceException e) { throw new TableNotExistException(identifier); @@ -397,17 +396,14 @@ public void dropPartition(Identifier identifier, Map partitions) checkNotSystemTable(identifier, "dropPartition"); dropPartitionMetadata(identifier, partitions); Table table = getTable(identifier); - if (table != null) { - cleanPartitionsInFileSystem(table, partitions); - } else { - throw new TableNotExistException(identifier); - } + cleanPartitionsInFileSystem(table, partitions); } @Override public List listPartitions(Identifier identifier) throws TableNotExistException { - boolean whetherSupportListPartitions = context.options().get(METASTORE_PARTITIONED); + boolean whetherSupportListPartitions = + context.options().get(CoreOptions.METASTORE_PARTITIONED_TABLE); if (whetherSupportListPartitions) { FileStoreTable table = (FileStoreTable) getTable(identifier); RowType rowType = table.schema().logicalPartitionType(); @@ -489,8 +485,7 @@ public List listPartitionsFromServer(Identifier identifier, RowT } @VisibleForTesting - PartitionEntry convertToPartitionEntry( - ListPartitionsResponse.Partition partition, RowType rowType) { + PartitionEntry convertToPartitionEntry(PartitionResponse partition, RowType rowType) { InternalRowSerializer serializer = new InternalRowSerializer(rowType); GenericRow row = convertSpecToInternalRow(partition.getSpec(), rowType, null); return new PartitionEntry( @@ -528,18 +523,18 @@ protected GetTableResponse getTableResponse(Identifier identifier) } } - protected SuccessResponse dropPartitionMetadata( - Identifier identifier, Map partitions) + protected boolean dropPartitionMetadata(Identifier identifier, Map partitions) throws TableNoPermissionException { try { DropPartitionRequest request = new DropPartitionRequest(partitions); - return client.delete( + client.delete( resourcePaths.partitions( identifier.getDatabaseName(), identifier.getTableName()), request, headers()); + return true; } catch (NoSuchResourceException ignore) { - return new SuccessResponse(); + return true; } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index d750654f94af..1af64def4f71 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -71,10 +71,4 @@ public class RESTCatalogOptions { .stringType() .noDefaultValue() .withDescription("REST Catalog auth token provider path."); - - public static final ConfigOption METASTORE_PARTITIONED = - ConfigOptions.key("metastore-partitioned") - .booleanType() - .defaultValue(false) - .withDescription("REST Catalog Server whether support list partitions."); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java index e2ce322fd7e3..1f194d208e99 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListPartitionsResponse.java @@ -26,7 +26,6 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; -import java.util.Map; /** Response for listing partitions. */ @JsonIgnoreProperties(ignoreUnknown = true) @@ -35,80 +34,16 @@ public class ListPartitionsResponse implements RESTResponse { public static final String FIELD_PARTITIONS = "partitions"; @JsonProperty(FIELD_PARTITIONS) - private final List partitions; + private final List partitions; @JsonCreator - public ListPartitionsResponse(@JsonProperty(FIELD_PARTITIONS) List partitions) { + public ListPartitionsResponse( + @JsonProperty(FIELD_PARTITIONS) List partitions) { this.partitions = partitions; } @JsonGetter(FIELD_PARTITIONS) - public List getPartitions() { + public List getPartitions() { return partitions; } - - /** Partition for rest api. */ - @JsonIgnoreProperties(ignoreUnknown = true) - public static class Partition implements RESTResponse { - - private static final String FIELD_SPEC = "spec"; - public static final String FIELD_RECORD_COUNT = "recordCount"; - public static final String FIELD_FILE_SIZE_IN_BYTES = "fileSizeInBytes"; - public static final String FIELD_FILE_COUNT = "fileCount"; - public static final String FIELD_LAST_FILE_CREATION_TIME = "lastFileCreationTime"; - - @JsonProperty(FIELD_SPEC) - private final Map spec; - - @JsonProperty(FIELD_RECORD_COUNT) - private final long recordCount; - - @JsonProperty(FIELD_FILE_SIZE_IN_BYTES) - private final long fileSizeInBytes; - - @JsonProperty(FIELD_FILE_COUNT) - private final long fileCount; - - @JsonProperty(FIELD_LAST_FILE_CREATION_TIME) - private final long lastFileCreationTime; - - @JsonCreator - public Partition( - @JsonProperty(FIELD_SPEC) Map spec, - @JsonProperty(FIELD_RECORD_COUNT) long recordCount, - @JsonProperty(FIELD_FILE_SIZE_IN_BYTES) long fileSizeInBytes, - @JsonProperty(FIELD_FILE_COUNT) long fileCount, - @JsonProperty(FIELD_LAST_FILE_CREATION_TIME) long lastFileCreationTime) { - this.spec = spec; - this.recordCount = recordCount; - this.fileSizeInBytes = fileSizeInBytes; - this.fileCount = fileCount; - this.lastFileCreationTime = lastFileCreationTime; - } - - @JsonGetter(FIELD_SPEC) - public Map getSpec() { - return spec; - } - - @JsonGetter(FIELD_RECORD_COUNT) - public long getRecordCount() { - return recordCount; - } - - @JsonGetter(FIELD_FILE_SIZE_IN_BYTES) - public long getFileSizeInBytes() { - return fileSizeInBytes; - } - - @JsonGetter(FIELD_FILE_COUNT) - public long getFileCount() { - return fileCount; - } - - @JsonGetter(FIELD_LAST_FILE_CREATION_TIME) - public long getLastFileCreationTime() { - return lastFileCreationTime; - } - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java new file mode 100644 index 000000000000..5c08c4d64f1c --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; + +/** Partition for rest api. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class PartitionResponse implements RESTResponse { + public static final String FIELD_SPEC = "spec"; + public static final String FIELD_RECORD_COUNT = "recordCount"; + public static final String FIELD_FILE_SIZE_IN_BYTES = "fileSizeInBytes"; + public static final String FIELD_FILE_COUNT = "fileCount"; + public static final String FIELD_LAST_FILE_CREATION_TIME = "lastFileCreationTime"; + + @JsonProperty(FIELD_SPEC) + private final Map spec; + + @JsonProperty(FIELD_RECORD_COUNT) + private final long recordCount; + + @JsonProperty(FIELD_FILE_SIZE_IN_BYTES) + private final long fileSizeInBytes; + + @JsonProperty(FIELD_FILE_COUNT) + private final long fileCount; + + @JsonProperty(FIELD_LAST_FILE_CREATION_TIME) + private final long lastFileCreationTime; + + @JsonCreator + public PartitionResponse( + @JsonProperty(FIELD_SPEC) Map spec, + @JsonProperty(FIELD_RECORD_COUNT) long recordCount, + @JsonProperty(FIELD_FILE_SIZE_IN_BYTES) long fileSizeInBytes, + @JsonProperty(FIELD_FILE_COUNT) long fileCount, + @JsonProperty(FIELD_LAST_FILE_CREATION_TIME) long lastFileCreationTime) { + this.spec = spec; + this.recordCount = recordCount; + this.fileSizeInBytes = fileSizeInBytes; + this.fileCount = fileCount; + this.lastFileCreationTime = lastFileCreationTime; + } + + @JsonGetter(FIELD_SPEC) + public Map getSpec() { + return spec; + } + + @JsonGetter(FIELD_RECORD_COUNT) + public long getRecordCount() { + return recordCount; + } + + @JsonGetter(FIELD_FILE_SIZE_IN_BYTES) + public long getFileSizeInBytes() { + return fileSizeInBytes; + } + + @JsonGetter(FIELD_FILE_COUNT) + public long getFileCount() { + return fileCount; + } + + @JsonGetter(FIELD_LAST_FILE_CREATION_TIME) + public long getLastFileCreationTime() { + return lastFileCreationTime; + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index 33f8451f3f6b..592ec90c3bf4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -34,6 +34,7 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; +import org.apache.paimon.rest.responses.PartitionResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.types.DataField; @@ -147,8 +148,7 @@ public static DropPartitionRequest dropPartitionRequest() { public static ListPartitionsResponse listPartitionsResponse() { Map spec = new HashMap<>(); spec.put("f0", "1"); - ListPartitionsResponse.Partition partition = - new ListPartitionsResponse.Partition(spec, 1, 1, 1, 1); + PartitionResponse partition = new PartitionResponse(spec, 1, 1, 1, 1); return new ListPartitionsResponse(ImmutableList.of(partition)); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index b52d81e8e7cd..2104d36d9d97 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -18,6 +18,7 @@ package org.apache.paimon.rest; +import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.Database; @@ -34,6 +35,7 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; +import org.apache.paimon.rest.responses.PartitionResponse; import org.apache.paimon.rest.responses.SuccessResponse; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.Table; @@ -472,7 +474,7 @@ public void testDropPartitionWhenTableNoExist() throws Exception { @Test public void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception { Options options = mockInitOptions(); - options = options.set(RESTCatalogOptions.METASTORE_PARTITIONED, true); + options = options.set(CoreOptions.METASTORE_PARTITIONED_TABLE, true); mockConfig(warehouseStr); RESTCatalog restCatalog = new RESTCatalog(CatalogContext.create(options)); RESTCatalog mockRestCatalog = spy(restCatalog); @@ -506,8 +508,7 @@ public void convertToPartitionEntryTest() { fields.add(new DataField(0, "a", DataTypes.INT())); fields.add(new DataField(1, "b", DataTypes.STRING())); RowType partitionRowType = new RowType(false, fields); - ListPartitionsResponse.Partition partition = - new ListPartitionsResponse.Partition(spec, 1, 1, 1, 1); + PartitionResponse partition = new PartitionResponse(spec, 1, 1, 1, 1); PartitionEntry partitionEntry = mockRestCatalog.convertToPartitionEntry(partition, partitionRowType); InternalRowPartitionComputer partitionComputer = diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml index 9664dcb41f9a..b1c6489855ab 100644 --- a/paimon-open-api/rest-catalog-open-api.yaml +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -471,7 +471,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/SuccessResponse' + $ref: '#/components/schemas/PartitionResponse' "404": description: Resource not found content: @@ -508,11 +508,7 @@ paths: $ref: '#/components/schemas/DropPartitionRequest' responses: "200": - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/SuccessResponse' + description: Success, no content "404": description: Resource not found content: @@ -880,8 +876,8 @@ components: partitions: type: array items: - $ref: '#/components/schemas/Partition' - Partition: + $ref: '#/components/schemas/PartitionResponse' + PartitionResponse: type: object properties: spec: diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java index b09c352099ba..87443d1d6245 100644 --- a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java @@ -35,7 +35,7 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.SuccessResponse; +import org.apache.paimon.rest.responses.PartitionResponse; import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; @@ -147,6 +147,7 @@ public GetDatabaseResponse getDatabases( summary = "Drop Database", tags = {"database"}) @ApiResponses({ + @ApiResponse(responseCode = "200", description = "Success, no content"), @ApiResponse( responseCode = "404", description = "Resource not found", @@ -305,6 +306,7 @@ public GetTableResponse alterTable( summary = "Drop table", tags = {"table"}) @ApiResponses({ + @ApiResponse(responseCode = "200", description = "Success, no content"), @ApiResponse( responseCode = "404", description = "Resource not found", @@ -375,8 +377,7 @@ public ListPartitionsResponse listPartitions( @PathVariable String table) { Map spec = new HashMap<>(); spec.put("f1", "1"); - ListPartitionsResponse.Partition partition = - new ListPartitionsResponse.Partition(spec, 1, 2, 3, 4); + PartitionResponse partition = new PartitionResponse(spec, 1, 2, 3, 4); return new ListPartitionsResponse(ImmutableList.of(partition)); } @@ -384,9 +385,7 @@ public ListPartitionsResponse listPartitions( summary = "Create partition", tags = {"partition"}) @ApiResponses({ - @ApiResponse( - responseCode = "200", - content = {@Content(schema = @Schema(implementation = SuccessResponse.class))}), + @ApiResponse(responseCode = "200", description = "Success, no content"), @ApiResponse( responseCode = "404", description = "Resource not found", @@ -396,21 +395,20 @@ public ListPartitionsResponse listPartitions( content = {@Content(schema = @Schema())}) }) @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions") - public SuccessResponse createPartition( + public PartitionResponse createPartition( @PathVariable String prefix, @PathVariable String database, @PathVariable String table, @RequestBody CreatePartitionRequest request) { - return new SuccessResponse(); + Map spec = new HashMap<>(); + spec.put("f1", "1"); + return new PartitionResponse(spec, 0, 0, 0, 4); } @Operation( summary = "Drop partition", tags = {"partition"}) @ApiResponses({ - @ApiResponse( - responseCode = "200", - content = {@Content(schema = @Schema(implementation = SuccessResponse.class))}), @ApiResponse( responseCode = "404", description = "Resource not found", @@ -420,11 +418,9 @@ public SuccessResponse createPartition( content = {@Content(schema = @Schema())}) }) @DeleteMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions") - public SuccessResponse dropPartition( + public void dropPartition( @PathVariable String prefix, @PathVariable String database, @PathVariable String table, - @RequestBody DropPartitionRequest request) { - return new SuccessResponse(); - } + @RequestBody DropPartitionRequest request) {} } From 3e5e7ecd062be2f168b2f2776a84c4d0e8694a3f Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 30 Dec 2024 14:47:25 +0800 Subject: [PATCH 12/17] delete SuccessResponse --- .../rest/responses/SuccessResponse.java | 27 ------------------- .../apache/paimon/rest/MockRESTMessage.java | 8 ++++-- .../apache/paimon/rest/RESTCatalogTest.java | 6 ++--- .../paimon/rest/RESTObjectMapperTest.java | 10 ++++--- paimon-open-api/rest-catalog-open-api.yaml | 2 -- 5 files changed, 15 insertions(+), 38 deletions(-) delete mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/responses/SuccessResponse.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/SuccessResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/SuccessResponse.java deleted file mode 100644 index 0169731779c6..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/SuccessResponse.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest.responses; - -import org.apache.paimon.rest.RESTResponse; - -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -/** Response for do action success. */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class SuccessResponse implements RESTResponse {} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index 592ec90c3bf4..013690e2e3b6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -145,10 +145,14 @@ public static DropPartitionRequest dropPartitionRequest() { return new DropPartitionRequest(Collections.singletonMap("pt", "1")); } - public static ListPartitionsResponse listPartitionsResponse() { + public static PartitionResponse partitionResponse() { Map spec = new HashMap<>(); spec.put("f0", "1"); - PartitionResponse partition = new PartitionResponse(spec, 1, 1, 1, 1); + return new PartitionResponse(spec, 1, 1, 1, 1); + } + + public static ListPartitionsResponse listPartitionsResponse() { + PartitionResponse partition = partitionResponse(); return new ListPartitionsResponse(ImmutableList.of(partition)); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 2104d36d9d97..5517dbc1d5e1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -36,7 +36,6 @@ import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; import org.apache.paimon.rest.responses.PartitionResponse; -import org.apache.paimon.rest.responses.SuccessResponse; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.Table; import org.apache.paimon.types.DataField; @@ -365,7 +364,8 @@ public void testCreatePartition() throws Exception { String databaseName = MockRESTMessage.databaseName(); Map partitionSpec = new HashMap<>(); partitionSpec.put("p1", "v1"); - mockResponse(mapper.writeValueAsString(new SuccessResponse()), 200); + PartitionResponse response = MockRESTMessage.partitionResponse(); + mockResponse(mapper.writeValueAsString(response), 200); assertDoesNotThrow( () -> mockRestCatalog.createPartition( @@ -458,7 +458,7 @@ public void testDropPartitionWhenTableNoExist() throws Exception { Map partitionSpec = new HashMap<>(); GetTableResponse response = MockRESTMessage.getTableResponse(); partitionSpec.put(response.getSchema().primaryKeys().get(0), "1"); - mockResponse(mapper.writeValueAsString(new SuccessResponse()), 200); + mockResponse(mapper.writeValueAsString(""), 200); mockResponse("", 404); doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any()); assertThrows( diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java index b3a5c378d812..6712b7b991f3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java @@ -34,7 +34,7 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.SuccessResponse; +import org.apache.paimon.rest.responses.PartitionResponse; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; @@ -230,10 +230,12 @@ public void listPartitionsResponseParseTest() throws Exception { } @Test - public void successResponseParseTest() throws Exception { - SuccessResponse response = new SuccessResponse(); + public void partitionResponseParseTest() throws Exception { + PartitionResponse response = MockRESTMessage.partitionResponse(); assertDoesNotThrow(() -> mapper.writeValueAsString(response)); assertDoesNotThrow( - () -> mapper.readValue(mapper.writeValueAsString(response), SuccessResponse.class)); + () -> + mapper.readValue( + mapper.writeValueAsString(response), PartitionResponse.class)); } } diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml index b1c6489855ab..68a0bfd3df4b 100644 --- a/paimon-open-api/rest-catalog-open-api.yaml +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -894,8 +894,6 @@ components: lastFileCreationTime: type: integer format: int64 - SuccessResponse: - type: object securitySchemes: BearerAuth: type: http From 58a0aa3510e36f863963f1e4948e3315b51fdd4e Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 30 Dec 2024 14:55:56 +0800 Subject: [PATCH 13/17] format PartitionResponse and update open api --- .../org/apache/paimon/rest/responses/PartitionResponse.java | 1 + paimon-open-api/rest-catalog-open-api.yaml | 4 ++++ .../org/apache/paimon/open/api/RESTCatalogController.java | 5 ++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java index 5c08c4d64f1c..2706b5d7daf9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java @@ -30,6 +30,7 @@ /** Partition for rest api. */ @JsonIgnoreProperties(ignoreUnknown = true) public class PartitionResponse implements RESTResponse { + public static final String FIELD_SPEC = "spec"; public static final String FIELD_RECORD_COUNT = "recordCount"; public static final String FIELD_FILE_SIZE_IN_BYTES = "fileSizeInBytes"; diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml index 68a0bfd3df4b..158305944134 100644 --- a/paimon-open-api/rest-catalog-open-api.yaml +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -144,6 +144,8 @@ paths: schema: type: string responses: + "200": + description: Success, no content "404": description: Resource not found content: @@ -346,6 +348,8 @@ paths: schema: type: string responses: + "200": + description: Success, no content "404": description: Resource not found content: diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java index 87443d1d6245..62f99876a397 100644 --- a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java @@ -385,7 +385,9 @@ public ListPartitionsResponse listPartitions( summary = "Create partition", tags = {"partition"}) @ApiResponses({ - @ApiResponse(responseCode = "200", description = "Success, no content"), + @ApiResponse( + responseCode = "200", + content = {@Content(schema = @Schema(implementation = PartitionResponse.class))}), @ApiResponse( responseCode = "404", description = "Resource not found", @@ -409,6 +411,7 @@ public PartitionResponse createPartition( summary = "Drop partition", tags = {"partition"}) @ApiResponses({ + @ApiResponse(responseCode = "200", description = "Success, no content"), @ApiResponse( responseCode = "404", description = "Resource not found", From 9c7139bd24924dae9486c9c7aa4dde6628d4426a Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 30 Dec 2024 15:04:48 +0800 Subject: [PATCH 14/17] update partitionSpec to spec, same with PartitionResponse --- .../apache/paimon/rest/requests/CreatePartitionRequest.java | 2 +- .../org/apache/paimon/rest/requests/DropPartitionRequest.java | 2 +- paimon-open-api/rest-catalog-open-api.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java index 094274514029..e8094ab821bf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java @@ -33,7 +33,7 @@ public class CreatePartitionRequest implements RESTRequest { private static final String FIELD_IDENTIFIER = "identifier"; - private static final String FIELD_PARTITION_SPEC = "partitionSpec"; + private static final String FIELD_PARTITION_SPEC = "spec"; @JsonProperty(FIELD_IDENTIFIER) private final Identifier identifier; diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java index 47fbf5651881..4fabf1163651 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java @@ -31,7 +31,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class DropPartitionRequest implements RESTRequest { - private static final String FIELD_PARTITION_SPEC = "partitionSpec"; + private static final String FIELD_PARTITION_SPEC = "spec"; @JsonProperty(FIELD_PARTITION_SPEC) private final Map partitionSpec; diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml index 158305944134..7a0c9663b4f2 100644 --- a/paimon-open-api/rest-catalog-open-api.yaml +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -537,12 +537,12 @@ components: properties: identifier: $ref: '#/components/schemas/Identifier' - partitionSpec: + spec: type: object DropPartitionRequest: type: object properties: - partitionSpec: + spec: type: object CreateDatabaseResponse: type: object From ccd53b229c52ce88a290b839b1c4b059a5be6391 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 30 Dec 2024 15:57:35 +0800 Subject: [PATCH 15/17] list partitions check options use table options --- .../org/apache/paimon/rest/RESTCatalog.java | 5 +++-- .../apache/paimon/rest/MockRESTMessage.java | 18 +++++++++++++----- .../apache/paimon/rest/RESTCatalogTest.java | 11 +++-------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 4cb5087e8340..84241dfb65a2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -402,10 +402,11 @@ public void dropPartition(Identifier identifier, Map partitions) @Override public List listPartitions(Identifier identifier) throws TableNotExistException { + FileStoreTable table = (FileStoreTable) getTable(identifier); boolean whetherSupportListPartitions = - context.options().get(CoreOptions.METASTORE_PARTITIONED_TABLE); + Boolean.parseBoolean( + table.options().get(CoreOptions.METASTORE_PARTITIONED_TABLE.key())); if (whetherSupportListPartitions) { - FileStoreTable table = (FileStoreTable) getTable(identifier); RowType rowType = table.schema().logicalPartitionType(); return listPartitionsFromServer(identifier, rowType); } else { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index 013690e2e3b6..9b686b683773 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -18,6 +18,7 @@ package org.apache.paimon.rest; +import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterTableRequest; @@ -227,20 +228,27 @@ public static List getChanges() { return schemaChanges; } + public static GetTableResponse getTableResponseEnablePartition() { + Map options = new HashMap<>(); + options.put("option-1", "value-1"); + options.put(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true"); + return new GetTableResponse("/tmp/2", 1, schema(options)); + } + public static GetTableResponse getTableResponse() { - return new GetTableResponse("/tmp/1", 1, schema()); + Map options = new HashMap<>(); + options.put("option-1", "value-1"); + options.put("option-2", "value-2"); + return new GetTableResponse("/tmp/1", 1, schema(options)); } - private static Schema schema() { + private static Schema schema(Map options) { List fields = Arrays.asList( new DataField(0, "f0", new IntType()), new DataField(1, "f1", new IntType())); List partitionKeys = Collections.singletonList("f0"); List primaryKeys = Arrays.asList("f0", "f1"); - Map options = new HashMap<>(); - options.put("option-1", "value-1"); - options.put("option-2", "value-2"); return new Schema(fields, partitionKeys, primaryKeys, options, "comment"); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 5517dbc1d5e1..db017a4cb976 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -18,7 +18,6 @@ package org.apache.paimon.rest; -import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.Database; @@ -473,13 +472,8 @@ public void testDropPartitionWhenTableNoExist() throws Exception { @Test public void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception { - Options options = mockInitOptions(); - options = options.set(CoreOptions.METASTORE_PARTITIONED_TABLE, true); - mockConfig(warehouseStr); - RESTCatalog restCatalog = new RESTCatalog(CatalogContext.create(options)); - RESTCatalog mockRestCatalog = spy(restCatalog); String databaseName = MockRESTMessage.databaseName(); - GetTableResponse getTableResponse = MockRESTMessage.getTableResponse(); + GetTableResponse getTableResponse = MockRESTMessage.getTableResponseEnablePartition(); mockResponse(mapper.writeValueAsString(getTableResponse), 200); ListPartitionsResponse response = MockRESTMessage.listPartitionsResponse(); mockResponse(mapper.writeValueAsString(response), 200); @@ -494,8 +488,9 @@ public void testListPartitionsFromFile() throws Exception { String databaseName = MockRESTMessage.databaseName(); GetTableResponse response = MockRESTMessage.getTableResponse(); mockResponse(mapper.writeValueAsString(response), 200); + mockResponse(mapper.writeValueAsString(response), 200); mockRestCatalog.listPartitions(Identifier.create(databaseName, "table")); - verify(mockRestCatalog, times(1)).getTable(any()); + verify(mockRestCatalog, times(2)).getTable(any()); verify(mockRestCatalog, times(0)).listPartitionsFromServer(any(), any()); } From d84198d6fedd8b1278a4f03d178ebcd6f8b4e590 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 30 Dec 2024 16:38:00 +0800 Subject: [PATCH 16/17] drop partition when 404 throw exception --- .../org/apache/paimon/rest/RESTCatalog.java | 40 ++++++++----------- .../apache/paimon/rest/RESTCatalogTest.java | 36 ++--------------- 2 files changed, 20 insertions(+), 56 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 84241dfb65a2..2a771a2c4fc3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -20,7 +20,6 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.TableType; -import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogUtils; @@ -429,16 +428,14 @@ public void close() throws Exception { } } - @VisibleForTesting - Map fetchOptionsFromServer( + protected Map fetchOptionsFromServer( Map headers, Map clientProperties) { ConfigResponse response = client.get(ResourcePaths.V1_CONFIG, ConfigResponse.class, headers); return response.merge(clientProperties); } - @VisibleForTesting - Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException { + protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException { Preconditions.checkArgument(identifier.getSystemTableName() == null); GetTableResponse response = getTableResponse(identifier); FileStoreTable table = @@ -461,8 +458,7 @@ Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException return table; } - @VisibleForTesting - public List listPartitionsFromServer(Identifier identifier, RowType rowType) + protected List listPartitionsFromServer(Identifier identifier, RowType rowType) throws TableNotExistException { try { ListPartitionsResponse response = @@ -485,20 +481,7 @@ public List listPartitionsFromServer(Identifier identifier, RowT } } - @VisibleForTesting - PartitionEntry convertToPartitionEntry(PartitionResponse partition, RowType rowType) { - InternalRowSerializer serializer = new InternalRowSerializer(rowType); - GenericRow row = convertSpecToInternalRow(partition.getSpec(), rowType, null); - return new PartitionEntry( - serializer.toBinaryRow(row).copy(), - partition.getRecordCount(), - partition.getFileSizeInBytes(), - partition.getFileCount(), - partition.getLastFileCreationTime()); - } - - @VisibleForTesting - void cleanPartitionsInFileSystem(Table table, Map partitions) { + protected void cleanPartitionsInFileSystem(Table table, Map partitions) { FileStoreTable fileStoreTable = (FileStoreTable) table; try (FileStoreCommit commit = fileStoreTable @@ -525,7 +508,7 @@ protected GetTableResponse getTableResponse(Identifier identifier) } protected boolean dropPartitionMetadata(Identifier identifier, Map partitions) - throws TableNoPermissionException { + throws TableNoPermissionException, PartitionNotExistException { try { DropPartitionRequest request = new DropPartitionRequest(partitions); client.delete( @@ -535,7 +518,7 @@ protected boolean dropPartitionMetadata(Identifier identifier, Map mockRestCatalog.dropPartition( Identifier.create(databaseName, "table"), partitionSpec)); verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any()); - verify(mockRestCatalog, times(1)).getTable(any()); - verify(mockRestCatalog, times(1)).cleanPartitionsInFileSystem(any(), any()); + verify(mockRestCatalog, times(0)).cleanPartitionsInFileSystem(any(), any()); } @Test @@ -494,31 +489,6 @@ public void testListPartitionsFromFile() throws Exception { verify(mockRestCatalog, times(0)).listPartitionsFromServer(any(), any()); } - @Test - public void convertToPartitionEntryTest() { - Map spec = new HashMap<>(); - spec.put("a", "1"); - spec.put("b", "2"); - List fields = new ArrayList<>(); - fields.add(new DataField(0, "a", DataTypes.INT())); - fields.add(new DataField(1, "b", DataTypes.STRING())); - RowType partitionRowType = new RowType(false, fields); - PartitionResponse partition = new PartitionResponse(spec, 1, 1, 1, 1); - PartitionEntry partitionEntry = - mockRestCatalog.convertToPartitionEntry(partition, partitionRowType); - InternalRowPartitionComputer partitionComputer = - FileStorePathFactory.getPartitionComputer(partitionRowType, null, false); - Map partValues = - partitionComputer.generatePartValues(partitionEntry.partition()); - for (Map.Entry entry : spec.entrySet()) { - assertEquals(entry.getValue(), partValues.get(entry.getKey())); - } - assertEquals(partitionEntry.recordCount(), partition.getRecordCount()); - assertEquals(partitionEntry.fileSizeInBytes(), partition.getFileSizeInBytes()); - assertEquals(partitionEntry.fileCount(), partition.getFileCount()); - assertEquals(partitionEntry.lastFileCreationTime(), partition.getLastFileCreationTime()); - } - private void mockResponse(String mockResponse, int httpCode) { MockResponse mockResponseObj = new MockResponse() From 67da13454a2a9fb23c9574ecbb09524d89816021 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 30 Dec 2024 18:06:10 +0800 Subject: [PATCH 17/17] RESTCatalogTest remove inner api dependencies --- .../org/apache/paimon/rest/RESTCatalog.java | 39 ++++---- .../apache/paimon/rest/RESTCatalogTest.java | 93 ++++++------------- 2 files changed, 46 insertions(+), 86 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 2a771a2c4fc3..c430e303b276 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -155,20 +155,6 @@ public RESTCatalog(CatalogContext catalogContext) { this.fileIO = getFileIOFromOptions(context); } - private static FileIO getFileIOFromOptions(CatalogContext context) { - try { - Options options = context.options(); - String warehouseStr = options.get(CatalogOptions.WAREHOUSE); - Path warehousePath = new Path(warehouseStr); - CatalogContext contextWithNewOptions = - CatalogContext.create(options, context.preferIO(), context.fallbackIO()); - return FileIO.get(warehousePath, contextWithNewOptions); - } catch (IOException e) { - LOG.warn("Can not get FileIO from options."); - throw new RuntimeException(e); - } - } - @Override public String warehouse() { return context.options().get(CatalogOptions.WAREHOUSE); @@ -435,7 +421,7 @@ protected Map fetchOptionsFromServer( return response.merge(clientProperties); } - protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException { + private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException { Preconditions.checkArgument(identifier.getSystemTableName() == null); GetTableResponse response = getTableResponse(identifier); FileStoreTable table = @@ -458,7 +444,7 @@ protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExist return table; } - protected List listPartitionsFromServer(Identifier identifier, RowType rowType) + private List listPartitionsFromServer(Identifier identifier, RowType rowType) throws TableNotExistException { try { ListPartitionsResponse response = @@ -481,7 +467,7 @@ protected List listPartitionsFromServer(Identifier identifier, R } } - protected void cleanPartitionsInFileSystem(Table table, Map partitions) { + private void cleanPartitionsInFileSystem(Table table, Map partitions) { FileStoreTable fileStoreTable = (FileStoreTable) table; try (FileStoreCommit commit = fileStoreTable @@ -493,8 +479,7 @@ protected void cleanPartitionsInFileSystem(Table table, Map part } } - protected GetTableResponse getTableResponse(Identifier identifier) - throws TableNotExistException { + private GetTableResponse getTableResponse(Identifier identifier) throws TableNotExistException { try { return client.get( resourcePaths.table(identifier.getDatabaseName(), identifier.getTableName()), @@ -507,7 +492,7 @@ protected GetTableResponse getTableResponse(Identifier identifier) } } - protected boolean dropPartitionMetadata(Identifier identifier, Map partitions) + private boolean dropPartitionMetadata(Identifier identifier, Map partitions) throws TableNoPermissionException, PartitionNotExistException { try { DropPartitionRequest request = new DropPartitionRequest(partitions); @@ -565,4 +550,18 @@ private PartitionEntry convertToPartitionEntry(PartitionResponse partition, RowT partition.getFileCount(), partition.getLastFileCreationTime()); } + + private static FileIO getFileIOFromOptions(CatalogContext context) { + try { + Options options = context.options(); + String warehouseStr = options.get(CatalogOptions.WAREHOUSE); + Path warehousePath = new Path(warehouseStr); + CatalogContext contextWithNewOptions = + CatalogContext.create(options, context.preferIO(), context.fallbackIO()); + return FileIO.get(warehousePath, contextWithNewOptions); + } catch (IOException e) { + LOG.warn("Can not get FileIO from options."); + throw new RuntimeException(e); + } + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index fd5793193fe8..67103aaa5204 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -58,14 +58,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; /** Test for REST Catalog. */ public class RESTCatalogTest { @@ -73,7 +65,6 @@ public class RESTCatalogTest { private final ObjectMapper mapper = RESTObjectMapper.create(); private MockWebServer mockWebServer; private RESTCatalog restCatalog; - private RESTCatalog mockRestCatalog; private String warehouseStr; private String serverUrl; @Rule public TemporaryFolder folder = new TemporaryFolder(); @@ -87,7 +78,6 @@ public void setUp() throws IOException { warehouseStr = folder.getRoot().getPath(); mockConfig(warehouseStr); restCatalog = new RESTCatalog(CatalogContext.create(options)); - mockRestCatalog = spy(restCatalog); } @After @@ -148,9 +138,7 @@ public void testGetDatabase() throws Exception { public void testDropDatabase() throws Exception { String name = MockRESTMessage.databaseName(); mockResponse("", 200); - assertDoesNotThrow(() -> mockRestCatalog.dropDatabase(name, false, true)); - verify(mockRestCatalog, times(1)).dropDatabase(eq(name), eq(false), eq(true)); - verify(mockRestCatalog, times(0)).listTables(eq(name)); + assertDoesNotThrow(() -> restCatalog.dropDatabase(name, false, true)); } @Test @@ -160,7 +148,7 @@ public void testDropDatabaseWhenNoExistAndIgnoreIfNotExistsIsFalse() throws Exce mockResponse(mapper.writeValueAsString(response), 404); assertThrows( Catalog.DatabaseNotExistException.class, - () -> mockRestCatalog.dropDatabase(name, false, true)); + () -> restCatalog.dropDatabase(name, false, true)); } @Test @@ -168,9 +156,7 @@ public void testDropDatabaseWhenNoExistAndIgnoreIfNotExistsIsTrue() throws Excep String name = MockRESTMessage.databaseName(); ErrorResponse response = MockRESTMessage.noSuchResourceExceptionErrorResponse(); mockResponse(mapper.writeValueAsString(response), 404); - assertDoesNotThrow(() -> mockRestCatalog.dropDatabase(name, true, true)); - verify(mockRestCatalog, times(1)).dropDatabase(eq(name), eq(true), eq(true)); - verify(mockRestCatalog, times(0)).listTables(eq(name)); + assertDoesNotThrow(() -> restCatalog.dropDatabase(name, true, true)); } @Test @@ -180,9 +166,7 @@ public void testDropDatabaseWhenCascadeIsFalseAndNoTables() throws Exception { ListTablesResponse response = MockRESTMessage.listTablesEmptyResponse(); mockResponse(mapper.writeValueAsString(response), 200); mockResponse("", 200); - assertDoesNotThrow(() -> mockRestCatalog.dropDatabase(name, false, cascade)); - verify(mockRestCatalog, times(1)).dropDatabase(eq(name), eq(false), eq(cascade)); - verify(mockRestCatalog, times(1)).listTables(eq(name)); + assertDoesNotThrow(() -> restCatalog.dropDatabase(name, false, cascade)); } @Test @@ -193,9 +177,7 @@ public void testDropDatabaseWhenCascadeIsFalseAndTablesExist() throws Exception mockResponse(mapper.writeValueAsString(response), 200); assertThrows( Catalog.DatabaseNotEmptyException.class, - () -> mockRestCatalog.dropDatabase(name, false, cascade)); - verify(mockRestCatalog, times(1)).dropDatabase(eq(name), eq(false), eq(cascade)); - verify(mockRestCatalog, times(1)).listTables(eq(name)); + () -> restCatalog.dropDatabase(name, false, cascade)); } @Test @@ -203,7 +185,7 @@ public void testAlterDatabase() throws Exception { String name = MockRESTMessage.databaseName(); AlterDatabaseResponse response = MockRESTMessage.alterDatabaseResponse(); mockResponse(mapper.writeValueAsString(response), 200); - assertDoesNotThrow(() -> mockRestCatalog.alterDatabase(name, new ArrayList<>(), true)); + assertDoesNotThrow(() -> restCatalog.alterDatabase(name, new ArrayList<>(), true)); } @Test @@ -214,7 +196,7 @@ public void testAlterDatabaseWhenDatabaseNotExistAndIgnoreIfNotExistsIsFalse() mockResponse(mapper.writeValueAsString(response), 404); assertThrows( Catalog.DatabaseNotExistException.class, - () -> mockRestCatalog.alterDatabase(name, new ArrayList<>(), false)); + () -> restCatalog.alterDatabase(name, new ArrayList<>(), false)); } @Test @@ -222,7 +204,7 @@ public void testAlterDatabaseWhenDatabaseNotExistAndIgnoreIfNotExistsIsTrue() th String name = MockRESTMessage.databaseName(); ErrorResponse response = MockRESTMessage.noSuchResourceExceptionErrorResponse(); mockResponse(mapper.writeValueAsString(response), 404); - assertDoesNotThrow(() -> mockRestCatalog.alterDatabase(name, new ArrayList<>(), true)); + assertDoesNotThrow(() -> restCatalog.alterDatabase(name, new ArrayList<>(), true)); } @Test @@ -239,10 +221,8 @@ public void testGetTable() throws Exception { String databaseName = MockRESTMessage.databaseName(); GetTableResponse response = MockRESTMessage.getTableResponse(); mockResponse(mapper.writeValueAsString(response), 200); - Table result = mockRestCatalog.getTable(Identifier.create(databaseName, "table")); - // catalog will add path option + Table result = restCatalog.getTable(Identifier.create(databaseName, "table")); assertEquals(response.getSchema().options().size() + 1, result.options().size()); - verify(mockRestCatalog, times(1)).getDataOrFormatTable(any()); } @Test @@ -272,11 +252,10 @@ public void testRenameTable() throws Exception { mockResponse(mapper.writeValueAsString(response), 200); assertDoesNotThrow( () -> - mockRestCatalog.renameTable( + restCatalog.renameTable( Identifier.create(databaseName, fromTableName), Identifier.create(databaseName, toTableName), true)); - verify(mockRestCatalog, times(1)).renameTable(any(), any(), anyBoolean()); } @Test @@ -288,7 +267,7 @@ public void testRenameTableWhenTableNotExistAndIgnoreIfNotExistsIsFalse() throws assertThrows( Catalog.TableNotExistException.class, () -> - mockRestCatalog.renameTable( + restCatalog.renameTable( Identifier.create(databaseName, fromTableName), Identifier.create(databaseName, toTableName), false)); @@ -303,7 +282,7 @@ public void testRenameTableWhenToTableAlreadyExist() throws Exception { assertThrows( Catalog.TableAlreadyExistException.class, () -> - mockRestCatalog.renameTable( + restCatalog.renameTable( Identifier.create(databaseName, fromTableName), Identifier.create(databaseName, toTableName), false)); @@ -316,10 +295,7 @@ public void testAlterTable() throws Exception { GetTableResponse response = MockRESTMessage.getTableResponse(); mockResponse(mapper.writeValueAsString(response), 200); assertDoesNotThrow( - () -> - mockRestCatalog.alterTable( - Identifier.create(databaseName, "t1"), changes, true)); - verify(mockRestCatalog, times(1)).alterTable(any(), anyList(), anyBoolean()); + () -> restCatalog.alterTable(Identifier.create(databaseName, "t1"), changes, true)); } @Test @@ -330,7 +306,7 @@ public void testAlterTableWhenTableNotExistAndIgnoreIfNotExistsIsFalse() throws assertThrows( Catalog.TableNotExistException.class, () -> - mockRestCatalog.alterTable( + restCatalog.alterTable( Identifier.create(databaseName, "t1"), changes, false)); } @@ -362,7 +338,7 @@ public void testCreatePartition() throws Exception { mockResponse(mapper.writeValueAsString(response), 200); assertDoesNotThrow( () -> - mockRestCatalog.createPartition( + restCatalog.createPartition( Identifier.create(databaseName, "table"), partitionSpec)); } @@ -375,7 +351,7 @@ public void testCreatePartitionWhenTableNotExist() throws Exception { assertThrows( Catalog.TableNotExistException.class, () -> - mockRestCatalog.createPartition( + restCatalog.createPartition( Identifier.create(databaseName, "table"), partitionSpec)); } @@ -388,7 +364,7 @@ public void testCreatePartitionWhenTableNoPermissionException() throws Exception assertThrows( Catalog.TableNoPermissionException.class, () -> - mockRestCatalog.createPartition( + restCatalog.createPartition( Identifier.create(databaseName, "table"), partitionSpec)); } @@ -400,14 +376,11 @@ public void testDropPartition() throws Exception { partitionSpec.put(response.getSchema().primaryKeys().get(0), "1"); mockResponse(mapper.writeValueAsString(""), 200); mockResponse(mapper.writeValueAsString(response), 200); - doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any()); - assertDoesNotThrow( + assertThrows( + RuntimeException.class, () -> - mockRestCatalog.dropPartition( + restCatalog.dropPartition( Identifier.create(databaseName, "table"), partitionSpec)); - verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any()); - verify(mockRestCatalog, times(1)).getTable(any()); - verify(mockRestCatalog, times(1)).cleanPartitionsInFileSystem(any(), any()); } @Test @@ -418,14 +391,11 @@ public void testDropPartitionWhenPartitionNoExist() throws Exception { partitionSpec.put(response.getSchema().primaryKeys().get(0), "1"); mockResponse(mapper.writeValueAsString(""), 404); mockResponse(mapper.writeValueAsString(response), 200); - doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any()); assertThrows( Catalog.PartitionNotExistException.class, () -> - mockRestCatalog.dropPartition( + restCatalog.dropPartition( Identifier.create(databaseName, "table"), partitionSpec)); - verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any()); - verify(mockRestCatalog, times(0)).cleanPartitionsInFileSystem(any(), any()); } @Test @@ -435,15 +405,11 @@ public void testDropPartitionWhenTableNoPermission() throws Exception { GetTableResponse response = MockRESTMessage.getTableResponse(); partitionSpec.put(response.getSchema().primaryKeys().get(0), "1"); mockResponse(mapper.writeValueAsString(""), 403); - doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any()); assertThrows( Catalog.TableNoPermissionException.class, () -> - mockRestCatalog.dropPartition( + restCatalog.dropPartition( Identifier.create(databaseName, "table"), partitionSpec)); - verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any()); - verify(mockRestCatalog, times(0)).getTable(any()); - verify(mockRestCatalog, times(0)).cleanPartitionsInFileSystem(any(), any()); } @Test @@ -454,15 +420,11 @@ public void testDropPartitionWhenTableNoExist() throws Exception { partitionSpec.put(response.getSchema().primaryKeys().get(0), "1"); mockResponse(mapper.writeValueAsString(""), 200); mockResponse("", 404); - doNothing().when(mockRestCatalog).cleanPartitionsInFileSystem(any(), any()); assertThrows( Catalog.TableNotExistException.class, () -> - mockRestCatalog.dropPartition( + restCatalog.dropPartition( Identifier.create(databaseName, "table"), partitionSpec)); - verify(mockRestCatalog, times(1)).dropPartitionMetadata(any(), any()); - verify(mockRestCatalog, times(1)).getTable(any()); - verify(mockRestCatalog, times(0)).cleanPartitionsInFileSystem(any(), any()); } @Test @@ -473,8 +435,7 @@ public void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception ListPartitionsResponse response = MockRESTMessage.listPartitionsResponse(); mockResponse(mapper.writeValueAsString(response), 200); List result = - mockRestCatalog.listPartitions(Identifier.create(databaseName, "table")); - verify(mockRestCatalog, times(1)).listPartitionsFromServer(any(), any()); + restCatalog.listPartitions(Identifier.create(databaseName, "table")); assertEquals(response.getPartitions().size(), result.size()); } @@ -484,9 +445,9 @@ public void testListPartitionsFromFile() throws Exception { GetTableResponse response = MockRESTMessage.getTableResponse(); mockResponse(mapper.writeValueAsString(response), 200); mockResponse(mapper.writeValueAsString(response), 200); - mockRestCatalog.listPartitions(Identifier.create(databaseName, "table")); - verify(mockRestCatalog, times(2)).getTable(any()); - verify(mockRestCatalog, times(0)).listPartitionsFromServer(any(), any()); + List partitionEntries = + restCatalog.listPartitions(Identifier.create(databaseName, "table")); + assertEquals(partitionEntries.size(), 0); } private void mockResponse(String mockResponse, int httpCode) {