diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index 702d5229cf20..dedfd5f11276 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -54,11 +54,14 @@ import java.util.Optional; import java.util.stream.Collectors; +import static org.apache.paimon.CoreOptions.OBJECT_LOCATION; import static org.apache.paimon.CoreOptions.TYPE; import static org.apache.paimon.CoreOptions.createCommitUser; +import static org.apache.paimon.catalog.CatalogUtils.buildFormatTableByTableSchema; import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable; +import static org.apache.paimon.catalog.CatalogUtils.getTableType; import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.listPartitionsFromFileSystem; import static org.apache.paimon.catalog.CatalogUtils.validateAutoCreateClose; @@ -309,7 +312,7 @@ private void createObjectTable(Identifier identifier, Schema schema) { ObjectTable.SCHEMA, rowType); checkArgument( - schema.options().containsKey(CoreOptions.OBJECT_LOCATION.key()), + schema.options().containsKey(OBJECT_LOCATION.key()), "Object table should have object-location option."); createTableImpl(identifier, schema.copy(ObjectTable.SCHEMA)); } @@ -383,9 +386,20 @@ public Table getTable(Identifier identifier) throws TableNotExistException { } } + // hive override this method. protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException { Preconditions.checkArgument(identifier.getSystemTableName() == null); TableMeta tableMeta = getDataTableMeta(identifier); + TableType tableType = getTableType(tableMeta.schema().options()); + if (tableType == TableType.FORMAT_TABLE) { + TableSchema schema = tableMeta.schema(); + return buildFormatTableByTableSchema( + identifier, + schema.options(), + schema.logicalRowType(), + schema.partitionKeys(), + schema.comment()); + } FileStoreTable table = FileStoreTableFactory.create( fileIO, @@ -399,9 +413,8 @@ protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExist lockContext().orElse(null), identifier), catalogLoader())); - CoreOptions options = table.coreOptions(); - if (options.type() == TableType.OBJECT_TABLE) { - String objectLocation = options.objectLocation(); + if (tableType == TableType.OBJECT_TABLE) { + String objectLocation = table.coreOptions().objectLocation(); checkNotNull(objectLocation, "Object location should not be null for object table."); table = ObjectTable.builder() diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index 9b69248d6d0e..3c245040ccd6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -19,16 +19,19 @@ package org.apache.paimon.catalog; import org.apache.paimon.CoreOptions; +import org.apache.paimon.TableType; import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.system.AllTableOptionsTable; import org.apache.paimon.table.system.CatalogOptionsTable; import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InternalRowPartitionComputer; import org.apache.paimon.utils.Preconditions; @@ -191,4 +194,33 @@ public static List listPartitionsFromFileSystem(Table table) { } return partitions; } + + public static TableType getTableType(Map options) { + return options.containsKey(CoreOptions.TYPE.key()) + ? TableType.fromString(options.get(CoreOptions.TYPE.key())) + : CoreOptions.TYPE.defaultValue(); + } + + public static FormatTable buildFormatTableByTableSchema( + Identifier identifier, + Map options, + RowType rowType, + List partitionKeys, + String comment) { + FormatTable.Format format = + FormatTable.parseFormat( + options.getOrDefault( + CoreOptions.FILE_FORMAT.key(), + CoreOptions.FILE_FORMAT.defaultValue())); + String location = options.get(CoreOptions.PATH.key()); + return FormatTable.builder() + .identifier(identifier) + .rowType(rowType) + .partitionKeys(partitionKeys) + .location(location) + .format(format) + .options(options) + .comment(comment) + .build(); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java index eee6abffb1ec..08d6c8a050a0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -89,6 +89,12 @@ public T get( return exec(request, responseType); } + @Override + public T post( + String path, RESTRequest body, Map headers) { + return post(path, body, null, headers); + } + @Override public T post( String path, RESTRequest body, Class responseType, Map headers) { diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 06eba96d1f13..4b4e90fb7c20 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -18,7 +18,6 @@ package org.apache.paimon.rest; -import org.apache.paimon.CoreOptions; import org.apache.paimon.TableType; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; @@ -29,6 +28,7 @@ import org.apache.paimon.catalog.PropertyChange; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.operation.FileStoreCommit; import org.apache.paimon.operation.Lock; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; @@ -40,9 +40,13 @@ import org.apache.paimon.rest.exceptions.NoSuchResourceException; import org.apache.paimon.rest.exceptions.ServiceFailureException; import org.apache.paimon.rest.requests.AlterDatabaseRequest; +import org.apache.paimon.rest.requests.AlterPartitionsRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; +import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.ConfigResponse; @@ -61,6 +65,7 @@ import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.Table; import org.apache.paimon.table.object.ObjectTable; +import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Preconditions; @@ -79,9 +84,12 @@ import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; import static org.apache.paimon.CoreOptions.PATH; +import static org.apache.paimon.CoreOptions.createCommitUser; +import static org.apache.paimon.catalog.CatalogUtils.buildFormatTableByTableSchema; import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable; +import static org.apache.paimon.catalog.CatalogUtils.getTableType; import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.listPartitionsFromFileSystem; import static org.apache.paimon.catalog.CatalogUtils.validateAutoCreateClose; @@ -390,32 +398,89 @@ public void dropTable(Identifier identifier, boolean ignoreIfNotExists) @Override public void createPartitions(Identifier identifier, List> partitions) throws TableNotExistException { - throw new UnsupportedOperationException(); + Table table = getTable(identifier); + if (isMetaStorePartitionedTable(table)) { + try { + CreatePartitionsRequest request = new CreatePartitionsRequest(partitions); + client.post( + resourcePaths.partitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + headers()); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } + } } @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { - throw new UnsupportedOperationException(); + Table table = getTable(identifier); + if (isMetaStorePartitionedTable(table)) { + try { + DropPartitionsRequest request = new DropPartitionsRequest(partitions); + client.post( + resourcePaths.dropPartitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + headers()); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } + } else { + FileStoreTable fileStoreTable = (FileStoreTable) table; + try (FileStoreCommit commit = + fileStoreTable + .store() + .newCommit( + createCommitUser( + fileStoreTable.coreOptions().toConfiguration()))) { + commit.dropPartitions(partitions, BatchWriteBuilder.COMMIT_IDENTIFIER); + } + } } @Override public void alterPartitions(Identifier identifier, List partitions) throws TableNotExistException { - throw new UnsupportedOperationException(); + Table table = getTable(identifier); + if (isMetaStorePartitionedTable(table)) { + try { + AlterPartitionsRequest request = new AlterPartitionsRequest(partitions); + client.post( + resourcePaths.alterPartitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + headers()); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } + } } @Override public void markDonePartitions(Identifier identifier, List> partitions) throws TableNotExistException { - throw new UnsupportedOperationException(); + Table table = getTable(identifier); + if (isMetaStorePartitionedTable(table)) { + try { + MarkDonePartitionsRequest request = new MarkDonePartitionsRequest(partitions); + client.post( + resourcePaths.markDonePartitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + headers()); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } + } } @Override public List listPartitions(Identifier identifier) throws TableNotExistException { Table table = getTable(identifier); - Options options = Options.fromMap(table.options()); - if (!options.get(METASTORE_PARTITIONED_TABLE)) { + if (!isMetaStorePartitionedTable(table)) { return listPartitionsFromFileSystem(table); } @@ -471,7 +536,16 @@ private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistEx } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); } - + TableType tableType = getTableType(response.getSchema().options()); + if (tableType == TableType.FORMAT_TABLE) { + Schema schema = response.getSchema(); + return buildFormatTableByTableSchema( + identifier, + schema.options(), + schema.rowType(), + schema.partitionKeys(), + schema.comment()); + } TableSchema schema = TableSchema.create(response.getSchemaId(), response.getSchema()); FileStoreTable table = FileStoreTableFactory.create( @@ -483,9 +557,8 @@ private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistEx response.getId(), Lock.emptyFactory(), catalogLoader())); - CoreOptions options = table.coreOptions(); - if (options.type() == TableType.OBJECT_TABLE) { - String objectLocation = options.objectLocation(); + if (tableType == TableType.OBJECT_TABLE) { + String objectLocation = table.coreOptions().objectLocation(); checkNotNull(objectLocation, "Object location should not be null for object table."); table = ObjectTable.builder() @@ -497,6 +570,11 @@ private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistEx return table; } + private boolean isMetaStorePartitionedTable(Table table) { + Options options = Options.fromMap(table.options()); + return Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE)); + } + private Map headers() { return catalogAuth.getHeaders(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java index d90cb5fa4ad9..d816f09ed0d9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java @@ -26,6 +26,8 @@ public interface RESTClient extends Closeable { T get(String path, Class responseType, Map headers); + T post(String path, RESTRequest body, Map headers); + T post( String path, RESTRequest body, Class responseType, Map headers); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java index f7d2f7116930..7b092196626f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -69,4 +69,19 @@ public String renameTable(String databaseName, String tableName) { public String partitions(String databaseName, String tableName) { return SLASH.join(V1, prefix, DATABASES, databaseName, TABLES, tableName, "partitions"); } + + public String dropPartitions(String databaseName, String tableName) { + return SLASH.join( + V1, prefix, DATABASES, databaseName, TABLES, tableName, "partitions", "drop"); + } + + public String alterPartitions(String databaseName, String tableName) { + return SLASH.join( + V1, prefix, DATABASES, databaseName, TABLES, tableName, "partitions", "alter"); + } + + public String markDonePartitions(String databaseName, String tableName) { + return SLASH.join( + V1, prefix, DATABASES, databaseName, TABLES, tableName, "partitions", "mark"); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java similarity index 70% rename from paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java rename to paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java index 4fabf1163651..e8f542eff578 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java @@ -18,6 +18,7 @@ package org.apache.paimon.rest.requests; +import org.apache.paimon.partition.Partition; import org.apache.paimon.rest.RESTRequest; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; @@ -25,25 +26,24 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Map; +import java.util.List; -/** Request for deleting partition. */ +/** Request for altering partitions. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class DropPartitionRequest implements RESTRequest { +public class AlterPartitionsRequest implements RESTRequest { - private static final String FIELD_PARTITION_SPEC = "spec"; + public static final String FIELD_PARTITIONS = "partitions"; - @JsonProperty(FIELD_PARTITION_SPEC) - private final Map partitionSpec; + @JsonProperty(FIELD_PARTITIONS) + private final List partitions; @JsonCreator - public DropPartitionRequest( - @JsonProperty(FIELD_PARTITION_SPEC) Map partitionSpec) { - this.partitionSpec = partitionSpec; + public AlterPartitionsRequest(@JsonProperty(FIELD_PARTITIONS) List partitions) { + this.partitions = partitions; } - @JsonGetter(FIELD_PARTITION_SPEC) - public Map getPartitionSpec() { - return partitionSpec; + @JsonGetter(FIELD_PARTITIONS) + public List getPartitions() { + return partitions; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/BasePartitionsRequest.java similarity index 63% rename from paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java rename to paimon-core/src/main/java/org/apache/paimon/rest/requests/BasePartitionsRequest.java index f4486b9260d0..25dc20061ebb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/BasePartitionsRequest.java @@ -16,32 +16,35 @@ * limitations under the License. */ -package org.apache.paimon.rest.responses; +package org.apache.paimon.rest.requests; -import org.apache.paimon.partition.Partition; -import org.apache.paimon.rest.RESTResponse; +import org.apache.paimon.rest.RESTRequest; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; -/** Partition for rest api. */ +import java.util.List; +import java.util.Map; + +/** Request for partitions action. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class PartitionResponse implements RESTResponse { +public abstract class BasePartitionsRequest implements RESTRequest { - public static final String FIELD_PARTITION = "partition"; + protected static final String FIELD_PARTITION_SPECS = "specs"; - @JsonProperty(FIELD_PARTITION) - private final Partition partition; + @JsonProperty(FIELD_PARTITION_SPECS) + private final List> partitionSpecs; @JsonCreator - public PartitionResponse(@JsonProperty(FIELD_PARTITION) Partition partition) { - this.partition = partition; + public BasePartitionsRequest( + @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs) { + this.partitionSpecs = partitionSpecs; } - @JsonGetter(FIELD_PARTITION) - public Partition getPartition() { - return partition; + @JsonGetter(FIELD_PARTITION_SPECS) + public List> getPartitionSpecs() { + return partitionSpecs; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java similarity index 54% rename from paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java rename to paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java index e8094ab821bf..420998c1fc5c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java @@ -18,44 +18,20 @@ package org.apache.paimon.rest.requests; -import org.apache.paimon.catalog.Identifier; -import org.apache.paimon.rest.RESTRequest; - import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import java.util.Map; /** Request for creating partition. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class CreatePartitionRequest implements RESTRequest { - - private static final String FIELD_IDENTIFIER = "identifier"; - private static final String FIELD_PARTITION_SPEC = "spec"; - - @JsonProperty(FIELD_IDENTIFIER) - private final Identifier identifier; - - @JsonProperty(FIELD_PARTITION_SPEC) - private final Map partitionSpec; +public class CreatePartitionsRequest extends BasePartitionsRequest { @JsonCreator - public CreatePartitionRequest( - @JsonProperty(FIELD_IDENTIFIER) Identifier identifier, - @JsonProperty(FIELD_PARTITION_SPEC) Map partitionSpec) { - this.identifier = identifier; - this.partitionSpec = partitionSpec; - } - - @JsonGetter(FIELD_IDENTIFIER) - public Identifier getIdentifier() { - return identifier; - } - - @JsonGetter(FIELD_PARTITION_SPEC) - public Map getPartitionSpec() { - return partitionSpec; + public CreatePartitionsRequest( + @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs) { + super(partitionSpecs); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java new file mode 100644 index 000000000000..d84c0d39008b --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Map; + +/** Request for dropping partition. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DropPartitionsRequest extends BasePartitionsRequest { + + @JsonCreator + public DropPartitionsRequest( + @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs) { + super(partitionSpecs); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/MarkDonePartitionsRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/MarkDonePartitionsRequest.java new file mode 100644 index 000000000000..88345e962027 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/MarkDonePartitionsRequest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Map; + +/** Request for marking done partition. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class MarkDonePartitionsRequest extends BasePartitionsRequest { + + @JsonCreator + public MarkDonePartitionsRequest( + @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs) { + super(partitionSpecs); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java index 6448972cde04..c8b9192c1c38 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java @@ -23,6 +23,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.FileStoreTable; @@ -43,12 +44,15 @@ import org.junit.jupiter.api.io.TempDir; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; +import static org.apache.paimon.CoreOptions.METASTORE_TAG_TO_PARTITION; import static org.apache.paimon.catalog.Catalog.SYSTEM_DATABASE_NAME; import static org.apache.paimon.table.system.AllTableOptionsTable.ALL_TABLE_OPTIONS; import static org.apache.paimon.table.system.CatalogOptionsTable.CATALOG_OPTIONS; @@ -57,6 +61,7 @@ import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -1025,6 +1030,102 @@ public void testTableUUID() throws Exception { .isGreaterThan(0); } + @Test + public void testPartitions() throws Exception { + if (!supportPartitions()) { + return; + } + String databaseName = "testPartitionTable"; + List> partitionSpecs = + Arrays.asList( + Collections.singletonMap("dt", "20250101"), + Collections.singletonMap("dt", "20250102")); + catalog.dropDatabase(databaseName, true, true); + catalog.createDatabase(databaseName, true); + Identifier identifier = Identifier.create(databaseName, "table"); + catalog.createTable( + identifier, + Schema.newBuilder() + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(METASTORE_TAG_TO_PARTITION.key(), "dt") + .column("col", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .build(), + true); + + catalog.createPartitions(identifier, partitionSpecs); + assertThat(catalog.listPartitions(identifier).stream().map(Partition::spec)) + .containsExactlyInAnyOrder(partitionSpecs.get(0), partitionSpecs.get(1)); + + assertDoesNotThrow(() -> catalog.markDonePartitions(identifier, partitionSpecs)); + + catalog.dropPartitions(identifier, partitionSpecs); + assertThat(catalog.listPartitions(identifier)).isEmpty(); + + // Test when table does not exist + assertThatExceptionOfType(Catalog.TableNotExistException.class) + .isThrownBy( + () -> + catalog.createPartitions( + Identifier.create(databaseName, "non_existing_table"), + partitionSpecs)); + assertThatExceptionOfType(Catalog.TableNotExistException.class) + .isThrownBy( + () -> + catalog.listPartitions( + Identifier.create(databaseName, "non_existing_table"))); + assertThatExceptionOfType(Catalog.TableNotExistException.class) + .isThrownBy( + () -> + catalog.markDonePartitions( + Identifier.create(databaseName, "non_existing_table"), + partitionSpecs)); + } + + @Test + public void testAlterPartitions() throws Exception { + if (!supportPartitions()) { + return; + } + String databaseName = "testAlterPartitionTable"; + catalog.dropDatabase(databaseName, true, true); + catalog.createDatabase(databaseName, true); + Identifier alterIdentifier = Identifier.create(databaseName, "alert_partitions"); + catalog.createTable( + alterIdentifier, + Schema.newBuilder() + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(METASTORE_TAG_TO_PARTITION.key(), "dt") + .column("col", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .build(), + true); + catalog.createPartitions( + alterIdentifier, Arrays.asList(Collections.singletonMap("dt", "20250101"))); + assertThat(catalog.listPartitions(alterIdentifier).stream().map(Partition::spec)) + .containsExactlyInAnyOrder(Collections.singletonMap("dt", "20250101")); + Partition partition = + new Partition( + Collections.singletonMap("dt", "20250101"), + 1, + 2, + 3, + System.currentTimeMillis()); + catalog.alterPartitions(alterIdentifier, Arrays.asList(partition)); + Partition partitionFromServer = catalog.listPartitions(alterIdentifier).get(0); + checkPartition(partition, partitionFromServer); + + // Test when table does not exist + assertThatExceptionOfType(Catalog.TableNotExistException.class) + .isThrownBy( + () -> + catalog.alterPartitions( + Identifier.create(databaseName, "non_existing_table"), + Arrays.asList(partition))); + } + protected boolean supportsAlterDatabase() { return false; } @@ -1036,4 +1137,12 @@ protected boolean supportsFormatTable() { protected boolean supportsView() { return false; } + + protected void checkPartition(Partition expected, Partition actual) { + assertThat(actual).isEqualTo(expected); + } + + protected boolean supportPartitions() { + return false; + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index ca6be9d00d82..822e06d7fbbc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -21,11 +21,12 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.partition.Partition; import org.apache.paimon.rest.requests.AlterDatabaseRequest; +import org.apache.paimon.rest.requests.AlterPartitionsRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; -import org.apache.paimon.rest.requests.CreatePartitionRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; -import org.apache.paimon.rest.requests.DropPartitionRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; @@ -34,7 +35,6 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.PartitionResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.types.DataField; @@ -46,8 +46,6 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; -import okhttp3.mockwebserver.MockResponse; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -135,23 +133,18 @@ public static AlterTableRequest alterTableRequest() { return new AlterTableRequest(getChanges()); } - public static CreatePartitionRequest createPartitionRequest(String tableName) { - Identifier identifier = Identifier.create(databaseName(), tableName); - return new CreatePartitionRequest(identifier, Collections.singletonMap("pt", "1")); + public static CreatePartitionsRequest createPartitionRequest() { + return new CreatePartitionsRequest(ImmutableList.of(Collections.singletonMap("pt", "1"))); } - public static DropPartitionRequest dropPartitionRequest() { - return new DropPartitionRequest(Collections.singletonMap("pt", "1")); + public static DropPartitionsRequest dropPartitionsRequest() { + return new DropPartitionsRequest(ImmutableList.of(Collections.singletonMap("pt", "1"))); } - public static PartitionResponse partitionResponse() { + public static ListPartitionsResponse listPartitionsResponse() { Map spec = new HashMap<>(); spec.put("f0", "1"); - return new PartitionResponse(new Partition(spec, 1, 1, 1, 1)); - } - - public static ListPartitionsResponse listPartitionsResponse() { - Partition partition = partitionResponse().getPartition(); + Partition partition = new Partition(spec, 1, 1, 1, 1); return new ListPartitionsResponse(ImmutableList.of(partition)); } @@ -233,11 +226,12 @@ public static GetTableResponse getTableResponse() { return new GetTableResponse(UUID.randomUUID().toString(), "", 1, schema(options)); } - public static MockResponse mockResponse(String body, int httpCode) { - return new MockResponse() - .setResponseCode(httpCode) - .setBody(body) - .addHeader("Content-Type", "application/json"); + public static AlterPartitionsRequest alterPartitionsRequest() { + return new AlterPartitionsRequest(ImmutableList.of(partition())); + } + + private static Partition partition() { + return new Partition(Collections.singletonMap("pt", "1"), 1, 1, 1, 1); } private static Schema schema(Map options) { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 4e0c911c382d..1c802cf4c0ef 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -20,15 +20,18 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; -import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.catalog.Database; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; +import org.apache.paimon.rest.requests.AlterPartitionsRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; +import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.ErrorResponse; @@ -38,7 +41,11 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; +import org.apache.paimon.schema.Schema; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FormatTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.DataField; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException; @@ -68,9 +75,7 @@ public RESTCatalogServer(String warehouse, String initToken) { authToken = initToken; Options conf = new Options(); conf.setString("warehouse", warehouse); - this.catalog = - CatalogFactory.createCatalog( - CatalogContext.create(conf), this.getClass().getClassLoader()); + this.catalog = TestRESTCatalog.create(CatalogContext.create(conf)); this.dispatcher = initDispatcher(catalog, authToken); MockWebServer mockWebServer = new MockWebServer(); mockWebServer.setDispatcher(dispatcher); @@ -119,13 +124,55 @@ public MockResponse dispatch(RecordedRequest request) { resources.length == 4 && "tables".equals(resources[1]) && "partitions".equals(resources[3]); - if (isPartitions) { + + boolean isDropPartitions = + resources.length == 5 + && "tables".equals(resources[1]) + && "partitions".equals(resources[3]) + && "drop".equals(resources[4]); + boolean isAlterPartitions = + resources.length == 5 + && "tables".equals(resources[1]) + && "partitions".equals(resources[3]) + && "alter".equals(resources[4]); + boolean isMarkDonePartitions = + resources.length == 5 + && "tables".equals(resources[1]) + && "partitions".equals(resources[3]) + && "mark".equals(resources[4]); + if (isDropPartitions) { + String tableName = resources[2]; + Identifier identifier = Identifier.create(databaseName, tableName); + DropPartitionsRequest dropPartitionsRequest = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), + DropPartitionsRequest.class); + catalog.dropPartitions( + identifier, dropPartitionsRequest.getPartitionSpecs()); + return new MockResponse().setResponseCode(200); + } else if (isAlterPartitions) { String tableName = resources[2]; - List partitions = - catalog.listPartitions( - Identifier.create(databaseName, tableName)); - response = new ListPartitionsResponse(partitions); - return mockResponse(response, 200); + Identifier identifier = Identifier.create(databaseName, tableName); + AlterPartitionsRequest alterPartitionsRequest = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), + AlterPartitionsRequest.class); + catalog.alterPartitions( + identifier, alterPartitionsRequest.getPartitions()); + return new MockResponse().setResponseCode(200); + } else if (isMarkDonePartitions) { + String tableName = resources[2]; + Identifier identifier = Identifier.create(databaseName, tableName); + MarkDonePartitionsRequest markDonePartitionsRequest = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), + MarkDonePartitionsRequest.class); + catalog.markDonePartitions( + identifier, markDonePartitionsRequest.getPartitionSpecs()); + return new MockResponse().setResponseCode(200); + } else if (isPartitions) { + String tableName = resources[2]; + return partitionsApiHandler(catalog, request, databaseName, tableName); } else if (isTableRename) { return renameTableApiHandler( catalog, request, databaseName, resources[2]); @@ -216,106 +263,142 @@ private static MockResponse renameTableApiHandler( OBJECT_MAPPER.readValue(request.getBody().readUtf8(), RenameTableRequest.class); catalog.renameTable( Identifier.create(databaseName, tableName), requestBody.getNewIdentifier(), false); - FileStoreTable table = (FileStoreTable) catalog.getTable(requestBody.getNewIdentifier()); - RESTResponse response = - new GetTableResponse( - UUID.randomUUID().toString(), - tableName, - table.schema().id(), - table.schema().toSchema()); + GetTableResponse response = + getTable( + catalog, + requestBody.getNewIdentifier().getDatabaseName(), + requestBody.getNewIdentifier().getTableName()); return mockResponse(response, 200); } private static MockResponse databasesApiHandler(Catalog catalog, RecordedRequest request) throws Exception { RESTResponse response; - if (request.getMethod().equals("GET")) { - List databaseNameList = catalog.listDatabases(); - response = new ListDatabasesResponse(databaseNameList); - return mockResponse(response, 200); - } else if (request.getMethod().equals("POST")) { - CreateDatabaseRequest requestBody = - OBJECT_MAPPER.readValue( - request.getBody().readUtf8(), CreateDatabaseRequest.class); - String databaseName = requestBody.getName(); - catalog.createDatabase(databaseName, false); - response = new CreateDatabaseResponse(databaseName, requestBody.getOptions()); - return mockResponse(response, 200); + switch (request.getMethod()) { + case "GET": + List databaseNameList = catalog.listDatabases(); + response = new ListDatabasesResponse(databaseNameList); + return mockResponse(response, 200); + case "POST": + CreateDatabaseRequest requestBody = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), CreateDatabaseRequest.class); + String databaseName = requestBody.getName(); + catalog.createDatabase(databaseName, false); + response = new CreateDatabaseResponse(databaseName, requestBody.getOptions()); + return mockResponse(response, 200); + default: + return new MockResponse().setResponseCode(404); } - return new MockResponse().setResponseCode(404); } private static MockResponse databaseApiHandler( Catalog catalog, RecordedRequest request, String databaseName) throws Exception { RESTResponse response; - if (request.getMethod().equals("GET")) { - Database database = catalog.getDatabase(databaseName); - response = - new GetDatabaseResponse( - UUID.randomUUID().toString(), database.name(), database.options()); - return mockResponse(response, 200); - } else if (request.getMethod().equals("DELETE")) { - catalog.dropDatabase(databaseName, false, true); - return new MockResponse().setResponseCode(200); + switch (request.getMethod()) { + case "GET": + Database database = catalog.getDatabase(databaseName); + response = + new GetDatabaseResponse( + UUID.randomUUID().toString(), database.name(), database.options()); + return mockResponse(response, 200); + case "DELETE": + catalog.dropDatabase(databaseName, false, true); + return new MockResponse().setResponseCode(200); + default: + return new MockResponse().setResponseCode(404); } - return new MockResponse().setResponseCode(404); } private static MockResponse tablesApiHandler( Catalog catalog, RecordedRequest request, String databaseName) throws Exception { RESTResponse response; - if (request.getMethod().equals("POST")) { - CreateTableRequest requestBody = - OBJECT_MAPPER.readValue(request.getBody().readUtf8(), CreateTableRequest.class); - catalog.createTable(requestBody.getIdentifier(), requestBody.getSchema(), false); - response = - new GetTableResponse( - UUID.randomUUID().toString(), - requestBody.getIdentifier().getTableName(), - 1L, - requestBody.getSchema()); - return mockResponse(response, 200); - } else if (request.getMethod().equals("GET")) { - catalog.listTables(databaseName); - response = new ListTablesResponse(catalog.listTables(databaseName)); - return mockResponse(response, 200); + switch (request.getMethod()) { + case "GET": + catalog.listTables(databaseName); + response = new ListTablesResponse(catalog.listTables(databaseName)); + return mockResponse(response, 200); + case "POST": + CreateTableRequest requestBody = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), CreateTableRequest.class); + catalog.createTable(requestBody.getIdentifier(), requestBody.getSchema(), false); + response = + new GetTableResponse( + UUID.randomUUID().toString(), "", 1L, requestBody.getSchema()); + return mockResponse(response, 200); + default: + return new MockResponse().setResponseCode(404); } - return new MockResponse().setResponseCode(404); } private static MockResponse tableApiHandler( Catalog catalog, RecordedRequest request, String databaseName, String tableName) throws Exception { RESTResponse response; - if (request.getMethod().equals("GET")) { - Identifier identifier = Identifier.create(databaseName, tableName); - FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); - response = - new GetTableResponse( - UUID.randomUUID().toString(), - tableName, - table.schema().id(), - table.schema().toSchema()); - return mockResponse(response, 200); - } else if (request.getMethod().equals("POST")) { - Identifier identifier = Identifier.create(databaseName, tableName); - AlterTableRequest requestBody = - OBJECT_MAPPER.readValue(request.getBody().readUtf8(), AlterTableRequest.class); - catalog.alterTable(identifier, requestBody.getChanges(), false); - FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); - response = - new GetTableResponse( - UUID.randomUUID().toString(), - tableName, - table.schema().id(), - table.schema().toSchema()); - return mockResponse(response, 200); - } else if (request.getMethod().equals("DELETE")) { - Identifier identifier = Identifier.create(databaseName, tableName); - catalog.dropTable(identifier, false); - return new MockResponse().setResponseCode(200); + Identifier identifier = Identifier.create(databaseName, tableName); + switch (request.getMethod()) { + case "GET": + response = getTable(catalog, databaseName, tableName); + return mockResponse(response, 200); + case "POST": + AlterTableRequest requestBody = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), AlterTableRequest.class); + catalog.alterTable(identifier, requestBody.getChanges(), false); + response = getTable(catalog, databaseName, tableName); + return mockResponse(response, 200); + case "DELETE": + catalog.dropTable(identifier, false); + return new MockResponse().setResponseCode(200); + default: + return new MockResponse().setResponseCode(404); + } + } + + private static MockResponse partitionsApiHandler( + Catalog catalog, RecordedRequest request, String databaseName, String tableName) + throws Exception { + RESTResponse response; + Identifier identifier = Identifier.create(databaseName, tableName); + switch (request.getMethod()) { + case "GET": + List partitions = catalog.listPartitions(identifier); + response = new ListPartitionsResponse(partitions); + return mockResponse(response, 200); + case "POST": + CreatePartitionsRequest requestBody = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), CreatePartitionsRequest.class); + catalog.createPartitions(identifier, requestBody.getPartitionSpecs()); + return new MockResponse().setResponseCode(200); + default: + return new MockResponse().setResponseCode(404); + } + } + + private static GetTableResponse getTable(Catalog catalog, String databaseName, String tableName) + throws Exception { + Identifier identifier = Identifier.create(databaseName, tableName); + Table table = catalog.getTable(identifier); + Schema schema; + Long schemaId = 1L; + if (table instanceof FileStoreTable) { + FileStoreTable fileStoreTable = (FileStoreTable) table; + schema = fileStoreTable.schema().toSchema(); + schemaId = fileStoreTable.schema().id(); + } else { + FormatTable formatTable = (FormatTable) table; + List fields = formatTable.rowType().getFields(); + schema = + new Schema( + fields, + table.partitionKeys(), + table.primaryKeys(), + table.options(), + table.comment().orElse(null)); } - return new MockResponse().setResponseCode(404); + return new GetTableResponse(table.uuid(), table.name(), schemaId, schema); } private static MockResponse mockResponse(RESTResponse response, int httpCode) { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index d9f3bd2a61c4..f63d70332650 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -29,6 +29,7 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; @@ -40,6 +41,7 @@ import java.util.List; import java.util.Map; +import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -89,7 +91,10 @@ void testAuthFail() { @Test void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception { Identifier identifier = Identifier.create("test_db", "test_table"); - createTable(identifier, Maps.newHashMap(), Lists.newArrayList("col1")); + createTable( + identifier, + ImmutableMap.of(METASTORE_PARTITIONED_TABLE.key(), "" + true), + Lists.newArrayList("col1")); List result = catalog.listPartitions(identifier); assertEquals(0, result.size()); } @@ -102,6 +107,16 @@ void testListPartitionsFromFile() throws Exception { assertEquals(0, result.size()); } + @Override + protected boolean supportsFormatTable() { + return true; + } + + @Override + protected boolean supportPartitions() { + return true; + } + private void createTable( Identifier identifier, Map options, List partitionKeys) throws Exception { @@ -109,7 +124,7 @@ private void createTable( catalog.createTable( identifier, new Schema( - Lists.newArrayList(new DataField(0, "col1", DataTypes.STRING())), + Lists.newArrayList(new DataField(0, "col1", DataTypes.INT())), partitionKeys, Collections.emptyList(), options, diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java index 57db06f0d160..6e260a5e7341 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java @@ -19,11 +19,12 @@ package org.apache.paimon.rest; import org.apache.paimon.rest.requests.AlterDatabaseRequest; +import org.apache.paimon.rest.requests.AlterPartitionsRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; -import org.apache.paimon.rest.requests.CreatePartitionRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; -import org.apache.paimon.rest.requests.DropPartitionRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.ConfigResponse; @@ -34,7 +35,6 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.PartitionResponse; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; @@ -49,7 +49,6 @@ import static org.apache.paimon.rest.RESTObjectMapper.OBJECT_MAPPER; import static org.junit.Assert.assertEquals; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; /** Test for {@link RESTObjectMapper}. */ public class RESTObjectMapperTest { @@ -208,21 +207,20 @@ public void alterTableRequestParseTest() throws Exception { @Test public void createPartitionRequestParseTest() throws JsonProcessingException { - CreatePartitionRequest request = MockRESTMessage.createPartitionRequest("t1"); + CreatePartitionsRequest request = MockRESTMessage.createPartitionRequest(); String requestStr = OBJECT_MAPPER.writeValueAsString(request); - CreatePartitionRequest parseData = - OBJECT_MAPPER.readValue(requestStr, CreatePartitionRequest.class); - assertEquals(parseData.getIdentifier(), parseData.getIdentifier()); - assertEquals(parseData.getPartitionSpec().size(), parseData.getPartitionSpec().size()); + CreatePartitionsRequest parseData = + OBJECT_MAPPER.readValue(requestStr, CreatePartitionsRequest.class); + assertEquals(parseData.getPartitionSpecs().size(), parseData.getPartitionSpecs().size()); } @Test public void dropPartitionRequestParseTest() throws JsonProcessingException { - DropPartitionRequest request = MockRESTMessage.dropPartitionRequest(); + DropPartitionsRequest request = MockRESTMessage.dropPartitionsRequest(); String requestStr = OBJECT_MAPPER.writeValueAsString(request); - DropPartitionRequest parseData = - OBJECT_MAPPER.readValue(requestStr, DropPartitionRequest.class); - assertEquals(parseData.getPartitionSpec().size(), parseData.getPartitionSpec().size()); + DropPartitionsRequest parseData = + OBJECT_MAPPER.readValue(requestStr, DropPartitionsRequest.class); + assertEquals(parseData.getPartitionSpecs().size(), parseData.getPartitionSpecs().size()); } @Test @@ -237,13 +235,11 @@ public void listPartitionsResponseParseTest() throws Exception { } @Test - public void partitionResponseParseTest() throws Exception { - PartitionResponse response = MockRESTMessage.partitionResponse(); - assertDoesNotThrow(() -> OBJECT_MAPPER.writeValueAsString(response)); - assertDoesNotThrow( - () -> - OBJECT_MAPPER.readValue( - OBJECT_MAPPER.writeValueAsString(response), - PartitionResponse.class)); + public void alterPartitionsRequestParseTest() throws Exception { + AlterPartitionsRequest request = MockRESTMessage.alterPartitionsRequest(); + String requestStr = OBJECT_MAPPER.writeValueAsString(request); + AlterPartitionsRequest parseData = + OBJECT_MAPPER.readValue(requestStr, AlterPartitionsRequest.class); + assertEquals(request.getPartitions(), parseData.getPartitions()); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java new file mode 100644 index 000000000000..a0f820e7ad0b --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.TableType; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.CatalogUtils; +import org.apache.paimon.catalog.FileSystemCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.Options; +import org.apache.paimon.partition.Partition; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.schema.TableSchema; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** A catalog for testing RESTCatalog. */ +public class TestRESTCatalog extends FileSystemCatalog { + + public Map tableFullName2Schema = new HashMap(); + public Map> tableFullName2Partitions = + new HashMap>(); + + public TestRESTCatalog(FileIO fileIO, Path warehouse, Options options) { + super(fileIO, warehouse, options); + } + + public static TestRESTCatalog create(CatalogContext context) { + String warehouse = CatalogFactory.warehouse(context).toUri().toString(); + + Path warehousePath = new Path(warehouse); + FileIO fileIO; + + try { + fileIO = FileIO.get(warehousePath, context); + fileIO.checkOrMkdirs(warehousePath); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + return new TestRESTCatalog(fileIO, warehousePath, context.options()); + } + + @Override + public void createPartitions(Identifier identifier, List> partitions) + throws TableNotExistException { + getTable(identifier); + tableFullName2Partitions.put( + identifier.getFullName(), + partitions.stream() + .map(partition -> spec2Partition(partition)) + .collect(Collectors.toList())); + } + + @Override + public void dropPartitions(Identifier identifier, List> partitions) + throws TableNotExistException { + getTable(identifier); + List existPartitions = tableFullName2Partitions.get(identifier.getFullName()); + partitions.forEach( + partition -> { + for (Map.Entry entry : partition.entrySet()) { + existPartitions.stream() + .filter( + p -> + p.spec().containsKey(entry.getKey()) + && p.spec() + .get(entry.getKey()) + .equals(entry.getValue())) + .findFirst() + .ifPresent( + existPartition -> existPartitions.remove(existPartition)); + } + }); + } + + @Override + public void alterPartitions(Identifier identifier, List partitions) + throws TableNotExistException { + getTable(identifier); + List existPartitions = tableFullName2Partitions.get(identifier.getFullName()); + partitions.forEach( + partition -> { + for (Map.Entry entry : partition.spec().entrySet()) { + existPartitions.stream() + .filter( + p -> + p.spec().containsKey(entry.getKey()) + && p.spec() + .get(entry.getKey()) + .equals(entry.getValue())) + .findFirst() + .ifPresent( + existPartition -> existPartitions.remove(existPartition)); + } + }); + existPartitions.addAll(partitions); + tableFullName2Partitions.put(identifier.getFullName(), existPartitions); + } + + @Override + public List listPartitions(Identifier identifier) throws TableNotExistException { + getTable(identifier); + return tableFullName2Partitions.get(identifier.getFullName()); + } + + @Override + protected List listTablesImpl(String databaseName) { + List tables = super.listTablesImpl(databaseName); + for (Map.Entry entry : tableFullName2Schema.entrySet()) { + Identifier identifier = Identifier.fromString(entry.getKey()); + if (databaseName.equals(identifier.getDatabaseName())) { + tables.add(identifier.getTableName()); + } + } + return tables; + } + + @Override + protected void dropTableImpl(Identifier identifier) { + if (tableFullName2Schema.containsKey(identifier.getFullName())) { + tableFullName2Schema.remove(identifier.getFullName()); + } else { + super.dropTableImpl(identifier); + } + } + + @Override + public void renameTableImpl(Identifier fromTable, Identifier toTable) { + if (tableFullName2Schema.containsKey(fromTable.getFullName())) { + TableSchema tableSchema = tableFullName2Schema.get(fromTable.getFullName()); + tableFullName2Schema.remove(fromTable.getFullName()); + tableFullName2Schema.put(toTable.getFullName(), tableSchema); + } else { + super.renameTableImpl(fromTable, toTable); + } + } + + @Override + protected void alterTableImpl(Identifier identifier, List changes) + throws TableNotExistException, ColumnAlreadyExistException, ColumnNotExistException { + if (tableFullName2Schema.containsKey(identifier.getFullName())) { + TableSchema schema = tableFullName2Schema.get(identifier.getFullName()); + if (CatalogUtils.getTableType(schema.options()) == TableType.FORMAT_TABLE) { + throw new UnsupportedOperationException("Only data table support alter table."); + } + } else { + super.alterTableImpl(identifier, changes); + } + } + + @Override + public void createFormatTable(Identifier identifier, Schema schema) { + TableSchema tableSchema = + new TableSchema( + 1L, + schema.fields(), + 1, + schema.partitionKeys(), + schema.primaryKeys(), + schema.options(), + schema.comment()); + tableFullName2Schema.put(identifier.getFullName(), tableSchema); + } + + @Override + protected TableMeta getDataTableMeta(Identifier identifier) throws TableNotExistException { + if (tableFullName2Schema.containsKey(identifier.getFullName())) { + TableSchema tableSchema = tableFullName2Schema.get(identifier.getFullName()); + return new TableMeta(tableSchema, "uuid"); + } + return super.getDataTableMeta(identifier); + } + + private Partition spec2Partition(Map spec) { + return new Partition(spec, 123, 456, 789, 123); + } +} diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index ec9420dca4ff..4e2d7b6acb95 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -71,6 +71,7 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.thrift.TException; import org.slf4j.Logger; @@ -492,6 +493,8 @@ public void markDonePartitions(Identifier identifier, List> }); } catch (NoSuchObjectException e) { // do nothing if the partition not exists + } catch (UnknownTableException e) { + throw new TableNotExistException(identifier); } catch (TException | InterruptedException e) { throw new RuntimeException(e); } @@ -513,15 +516,30 @@ public List listPartitions(Identifier ide Short.MAX_VALUE)); return partitions.stream() .map( - part -> - new org.apache.paimon.partition.Partition( - Collections.singletonMap( - tagToPartitionField, - part.getValues().get(0)), - 1L, - 1L, - 1L, - System.currentTimeMillis())) + part -> { + Map parameters = part.getParameters(); + long recordCount = + Long.parseLong( + parameters.getOrDefault(NUM_ROWS_PROP, "1")); + long fileSizeInBytes = + Long.parseLong( + parameters.getOrDefault(TOTAL_SIZE_PROP, "1")); + long fileCount = + Long.parseLong( + parameters.getOrDefault(NUM_FILES_PROP, "1")); + long lastFileCreationTime = + Long.parseLong( + parameters.getOrDefault( + LAST_UPDATE_TIME_PROP, + System.currentTimeMillis() + "")); + return new org.apache.paimon.partition.Partition( + Collections.singletonMap( + tagToPartitionField, part.getValues().get(0)), + recordCount, + fileSizeInBytes, + fileCount, + lastFileCreationTime); + }) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException(e); diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java index e185e5acbf50..ea669d254fbd 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java @@ -56,7 +56,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORECONNECTURLKEY; -import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; import static org.apache.paimon.CoreOptions.METASTORE_TAG_TO_PARTITION; import static org.apache.paimon.hive.HiveCatalog.PAIMON_TABLE_IDENTIFIER; import static org.apache.paimon.hive.HiveCatalog.TABLE_TYPE_PROP; @@ -410,11 +409,22 @@ protected boolean supportsView() { return true; } + @Override + protected boolean supportPartitions() { + return true; + } + @Override protected boolean supportsFormatTable() { return true; } + @Override + protected void checkPartition(Partition expected, Partition actual) { + assertThat(actual.recordCount()).isEqualTo(expected.recordCount()); + assertThat(actual.lastFileCreationTime()).isEqualTo(expected.lastFileCreationTime() / 1000); + } + @Test public void testCreateExternalTableWithLocation(@TempDir java.nio.file.Path tempDir) throws Exception { @@ -473,32 +483,6 @@ public void testTagToPartitionTable() throws Exception { Collections.singletonMap("dt", "20250101")); } - @Test - public void testPartitionTable() throws Exception { - String databaseName = "testPartitionTable"; - catalog.dropDatabase(databaseName, true, true); - catalog.createDatabase(databaseName, true); - Identifier identifier = Identifier.create(databaseName, "table"); - catalog.createTable( - identifier, - Schema.newBuilder() - .option(METASTORE_PARTITIONED_TABLE.key(), "true") - .column("col", DataTypes.INT()) - .column("dt", DataTypes.STRING()) - .partitionKeys("dt") - .build(), - true); - - catalog.createPartitions( - identifier, - Arrays.asList( - Collections.singletonMap("dt", "20250101"), - Collections.singletonMap("dt", "20250102"))); - - // hive catalog list partitions from filesystem, so here return empty. - assertThat(catalog.listPartitions(identifier)).isEmpty(); - } - @Override protected boolean supportsAlterDatabase() { return true; diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml index 4bf2b879d8b5..41d2632454db 100644 --- a/paimon-open-api/rest-catalog-open-api.yaml +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -446,8 +446,8 @@ paths: post: tags: - partition - summary: Create partition - operationId: createPartition + summary: Create partitions + operationId: createPartitions parameters: - name: prefix in: path @@ -468,14 +468,48 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/CreatePartitionRequest' + $ref: '#/components/schemas/CreatePartitionsRequest' responses: "200": - description: OK + description: Success, no content + "404": + description: Resource not found content: application/json: schema: - $ref: '#/components/schemas/PartitionResponse' + $ref: '#/components/schemas/ErrorResponse' + "500": + description: Internal Server Error + /v1/{prefix}/databases/{database}/tables/{table}/partitions/drop: + post: + tags: + - partition + summary: Drop partitions + operationId: dropPartitions + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: table + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DropPartitionsRequest' + responses: + "200": + description: Success, no content "404": description: Resource not found content: @@ -484,11 +518,50 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": description: Internal Server Error - delete: + /v1/{prefix}/databases/{database}/tables/{table}/partitions/alter: + post: + tags: + - partition + summary: Alter partitions + operationId: alterPartitions + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: table + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/AlterPartitionsRequest' + responses: + "200": + description: Success, no content + "404": + description: Resource not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + description: Internal Server Error + /v1/{prefix}/databases/{database}/tables/{table}/partitions/mark: + post: tags: - partition - summary: Drop partition - operationId: dropPartition + summary: MarkDone partitions + operationId: markDonePartitions parameters: - name: prefix in: path @@ -509,7 +582,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/DropPartitionRequest' + $ref: '#/components/schemas/MarkDonePartitionsRequest' responses: "200": description: Success, no content @@ -532,18 +605,34 @@ components: type: object additionalProperties: type: string - CreatePartitionRequest: + CreatePartitionsRequest: type: object properties: - identifier: - $ref: '#/components/schemas/Identifier' - spec: - type: object - DropPartitionRequest: + specs: + type: array + items: + type: object + DropPartitionsRequest: type: object properties: - spec: - type: object + specs: + type: array + items: + type: object + AlterTableRequest: + type: object + properties: + changes: + type: array + items: + $ref: '#/components/schemas/SchemaChange' + MarkDonePartitionsRequest: + type: object + properties: + specs: + type: array + items: + type: object CreateDatabaseResponse: type: object properties: @@ -673,13 +762,8 @@ components: format: int64 schema: $ref: '#/components/schemas/Schema' - AlterTableRequest: - type: object - properties: - changes: - type: array - items: - $ref: '#/components/schemas/SchemaChange' + uuid: + type: string SchemaChange: anyOf: - $ref: '#/components/schemas/SetOption' @@ -827,6 +911,13 @@ components: type: object additionalProperties: type: string + AlterPartitionsRequest: + type: object + properties: + partitions: + type: array + items: + $ref: '#/components/schemas/Partition' AlterDatabaseResponse: type: object properties: @@ -885,11 +976,6 @@ components: type: array items: $ref: '#/components/schemas/Partition' - PartitionResponse: - type: object - properties: - partition: - $ref: '#/components/schemas/Partition' Partition: type: object properties: diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java index e2bd33769280..a9f3d02f5442 100644 --- a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java @@ -21,11 +21,13 @@ import org.apache.paimon.partition.Partition; import org.apache.paimon.rest.ResourcePaths; import org.apache.paimon.rest.requests.AlterDatabaseRequest; +import org.apache.paimon.rest.requests.AlterPartitionsRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; -import org.apache.paimon.rest.requests.CreatePartitionRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; -import org.apache.paimon.rest.requests.DropPartitionRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; +import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.ConfigResponse; @@ -36,7 +38,6 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.PartitionResponse; import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; @@ -391,9 +392,7 @@ public ListPartitionsResponse listPartitions( summary = "Create partition", tags = {"partition"}) @ApiResponses({ - @ApiResponse( - responseCode = "200", - content = {@Content(schema = @Schema(implementation = PartitionResponse.class))}), + @ApiResponse(responseCode = "200", description = "Success, no content"), @ApiResponse( responseCode = "404", description = "Resource not found", @@ -403,18 +402,54 @@ public ListPartitionsResponse listPartitions( content = {@Content(schema = @Schema())}) }) @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions") - public PartitionResponse createPartition( + public void createPartitions( @PathVariable String prefix, @PathVariable String database, @PathVariable String table, - @RequestBody CreatePartitionRequest request) { - Map spec = new HashMap<>(); - spec.put("f1", "1"); - return new PartitionResponse(new Partition(spec, 0, 0, 0, 4)); - } + @RequestBody CreatePartitionsRequest request) {} + + @Operation( + summary = "Drop partitions", + tags = {"partition"}) + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "Success, no content"), + @ApiResponse( + responseCode = "404", + description = "Resource not found", + content = {@Content(schema = @Schema(implementation = ErrorResponse.class))}), + @ApiResponse( + responseCode = "500", + content = {@Content(schema = @Schema())}) + }) + @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions/drop") + public void dropPartitions( + @PathVariable String prefix, + @PathVariable String database, + @PathVariable String table, + @RequestBody DropPartitionsRequest request) {} + + @Operation( + summary = "Alter partitions", + tags = {"partition"}) + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "Success, no content"), + @ApiResponse( + responseCode = "404", + description = "Resource not found", + content = {@Content(schema = @Schema(implementation = ErrorResponse.class))}), + @ApiResponse( + responseCode = "500", + content = {@Content(schema = @Schema())}) + }) + @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions/alter") + public void alterPartitions( + @PathVariable String prefix, + @PathVariable String database, + @PathVariable String table, + @RequestBody AlterPartitionsRequest request) {} @Operation( - summary = "Drop partition", + summary = "MarkDone partitions", tags = {"partition"}) @ApiResponses({ @ApiResponse(responseCode = "200", description = "Success, no content"), @@ -426,10 +461,10 @@ public PartitionResponse createPartition( responseCode = "500", content = {@Content(schema = @Schema())}) }) - @DeleteMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions") - public void dropPartition( + @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions/mark") + public void markDonePartitions( @PathVariable String prefix, @PathVariable String database, @PathVariable String table, - @RequestBody DropPartitionRequest request) {} + @RequestBody MarkDonePartitionsRequest request) {} }