From 8802caed19d592c73f96195065abbf77ca4e7013 Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 10 Jan 2025 14:17:52 +0800 Subject: [PATCH 01/15] get table add uuid and fix format table --- .../paimon/catalog/AbstractCatalog.java | 20 +++++++++--- .../apache/paimon/catalog/CatalogUtils.java | 32 +++++++++++++++++++ .../org/apache/paimon/rest/RESTCatalog.java | 19 +++++++++-- .../rest/responses/GetTableResponse.java | 13 +++++++- .../apache/paimon/rest/MockRESTMessage.java | 5 +-- .../apache/paimon/rest/RESTCatalogServer.java | 15 ++++++--- paimon-open-api/rest-catalog-open-api.yaml | 2 ++ .../open/api/RESTCatalogController.java | 12 ++++--- 8 files changed, 100 insertions(+), 18 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index 702d5229cf20..a257c95e0033 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -54,11 +54,14 @@ import java.util.Optional; import java.util.stream.Collectors; +import static org.apache.paimon.CoreOptions.OBJECT_LOCATION; import static org.apache.paimon.CoreOptions.TYPE; import static org.apache.paimon.CoreOptions.createCommitUser; +import static org.apache.paimon.catalog.CatalogUtils.buildFormatTableByTableSchema; import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable; +import static org.apache.paimon.catalog.CatalogUtils.getTableType; import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.listPartitionsFromFileSystem; import static org.apache.paimon.catalog.CatalogUtils.validateAutoCreateClose; @@ -309,7 +312,7 @@ private void createObjectTable(Identifier identifier, Schema schema) { ObjectTable.SCHEMA, rowType); checkArgument( - schema.options().containsKey(CoreOptions.OBJECT_LOCATION.key()), + schema.options().containsKey(OBJECT_LOCATION.key()), "Object table should have object-location option."); createTableImpl(identifier, schema.copy(ObjectTable.SCHEMA)); } @@ -386,6 +389,16 @@ public Table getTable(Identifier identifier) throws TableNotExistException { protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException { Preconditions.checkArgument(identifier.getSystemTableName() == null); TableMeta tableMeta = getDataTableMeta(identifier); + TableType tableType = getTableType(tableMeta.schema().options()); + if (tableType == TableType.FORMAT_TABLE) { + TableSchema schema = tableMeta.schema(); + return buildFormatTableByTableSchema( + identifier, + schema.options(), + schema.logicalRowType(), + schema.partitionKeys(), + schema.comment()); + } FileStoreTable table = FileStoreTableFactory.create( fileIO, @@ -399,9 +412,8 @@ protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExist lockContext().orElse(null), identifier), catalogLoader())); - CoreOptions options = table.coreOptions(); - if (options.type() == TableType.OBJECT_TABLE) { - String objectLocation = options.objectLocation(); + if (tableType == TableType.OBJECT_TABLE) { + String objectLocation = table.coreOptions().objectLocation(); checkNotNull(objectLocation, "Object location should not be null for object table."); table = ObjectTable.builder() diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index 9b69248d6d0e..3c245040ccd6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -19,16 +19,19 @@ package org.apache.paimon.catalog; import org.apache.paimon.CoreOptions; +import org.apache.paimon.TableType; import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.system.AllTableOptionsTable; import org.apache.paimon.table.system.CatalogOptionsTable; import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InternalRowPartitionComputer; import org.apache.paimon.utils.Preconditions; @@ -191,4 +194,33 @@ public static List listPartitionsFromFileSystem(Table table) { } return partitions; } + + public static TableType getTableType(Map options) { + return options.containsKey(CoreOptions.TYPE.key()) + ? TableType.fromString(options.get(CoreOptions.TYPE.key())) + : CoreOptions.TYPE.defaultValue(); + } + + public static FormatTable buildFormatTableByTableSchema( + Identifier identifier, + Map options, + RowType rowType, + List partitionKeys, + String comment) { + FormatTable.Format format = + FormatTable.parseFormat( + options.getOrDefault( + CoreOptions.FILE_FORMAT.key(), + CoreOptions.FILE_FORMAT.defaultValue())); + String location = options.get(CoreOptions.PATH.key()); + return FormatTable.builder() + .identifier(identifier) + .rowType(rowType) + .partitionKeys(partitionKeys) + .location(location) + .format(format) + .options(options) + .comment(comment) + .build(); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index a807ad2c9d20..ca498830dc8c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -78,9 +78,11 @@ import java.util.concurrent.ScheduledExecutorService; import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; +import static org.apache.paimon.catalog.CatalogUtils.buildFormatTableByTableSchema; import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable; +import static org.apache.paimon.catalog.CatalogUtils.getTableType; import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.listPartitionsFromFileSystem; import static org.apache.paimon.catalog.CatalogUtils.validateAutoCreateClose; @@ -470,15 +472,26 @@ private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistEx } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); } - + TableType tableType = getTableType(response.getSchema().options()); + if (tableType == TableType.FORMAT_TABLE) { + Schema schema = response.getSchema(); + return buildFormatTableByTableSchema( + identifier, + schema.options(), + schema.rowType(), + schema.partitionKeys(), + schema.comment()); + } FileStoreTable table = FileStoreTableFactory.create( fileIO(), new Path(response.getPath()), TableSchema.create(response.getSchemaId(), response.getSchema()), - // TODO add uuid from server new CatalogEnvironment( - identifier, null, Lock.emptyFactory(), catalogLoader())); + identifier, + response.getUuid(), + Lock.emptyFactory(), + catalogLoader())); CoreOptions options = table.coreOptions(); if (options.type() == TableType.OBJECT_TABLE) { String objectLocation = options.objectLocation(); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetTableResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetTableResponse.java index 11f7f3a50d20..7606b4d06b4c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetTableResponse.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetTableResponse.java @@ -33,6 +33,7 @@ public class GetTableResponse implements RESTResponse { private static final String FIELD_PATH = "path"; private static final String FIELD_SCHEMA_ID = "schemaId"; private static final String FIELD_SCHEMA = "schema"; + private static final String FIELD_UUID = "uuid"; @JsonProperty(FIELD_PATH) private final String path; @@ -43,14 +44,19 @@ public class GetTableResponse implements RESTResponse { @JsonProperty(FIELD_SCHEMA) private final Schema schema; + @JsonProperty(FIELD_UUID) + private final String uuid; + @JsonCreator public GetTableResponse( @JsonProperty(FIELD_PATH) String path, @JsonProperty(FIELD_SCHEMA_ID) long schemaId, - @JsonProperty(FIELD_SCHEMA) Schema schema) { + @JsonProperty(FIELD_SCHEMA) Schema schema, + @JsonProperty(FIELD_UUID) String uuid) { this.path = path; this.schemaId = schemaId; this.schema = schema; + this.uuid = uuid; } @JsonGetter(FIELD_PATH) @@ -67,4 +73,9 @@ public long getSchemaId() { public Schema getSchema() { return this.schema; } + + @JsonGetter(FIELD_UUID) + public String getUuid() { + return uuid; + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index 58a73bbfcbaa..e845936d3535 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -55,6 +55,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import static org.apache.paimon.rest.RESTCatalogInternalOptions.DATABASE_COMMENT; @@ -230,14 +231,14 @@ public static GetTableResponse getTableResponseEnablePartition() { Map options = new HashMap<>(); options.put("option-1", "value-1"); options.put(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true"); - return new GetTableResponse("/tmp/2", 1, schema(options)); + return new GetTableResponse("/tmp/2", 1, schema(options), UUID.randomUUID().toString()); } public static GetTableResponse getTableResponse() { Map options = new HashMap<>(); options.put("option-1", "value-1"); options.put("option-2", "value-2"); - return new GetTableResponse("/tmp/1", 1, schema(options)); + return new GetTableResponse("/tmp/1", 1, schema(options), UUID.randomUUID().toString()); } public static MockResponse mockResponse(String body, int httpCode) { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 4fe20291135c..8013819b496d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -51,6 +51,7 @@ import java.io.IOException; import java.util.List; +import java.util.UUID; /** Mock REST server for testing. */ public class RESTCatalogServer { @@ -223,7 +224,8 @@ private static MockResponse renameTableApiHandler( catalog.warehouse(), requestBody.getNewIdentifier()) .toString(), table.schema().id(), - table.schema().toSchema()); + table.schema().toSchema(), + table.uuid()); return mockResponse(response, 200); } @@ -267,7 +269,9 @@ private static MockResponse tablesApiHandler( CreateTableRequest requestBody = OBJECT_MAPPER.readValue(request.getBody().readUtf8(), CreateTableRequest.class); catalog.createTable(requestBody.getIdentifier(), requestBody.getSchema(), false); - response = new GetTableResponse("", 1L, requestBody.getSchema()); + response = + new GetTableResponse( + "", 1L, requestBody.getSchema(), UUID.randomUUID().toString()); return mockResponse(response, 200); } else if (request.getMethod().equals("GET")) { catalog.listTables(databaseName); @@ -289,7 +293,8 @@ private static MockResponse tableApiHandler( AbstractCatalog.newTableLocation(catalog.warehouse(), identifier) .toString(), table.schema().id(), - table.schema().toSchema()); + table.schema().toSchema(), + table.uuid()); return mockResponse(response, 200); } else if (request.getMethod().equals("POST")) { Identifier identifier = Identifier.create(databaseName, tableName); @@ -297,7 +302,9 @@ private static MockResponse tableApiHandler( OBJECT_MAPPER.readValue(request.getBody().readUtf8(), AlterTableRequest.class); catalog.alterTable(identifier, requestBody.getChanges(), false); FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); - response = new GetTableResponse("", table.schema().id(), table.schema().toSchema()); + response = + new GetTableResponse( + "", table.schema().id(), table.schema().toSchema(), table.uuid()); return mockResponse(response, 200); } else if (request.getMethod().equals("DELETE")) { Identifier identifier = Identifier.create(databaseName, tableName); diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml index 9e4d13b44c35..4e5d45d2fbb8 100644 --- a/paimon-open-api/rest-catalog-open-api.yaml +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -671,6 +671,8 @@ components: format: int64 schema: $ref: '#/components/schemas/Schema' + uuid: + type: string AlterTableRequest: type: object properties: diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java index 98f02784d91e..014bd31b12da 100644 --- a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java @@ -241,7 +241,8 @@ public GetTableResponse getTable( ImmutableList.of(), ImmutableList.of(), new HashMap<>(), - "comment")); + "comment"), + "uuid"); } @Operation( @@ -268,7 +269,8 @@ public GetTableResponse createTable( ImmutableList.of(), ImmutableList.of(), new HashMap<>(), - "comment")); + "comment"), + "uuid"); } @Operation( @@ -300,7 +302,8 @@ public GetTableResponse alterTable( ImmutableList.of(), ImmutableList.of(), new HashMap<>(), - "comment")); + "comment"), + "uuid"); } @Operation( @@ -351,7 +354,8 @@ public GetTableResponse renameTable( ImmutableList.of(), ImmutableList.of(), new HashMap<>(), - "comment")); + "comment"), + "uuid"); } @Operation( From bd422cc465174b2596b85243e36077701bcf4579 Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 10 Jan 2025 14:47:28 +0800 Subject: [PATCH 02/15] move format table logic into RESTCatalog --- .../paimon/catalog/AbstractCatalog.java | 18 +++------- .../apache/paimon/catalog/CatalogUtils.java | 32 ----------------- .../org/apache/paimon/rest/RESTCatalog.java | 36 +++++++++++++------ .../apache/paimon/rest/RESTCatalogTest.java | 5 +++ 4 files changed, 34 insertions(+), 57 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index a257c95e0033..a9db88f9e6a3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -57,11 +57,9 @@ import static org.apache.paimon.CoreOptions.OBJECT_LOCATION; import static org.apache.paimon.CoreOptions.TYPE; import static org.apache.paimon.CoreOptions.createCommitUser; -import static org.apache.paimon.catalog.CatalogUtils.buildFormatTableByTableSchema; import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable; -import static org.apache.paimon.catalog.CatalogUtils.getTableType; import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.listPartitionsFromFileSystem; import static org.apache.paimon.catalog.CatalogUtils.validateAutoCreateClose; @@ -386,19 +384,10 @@ public Table getTable(Identifier identifier) throws TableNotExistException { } } + // here just data table, hive override this method. protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException { Preconditions.checkArgument(identifier.getSystemTableName() == null); TableMeta tableMeta = getDataTableMeta(identifier); - TableType tableType = getTableType(tableMeta.schema().options()); - if (tableType == TableType.FORMAT_TABLE) { - TableSchema schema = tableMeta.schema(); - return buildFormatTableByTableSchema( - identifier, - schema.options(), - schema.logicalRowType(), - schema.partitionKeys(), - schema.comment()); - } FileStoreTable table = FileStoreTableFactory.create( fileIO, @@ -412,8 +401,9 @@ protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExist lockContext().orElse(null), identifier), catalogLoader())); - if (tableType == TableType.OBJECT_TABLE) { - String objectLocation = table.coreOptions().objectLocation(); + CoreOptions options = table.coreOptions(); + if (options.type() == TableType.OBJECT_TABLE) { + String objectLocation = options.objectLocation(); checkNotNull(objectLocation, "Object location should not be null for object table."); table = ObjectTable.builder() diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index 3c245040ccd6..9b69248d6d0e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -19,19 +19,16 @@ package org.apache.paimon.catalog; import org.apache.paimon.CoreOptions; -import org.apache.paimon.TableType; import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; -import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.system.AllTableOptionsTable; import org.apache.paimon.table.system.CatalogOptionsTable; import org.apache.paimon.table.system.SystemTableLoader; -import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InternalRowPartitionComputer; import org.apache.paimon.utils.Preconditions; @@ -194,33 +191,4 @@ public static List listPartitionsFromFileSystem(Table table) { } return partitions; } - - public static TableType getTableType(Map options) { - return options.containsKey(CoreOptions.TYPE.key()) - ? TableType.fromString(options.get(CoreOptions.TYPE.key())) - : CoreOptions.TYPE.defaultValue(); - } - - public static FormatTable buildFormatTableByTableSchema( - Identifier identifier, - Map options, - RowType rowType, - List partitionKeys, - String comment) { - FormatTable.Format format = - FormatTable.parseFormat( - options.getOrDefault( - CoreOptions.FILE_FORMAT.key(), - CoreOptions.FILE_FORMAT.defaultValue())); - String location = options.get(CoreOptions.PATH.key()); - return FormatTable.builder() - .identifier(identifier) - .rowType(rowType) - .partitionKeys(partitionKeys) - .location(location) - .format(format) - .options(options) - .comment(comment) - .build(); - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index ca498830dc8c..0cba34ae6bfc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -59,6 +59,7 @@ import org.apache.paimon.table.CatalogEnvironment; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.object.ObjectTable; import org.apache.paimon.utils.Pair; @@ -78,11 +79,9 @@ import java.util.concurrent.ScheduledExecutorService; import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; -import static org.apache.paimon.catalog.CatalogUtils.buildFormatTableByTableSchema; import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable; -import static org.apache.paimon.catalog.CatalogUtils.getTableType; import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.listPartitionsFromFileSystem; import static org.apache.paimon.catalog.CatalogUtils.validateAutoCreateClose; @@ -475,12 +474,22 @@ private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistEx TableType tableType = getTableType(response.getSchema().options()); if (tableType == TableType.FORMAT_TABLE) { Schema schema = response.getSchema(); - return buildFormatTableByTableSchema( - identifier, - schema.options(), - schema.rowType(), - schema.partitionKeys(), - schema.comment()); + FormatTable.Format format = + FormatTable.parseFormat( + schema.options() + .getOrDefault( + CoreOptions.FILE_FORMAT.key(), + CoreOptions.FILE_FORMAT.defaultValue())); + String location = options.get(CoreOptions.PATH.key()); + return FormatTable.builder() + .identifier(identifier) + .rowType(schema.rowType()) + .partitionKeys(schema.partitionKeys()) + .location(location) + .format(format) + .options(schema.options()) + .comment(schema.comment()) + .build(); } FileStoreTable table = FileStoreTableFactory.create( @@ -492,9 +501,8 @@ private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistEx response.getUuid(), Lock.emptyFactory(), catalogLoader())); - CoreOptions options = table.coreOptions(); - if (options.type() == TableType.OBJECT_TABLE) { - String objectLocation = options.objectLocation(); + if (tableType == TableType.OBJECT_TABLE) { + String objectLocation = table.coreOptions().objectLocation(); checkNotNull(objectLocation, "Object location should not be null for object table."); table = ObjectTable.builder() @@ -532,4 +540,10 @@ private ScheduledExecutorService tokenRefreshExecutor() { return refreshExecutor; } + + private static TableType getTableType(Map options) { + return options.containsKey(CoreOptions.TYPE.key()) + ? TableType.fromString(options.get(CoreOptions.TYPE.key())) + : CoreOptions.TYPE.defaultValue(); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 4bbfcde21544..7f08e8775394 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -102,6 +102,11 @@ void testListPartitionsFromFile() throws Exception { assertEquals(0, result.size()); } + @Override + protected boolean supportGetFromSystemDatabase() { + return false; + } + private void createTable( Identifier identifier, Map options, List partitionKeys) throws Exception { From 51e296d2ebc4a25c1e2b2f9100aa45424222bdea Mon Sep 17 00:00:00 2001 From: yantian Date: Fri, 10 Jan 2025 17:35:55 +0800 Subject: [PATCH 03/15] add TestRESTCatalog so could test format table for RESTCatalog --- .../paimon/catalog/AbstractCatalog.java | 17 ++- .../apache/paimon/catalog/CatalogUtils.java | 32 +++++ .../org/apache/paimon/rest/RESTCatalog.java | 32 ++--- .../apache/paimon/rest/RESTCatalogServer.java | 66 +++++---- .../apache/paimon/rest/RESTCatalogTest.java | 5 + .../apache/paimon/rest/TestRESTCatalog.java | 131 ++++++++++++++++++ 6 files changed, 230 insertions(+), 53 deletions(-) create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index a9db88f9e6a3..35c1fbb2f94a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -57,9 +57,11 @@ import static org.apache.paimon.CoreOptions.OBJECT_LOCATION; import static org.apache.paimon.CoreOptions.TYPE; import static org.apache.paimon.CoreOptions.createCommitUser; +import static org.apache.paimon.catalog.CatalogUtils.buildFormatTableByTableSchema; import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable; +import static org.apache.paimon.catalog.CatalogUtils.getTableType; import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.listPartitionsFromFileSystem; import static org.apache.paimon.catalog.CatalogUtils.validateAutoCreateClose; @@ -388,6 +390,16 @@ public Table getTable(Identifier identifier) throws TableNotExistException { protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException { Preconditions.checkArgument(identifier.getSystemTableName() == null); TableMeta tableMeta = getDataTableMeta(identifier); + TableType tableType = getTableType(tableMeta.schema().options()); + if (tableType == TableType.FORMAT_TABLE) { + TableSchema schema = tableMeta.schema(); + return buildFormatTableByTableSchema( + identifier, + schema.options(), + schema.logicalRowType(), + schema.partitionKeys(), + schema.comment()); + } FileStoreTable table = FileStoreTableFactory.create( fileIO, @@ -401,9 +413,8 @@ protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExist lockContext().orElse(null), identifier), catalogLoader())); - CoreOptions options = table.coreOptions(); - if (options.type() == TableType.OBJECT_TABLE) { - String objectLocation = options.objectLocation(); + if (tableType == TableType.OBJECT_TABLE) { + String objectLocation = table.coreOptions().objectLocation(); checkNotNull(objectLocation, "Object location should not be null for object table."); table = ObjectTable.builder() diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index 9b69248d6d0e..3c245040ccd6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -19,16 +19,19 @@ package org.apache.paimon.catalog; import org.apache.paimon.CoreOptions; +import org.apache.paimon.TableType; import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.system.AllTableOptionsTable; import org.apache.paimon.table.system.CatalogOptionsTable; import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InternalRowPartitionComputer; import org.apache.paimon.utils.Preconditions; @@ -191,4 +194,33 @@ public static List listPartitionsFromFileSystem(Table table) { } return partitions; } + + public static TableType getTableType(Map options) { + return options.containsKey(CoreOptions.TYPE.key()) + ? TableType.fromString(options.get(CoreOptions.TYPE.key())) + : CoreOptions.TYPE.defaultValue(); + } + + public static FormatTable buildFormatTableByTableSchema( + Identifier identifier, + Map options, + RowType rowType, + List partitionKeys, + String comment) { + FormatTable.Format format = + FormatTable.parseFormat( + options.getOrDefault( + CoreOptions.FILE_FORMAT.key(), + CoreOptions.FILE_FORMAT.defaultValue())); + String location = options.get(CoreOptions.PATH.key()); + return FormatTable.builder() + .identifier(identifier) + .rowType(rowType) + .partitionKeys(partitionKeys) + .location(location) + .format(format) + .options(options) + .comment(comment) + .build(); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 0cba34ae6bfc..d2cb6a98d50d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -18,7 +18,6 @@ package org.apache.paimon.rest; -import org.apache.paimon.CoreOptions; import org.apache.paimon.TableType; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; @@ -59,7 +58,6 @@ import org.apache.paimon.table.CatalogEnvironment; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; -import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.object.ObjectTable; import org.apache.paimon.utils.Pair; @@ -79,9 +77,11 @@ import java.util.concurrent.ScheduledExecutorService; import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; +import static org.apache.paimon.catalog.CatalogUtils.buildFormatTableByTableSchema; import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable; +import static org.apache.paimon.catalog.CatalogUtils.getTableType; import static org.apache.paimon.catalog.CatalogUtils.isSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.listPartitionsFromFileSystem; import static org.apache.paimon.catalog.CatalogUtils.validateAutoCreateClose; @@ -474,22 +474,12 @@ private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistEx TableType tableType = getTableType(response.getSchema().options()); if (tableType == TableType.FORMAT_TABLE) { Schema schema = response.getSchema(); - FormatTable.Format format = - FormatTable.parseFormat( - schema.options() - .getOrDefault( - CoreOptions.FILE_FORMAT.key(), - CoreOptions.FILE_FORMAT.defaultValue())); - String location = options.get(CoreOptions.PATH.key()); - return FormatTable.builder() - .identifier(identifier) - .rowType(schema.rowType()) - .partitionKeys(schema.partitionKeys()) - .location(location) - .format(format) - .options(schema.options()) - .comment(schema.comment()) - .build(); + return buildFormatTableByTableSchema( + identifier, + schema.options(), + schema.rowType(), + schema.partitionKeys(), + schema.comment()); } FileStoreTable table = FileStoreTableFactory.create( @@ -540,10 +530,4 @@ private ScheduledExecutorService tokenRefreshExecutor() { return refreshExecutor; } - - private static TableType getTableType(Map options) { - return options.containsKey(CoreOptions.TYPE.key()) - ? TableType.fromString(options.get(CoreOptions.TYPE.key())) - : CoreOptions.TYPE.defaultValue(); - } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 8013819b496d..c661b4d16d42 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -21,7 +21,6 @@ import org.apache.paimon.catalog.AbstractCatalog; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; -import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.catalog.Database; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.options.CatalogOptions; @@ -39,7 +38,11 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; +import org.apache.paimon.schema.Schema; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FormatTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.DataField; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; @@ -69,9 +72,7 @@ public RESTCatalogServer(String warehouse, String initToken) { authToken = initToken; Options conf = new Options(); conf.setString("warehouse", warehouse); - this.catalog = - CatalogFactory.createCatalog( - CatalogContext.create(conf), this.getClass().getClassLoader()); + this.catalog = TestRESTCatalog.create(CatalogContext.create(conf)); this.dispatcher = initDispatcher(catalog, authToken); MockWebServer mockWebServer = new MockWebServer(); mockWebServer.setDispatcher(dispatcher); @@ -217,15 +218,11 @@ private static MockResponse renameTableApiHandler( OBJECT_MAPPER.readValue(request.getBody().readUtf8(), RenameTableRequest.class); catalog.renameTable( Identifier.create(databaseName, tableName), requestBody.getNewIdentifier(), false); - FileStoreTable table = (FileStoreTable) catalog.getTable(requestBody.getNewIdentifier()); - RESTResponse response = - new GetTableResponse( - AbstractCatalog.newTableLocation( - catalog.warehouse(), requestBody.getNewIdentifier()) - .toString(), - table.schema().id(), - table.schema().toSchema(), - table.uuid()); + GetTableResponse response = + getTable( + catalog, + requestBody.getNewIdentifier().getDatabaseName(), + requestBody.getNewIdentifier().getTableName()); return mockResponse(response, 200); } @@ -286,25 +283,14 @@ private static MockResponse tableApiHandler( throws Exception { RESTResponse response; if (request.getMethod().equals("GET")) { - Identifier identifier = Identifier.create(databaseName, tableName); - FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); - response = - new GetTableResponse( - AbstractCatalog.newTableLocation(catalog.warehouse(), identifier) - .toString(), - table.schema().id(), - table.schema().toSchema(), - table.uuid()); + response = getTable(catalog, databaseName, tableName); return mockResponse(response, 200); } else if (request.getMethod().equals("POST")) { Identifier identifier = Identifier.create(databaseName, tableName); AlterTableRequest requestBody = OBJECT_MAPPER.readValue(request.getBody().readUtf8(), AlterTableRequest.class); catalog.alterTable(identifier, requestBody.getChanges(), false); - FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); - response = - new GetTableResponse( - "", table.schema().id(), table.schema().toSchema(), table.uuid()); + response = getTable(catalog, databaseName, tableName); return mockResponse(response, 200); } else if (request.getMethod().equals("DELETE")) { Identifier identifier = Identifier.create(databaseName, tableName); @@ -314,6 +300,34 @@ private static MockResponse tableApiHandler( return new MockResponse().setResponseCode(404); } + private static GetTableResponse getTable(Catalog catalog, String databaseName, String tableName) + throws Exception { + Identifier identifier = Identifier.create(databaseName, tableName); + Table table = catalog.getTable(identifier); + Schema schema; + Long schemaId = 1L; + if (table instanceof FileStoreTable) { + FileStoreTable fileStoreTable = (FileStoreTable) table; + schema = fileStoreTable.schema().toSchema(); + schemaId = fileStoreTable.schema().id(); + } else { + FormatTable formatTable = (FormatTable) table; + List fields = formatTable.rowType().getFields(); + schema = + new Schema( + fields, + table.partitionKeys(), + table.primaryKeys(), + table.options(), + table.comment().orElse(null)); + } + return new GetTableResponse( + AbstractCatalog.newTableLocation(catalog.warehouse(), identifier).toString(), + schemaId, + schema, + table.uuid()); + } + private static MockResponse mockResponse(RESTResponse response, int httpCode) { try { return new MockResponse() diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 7f08e8775394..36368d0c56d9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -107,6 +107,11 @@ protected boolean supportGetFromSystemDatabase() { return false; } + @Override + protected boolean supportsFormatTable() { + return true; + } + private void createTable( Identifier identifier, Map options, List partitionKeys) throws Exception { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java new file mode 100644 index 000000000000..20fa19c8152e --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.TableType; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.CatalogUtils; +import org.apache.paimon.catalog.FileSystemCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.schema.TableSchema; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** A catalog for testing RESTCatalog. */ +public class TestRESTCatalog extends FileSystemCatalog { + public Map tableFullName2Schema = new HashMap(); + + public TestRESTCatalog(FileIO fileIO, Path warehouse, Options options) { + super(fileIO, warehouse, options); + } + + public static TestRESTCatalog create(CatalogContext context) { + String warehouse = CatalogFactory.warehouse(context).toUri().toString(); + + Path warehousePath = new Path(warehouse); + FileIO fileIO; + + try { + fileIO = FileIO.get(warehousePath, context); + fileIO.checkOrMkdirs(warehousePath); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + return new TestRESTCatalog(fileIO, warehousePath, context.options()); + } + + @Override + protected List listTablesImpl(String databaseName) { + List tables = super.listTablesImpl(databaseName); + for (Map.Entry entry : tableFullName2Schema.entrySet()) { + Identifier identifier = Identifier.fromString(entry.getKey()); + if (databaseName.equals(identifier.getDatabaseName())) { + tables.add(identifier.getTableName()); + } + } + return tables; + } + + @Override + protected void dropTableImpl(Identifier identifier) { + if (tableFullName2Schema.containsKey(identifier.getFullName())) { + tableFullName2Schema.remove(identifier.getFullName()); + } else { + super.dropTableImpl(identifier); + } + } + + @Override + public void renameTableImpl(Identifier fromTable, Identifier toTable) { + if (tableFullName2Schema.containsKey(fromTable.getFullName())) { + TableSchema tableSchema = tableFullName2Schema.get(fromTable.getFullName()); + tableFullName2Schema.remove(fromTable.getFullName()); + tableFullName2Schema.put(toTable.getFullName(), tableSchema); + } else { + super.renameTableImpl(fromTable, toTable); + } + } + + @Override + protected void alterTableImpl(Identifier identifier, List changes) + throws TableNotExistException, ColumnAlreadyExistException, ColumnNotExistException { + if (tableFullName2Schema.containsKey(identifier.getFullName())) { + TableSchema schema = tableFullName2Schema.get(identifier.getFullName()); + if (CatalogUtils.getTableType(schema.options()) == TableType.FORMAT_TABLE) { + throw new UnsupportedOperationException("Only data table support alter table."); + } + } else { + super.alterTableImpl(identifier, changes); + } + } + + @Override + public void createFormatTable(Identifier identifier, Schema schema) { + TableSchema tableSchema = + new TableSchema( + 1L, + schema.fields(), + 1, + schema.partitionKeys(), + schema.primaryKeys(), + schema.options(), + schema.comment()); + tableFullName2Schema.put(identifier.getFullName(), tableSchema); + } + + @Override + protected TableMeta getDataTableMeta(Identifier identifier) throws TableNotExistException { + if (tableFullName2Schema.containsKey(identifier.getFullName())) { + TableSchema tableSchema = tableFullName2Schema.get(identifier.getFullName()); + return new TableMeta(tableSchema, "uuid"); + } + return super.getDataTableMeta(identifier); + } +} From fd48030d364255bacbcd8b5d0626178895047a4d Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 13 Jan 2025 11:27:55 +0800 Subject: [PATCH 04/15] support createPartitions in RESTCatalog --- .../org/apache/paimon/rest/RESTCatalog.java | 20 ++++++++++- ...uest.java => CreatePartitionsRequest.java} | 21 +++++------ ...nResponse.java => PartitionsResponse.java} | 36 +++++++++++++------ .../paimon/catalog/CatalogTestBase.java | 35 ++++++++++++++++++ .../apache/paimon/rest/MockRESTMessage.java | 17 +++++---- .../apache/paimon/rest/RESTCatalogTest.java | 6 ++-- .../paimon/rest/RESTObjectMapperTest.java | 16 ++++----- .../apache/paimon/hive/HiveCatalogTest.java | 32 +++-------------- paimon-open-api/rest-catalog-open-api.yaml | 25 +++++++------ .../open/api/RESTCatalogController.java | 12 +++---- 10 files changed, 138 insertions(+), 82 deletions(-) rename paimon-core/src/main/java/org/apache/paimon/rest/requests/{CreatePartitionRequest.java => CreatePartitionsRequest.java} (76%) rename paimon-core/src/main/java/org/apache/paimon/rest/responses/{PartitionResponse.java => PartitionsResponse.java} (53%) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index d2cb6a98d50d..3dfd9fa198a7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -41,6 +41,7 @@ import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; @@ -52,6 +53,7 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; +import org.apache.paimon.rest.responses.PartitionsResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.TableSchema; @@ -390,7 +392,23 @@ public void dropTable(Identifier identifier, boolean ignoreIfNotExists) @Override public void createPartitions(Identifier identifier, List> partitions) throws TableNotExistException { - throw new UnsupportedOperationException(); + try { + CreatePartitionsRequest request = new CreatePartitionsRequest(identifier, partitions); + PartitionsResponse response = + client.post( + resourcePaths.partitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + PartitionsResponse.class, + headers()); + if (response.getFailPartitionSpecs() != null + && !response.getFailPartitionSpecs().isEmpty()) { + throw new RuntimeException( + "Create partitions failed: " + response.getFailPartitionSpecs()); + } + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java similarity index 76% rename from paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java rename to paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java index e8094ab821bf..7bc6fc5686eb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionRequest.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java @@ -26,27 +26,28 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import java.util.Map; /** Request for creating partition. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class CreatePartitionRequest implements RESTRequest { +public class CreatePartitionsRequest implements RESTRequest { private static final String FIELD_IDENTIFIER = "identifier"; - private static final String FIELD_PARTITION_SPEC = "spec"; + private static final String FIELD_PARTITION_SPECS = "specs"; @JsonProperty(FIELD_IDENTIFIER) private final Identifier identifier; - @JsonProperty(FIELD_PARTITION_SPEC) - private final Map partitionSpec; + @JsonProperty(FIELD_PARTITION_SPECS) + private final List> partitionSpecs; @JsonCreator - public CreatePartitionRequest( + public CreatePartitionsRequest( @JsonProperty(FIELD_IDENTIFIER) Identifier identifier, - @JsonProperty(FIELD_PARTITION_SPEC) Map partitionSpec) { + @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs) { this.identifier = identifier; - this.partitionSpec = partitionSpec; + this.partitionSpecs = partitionSpecs; } @JsonGetter(FIELD_IDENTIFIER) @@ -54,8 +55,8 @@ public Identifier getIdentifier() { return identifier; } - @JsonGetter(FIELD_PARTITION_SPEC) - public Map getPartitionSpec() { - return partitionSpec; + @JsonGetter(FIELD_PARTITION_SPECS) + public List> getPartitionSpecs() { + return partitionSpecs; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionsResponse.java similarity index 53% rename from paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java rename to paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionsResponse.java index f4486b9260d0..dedf2fb5e481 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionResponse.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionsResponse.java @@ -18,7 +18,6 @@ package org.apache.paimon.rest.responses; -import org.apache.paimon.partition.Partition; import org.apache.paimon.rest.RESTResponse; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; @@ -26,22 +25,39 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import java.util.Map; + /** Partition for rest api. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class PartitionResponse implements RESTResponse { +public class PartitionsResponse implements RESTResponse { + + public static final String FIELD_SUCCESS_PARTITION_SPECS = "successPartitionSpecs"; + public static final String FIELD_FAIL_PARTITIONS_SPECS = "failPartitionSpecs"; - public static final String FIELD_PARTITION = "partition"; + @JsonProperty(FIELD_SUCCESS_PARTITION_SPECS) + private final List> successPartitionSpecs; - @JsonProperty(FIELD_PARTITION) - private final Partition partition; + @JsonProperty(FIELD_FAIL_PARTITIONS_SPECS) + private final List> failPartitionSpecs; @JsonCreator - public PartitionResponse(@JsonProperty(FIELD_PARTITION) Partition partition) { - this.partition = partition; + public PartitionsResponse( + @JsonProperty(FIELD_SUCCESS_PARTITION_SPECS) + List> successPartitionSpecs, + @JsonProperty(FIELD_FAIL_PARTITIONS_SPECS) + List> failPartitionSpecs) { + this.successPartitionSpecs = successPartitionSpecs; + this.failPartitionSpecs = failPartitionSpecs; + } + + @JsonGetter(FIELD_SUCCESS_PARTITION_SPECS) + public List> getSuccessPartitionSpecs() { + return successPartitionSpecs; } - @JsonGetter(FIELD_PARTITION) - public Partition getPartition() { - return partition; + @JsonGetter(FIELD_FAIL_PARTITIONS_SPECS) + public List> getFailPartitionSpecs() { + return failPartitionSpecs; } } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java index 6448972cde04..b75ee0911c0e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java @@ -43,12 +43,14 @@ import org.junit.jupiter.api.io.TempDir; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; import static org.apache.paimon.catalog.Catalog.SYSTEM_DATABASE_NAME; import static org.apache.paimon.table.system.AllTableOptionsTable.ALL_TABLE_OPTIONS; import static org.apache.paimon.table.system.CatalogOptionsTable.CATALOG_OPTIONS; @@ -1025,6 +1027,35 @@ public void testTableUUID() throws Exception { .isGreaterThan(0); } + @Test + public void testCreatePartitions() throws Exception { + if (!supportPartitions()) { + return; + } + String databaseName = "testPartitionTable"; + catalog.dropDatabase(databaseName, true, true); + catalog.createDatabase(databaseName, true); + Identifier identifier = Identifier.create(databaseName, "table"); + catalog.createTable( + identifier, + Schema.newBuilder() + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .column("col", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .build(), + true); + + catalog.createPartitions( + identifier, + Arrays.asList( + Collections.singletonMap("dt", "20250101"), + Collections.singletonMap("dt", "20250102"))); + + // list partitions from filesystem, so here return empty. + assertThat(catalog.listPartitions(identifier)).isEmpty(); + } + protected boolean supportsAlterDatabase() { return false; } @@ -1036,4 +1067,8 @@ protected boolean supportsFormatTable() { protected boolean supportsView() { return false; } + + protected boolean supportPartitions() { + return false; + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index e845936d3535..a3d5faf5f13b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -24,7 +24,7 @@ import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; -import org.apache.paimon.rest.requests.CreatePartitionRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.DropPartitionRequest; import org.apache.paimon.rest.requests.RenameTableRequest; @@ -35,7 +35,7 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.PartitionResponse; +import org.apache.paimon.rest.responses.PartitionsResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.types.DataField; @@ -136,23 +136,26 @@ public static AlterTableRequest alterTableRequest() { return new AlterTableRequest(getChanges()); } - public static CreatePartitionRequest createPartitionRequest(String tableName) { + public static CreatePartitionsRequest createPartitionRequest(String tableName) { Identifier identifier = Identifier.create(databaseName(), tableName); - return new CreatePartitionRequest(identifier, Collections.singletonMap("pt", "1")); + return new CreatePartitionsRequest( + identifier, ImmutableList.of(Collections.singletonMap("pt", "1"))); } public static DropPartitionRequest dropPartitionRequest() { return new DropPartitionRequest(Collections.singletonMap("pt", "1")); } - public static PartitionResponse partitionResponse() { + public static PartitionsResponse partitionResponse() { Map spec = new HashMap<>(); spec.put("f0", "1"); - return new PartitionResponse(new Partition(spec, 1, 1, 1, 1)); + return new PartitionsResponse(ImmutableList.of(spec), ImmutableList.of()); } public static ListPartitionsResponse listPartitionsResponse() { - Partition partition = partitionResponse().getPartition(); + Map spec = new HashMap<>(); + spec.put("f0", "1"); + Partition partition = new Partition(spec, 1, 1, 1, 1); return new ListPartitionsResponse(ImmutableList.of(partition)); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 36368d0c56d9..3c1a5bfac871 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -103,12 +103,12 @@ void testListPartitionsFromFile() throws Exception { } @Override - protected boolean supportGetFromSystemDatabase() { - return false; + protected boolean supportsFormatTable() { + return true; } @Override - protected boolean supportsFormatTable() { + protected boolean supportPartitions() { return true; } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java index 354efe69d5d2..295b3d80a3ba 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java @@ -21,7 +21,7 @@ import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; -import org.apache.paimon.rest.requests.CreatePartitionRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.DropPartitionRequest; import org.apache.paimon.rest.requests.RenameTableRequest; @@ -34,7 +34,7 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.PartitionResponse; +import org.apache.paimon.rest.responses.PartitionsResponse; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; @@ -210,12 +210,12 @@ public void alterTableRequestParseTest() throws Exception { @Test public void createPartitionRequestParseTest() throws JsonProcessingException { - CreatePartitionRequest request = MockRESTMessage.createPartitionRequest("t1"); + CreatePartitionsRequest request = MockRESTMessage.createPartitionRequest("t1"); String requestStr = OBJECT_MAPPER.writeValueAsString(request); - CreatePartitionRequest parseData = - OBJECT_MAPPER.readValue(requestStr, CreatePartitionRequest.class); + CreatePartitionsRequest parseData = + OBJECT_MAPPER.readValue(requestStr, CreatePartitionsRequest.class); assertEquals(parseData.getIdentifier(), parseData.getIdentifier()); - assertEquals(parseData.getPartitionSpec().size(), parseData.getPartitionSpec().size()); + assertEquals(parseData.getPartitionSpecs().size(), parseData.getPartitionSpecs().size()); } @Test @@ -240,12 +240,12 @@ public void listPartitionsResponseParseTest() throws Exception { @Test public void partitionResponseParseTest() throws Exception { - PartitionResponse response = MockRESTMessage.partitionResponse(); + PartitionsResponse response = MockRESTMessage.partitionResponse(); assertDoesNotThrow(() -> OBJECT_MAPPER.writeValueAsString(response)); assertDoesNotThrow( () -> OBJECT_MAPPER.readValue( OBJECT_MAPPER.writeValueAsString(response), - PartitionResponse.class)); + PartitionsResponse.class)); } } diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java index e185e5acbf50..e30fb1c7bab5 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java @@ -56,7 +56,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORECONNECTURLKEY; -import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; import static org.apache.paimon.CoreOptions.METASTORE_TAG_TO_PARTITION; import static org.apache.paimon.hive.HiveCatalog.PAIMON_TABLE_IDENTIFIER; import static org.apache.paimon.hive.HiveCatalog.TABLE_TYPE_PROP; @@ -410,6 +409,11 @@ protected boolean supportsView() { return true; } + @Override + protected boolean supportPartitions() { + return true; + } + @Override protected boolean supportsFormatTable() { return true; @@ -473,32 +477,6 @@ public void testTagToPartitionTable() throws Exception { Collections.singletonMap("dt", "20250101")); } - @Test - public void testPartitionTable() throws Exception { - String databaseName = "testPartitionTable"; - catalog.dropDatabase(databaseName, true, true); - catalog.createDatabase(databaseName, true); - Identifier identifier = Identifier.create(databaseName, "table"); - catalog.createTable( - identifier, - Schema.newBuilder() - .option(METASTORE_PARTITIONED_TABLE.key(), "true") - .column("col", DataTypes.INT()) - .column("dt", DataTypes.STRING()) - .partitionKeys("dt") - .build(), - true); - - catalog.createPartitions( - identifier, - Arrays.asList( - Collections.singletonMap("dt", "20250101"), - Collections.singletonMap("dt", "20250102"))); - - // hive catalog list partitions from filesystem, so here return empty. - assertThat(catalog.listPartitions(identifier)).isEmpty(); - } - @Override protected boolean supportsAlterDatabase() { return true; diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml index 4e5d45d2fbb8..42e2c836c5e5 100644 --- a/paimon-open-api/rest-catalog-open-api.yaml +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -446,8 +446,8 @@ paths: post: tags: - partition - summary: Create partition - operationId: createPartition + summary: Create partitions + operationId: createPartitions parameters: - name: prefix in: path @@ -468,14 +468,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/CreatePartitionRequest' + $ref: '#/components/schemas/CreatePartitionsRequest' responses: "200": description: OK content: application/json: schema: - $ref: '#/components/schemas/PartitionResponse' + $ref: '#/components/schemas/PartitionsResponse' "404": description: Resource not found content: @@ -532,13 +532,14 @@ components: type: object additionalProperties: type: string - CreatePartitionRequest: + CreatePartitionsRequest: type: object properties: identifier: $ref: '#/components/schemas/Identifier' - spec: - type: object + specs: + type: array + items: object DropPartitionRequest: type: object properties: @@ -883,11 +884,15 @@ components: type: array items: $ref: '#/components/schemas/Partition' - PartitionResponse: + PartitionsResponse: type: object properties: - partition: - $ref: '#/components/schemas/Partition' + successPartitionSpecs: + type: array + items: object + failPartitionSpecs: + type: array + items: object Partition: type: object properties: diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java index 014bd31b12da..cb8aaff9878e 100644 --- a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java @@ -23,7 +23,7 @@ import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; -import org.apache.paimon.rest.requests.CreatePartitionRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.DropPartitionRequest; import org.apache.paimon.rest.requests.RenameTableRequest; @@ -36,7 +36,7 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.PartitionResponse; +import org.apache.paimon.rest.responses.PartitionsResponse; import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; @@ -392,7 +392,7 @@ public ListPartitionsResponse listPartitions( @ApiResponses({ @ApiResponse( responseCode = "200", - content = {@Content(schema = @Schema(implementation = PartitionResponse.class))}), + content = {@Content(schema = @Schema(implementation = PartitionsResponse.class))}), @ApiResponse( responseCode = "404", description = "Resource not found", @@ -402,14 +402,14 @@ public ListPartitionsResponse listPartitions( content = {@Content(schema = @Schema())}) }) @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions") - public PartitionResponse createPartition( + public PartitionsResponse createPartitions( @PathVariable String prefix, @PathVariable String database, @PathVariable String table, - @RequestBody CreatePartitionRequest request) { + @RequestBody CreatePartitionsRequest request) { Map spec = new HashMap<>(); spec.put("f1", "1"); - return new PartitionResponse(new Partition(spec, 0, 0, 0, 4)); + return new PartitionsResponse(ImmutableList.of(spec), ImmutableList.of()); } @Operation( From d4246b2f5b3ae7c21d43016f638888884469f094 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 13 Jan 2025 12:32:06 +0800 Subject: [PATCH 05/15] add partition test in CatalogTestBase --- .../paimon/catalog/CatalogTestBase.java | 10 +- .../apache/paimon/rest/RESTCatalogServer.java | 140 +++++++++++------- .../apache/paimon/rest/TestRESTCatalog.java | 35 +++++ 3 files changed, 128 insertions(+), 57 deletions(-) diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java index b75ee0911c0e..6011cc1876c7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java @@ -23,6 +23,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.FileStoreTable; @@ -51,6 +52,7 @@ import java.util.stream.Collectors; import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; +import static org.apache.paimon.CoreOptions.METASTORE_TAG_TO_PARTITION; import static org.apache.paimon.catalog.Catalog.SYSTEM_DATABASE_NAME; import static org.apache.paimon.table.system.AllTableOptionsTable.ALL_TABLE_OPTIONS; import static org.apache.paimon.table.system.CatalogOptionsTable.CATALOG_OPTIONS; @@ -1040,6 +1042,7 @@ public void testCreatePartitions() throws Exception { identifier, Schema.newBuilder() .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(METASTORE_TAG_TO_PARTITION.key(), "dt") .column("col", DataTypes.INT()) .column("dt", DataTypes.STRING()) .partitionKeys("dt") @@ -1051,9 +1054,10 @@ public void testCreatePartitions() throws Exception { Arrays.asList( Collections.singletonMap("dt", "20250101"), Collections.singletonMap("dt", "20250102"))); - - // list partitions from filesystem, so here return empty. - assertThat(catalog.listPartitions(identifier)).isEmpty(); + assertThat(catalog.listPartitions(identifier).stream().map(Partition::spec)) + .containsExactlyInAnyOrder( + Collections.singletonMap("dt", "20250102"), + Collections.singletonMap("dt", "20250101")); } protected boolean supportsAlterDatabase() { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index c661b4d16d42..e911dc1654e2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -28,6 +28,7 @@ import org.apache.paimon.partition.Partition; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.CreateDatabaseResponse; @@ -38,6 +39,7 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; +import org.apache.paimon.rest.responses.PartitionsResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FormatTable; @@ -51,6 +53,7 @@ import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.RecordedRequest; +import org.testcontainers.shaded.com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.List; @@ -123,11 +126,7 @@ public MockResponse dispatch(RecordedRequest request) { && "partitions".equals(resources[3]); if (isPartitions) { String tableName = resources[2]; - List partitions = - catalog.listPartitions( - Identifier.create(databaseName, tableName)); - response = new ListPartitionsResponse(partitions); - return mockResponse(response, 200); + return partitionsApiHandler(catalog, request, databaseName, tableName); } else if (isTableRename) { return renameTableApiHandler( catalog, request, databaseName, resources[2]); @@ -229,75 +228,108 @@ private static MockResponse renameTableApiHandler( private static MockResponse databasesApiHandler(Catalog catalog, RecordedRequest request) throws Exception { RESTResponse response; - if (request.getMethod().equals("GET")) { - List databaseNameList = catalog.listDatabases(); - response = new ListDatabasesResponse(databaseNameList); - return mockResponse(response, 200); - } else if (request.getMethod().equals("POST")) { - CreateDatabaseRequest requestBody = - OBJECT_MAPPER.readValue( - request.getBody().readUtf8(), CreateDatabaseRequest.class); - String databaseName = requestBody.getName(); - catalog.createDatabase(databaseName, false); - response = new CreateDatabaseResponse(databaseName, requestBody.getOptions()); - return mockResponse(response, 200); + switch (request.getMethod()) { + case "GET": + List databaseNameList = catalog.listDatabases(); + response = new ListDatabasesResponse(databaseNameList); + return mockResponse(response, 200); + case "POST": + CreateDatabaseRequest requestBody = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), CreateDatabaseRequest.class); + String databaseName = requestBody.getName(); + catalog.createDatabase(databaseName, false); + response = new CreateDatabaseResponse(databaseName, requestBody.getOptions()); + return mockResponse(response, 200); + default: + return new MockResponse().setResponseCode(404); } - return new MockResponse().setResponseCode(404); } private static MockResponse databaseApiHandler( Catalog catalog, RecordedRequest request, String databaseName) throws Exception { RESTResponse response; - if (request.getMethod().equals("GET")) { - Database database = catalog.getDatabase(databaseName); - response = new GetDatabaseResponse(database.name(), database.options()); - return mockResponse(response, 200); - } else if (request.getMethod().equals("DELETE")) { - catalog.dropDatabase(databaseName, false, true); - return new MockResponse().setResponseCode(200); + switch (request.getMethod()) { + case "GET": + Database database = catalog.getDatabase(databaseName); + response = new GetDatabaseResponse(database.name(), database.options()); + return mockResponse(response, 200); + case "DELETE": + catalog.dropDatabase(databaseName, false, true); + return new MockResponse().setResponseCode(200); + default: + return new MockResponse().setResponseCode(404); } - return new MockResponse().setResponseCode(404); } private static MockResponse tablesApiHandler( Catalog catalog, RecordedRequest request, String databaseName) throws Exception { RESTResponse response; - if (request.getMethod().equals("POST")) { - CreateTableRequest requestBody = - OBJECT_MAPPER.readValue(request.getBody().readUtf8(), CreateTableRequest.class); - catalog.createTable(requestBody.getIdentifier(), requestBody.getSchema(), false); - response = - new GetTableResponse( - "", 1L, requestBody.getSchema(), UUID.randomUUID().toString()); - return mockResponse(response, 200); - } else if (request.getMethod().equals("GET")) { - catalog.listTables(databaseName); - response = new ListTablesResponse(catalog.listTables(databaseName)); - return mockResponse(response, 200); + switch (request.getMethod()) { + case "GET": + catalog.listTables(databaseName); + response = new ListTablesResponse(catalog.listTables(databaseName)); + return mockResponse(response, 200); + case "POST": + CreateTableRequest requestBody = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), CreateTableRequest.class); + catalog.createTable(requestBody.getIdentifier(), requestBody.getSchema(), false); + response = + new GetTableResponse( + "", 1L, requestBody.getSchema(), UUID.randomUUID().toString()); + return mockResponse(response, 200); + default: + return new MockResponse().setResponseCode(404); } - return new MockResponse().setResponseCode(404); } private static MockResponse tableApiHandler( Catalog catalog, RecordedRequest request, String databaseName, String tableName) throws Exception { RESTResponse response; - if (request.getMethod().equals("GET")) { - response = getTable(catalog, databaseName, tableName); - return mockResponse(response, 200); - } else if (request.getMethod().equals("POST")) { - Identifier identifier = Identifier.create(databaseName, tableName); - AlterTableRequest requestBody = - OBJECT_MAPPER.readValue(request.getBody().readUtf8(), AlterTableRequest.class); - catalog.alterTable(identifier, requestBody.getChanges(), false); - response = getTable(catalog, databaseName, tableName); - return mockResponse(response, 200); - } else if (request.getMethod().equals("DELETE")) { - Identifier identifier = Identifier.create(databaseName, tableName); - catalog.dropTable(identifier, false); - return new MockResponse().setResponseCode(200); + Identifier identifier = Identifier.create(databaseName, tableName); + switch (request.getMethod()) { + case "GET": + response = getTable(catalog, databaseName, tableName); + return mockResponse(response, 200); + case "POST": + AlterTableRequest requestBody = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), AlterTableRequest.class); + catalog.alterTable(identifier, requestBody.getChanges(), false); + response = getTable(catalog, databaseName, tableName); + return mockResponse(response, 200); + case "DELETE": + catalog.dropTable(identifier, false); + return new MockResponse().setResponseCode(200); + default: + return new MockResponse().setResponseCode(404); + } + } + + private static MockResponse partitionsApiHandler( + Catalog catalog, RecordedRequest request, String databaseName, String tableName) + throws Exception { + RESTResponse response; + switch (request.getMethod()) { + case "GET": + List partitions = + catalog.listPartitions(Identifier.create(databaseName, tableName)); + response = new ListPartitionsResponse(partitions); + return mockResponse(response, 200); + case "POST": + CreatePartitionsRequest requestBody = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), CreatePartitionsRequest.class); + catalog.createPartitions( + requestBody.getIdentifier(), requestBody.getPartitionSpecs()); + response = + new PartitionsResponse(requestBody.getPartitionSpecs(), ImmutableList.of()); + return mockResponse(response, 200); + default: + return new MockResponse().setResponseCode(404); } - return new MockResponse().setResponseCode(404); } private static GetTableResponse getTable(Catalog catalog, String databaseName, String tableName) diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java index 20fa19c8152e..25394dca1a2e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java @@ -27,6 +27,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; +import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.TableSchema; @@ -36,10 +37,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** A catalog for testing RESTCatalog. */ public class TestRESTCatalog extends FileSystemCatalog { public Map tableFullName2Schema = new HashMap(); + public Map> tableFullName2Partitions = + new HashMap>(); public TestRESTCatalog(FileIO fileIO, Path warehouse, Options options) { super(fileIO, warehouse, options); @@ -61,6 +65,33 @@ public static TestRESTCatalog create(CatalogContext context) { return new TestRESTCatalog(fileIO, warehousePath, context.options()); } + @Override + public void createPartitions(Identifier identifier, List> partitions) + throws TableNotExistException { + tableFullName2Partitions.put( + identifier.getFullName(), + partitions.stream() + .map(partition -> spec2Partition(partition)) + .collect(Collectors.toList())); + } + + @Override + public void dropPartitions(Identifier identifier, List> partitions) + throws TableNotExistException { + tableFullName2Partitions.remove(identifier.getFullName()); + } + + @Override + public void alterPartitions(Identifier identifier, List partitions) + throws TableNotExistException { + tableFullName2Partitions.put(identifier.getFullName(), partitions); + } + + @Override + public List listPartitions(Identifier identifier) throws TableNotExistException { + return tableFullName2Partitions.get(identifier.getFullName()); + } + @Override protected List listTablesImpl(String databaseName) { List tables = super.listTablesImpl(databaseName); @@ -128,4 +159,8 @@ protected TableMeta getDataTableMeta(Identifier identifier) throws TableNotExist } return super.getDataTableMeta(identifier); } + + private Partition spec2Partition(Map spec) { + return new Partition(spec, 123, 456, 789, 123); + } } From 88f9456514bb29493b3d55ef13d683b56ca0876c Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 13 Jan 2025 13:45:40 +0800 Subject: [PATCH 06/15] check metastore. partitioned-table when create partition in RESTCatalog --- .../org/apache/paimon/rest/RESTCatalog.java | 38 +++++++++++-------- .../apache/paimon/rest/RESTCatalogTest.java | 25 +++++++++++- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 3dfd9fa198a7..d318c392cb4c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -392,22 +392,30 @@ public void dropTable(Identifier identifier, boolean ignoreIfNotExists) @Override public void createPartitions(Identifier identifier, List> partitions) throws TableNotExistException { - try { - CreatePartitionsRequest request = new CreatePartitionsRequest(identifier, partitions); - PartitionsResponse response = - client.post( - resourcePaths.partitions( - identifier.getDatabaseName(), identifier.getTableName()), - request, - PartitionsResponse.class, - headers()); - if (response.getFailPartitionSpecs() != null - && !response.getFailPartitionSpecs().isEmpty()) { - throw new RuntimeException( - "Create partitions failed: " + response.getFailPartitionSpecs()); + Table table = getTable(identifier); + Options options = Options.fromMap(table.options()); + if (Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE))) { + try { + CreatePartitionsRequest request = + new CreatePartitionsRequest(identifier, partitions); + PartitionsResponse response = + client.post( + resourcePaths.partitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + PartitionsResponse.class, + headers()); + if (response.getFailPartitionSpecs() != null + && !response.getFailPartitionSpecs().isEmpty()) { + // todo: whether this exception is ok? + throw new RuntimeException( + "Create partitions failed: " + response.getFailPartitionSpecs()); + } + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); } - } catch (NoSuchResourceException e) { - throw new TableNotExistException(identifier); + } else { + throw new UnsupportedOperationException(); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 3c1a5bfac871..02bd6e3e853e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -29,6 +29,7 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; @@ -36,10 +37,12 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -89,7 +92,10 @@ void testAuthFail() { @Test void testListPartitionsWhenMetastorePartitionedIsTrue() throws Exception { Identifier identifier = Identifier.create("test_db", "test_table"); - createTable(identifier, Maps.newHashMap(), Lists.newArrayList("col1")); + createTable( + identifier, + ImmutableMap.of(METASTORE_PARTITIONED_TABLE.key(), "" + true), + Lists.newArrayList("col1")); List result = catalog.listPartitions(identifier); assertEquals(0, result.size()); } @@ -102,6 +108,21 @@ void testListPartitionsFromFile() throws Exception { assertEquals(0, result.size()); } + @Test + void testCreatePartitionsWhenMetastorePartitionedIsFalse() throws Exception { + Identifier identifier = Identifier.create("test_db", "test_table"); + createTable( + identifier, + ImmutableMap.of(METASTORE_PARTITIONED_TABLE.key(), "" + false), + Lists.newArrayList("col1")); + assertThatThrownBy( + () -> + catalog.createPartitions( + identifier, + Arrays.asList(Collections.singletonMap("col1", "1")))) + .isInstanceOf(UnsupportedOperationException.class); + } + @Override protected boolean supportsFormatTable() { return true; @@ -119,7 +140,7 @@ private void createTable( catalog.createTable( identifier, new Schema( - Lists.newArrayList(new DataField(0, "col1", DataTypes.STRING())), + Lists.newArrayList(new DataField(0, "col1", DataTypes.INT())), partitionKeys, Collections.emptyList(), options, From 3339cfe23919a2d208a619b979df648c48c9125d Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 13 Jan 2025 14:42:55 +0800 Subject: [PATCH 07/15] support drop partitions in RESTCatalog --- .../org/apache/paimon/rest/RESTCatalog.java | 39 +++++++++++++++++-- .../org/apache/paimon/rest/ResourcePaths.java | 5 +++ ...equest.java => BasePartitionsRequest.java} | 23 +++++------ .../requests/CreatePartitionsRequest.java | 29 +------------- .../rest/requests/DropPartitionsRequest.java | 37 ++++++++++++++++++ .../paimon/catalog/CatalogTestBase.java | 8 +++- .../apache/paimon/rest/MockRESTMessage.java | 12 +++--- .../apache/paimon/rest/RESTCatalogServer.java | 30 +++++++++++--- .../paimon/rest/RESTObjectMapperTest.java | 13 +++---- .../apache/paimon/rest/TestRESTCatalog.java | 17 +++++++- paimon-open-api/rest-catalog-open-api.yaml | 28 +++++++------ .../open/api/RESTCatalogController.java | 14 ++++--- 12 files changed, 176 insertions(+), 79 deletions(-) rename paimon-core/src/main/java/org/apache/paimon/rest/requests/{DropPartitionRequest.java => BasePartitionsRequest.java} (69%) create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index d318c392cb4c..62886f9cc865 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -28,6 +28,7 @@ import org.apache.paimon.catalog.PropertyChange; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.operation.FileStoreCommit; import org.apache.paimon.operation.Lock; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; @@ -43,6 +44,7 @@ import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.ConfigResponse; @@ -62,6 +64,7 @@ import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.Table; import org.apache.paimon.table.object.ObjectTable; +import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Preconditions; @@ -79,6 +82,7 @@ import java.util.concurrent.ScheduledExecutorService; import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; +import static org.apache.paimon.CoreOptions.createCommitUser; import static org.apache.paimon.catalog.CatalogUtils.buildFormatTableByTableSchema; import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase; @@ -396,8 +400,7 @@ public void createPartitions(Identifier identifier, List> pa Options options = Options.fromMap(table.options()); if (Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE))) { try { - CreatePartitionsRequest request = - new CreatePartitionsRequest(identifier, partitions); + CreatePartitionsRequest request = new CreatePartitionsRequest(partitions); PartitionsResponse response = client.post( resourcePaths.partitions( @@ -422,7 +425,37 @@ public void createPartitions(Identifier identifier, List> pa @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { - throw new UnsupportedOperationException(); + Table table = getTable(identifier); + Options options = Options.fromMap(table.options()); + if (Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE))) { + try { + DropPartitionsRequest request = new DropPartitionsRequest(partitions); + PartitionsResponse response = + client.post( + resourcePaths.dropPartitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + PartitionsResponse.class, + headers()); + if (response.getFailPartitionSpecs() != null + && !response.getFailPartitionSpecs().isEmpty()) { + throw new RuntimeException( + "Drop partitions failed: " + response.getFailPartitionSpecs()); + } + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } + } else { + FileStoreTable fileStoreTable = (FileStoreTable) table; + try (FileStoreCommit commit = + fileStoreTable + .store() + .newCommit( + createCommitUser( + fileStoreTable.coreOptions().toConfiguration()))) { + commit.dropPartitions(partitions, BatchWriteBuilder.COMMIT_IDENTIFIER); + } + } } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java index f7d2f7116930..3cb20bf68f99 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -69,4 +69,9 @@ public String renameTable(String databaseName, String tableName) { public String partitions(String databaseName, String tableName) { return SLASH.join(V1, prefix, DATABASES, databaseName, TABLES, tableName, "partitions"); } + + public String dropPartitions(String databaseName, String tableName) { + return SLASH.join( + V1, prefix, DATABASES, databaseName, TABLES, tableName, "partitions", "drop"); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/BasePartitionsRequest.java similarity index 69% rename from paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java rename to paimon-core/src/main/java/org/apache/paimon/rest/requests/BasePartitionsRequest.java index 4fabf1163651..25dc20061ebb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionRequest.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/BasePartitionsRequest.java @@ -25,25 +25,26 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import java.util.Map; -/** Request for deleting partition. */ +/** Request for partitions action. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class DropPartitionRequest implements RESTRequest { +public abstract class BasePartitionsRequest implements RESTRequest { - private static final String FIELD_PARTITION_SPEC = "spec"; + protected static final String FIELD_PARTITION_SPECS = "specs"; - @JsonProperty(FIELD_PARTITION_SPEC) - private final Map partitionSpec; + @JsonProperty(FIELD_PARTITION_SPECS) + private final List> partitionSpecs; @JsonCreator - public DropPartitionRequest( - @JsonProperty(FIELD_PARTITION_SPEC) Map partitionSpec) { - this.partitionSpec = partitionSpec; + public BasePartitionsRequest( + @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs) { + this.partitionSpecs = partitionSpecs; } - @JsonGetter(FIELD_PARTITION_SPEC) - public Map getPartitionSpec() { - return partitionSpec; + @JsonGetter(FIELD_PARTITION_SPECS) + public List> getPartitionSpecs() { + return partitionSpecs; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java index 7bc6fc5686eb..420998c1fc5c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java @@ -18,11 +18,7 @@ package org.apache.paimon.rest.requests; -import org.apache.paimon.catalog.Identifier; -import org.apache.paimon.rest.RESTRequest; - import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; @@ -31,32 +27,11 @@ /** Request for creating partition. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class CreatePartitionsRequest implements RESTRequest { - - private static final String FIELD_IDENTIFIER = "identifier"; - private static final String FIELD_PARTITION_SPECS = "specs"; - - @JsonProperty(FIELD_IDENTIFIER) - private final Identifier identifier; - - @JsonProperty(FIELD_PARTITION_SPECS) - private final List> partitionSpecs; +public class CreatePartitionsRequest extends BasePartitionsRequest { @JsonCreator public CreatePartitionsRequest( - @JsonProperty(FIELD_IDENTIFIER) Identifier identifier, @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs) { - this.identifier = identifier; - this.partitionSpecs = partitionSpecs; - } - - @JsonGetter(FIELD_IDENTIFIER) - public Identifier getIdentifier() { - return identifier; - } - - @JsonGetter(FIELD_PARTITION_SPECS) - public List> getPartitionSpecs() { - return partitionSpecs; + super(partitionSpecs); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java new file mode 100644 index 000000000000..d84c0d39008b --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Map; + +/** Request for dropping partition. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DropPartitionsRequest extends BasePartitionsRequest { + + @JsonCreator + public DropPartitionsRequest( + @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs) { + super(partitionSpecs); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java index 6011cc1876c7..829913d6ebb9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java @@ -1030,7 +1030,7 @@ public void testTableUUID() throws Exception { } @Test - public void testCreatePartitions() throws Exception { + public void testPartitions() throws Exception { if (!supportPartitions()) { return; } @@ -1058,6 +1058,12 @@ public void testCreatePartitions() throws Exception { .containsExactlyInAnyOrder( Collections.singletonMap("dt", "20250102"), Collections.singletonMap("dt", "20250101")); + catalog.dropPartitions( + identifier, + Arrays.asList( + Collections.singletonMap("dt", "20250102"), + Collections.singletonMap("dt", "20250101"))); + assertThat(catalog.listPartitions(identifier).isEmpty()).isTrue(); } protected boolean supportsAlterDatabase() { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index a3d5faf5f13b..fe4ba0f64c04 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -26,7 +26,7 @@ import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; -import org.apache.paimon.rest.requests.DropPartitionRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; @@ -136,14 +136,12 @@ public static AlterTableRequest alterTableRequest() { return new AlterTableRequest(getChanges()); } - public static CreatePartitionsRequest createPartitionRequest(String tableName) { - Identifier identifier = Identifier.create(databaseName(), tableName); - return new CreatePartitionsRequest( - identifier, ImmutableList.of(Collections.singletonMap("pt", "1"))); + public static CreatePartitionsRequest createPartitionRequest() { + return new CreatePartitionsRequest(ImmutableList.of(Collections.singletonMap("pt", "1"))); } - public static DropPartitionRequest dropPartitionRequest() { - return new DropPartitionRequest(Collections.singletonMap("pt", "1")); + public static DropPartitionsRequest dropPartitionsRequest() { + return new DropPartitionsRequest(ImmutableList.of(Collections.singletonMap("pt", "1"))); } public static PartitionsResponse partitionResponse() { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index e911dc1654e2..3607a7857b8f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -30,6 +30,7 @@ import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.ErrorResponse; @@ -124,7 +125,27 @@ public MockResponse dispatch(RecordedRequest request) { resources.length == 4 && "tables".equals(resources[1]) && "partitions".equals(resources[3]); - if (isPartitions) { + + boolean isDropPartitions = + resources.length == 5 + && "tables".equals(resources[1]) + && "partitions".equals(resources[3]) + && "drop".equals(resources[4]); + if (isDropPartitions) { + String tableName = resources[2]; + Identifier identifier = Identifier.create(databaseName, tableName); + DropPartitionsRequest dropPartitionsRequest = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), + DropPartitionsRequest.class); + catalog.dropPartitions( + identifier, dropPartitionsRequest.getPartitionSpecs()); + response = + new PartitionsResponse( + dropPartitionsRequest.getPartitionSpecs(), + ImmutableList.of()); + return mockResponse(response, 200); + } else if (isPartitions) { String tableName = resources[2]; return partitionsApiHandler(catalog, request, databaseName, tableName); } else if (isTableRename) { @@ -312,18 +333,17 @@ private static MockResponse partitionsApiHandler( Catalog catalog, RecordedRequest request, String databaseName, String tableName) throws Exception { RESTResponse response; + Identifier identifier = Identifier.create(databaseName, tableName); switch (request.getMethod()) { case "GET": - List partitions = - catalog.listPartitions(Identifier.create(databaseName, tableName)); + List partitions = catalog.listPartitions(identifier); response = new ListPartitionsResponse(partitions); return mockResponse(response, 200); case "POST": CreatePartitionsRequest requestBody = OBJECT_MAPPER.readValue( request.getBody().readUtf8(), CreatePartitionsRequest.class); - catalog.createPartitions( - requestBody.getIdentifier(), requestBody.getPartitionSpecs()); + catalog.createPartitions(identifier, requestBody.getPartitionSpecs()); response = new PartitionsResponse(requestBody.getPartitionSpecs(), ImmutableList.of()); return mockResponse(response, 200); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java index 295b3d80a3ba..091560ffd8d9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java @@ -23,7 +23,7 @@ import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; -import org.apache.paimon.rest.requests.DropPartitionRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.ConfigResponse; @@ -210,21 +210,20 @@ public void alterTableRequestParseTest() throws Exception { @Test public void createPartitionRequestParseTest() throws JsonProcessingException { - CreatePartitionsRequest request = MockRESTMessage.createPartitionRequest("t1"); + CreatePartitionsRequest request = MockRESTMessage.createPartitionRequest(); String requestStr = OBJECT_MAPPER.writeValueAsString(request); CreatePartitionsRequest parseData = OBJECT_MAPPER.readValue(requestStr, CreatePartitionsRequest.class); - assertEquals(parseData.getIdentifier(), parseData.getIdentifier()); assertEquals(parseData.getPartitionSpecs().size(), parseData.getPartitionSpecs().size()); } @Test public void dropPartitionRequestParseTest() throws JsonProcessingException { - DropPartitionRequest request = MockRESTMessage.dropPartitionRequest(); + DropPartitionsRequest request = MockRESTMessage.dropPartitionsRequest(); String requestStr = OBJECT_MAPPER.writeValueAsString(request); - DropPartitionRequest parseData = - OBJECT_MAPPER.readValue(requestStr, DropPartitionRequest.class); - assertEquals(parseData.getPartitionSpec().size(), parseData.getPartitionSpec().size()); + DropPartitionsRequest parseData = + OBJECT_MAPPER.readValue(requestStr, DropPartitionsRequest.class); + assertEquals(parseData.getPartitionSpecs().size(), parseData.getPartitionSpecs().size()); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java index 25394dca1a2e..5986ab7c3da1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java @@ -78,7 +78,22 @@ public void createPartitions(Identifier identifier, List> pa @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { - tableFullName2Partitions.remove(identifier.getFullName()); + List existPartitions = tableFullName2Partitions.get(identifier.getFullName()); + partitions.forEach( + partition -> { + for (Map.Entry entry : partition.entrySet()) { + existPartitions.stream() + .filter( + p -> + p.spec().containsKey(entry.getKey()) + && p.spec() + .get(entry.getKey()) + .equals(entry.getValue())) + .findFirst() + .ifPresent( + existPartition -> existPartitions.remove(existPartition)); + } + }); } @Override diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml index 42e2c836c5e5..ecb070a3a902 100644 --- a/paimon-open-api/rest-catalog-open-api.yaml +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -484,11 +484,12 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": description: Internal Server Error - delete: + /v1/{prefix}/databases/{database}/tables/{table}/partitions/drop: + post: tags: - partition - summary: Drop partition - operationId: dropPartition + summary: Drop partitions + operationId: dropPartitions parameters: - name: prefix in: path @@ -509,10 +510,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/DropPartitionRequest' + $ref: '#/components/schemas/DropPartitionsRequest' responses: "200": - description: Success, no content + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/PartitionsResponse' "404": description: Resource not found content: @@ -535,16 +540,17 @@ components: CreatePartitionsRequest: type: object properties: - identifier: - $ref: '#/components/schemas/Identifier' specs: type: array - items: object - DropPartitionRequest: + items: + type: object + DropPartitionsRequest: type: object properties: - spec: - type: object + specs: + type: array + items: + type: object CreateDatabaseResponse: type: object properties: diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java index cb8aaff9878e..4422969a761d 100644 --- a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java @@ -25,7 +25,7 @@ import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; -import org.apache.paimon.rest.requests.DropPartitionRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.ConfigResponse; @@ -413,10 +413,12 @@ public PartitionsResponse createPartitions( } @Operation( - summary = "Drop partition", + summary = "Drop partitions", tags = {"partition"}) @ApiResponses({ - @ApiResponse(responseCode = "200", description = "Success, no content"), + @ApiResponse( + responseCode = "200", + content = {@Content(schema = @Schema(implementation = PartitionsResponse.class))}), @ApiResponse( responseCode = "404", description = "Resource not found", @@ -425,10 +427,10 @@ public PartitionsResponse createPartitions( responseCode = "500", content = {@Content(schema = @Schema())}) }) - @DeleteMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions") - public void dropPartition( + @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions/drop") + public void dropPartitions( @PathVariable String prefix, @PathVariable String database, @PathVariable String table, - @RequestBody DropPartitionRequest request) {} + @RequestBody DropPartitionsRequest request) {} } From ba47d9ac1dd29af1befa5ec6c424532a6091708c Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 13 Jan 2025 16:17:32 +0800 Subject: [PATCH 08/15] support alter partitions in RESTCatalog --- .../org/apache/paimon/rest/RESTCatalog.java | 24 +++++++- .../org/apache/paimon/rest/ResourcePaths.java | 5 ++ .../rest/requests/AlterPartitionsRequest.java | 48 +++++++++++++++ .../responses/AlterPartitionsResponse.java | 61 +++++++++++++++++++ .../rest/responses/PartitionsResponse.java | 8 +-- .../paimon/catalog/CatalogTestBase.java | 41 ++++++++++++- .../apache/paimon/rest/RESTCatalogServer.java | 21 +++++++ .../apache/paimon/rest/TestRESTCatalog.java | 19 +++++- .../org/apache/paimon/hive/HiveCatalog.java | 33 +++++++--- .../apache/paimon/hive/HiveCatalogTest.java | 6 ++ paimon-open-api/rest-catalog-open-api.yaml | 60 ++++++++++++++++++ .../open/api/RESTCatalogController.java | 32 +++++++++- 12 files changed, 340 insertions(+), 18 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/responses/AlterPartitionsResponse.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 62886f9cc865..d5d681a4cb70 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -40,6 +40,7 @@ import org.apache.paimon.rest.exceptions.NoSuchResourceException; import org.apache.paimon.rest.exceptions.ServiceFailureException; import org.apache.paimon.rest.requests.AlterDatabaseRequest; +import org.apache.paimon.rest.requests.AlterPartitionsRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; @@ -47,6 +48,7 @@ import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; +import org.apache.paimon.rest.responses.AlterPartitionsResponse; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.ErrorResponseResourceType; @@ -461,7 +463,27 @@ public void dropPartitions(Identifier identifier, List> part @Override public void alterPartitions(Identifier identifier, List partitions) throws TableNotExistException { - throw new UnsupportedOperationException(); + Table table = getTable(identifier); + Options options = Options.fromMap(table.options()); + if (Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE))) { + try { + AlterPartitionsRequest request = new AlterPartitionsRequest(partitions); + AlterPartitionsResponse response = + client.post( + resourcePaths.alterPartitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + AlterPartitionsResponse.class, + headers()); + if (response.getFailPartitions() != null + && !response.getFailPartitions().isEmpty()) { + throw new RuntimeException( + "Alter partitions failed: " + response.getFailPartitions()); + } + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } + } } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 3cb20bf68f99..a64153d7fd8d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -74,4 +74,9 @@ public String dropPartitions(String databaseName, String tableName) { return SLASH.join( V1, prefix, DATABASES, databaseName, TABLES, tableName, "partitions", "drop"); } + + public String alterPartitions(String databaseName, String tableName) { + return SLASH.join( + V1, prefix, DATABASES, databaseName, TABLES, tableName, "partitions", "alter"); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java new file mode 100644 index 000000000000..d3ac8fc94391 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.partition.Partition; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** Request for altering partitions. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class AlterPartitionsRequest implements RESTRequest { + public static final String FIELD_PARTITIONS = "partitions"; + + @JsonProperty(FIELD_PARTITIONS) + private List partitions; + + @JsonCreator + public AlterPartitionsRequest(@JsonProperty(FIELD_PARTITIONS) List partitions) { + this.partitions = partitions; + } + + @JsonGetter(FIELD_PARTITIONS) + public List getPartitions() { + return partitions; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/AlterPartitionsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/AlterPartitionsResponse.java new file mode 100644 index 000000000000..62ac0e25663d --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/AlterPartitionsResponse.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.partition.Partition; +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** response for altering partitions. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class AlterPartitionsResponse implements RESTResponse { + + public static final String FIELD_SUCCESS_PARTITIONS = "successPartitions"; + public static final String FIELD_FAIL_PARTITIONS = "failPartitions"; + + @JsonProperty(FIELD_SUCCESS_PARTITIONS) + private final List successPartitions; + + @JsonProperty(FIELD_FAIL_PARTITIONS) + private final List failPartitions; + + @JsonCreator + public AlterPartitionsResponse( + @JsonProperty(FIELD_SUCCESS_PARTITIONS) List successPartitions, + @JsonProperty(FIELD_FAIL_PARTITIONS) List failPartitions) { + this.successPartitions = successPartitions; + this.failPartitions = failPartitions; + } + + @JsonGetter(FIELD_SUCCESS_PARTITIONS) + public List getSuccessPartitions() { + return successPartitions; + } + + @JsonGetter(FIELD_FAIL_PARTITIONS) + public List getFailPartitions() { + return failPartitions; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionsResponse.java index dedf2fb5e481..3f78667ddcda 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionsResponse.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionsResponse.java @@ -33,19 +33,19 @@ public class PartitionsResponse implements RESTResponse { public static final String FIELD_SUCCESS_PARTITION_SPECS = "successPartitionSpecs"; - public static final String FIELD_FAIL_PARTITIONS_SPECS = "failPartitionSpecs"; + public static final String FIELD_FAIL_PARTITION_SPECS = "failPartitionSpecs"; @JsonProperty(FIELD_SUCCESS_PARTITION_SPECS) private final List> successPartitionSpecs; - @JsonProperty(FIELD_FAIL_PARTITIONS_SPECS) + @JsonProperty(FIELD_FAIL_PARTITION_SPECS) private final List> failPartitionSpecs; @JsonCreator public PartitionsResponse( @JsonProperty(FIELD_SUCCESS_PARTITION_SPECS) List> successPartitionSpecs, - @JsonProperty(FIELD_FAIL_PARTITIONS_SPECS) + @JsonProperty(FIELD_FAIL_PARTITION_SPECS) List> failPartitionSpecs) { this.successPartitionSpecs = successPartitionSpecs; this.failPartitionSpecs = failPartitionSpecs; @@ -56,7 +56,7 @@ public List> getSuccessPartitionSpecs() { return successPartitionSpecs; } - @JsonGetter(FIELD_FAIL_PARTITIONS_SPECS) + @JsonGetter(FIELD_FAIL_PARTITION_SPECS) public List> getFailPartitionSpecs() { return failPartitionSpecs; } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java index 829913d6ebb9..c0fefa788bc2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java @@ -1063,7 +1063,42 @@ public void testPartitions() throws Exception { Arrays.asList( Collections.singletonMap("dt", "20250102"), Collections.singletonMap("dt", "20250101"))); - assertThat(catalog.listPartitions(identifier).isEmpty()).isTrue(); + assertThat(catalog.listPartitions(identifier)).isEmpty(); + } + + @Test + public void testAlterPartitions() throws Exception { + if (!supportPartitions()) { + return; + } + String databaseName = "testAlterPartitionTable"; + catalog.dropDatabase(databaseName, true, true); + catalog.createDatabase(databaseName, true); + Identifier alterIdentifier = Identifier.create(databaseName, "alert_partitions"); + catalog.createTable( + alterIdentifier, + Schema.newBuilder() + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(METASTORE_TAG_TO_PARTITION.key(), "dt") + .column("col", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .build(), + true); + catalog.createPartitions( + alterIdentifier, Arrays.asList(Collections.singletonMap("dt", "20250101"))); + assertThat(catalog.listPartitions(alterIdentifier).stream().map(Partition::spec)) + .containsExactlyInAnyOrder(Collections.singletonMap("dt", "20250101")); + Partition partition = + new Partition( + Collections.singletonMap("dt", "20250101"), + 1, + 2, + 3, + System.currentTimeMillis()); + catalog.alterPartitions(alterIdentifier, Arrays.asList(partition)); + Partition partitionFromServer = catalog.listPartitions(alterIdentifier).get(0); + checkPartition(partition, partitionFromServer); } protected boolean supportsAlterDatabase() { @@ -1078,6 +1113,10 @@ protected boolean supportsView() { return false; } + protected void checkPartition(Partition expected, Partition actual) { + assertThat(actual).isEqualTo(expected); + } + protected boolean supportPartitions() { return false; } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 3607a7857b8f..0253466c2d73 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -26,12 +26,14 @@ import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; +import org.apache.paimon.rest.requests.AlterPartitionsRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; +import org.apache.paimon.rest.responses.AlterPartitionsResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.ErrorResponseResourceType; @@ -131,6 +133,11 @@ public MockResponse dispatch(RecordedRequest request) { && "tables".equals(resources[1]) && "partitions".equals(resources[3]) && "drop".equals(resources[4]); + boolean isAlterPartitions = + resources.length == 5 + && "tables".equals(resources[1]) + && "partitions".equals(resources[3]) + && "alter".equals(resources[4]); if (isDropPartitions) { String tableName = resources[2]; Identifier identifier = Identifier.create(databaseName, tableName); @@ -145,6 +152,20 @@ public MockResponse dispatch(RecordedRequest request) { dropPartitionsRequest.getPartitionSpecs(), ImmutableList.of()); return mockResponse(response, 200); + } else if (isAlterPartitions) { + String tableName = resources[2]; + Identifier identifier = Identifier.create(databaseName, tableName); + AlterPartitionsRequest alterPartitionsRequest = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), + AlterPartitionsRequest.class); + catalog.alterPartitions( + identifier, alterPartitionsRequest.getPartitions()); + response = + new AlterPartitionsResponse( + alterPartitionsRequest.getPartitions(), + ImmutableList.of()); + return mockResponse(response, 200); } else if (isPartitions) { String tableName = resources[2]; return partitionsApiHandler(catalog, request, databaseName, tableName); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java index 5986ab7c3da1..85717470c9ba 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java @@ -99,7 +99,24 @@ public void dropPartitions(Identifier identifier, List> part @Override public void alterPartitions(Identifier identifier, List partitions) throws TableNotExistException { - tableFullName2Partitions.put(identifier.getFullName(), partitions); + List existPartitions = tableFullName2Partitions.get(identifier.getFullName()); + partitions.forEach( + partition -> { + for (Map.Entry entry : partition.spec().entrySet()) { + existPartitions.stream() + .filter( + p -> + p.spec().containsKey(entry.getKey()) + && p.spec() + .get(entry.getKey()) + .equals(entry.getValue())) + .findFirst() + .ifPresent( + existPartition -> existPartitions.remove(existPartition)); + } + }); + existPartitions.addAll(partitions); + tableFullName2Partitions.put(identifier.getFullName(), existPartitions); } @Override diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index ec9420dca4ff..d314b5c62386 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -513,15 +513,30 @@ public List listPartitions(Identifier ide Short.MAX_VALUE)); return partitions.stream() .map( - part -> - new org.apache.paimon.partition.Partition( - Collections.singletonMap( - tagToPartitionField, - part.getValues().get(0)), - 1L, - 1L, - 1L, - System.currentTimeMillis())) + part -> { + Map parameters = part.getParameters(); + long recordCount = + Long.parseLong( + parameters.getOrDefault(NUM_ROWS_PROP, "1")); + long fileSizeInBytes = + Long.parseLong( + parameters.getOrDefault(TOTAL_SIZE_PROP, "1")); + long fileCount = + Long.parseLong( + parameters.getOrDefault(NUM_FILES_PROP, "1")); + long lastFileCreationTime = + Long.parseLong( + parameters.getOrDefault( + LAST_UPDATE_TIME_PROP, + System.currentTimeMillis() + "")); + return new org.apache.paimon.partition.Partition( + Collections.singletonMap( + tagToPartitionField, part.getValues().get(0)), + recordCount, + fileSizeInBytes, + fileCount, + lastFileCreationTime); + }) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException(e); diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java index e30fb1c7bab5..66e45427a9dc 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java @@ -419,6 +419,12 @@ protected boolean supportsFormatTable() { return true; } + @Override + protected void checkPartition(Partition expected, Partition actual) { + assertThat(actual.recordCount()).isEqualTo(expected.recordCount()); + assertThat(actual.lastFileCreationTime() / 1000).isEqualTo(expected.lastFileCreationTime()); + } + @Test public void testCreateExternalTableWithLocation(@TempDir java.nio.file.Path tempDir) throws Exception { diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml index ecb070a3a902..7593e60b31ce 100644 --- a/paimon-open-api/rest-catalog-open-api.yaml +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -526,6 +526,48 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": description: Internal Server Error + /v1/{prefix}/databases/{database}/tables/{table}/partitions/alter: + post: + tags: + - partition + summary: Alter partitions + operationId: alterPartitions + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: table + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/AlterPartitionsRequest' + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/AlterPartitionsResponse' + "404": + description: Resource not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + description: Internal Server Error components: schemas: CreateDatabaseRequest: @@ -834,6 +876,13 @@ components: type: object additionalProperties: type: string + AlterPartitionsRequest: + type: object + properties: + partitions: + type: array + items: + $ref: '#/components/schemas/Partition' AlterDatabaseResponse: type: object properties: @@ -916,6 +965,17 @@ components: lastFileCreationTime: type: integer format: int64 + AlterPartitionsResponse: + type: object + properties: + successPartitions: + type: array + items: + $ref: '#/components/schemas/Partition' + failPartitions: + type: array + items: + $ref: '#/components/schemas/Partition' securitySchemes: BearerAuth: type: http diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java index 4422969a761d..0cec5cae8ce6 100644 --- a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java @@ -21,6 +21,7 @@ import org.apache.paimon.partition.Partition; import org.apache.paimon.rest.ResourcePaths; import org.apache.paimon.rest.requests.AlterDatabaseRequest; +import org.apache.paimon.rest.requests.AlterPartitionsRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; @@ -28,6 +29,7 @@ import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; +import org.apache.paimon.rest.responses.AlterPartitionsResponse; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.ErrorResponse; @@ -428,9 +430,35 @@ public PartitionsResponse createPartitions( content = {@Content(schema = @Schema())}) }) @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions/drop") - public void dropPartitions( + public PartitionsResponse dropPartitions( @PathVariable String prefix, @PathVariable String database, @PathVariable String table, - @RequestBody DropPartitionsRequest request) {} + @RequestBody DropPartitionsRequest request) { + return new PartitionsResponse(ImmutableList.of(), ImmutableList.of()); + } + + @Operation( + summary = "Drop partitions", + tags = {"partition"}) + @ApiResponses({ + @ApiResponse( + responseCode = "200", + content = {@Content(schema = @Schema(implementation = PartitionsResponse.class))}), + @ApiResponse( + responseCode = "404", + description = "Resource not found", + content = {@Content(schema = @Schema(implementation = ErrorResponse.class))}), + @ApiResponse( + responseCode = "500", + content = {@Content(schema = @Schema())}) + }) + @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions/alter") + public AlterPartitionsResponse alterPartitions( + @PathVariable String prefix, + @PathVariable String database, + @PathVariable String table, + @RequestBody AlterPartitionsRequest request) { + return new AlterPartitionsResponse(ImmutableList.of(), ImmutableList.of()); + } } From e0011729aef9ab6dd9f6c4150b6eae3a06eb4e44 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 13 Jan 2025 16:33:05 +0800 Subject: [PATCH 09/15] add test for alter partitions --- .../rest/requests/AlterPartitionsRequest.java | 2 +- .../apache/paimon/rest/MockRESTMessage.java | 28 +++++++++---------- .../paimon/rest/RESTObjectMapperTest.java | 21 ++++++++++++++ 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java index d3ac8fc94391..fd1b3afdedb7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java @@ -34,7 +34,7 @@ public class AlterPartitionsRequest implements RESTRequest { public static final String FIELD_PARTITIONS = "partitions"; @JsonProperty(FIELD_PARTITIONS) - private List partitions; + private final List partitions; @JsonCreator public AlterPartitionsRequest(@JsonProperty(FIELD_PARTITIONS) List partitions) { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index fe4ba0f64c04..8370334d0470 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -18,10 +18,10 @@ package org.apache.paimon.rest; -import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.partition.Partition; import org.apache.paimon.rest.requests.AlterDatabaseRequest; +import org.apache.paimon.rest.requests.AlterPartitionsRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; @@ -29,6 +29,7 @@ import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; +import org.apache.paimon.rest.responses.AlterPartitionsResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetTableResponse; @@ -47,8 +48,6 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; -import okhttp3.mockwebserver.MockResponse; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -228,13 +227,6 @@ public static List getChanges() { return schemaChanges; } - public static GetTableResponse getTableResponseEnablePartition() { - Map options = new HashMap<>(); - options.put("option-1", "value-1"); - options.put(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true"); - return new GetTableResponse("/tmp/2", 1, schema(options), UUID.randomUUID().toString()); - } - public static GetTableResponse getTableResponse() { Map options = new HashMap<>(); options.put("option-1", "value-1"); @@ -242,11 +234,17 @@ public static GetTableResponse getTableResponse() { return new GetTableResponse("/tmp/1", 1, schema(options), UUID.randomUUID().toString()); } - public static MockResponse mockResponse(String body, int httpCode) { - return new MockResponse() - .setResponseCode(httpCode) - .setBody(body) - .addHeader("Content-Type", "application/json"); + public static AlterPartitionsRequest alterPartitionsRequest() { + return new AlterPartitionsRequest(ImmutableList.of(partition())); + } + + public static AlterPartitionsResponse alterPartitionsResponse() { + return new AlterPartitionsResponse( + ImmutableList.of(partition()), ImmutableList.of(partition())); + } + + private static Partition partition() { + return new Partition(Collections.singletonMap("pt", "1"), 1, 1, 1, 1); } private static Schema schema(Map options) { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java index 091560ffd8d9..67db90870092 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.rest; import org.apache.paimon.rest.requests.AlterDatabaseRequest; +import org.apache.paimon.rest.requests.AlterPartitionsRequest; import org.apache.paimon.rest.requests.AlterTableRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; @@ -26,6 +27,7 @@ import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; +import org.apache.paimon.rest.responses.AlterPartitionsResponse; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.ErrorResponse; @@ -247,4 +249,23 @@ public void partitionResponseParseTest() throws Exception { OBJECT_MAPPER.writeValueAsString(response), PartitionsResponse.class)); } + + @Test + public void alterPartitionsRequestParseTest() throws Exception { + AlterPartitionsRequest request = MockRESTMessage.alterPartitionsRequest(); + String requestStr = OBJECT_MAPPER.writeValueAsString(request); + AlterPartitionsRequest parseData = + OBJECT_MAPPER.readValue(requestStr, AlterPartitionsRequest.class); + assertEquals(request.getPartitions(), parseData.getPartitions()); + } + + @Test + public void alterPartitionsResponseParseTest() throws Exception { + AlterPartitionsResponse response = MockRESTMessage.alterPartitionsResponse(); + String responseStr = OBJECT_MAPPER.writeValueAsString(response); + AlterPartitionsResponse parseData = + OBJECT_MAPPER.readValue(responseStr, AlterPartitionsResponse.class); + assertEquals(response.getFailPartitions(), parseData.getFailPartitions()); + assertEquals(response.getSuccessPartitions(), parseData.getSuccessPartitions()); + } } From 6f570ce44f9fe4cb5a499fdb344a356299886c91 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 13 Jan 2025 17:09:14 +0800 Subject: [PATCH 10/15] merge master fix conflict --- .../java/org/apache/paimon/rest/RESTCatalog.java | 2 +- .../paimon/rest/responses/GetTableResponse.java | 1 - .../org/apache/paimon/rest/RESTCatalogServer.java | 12 +++++------- .../paimon/open/api/RESTCatalogController.java | 12 ++++-------- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index c712a5d13dbb..d64935a3570a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -84,9 +84,9 @@ import java.util.concurrent.ScheduledExecutorService; import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE; +import static org.apache.paimon.CoreOptions.PATH; import static org.apache.paimon.CoreOptions.createCommitUser; import static org.apache.paimon.catalog.CatalogUtils.buildFormatTableByTableSchema; -import static org.apache.paimon.CoreOptions.PATH; import static org.apache.paimon.catalog.CatalogUtils.checkNotBranch; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemDatabase; import static org.apache.paimon.catalog.CatalogUtils.checkNotSystemTable; diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetTableResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetTableResponse.java index 6101215b2da8..aeed03044583 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetTableResponse.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetTableResponse.java @@ -57,7 +57,6 @@ public GetTableResponse( this.name = name; this.schemaId = schemaId; this.schema = schema; - this.uuid = uuid; } @JsonGetter(FIELD_ID) diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index da29fdccc95b..c99e9130eb35 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -293,7 +293,9 @@ private static MockResponse databaseApiHandler( switch (request.getMethod()) { case "GET": Database database = catalog.getDatabase(databaseName); - response = new GetDatabaseResponse(database.name(), database.options()); + response = + new GetDatabaseResponse( + UUID.randomUUID().toString(), database.name(), database.options()); return mockResponse(response, 200); case "DELETE": catalog.dropDatabase(databaseName, false, true); @@ -318,7 +320,7 @@ private static MockResponse tablesApiHandler( catalog.createTable(requestBody.getIdentifier(), requestBody.getSchema(), false); response = new GetTableResponse( - "", 1L, requestBody.getSchema(), UUID.randomUUID().toString()); + UUID.randomUUID().toString(), "", 1L, requestBody.getSchema()); return mockResponse(response, 200); default: return new MockResponse().setResponseCode(404); @@ -393,11 +395,7 @@ private static GetTableResponse getTable(Catalog catalog, String databaseName, S table.options(), table.comment().orElse(null)); } - return new GetTableResponse( - table.id(), - table.name(), - schemaId, - schema); + return new GetTableResponse(table.uuid(), table.name(), schemaId, schema); } private static MockResponse mockResponse(RESTResponse response, int httpCode) { diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java index 14005bcc6564..a5dc8b3cdb06 100644 --- a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java @@ -245,8 +245,7 @@ public GetTableResponse getTable( ImmutableList.of(), ImmutableList.of(), new HashMap<>(), - "comment"), - "uuid"); + "comment")); } @Operation( @@ -274,8 +273,7 @@ public GetTableResponse createTable( ImmutableList.of(), ImmutableList.of(), new HashMap<>(), - "comment"), - "uuid"); + "comment")); } @Operation( @@ -308,8 +306,7 @@ public GetTableResponse alterTable( ImmutableList.of(), ImmutableList.of(), new HashMap<>(), - "comment"), - "uuid"); + "comment")); } @Operation( @@ -361,8 +358,7 @@ public GetTableResponse renameTable( ImmutableList.of(), ImmutableList.of(), new HashMap<>(), - "comment"), - "uuid"); + "comment")); } @Operation( From f1c9ab757a4f20c70428bbb621ffdbab51f324c8 Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 13 Jan 2025 17:40:36 +0800 Subject: [PATCH 11/15] fix fail test --- .../src/test/java/org/apache/paimon/hive/HiveCatalogTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java index 66e45427a9dc..ea669d254fbd 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java @@ -422,7 +422,7 @@ protected boolean supportsFormatTable() { @Override protected void checkPartition(Partition expected, Partition actual) { assertThat(actual.recordCount()).isEqualTo(expected.recordCount()); - assertThat(actual.lastFileCreationTime() / 1000).isEqualTo(expected.lastFileCreationTime()); + assertThat(actual.lastFileCreationTime()).isEqualTo(expected.lastFileCreationTime() / 1000); } @Test From 974411bd666a5043386eb650577bbc8c13588e0e Mon Sep 17 00:00:00 2001 From: yantian Date: Mon, 13 Jan 2025 18:23:54 +0800 Subject: [PATCH 12/15] add mark done partitions for RESTCatalog --- .../org/apache/paimon/rest/HttpClient.java | 6 ++ .../org/apache/paimon/rest/RESTCatalog.java | 69 ++++++-------- .../org/apache/paimon/rest/RESTClient.java | 2 + .../org/apache/paimon/rest/ResourcePaths.java | 5 + .../requests/MarkDonePartitionsRequest.java | 37 ++++++++ .../responses/AlterPartitionsResponse.java | 61 ------------- .../rest/responses/PartitionsResponse.java | 63 ------------- .../apache/paimon/rest/MockRESTMessage.java | 13 --- .../apache/paimon/rest/RESTCatalogServer.java | 20 ++++ .../apache/paimon/rest/RESTCatalogTest.java | 15 +++ .../paimon/rest/RESTObjectMapperTest.java | 24 ----- paimon-open-api/rest-catalog-open-api.yaml | 91 +++++++++++-------- .../open/api/RESTCatalogController.java | 57 ++++++------ 13 files changed, 198 insertions(+), 265 deletions(-) create mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/requests/MarkDonePartitionsRequest.java delete mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/responses/AlterPartitionsResponse.java delete mode 100644 paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionsResponse.java diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java index eee6abffb1ec..08d6c8a050a0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -89,6 +89,12 @@ public T get( return exec(request, responseType); } + @Override + public T post( + String path, RESTRequest body, Map headers) { + return post(path, body, null, headers); + } + @Override public T post( String path, RESTRequest body, Class responseType, Map headers) { diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index d64935a3570a..4ee69084f590 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -46,9 +46,9 @@ import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.DropPartitionsRequest; +import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; -import org.apache.paimon.rest.responses.AlterPartitionsResponse; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.ErrorResponseResourceType; @@ -57,7 +57,6 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.PartitionsResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.TableSchema; @@ -404,19 +403,11 @@ public void createPartitions(Identifier identifier, List> pa if (Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE))) { try { CreatePartitionsRequest request = new CreatePartitionsRequest(partitions); - PartitionsResponse response = - client.post( - resourcePaths.partitions( - identifier.getDatabaseName(), identifier.getTableName()), - request, - PartitionsResponse.class, - headers()); - if (response.getFailPartitionSpecs() != null - && !response.getFailPartitionSpecs().isEmpty()) { - // todo: whether this exception is ok? - throw new RuntimeException( - "Create partitions failed: " + response.getFailPartitionSpecs()); - } + client.post( + resourcePaths.partitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + headers()); } catch (NoSuchResourceException e) { throw new TableNotExistException(identifier); } @@ -433,18 +424,11 @@ public void dropPartitions(Identifier identifier, List> part if (Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE))) { try { DropPartitionsRequest request = new DropPartitionsRequest(partitions); - PartitionsResponse response = - client.post( - resourcePaths.dropPartitions( - identifier.getDatabaseName(), identifier.getTableName()), - request, - PartitionsResponse.class, - headers()); - if (response.getFailPartitionSpecs() != null - && !response.getFailPartitionSpecs().isEmpty()) { - throw new RuntimeException( - "Drop partitions failed: " + response.getFailPartitionSpecs()); - } + client.post( + resourcePaths.dropPartitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + headers()); } catch (NoSuchResourceException e) { throw new TableNotExistException(identifier); } @@ -469,18 +453,11 @@ public void alterPartitions(Identifier identifier, List partitions) if (Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE))) { try { AlterPartitionsRequest request = new AlterPartitionsRequest(partitions); - AlterPartitionsResponse response = - client.post( - resourcePaths.alterPartitions( - identifier.getDatabaseName(), identifier.getTableName()), - request, - AlterPartitionsResponse.class, - headers()); - if (response.getFailPartitions() != null - && !response.getFailPartitions().isEmpty()) { - throw new RuntimeException( - "Alter partitions failed: " + response.getFailPartitions()); - } + client.post( + resourcePaths.alterPartitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + headers()); } catch (NoSuchResourceException e) { throw new TableNotExistException(identifier); } @@ -490,6 +467,20 @@ public void alterPartitions(Identifier identifier, List partitions) @Override public void markDonePartitions(Identifier identifier, List> partitions) throws TableNotExistException { + Table table = getTable(identifier); + Options options = Options.fromMap(table.options()); + if (Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE))) { + try { + MarkDonePartitionsRequest request = new MarkDonePartitionsRequest(partitions); + client.post( + resourcePaths.markDonePartitions( + identifier.getDatabaseName(), identifier.getTableName()), + request, + headers()); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } + } throw new UnsupportedOperationException(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java index d90cb5fa4ad9..d816f09ed0d9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java @@ -26,6 +26,8 @@ public interface RESTClient extends Closeable { T get(String path, Class responseType, Map headers); + T post(String path, RESTRequest body, Map headers); + T post( String path, RESTRequest body, Class responseType, Map headers); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java index a64153d7fd8d..7b092196626f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -79,4 +79,9 @@ public String alterPartitions(String databaseName, String tableName) { return SLASH.join( V1, prefix, DATABASES, databaseName, TABLES, tableName, "partitions", "alter"); } + + public String markDonePartitions(String databaseName, String tableName) { + return SLASH.join( + V1, prefix, DATABASES, databaseName, TABLES, tableName, "partitions", "mark"); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/MarkDonePartitionsRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/MarkDonePartitionsRequest.java new file mode 100644 index 000000000000..88345e962027 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/MarkDonePartitionsRequest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Map; + +/** Request for marking done partition. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class MarkDonePartitionsRequest extends BasePartitionsRequest { + + @JsonCreator + public MarkDonePartitionsRequest( + @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs) { + super(partitionSpecs); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/AlterPartitionsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/AlterPartitionsResponse.java deleted file mode 100644 index 62ac0e25663d..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/AlterPartitionsResponse.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest.responses; - -import org.apache.paimon.partition.Partition; -import org.apache.paimon.rest.RESTResponse; - -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.List; - -/** response for altering partitions. */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class AlterPartitionsResponse implements RESTResponse { - - public static final String FIELD_SUCCESS_PARTITIONS = "successPartitions"; - public static final String FIELD_FAIL_PARTITIONS = "failPartitions"; - - @JsonProperty(FIELD_SUCCESS_PARTITIONS) - private final List successPartitions; - - @JsonProperty(FIELD_FAIL_PARTITIONS) - private final List failPartitions; - - @JsonCreator - public AlterPartitionsResponse( - @JsonProperty(FIELD_SUCCESS_PARTITIONS) List successPartitions, - @JsonProperty(FIELD_FAIL_PARTITIONS) List failPartitions) { - this.successPartitions = successPartitions; - this.failPartitions = failPartitions; - } - - @JsonGetter(FIELD_SUCCESS_PARTITIONS) - public List getSuccessPartitions() { - return successPartitions; - } - - @JsonGetter(FIELD_FAIL_PARTITIONS) - public List getFailPartitions() { - return failPartitions; - } -} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionsResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionsResponse.java deleted file mode 100644 index 3f78667ddcda..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/rest/responses/PartitionsResponse.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest.responses; - -import org.apache.paimon.rest.RESTResponse; - -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.List; -import java.util.Map; - -/** Partition for rest api. */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class PartitionsResponse implements RESTResponse { - - public static final String FIELD_SUCCESS_PARTITION_SPECS = "successPartitionSpecs"; - public static final String FIELD_FAIL_PARTITION_SPECS = "failPartitionSpecs"; - - @JsonProperty(FIELD_SUCCESS_PARTITION_SPECS) - private final List> successPartitionSpecs; - - @JsonProperty(FIELD_FAIL_PARTITION_SPECS) - private final List> failPartitionSpecs; - - @JsonCreator - public PartitionsResponse( - @JsonProperty(FIELD_SUCCESS_PARTITION_SPECS) - List> successPartitionSpecs, - @JsonProperty(FIELD_FAIL_PARTITION_SPECS) - List> failPartitionSpecs) { - this.successPartitionSpecs = successPartitionSpecs; - this.failPartitionSpecs = failPartitionSpecs; - } - - @JsonGetter(FIELD_SUCCESS_PARTITION_SPECS) - public List> getSuccessPartitionSpecs() { - return successPartitionSpecs; - } - - @JsonGetter(FIELD_FAIL_PARTITION_SPECS) - public List> getFailPartitionSpecs() { - return failPartitionSpecs; - } -} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java index d32111c99d86..822e06d7fbbc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -29,14 +29,12 @@ import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; -import org.apache.paimon.rest.responses.AlterPartitionsResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetTableResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.PartitionsResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.types.DataField; @@ -143,12 +141,6 @@ public static DropPartitionsRequest dropPartitionsRequest() { return new DropPartitionsRequest(ImmutableList.of(Collections.singletonMap("pt", "1"))); } - public static PartitionsResponse partitionResponse() { - Map spec = new HashMap<>(); - spec.put("f0", "1"); - return new PartitionsResponse(ImmutableList.of(spec), ImmutableList.of()); - } - public static ListPartitionsResponse listPartitionsResponse() { Map spec = new HashMap<>(); spec.put("f0", "1"); @@ -238,11 +230,6 @@ public static AlterPartitionsRequest alterPartitionsRequest() { return new AlterPartitionsRequest(ImmutableList.of(partition())); } - public static AlterPartitionsResponse alterPartitionsResponse() { - return new AlterPartitionsResponse( - ImmutableList.of(partition()), ImmutableList.of(partition())); - } - private static Partition partition() { return new Partition(Collections.singletonMap("pt", "1"), 1, 1, 1, 1); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index c99e9130eb35..d80fc3f95a6d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -31,6 +31,7 @@ import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.DropPartitionsRequest; +import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterPartitionsResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; @@ -137,6 +138,11 @@ public MockResponse dispatch(RecordedRequest request) { && "tables".equals(resources[1]) && "partitions".equals(resources[3]) && "alter".equals(resources[4]); + boolean isMarkDonePartitions = + resources.length == 5 + && "tables".equals(resources[1]) + && "partitions".equals(resources[3]) + && "mark".equals(resources[4]); if (isDropPartitions) { String tableName = resources[2]; Identifier identifier = Identifier.create(databaseName, tableName); @@ -165,6 +171,20 @@ public MockResponse dispatch(RecordedRequest request) { alterPartitionsRequest.getPartitions(), ImmutableList.of()); return mockResponse(response, 200); + } else if (isMarkDonePartitions) { + String tableName = resources[2]; + Identifier identifier = Identifier.create(databaseName, tableName); + MarkDonePartitionsRequest markDonePartitionsRequest = + OBJECT_MAPPER.readValue( + request.getBody().readUtf8(), + MarkDonePartitionsRequest.class); + catalog.markDonePartitions( + identifier, markDonePartitionsRequest.getPartitionSpecs()); + response = + new PartitionsResponse( + markDonePartitionsRequest.getPartitionSpecs(), + ImmutableList.of()); + return mockResponse(response, 200); } else if (isPartitions) { String tableName = resources[2]; return partitionsApiHandler(catalog, request, databaseName, tableName); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 387fc9ee51c9..379daa628a93 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -123,6 +123,21 @@ void testCreatePartitionsWhenMetastorePartitionedIsFalse() throws Exception { .isInstanceOf(UnsupportedOperationException.class); } + @Test + void testMarkDonePartitionsWhenMetastorePartitionedIsFalse() throws Exception { + Identifier identifier = Identifier.create("test_db", "test_table"); + createTable( + identifier, + ImmutableMap.of(METASTORE_PARTITIONED_TABLE.key(), "" + false), + Lists.newArrayList("col1")); + assertThatThrownBy( + () -> + catalog.markDonePartitions( + identifier, + Arrays.asList(Collections.singletonMap("col1", "1")))) + .isInstanceOf(UnsupportedOperationException.class); + } + @Override protected boolean supportsFormatTable() { return true; diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java index 726d1715fa3e..6e260a5e7341 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java @@ -27,7 +27,6 @@ import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; -import org.apache.paimon.rest.responses.AlterPartitionsResponse; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.ErrorResponse; @@ -36,7 +35,6 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.PartitionsResponse; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; @@ -51,7 +49,6 @@ import static org.apache.paimon.rest.RESTObjectMapper.OBJECT_MAPPER; import static org.junit.Assert.assertEquals; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; /** Test for {@link RESTObjectMapper}. */ public class RESTObjectMapperTest { @@ -237,17 +234,6 @@ public void listPartitionsResponseParseTest() throws Exception { parseData.getPartitions().get(0).fileCount()); } - @Test - public void partitionResponseParseTest() throws Exception { - PartitionsResponse response = MockRESTMessage.partitionResponse(); - assertDoesNotThrow(() -> OBJECT_MAPPER.writeValueAsString(response)); - assertDoesNotThrow( - () -> - OBJECT_MAPPER.readValue( - OBJECT_MAPPER.writeValueAsString(response), - PartitionsResponse.class)); - } - @Test public void alterPartitionsRequestParseTest() throws Exception { AlterPartitionsRequest request = MockRESTMessage.alterPartitionsRequest(); @@ -256,14 +242,4 @@ public void alterPartitionsRequestParseTest() throws Exception { OBJECT_MAPPER.readValue(requestStr, AlterPartitionsRequest.class); assertEquals(request.getPartitions(), parseData.getPartitions()); } - - @Test - public void alterPartitionsResponseParseTest() throws Exception { - AlterPartitionsResponse response = MockRESTMessage.alterPartitionsResponse(); - String responseStr = OBJECT_MAPPER.writeValueAsString(response); - AlterPartitionsResponse parseData = - OBJECT_MAPPER.readValue(responseStr, AlterPartitionsResponse.class); - assertEquals(response.getFailPartitions(), parseData.getFailPartitions()); - assertEquals(response.getSuccessPartitions(), parseData.getSuccessPartitions()); - } } diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml index c51487c1d074..41d2632454db 100644 --- a/paimon-open-api/rest-catalog-open-api.yaml +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -471,11 +471,7 @@ paths: $ref: '#/components/schemas/CreatePartitionsRequest' responses: "200": - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/PartitionsResponse' + description: Success, no content "404": description: Resource not found content: @@ -513,11 +509,7 @@ paths: $ref: '#/components/schemas/DropPartitionsRequest' responses: "200": - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/PartitionsResponse' + description: Success, no content "404": description: Resource not found content: @@ -555,11 +547,45 @@ paths: $ref: '#/components/schemas/AlterPartitionsRequest' responses: "200": - description: OK + description: Success, no content + "404": + description: Resource not found content: application/json: schema: - $ref: '#/components/schemas/AlterPartitionsResponse' + $ref: '#/components/schemas/ErrorResponse' + "500": + description: Internal Server Error + /v1/{prefix}/databases/{database}/tables/{table}/partitions/mark: + post: + tags: + - partition + summary: MarkDone partitions + operationId: markDonePartitions + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: table + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/MarkDonePartitionsRequest' + responses: + "200": + description: Success, no content "404": description: Resource not found content: @@ -593,6 +619,20 @@ components: type: array items: type: object + AlterTableRequest: + type: object + properties: + changes: + type: array + items: + $ref: '#/components/schemas/SchemaChange' + MarkDonePartitionsRequest: + type: object + properties: + specs: + type: array + items: + type: object CreateDatabaseResponse: type: object properties: @@ -724,13 +764,6 @@ components: $ref: '#/components/schemas/Schema' uuid: type: string - AlterTableRequest: - type: object - properties: - changes: - type: array - items: - $ref: '#/components/schemas/SchemaChange' SchemaChange: anyOf: - $ref: '#/components/schemas/SetOption' @@ -943,15 +976,6 @@ components: type: array items: $ref: '#/components/schemas/Partition' - PartitionsResponse: - type: object - properties: - successPartitionSpecs: - type: array - items: object - failPartitionSpecs: - type: array - items: object Partition: type: object properties: @@ -969,17 +993,6 @@ components: lastFileCreationTime: type: integer format: int64 - AlterPartitionsResponse: - type: object - properties: - successPartitions: - type: array - items: - $ref: '#/components/schemas/Partition' - failPartitions: - type: array - items: - $ref: '#/components/schemas/Partition' securitySchemes: BearerAuth: type: http diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java index a5dc8b3cdb06..a9f3d02f5442 100644 --- a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java @@ -27,9 +27,9 @@ import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.DropPartitionsRequest; +import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; -import org.apache.paimon.rest.responses.AlterPartitionsResponse; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.ErrorResponse; @@ -38,7 +38,6 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.PartitionsResponse; import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; @@ -393,9 +392,7 @@ public ListPartitionsResponse listPartitions( summary = "Create partition", tags = {"partition"}) @ApiResponses({ - @ApiResponse( - responseCode = "200", - content = {@Content(schema = @Schema(implementation = PartitionsResponse.class))}), + @ApiResponse(responseCode = "200", description = "Success, no content"), @ApiResponse( responseCode = "404", description = "Resource not found", @@ -405,23 +402,17 @@ public ListPartitionsResponse listPartitions( content = {@Content(schema = @Schema())}) }) @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions") - public PartitionsResponse createPartitions( + public void createPartitions( @PathVariable String prefix, @PathVariable String database, @PathVariable String table, - @RequestBody CreatePartitionsRequest request) { - Map spec = new HashMap<>(); - spec.put("f1", "1"); - return new PartitionsResponse(ImmutableList.of(spec), ImmutableList.of()); - } + @RequestBody CreatePartitionsRequest request) {} @Operation( summary = "Drop partitions", tags = {"partition"}) @ApiResponses({ - @ApiResponse( - responseCode = "200", - content = {@Content(schema = @Schema(implementation = PartitionsResponse.class))}), + @ApiResponse(responseCode = "200", description = "Success, no content"), @ApiResponse( responseCode = "404", description = "Resource not found", @@ -431,21 +422,17 @@ public PartitionsResponse createPartitions( content = {@Content(schema = @Schema())}) }) @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions/drop") - public PartitionsResponse dropPartitions( + public void dropPartitions( @PathVariable String prefix, @PathVariable String database, @PathVariable String table, - @RequestBody DropPartitionsRequest request) { - return new PartitionsResponse(ImmutableList.of(), ImmutableList.of()); - } + @RequestBody DropPartitionsRequest request) {} @Operation( - summary = "Drop partitions", + summary = "Alter partitions", tags = {"partition"}) @ApiResponses({ - @ApiResponse( - responseCode = "200", - content = {@Content(schema = @Schema(implementation = PartitionsResponse.class))}), + @ApiResponse(responseCode = "200", description = "Success, no content"), @ApiResponse( responseCode = "404", description = "Resource not found", @@ -455,11 +442,29 @@ public PartitionsResponse dropPartitions( content = {@Content(schema = @Schema())}) }) @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions/alter") - public AlterPartitionsResponse alterPartitions( + public void alterPartitions( @PathVariable String prefix, @PathVariable String database, @PathVariable String table, - @RequestBody AlterPartitionsRequest request) { - return new AlterPartitionsResponse(ImmutableList.of(), ImmutableList.of()); - } + @RequestBody AlterPartitionsRequest request) {} + + @Operation( + summary = "MarkDone partitions", + tags = {"partition"}) + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "Success, no content"), + @ApiResponse( + responseCode = "404", + description = "Resource not found", + content = {@Content(schema = @Schema(implementation = ErrorResponse.class))}), + @ApiResponse( + responseCode = "500", + content = {@Content(schema = @Schema())}) + }) + @PostMapping("/v1/{prefix}/databases/{database}/tables/{table}/partitions/mark") + public void markDonePartitions( + @PathVariable String prefix, + @PathVariable String database, + @PathVariable String table, + @RequestBody MarkDonePartitionsRequest request) {} } From 4812d03d77afbe58ea5f31d82437480d493f0b6f Mon Sep 17 00:00:00 2001 From: yantian Date: Tue, 14 Jan 2025 10:56:44 +0800 Subject: [PATCH 13/15] remove UnsupportedOperationException same with AbstractCatalog --- .../paimon/catalog/AbstractCatalog.java | 2 +- .../org/apache/paimon/rest/RESTCatalog.java | 23 ++++++-------- .../paimon/catalog/CatalogTestBase.java | 9 ++++++ .../apache/paimon/rest/RESTCatalogServer.java | 25 +++------------ .../apache/paimon/rest/RESTCatalogTest.java | 31 ------------------- 5 files changed, 24 insertions(+), 66 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index 35c1fbb2f94a..dedfd5f11276 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -386,7 +386,7 @@ public Table getTable(Identifier identifier) throws TableNotExistException { } } - // here just data table, hive override this method. + // hive override this method. protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException { Preconditions.checkArgument(identifier.getSystemTableName() == null); TableMeta tableMeta = getDataTableMeta(identifier); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 4ee69084f590..3fd55331e809 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -399,8 +399,7 @@ public void dropTable(Identifier identifier, boolean ignoreIfNotExists) public void createPartitions(Identifier identifier, List> partitions) throws TableNotExistException { Table table = getTable(identifier); - Options options = Options.fromMap(table.options()); - if (Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE))) { + if (tableSupportPartitioned(table)) { try { CreatePartitionsRequest request = new CreatePartitionsRequest(partitions); client.post( @@ -411,8 +410,6 @@ public void createPartitions(Identifier identifier, List> pa } catch (NoSuchResourceException e) { throw new TableNotExistException(identifier); } - } else { - throw new UnsupportedOperationException(); } } @@ -420,8 +417,7 @@ public void createPartitions(Identifier identifier, List> pa public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { Table table = getTable(identifier); - Options options = Options.fromMap(table.options()); - if (Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE))) { + if (tableSupportPartitioned(table)) { try { DropPartitionsRequest request = new DropPartitionsRequest(partitions); client.post( @@ -449,8 +445,7 @@ public void dropPartitions(Identifier identifier, List> part public void alterPartitions(Identifier identifier, List partitions) throws TableNotExistException { Table table = getTable(identifier); - Options options = Options.fromMap(table.options()); - if (Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE))) { + if (tableSupportPartitioned(table)) { try { AlterPartitionsRequest request = new AlterPartitionsRequest(partitions); client.post( @@ -468,8 +463,7 @@ public void alterPartitions(Identifier identifier, List partitions) public void markDonePartitions(Identifier identifier, List> partitions) throws TableNotExistException { Table table = getTable(identifier); - Options options = Options.fromMap(table.options()); - if (Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE))) { + if (tableSupportPartitioned(table)) { try { MarkDonePartitionsRequest request = new MarkDonePartitionsRequest(partitions); client.post( @@ -481,14 +475,12 @@ public void markDonePartitions(Identifier identifier, List> throw new TableNotExistException(identifier); } } - throw new UnsupportedOperationException(); } @Override public List listPartitions(Identifier identifier) throws TableNotExistException { Table table = getTable(identifier); - Options options = Options.fromMap(table.options()); - if (!options.get(METASTORE_PARTITIONED_TABLE)) { + if (!tableSupportPartitioned(table)) { return listPartitionsFromFileSystem(table); } @@ -578,6 +570,11 @@ private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistEx return table; } + private boolean tableSupportPartitioned(Table table) { + Options options = Options.fromMap(table.options()); + return Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE)); + } + private Map headers() { return catalogAuth.getHeaders(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java index c0fefa788bc2..bf0f0e9ff7f0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java @@ -35,6 +35,7 @@ import org.apache.paimon.view.View; import org.apache.paimon.view.ViewImpl; +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; @@ -61,6 +62,7 @@ import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -1058,6 +1060,13 @@ public void testPartitions() throws Exception { .containsExactlyInAnyOrder( Collections.singletonMap("dt", "20250102"), Collections.singletonMap("dt", "20250101")); + + assertDoesNotThrow( + () -> + catalog.markDonePartitions( + identifier, + ImmutableList.of(Collections.singletonMap("dt", "20250102")))); + catalog.dropPartitions( identifier, Arrays.asList( diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index d80fc3f95a6d..1c802cf4c0ef 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -33,7 +33,6 @@ import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; -import org.apache.paimon.rest.responses.AlterPartitionsResponse; import org.apache.paimon.rest.responses.CreateDatabaseResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.ErrorResponseResourceType; @@ -42,7 +41,6 @@ import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListTablesResponse; -import org.apache.paimon.rest.responses.PartitionsResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FormatTable; @@ -55,7 +53,6 @@ import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.RecordedRequest; -import org.testcontainers.shaded.com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.List; @@ -152,11 +149,7 @@ public MockResponse dispatch(RecordedRequest request) { DropPartitionsRequest.class); catalog.dropPartitions( identifier, dropPartitionsRequest.getPartitionSpecs()); - response = - new PartitionsResponse( - dropPartitionsRequest.getPartitionSpecs(), - ImmutableList.of()); - return mockResponse(response, 200); + return new MockResponse().setResponseCode(200); } else if (isAlterPartitions) { String tableName = resources[2]; Identifier identifier = Identifier.create(databaseName, tableName); @@ -166,11 +159,7 @@ public MockResponse dispatch(RecordedRequest request) { AlterPartitionsRequest.class); catalog.alterPartitions( identifier, alterPartitionsRequest.getPartitions()); - response = - new AlterPartitionsResponse( - alterPartitionsRequest.getPartitions(), - ImmutableList.of()); - return mockResponse(response, 200); + return new MockResponse().setResponseCode(200); } else if (isMarkDonePartitions) { String tableName = resources[2]; Identifier identifier = Identifier.create(databaseName, tableName); @@ -180,11 +169,7 @@ public MockResponse dispatch(RecordedRequest request) { MarkDonePartitionsRequest.class); catalog.markDonePartitions( identifier, markDonePartitionsRequest.getPartitionSpecs()); - response = - new PartitionsResponse( - markDonePartitionsRequest.getPartitionSpecs(), - ImmutableList.of()); - return mockResponse(response, 200); + return new MockResponse().setResponseCode(200); } else if (isPartitions) { String tableName = resources[2]; return partitionsApiHandler(catalog, request, databaseName, tableName); @@ -386,9 +371,7 @@ private static MockResponse partitionsApiHandler( OBJECT_MAPPER.readValue( request.getBody().readUtf8(), CreatePartitionsRequest.class); catalog.createPartitions(identifier, requestBody.getPartitionSpecs()); - response = - new PartitionsResponse(requestBody.getPartitionSpecs(), ImmutableList.of()); - return mockResponse(response, 200); + return new MockResponse().setResponseCode(200); default: return new MockResponse().setResponseCode(404); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 379daa628a93..f63d70332650 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -37,7 +37,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -108,36 +107,6 @@ void testListPartitionsFromFile() throws Exception { assertEquals(0, result.size()); } - @Test - void testCreatePartitionsWhenMetastorePartitionedIsFalse() throws Exception { - Identifier identifier = Identifier.create("test_db", "test_table"); - createTable( - identifier, - ImmutableMap.of(METASTORE_PARTITIONED_TABLE.key(), "" + false), - Lists.newArrayList("col1")); - assertThatThrownBy( - () -> - catalog.createPartitions( - identifier, - Arrays.asList(Collections.singletonMap("col1", "1")))) - .isInstanceOf(UnsupportedOperationException.class); - } - - @Test - void testMarkDonePartitionsWhenMetastorePartitionedIsFalse() throws Exception { - Identifier identifier = Identifier.create("test_db", "test_table"); - createTable( - identifier, - ImmutableMap.of(METASTORE_PARTITIONED_TABLE.key(), "" + false), - Lists.newArrayList("col1")); - assertThatThrownBy( - () -> - catalog.markDonePartitions( - identifier, - Arrays.asList(Collections.singletonMap("col1", "1")))) - .isInstanceOf(UnsupportedOperationException.class); - } - @Override protected boolean supportsFormatTable() { return true; From 673e8df3e3183b605d603edfe72ae62bb1d78051 Mon Sep 17 00:00:00 2001 From: yantian Date: Tue, 14 Jan 2025 11:16:55 +0800 Subject: [PATCH 14/15] change tableSupportPartitioned to isMetaStorePartitionedTable --- .../java/org/apache/paimon/rest/RESTCatalog.java | 12 ++++++------ .../paimon/rest/requests/AlterPartitionsRequest.java | 1 + .../java/org/apache/paimon/rest/TestRESTCatalog.java | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 3fd55331e809..4b4e90fb7c20 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -399,7 +399,7 @@ public void dropTable(Identifier identifier, boolean ignoreIfNotExists) public void createPartitions(Identifier identifier, List> partitions) throws TableNotExistException { Table table = getTable(identifier); - if (tableSupportPartitioned(table)) { + if (isMetaStorePartitionedTable(table)) { try { CreatePartitionsRequest request = new CreatePartitionsRequest(partitions); client.post( @@ -417,7 +417,7 @@ public void createPartitions(Identifier identifier, List> pa public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { Table table = getTable(identifier); - if (tableSupportPartitioned(table)) { + if (isMetaStorePartitionedTable(table)) { try { DropPartitionsRequest request = new DropPartitionsRequest(partitions); client.post( @@ -445,7 +445,7 @@ public void dropPartitions(Identifier identifier, List> part public void alterPartitions(Identifier identifier, List partitions) throws TableNotExistException { Table table = getTable(identifier); - if (tableSupportPartitioned(table)) { + if (isMetaStorePartitionedTable(table)) { try { AlterPartitionsRequest request = new AlterPartitionsRequest(partitions); client.post( @@ -463,7 +463,7 @@ public void alterPartitions(Identifier identifier, List partitions) public void markDonePartitions(Identifier identifier, List> partitions) throws TableNotExistException { Table table = getTable(identifier); - if (tableSupportPartitioned(table)) { + if (isMetaStorePartitionedTable(table)) { try { MarkDonePartitionsRequest request = new MarkDonePartitionsRequest(partitions); client.post( @@ -480,7 +480,7 @@ public void markDonePartitions(Identifier identifier, List> @Override public List listPartitions(Identifier identifier) throws TableNotExistException { Table table = getTable(identifier); - if (!tableSupportPartitioned(table)) { + if (!isMetaStorePartitionedTable(table)) { return listPartitionsFromFileSystem(table); } @@ -570,7 +570,7 @@ private Table getDataOrFormatTable(Identifier identifier) throws TableNotExistEx return table; } - private boolean tableSupportPartitioned(Table table) { + private boolean isMetaStorePartitionedTable(Table table) { Options options = Options.fromMap(table.options()); return Boolean.TRUE.equals(options.get(METASTORE_PARTITIONED_TABLE)); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java index fd1b3afdedb7..e8f542eff578 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/AlterPartitionsRequest.java @@ -31,6 +31,7 @@ /** Request for altering partitions. */ @JsonIgnoreProperties(ignoreUnknown = true) public class AlterPartitionsRequest implements RESTRequest { + public static final String FIELD_PARTITIONS = "partitions"; @JsonProperty(FIELD_PARTITIONS) diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java index 85717470c9ba..a9181ec80f47 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java @@ -41,6 +41,7 @@ /** A catalog for testing RESTCatalog. */ public class TestRESTCatalog extends FileSystemCatalog { + public Map tableFullName2Schema = new HashMap(); public Map> tableFullName2Partitions = new HashMap>(); From 9505948c715dd0ac80687a04d55de04a21736781 Mon Sep 17 00:00:00 2001 From: yantian Date: Tue, 14 Jan 2025 11:44:43 +0800 Subject: [PATCH 15/15] partition ut add when table not exist --- .../paimon/catalog/CatalogTestBase.java | 54 ++++++++++++------- .../apache/paimon/rest/TestRESTCatalog.java | 4 ++ .../org/apache/paimon/hive/HiveCatalog.java | 3 ++ 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java index bf0f0e9ff7f0..c8b9192c1c38 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java @@ -35,7 +35,6 @@ import org.apache.paimon.view.View; import org.apache.paimon.view.ViewImpl; -import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; @@ -1037,6 +1036,10 @@ public void testPartitions() throws Exception { return; } String databaseName = "testPartitionTable"; + List> partitionSpecs = + Arrays.asList( + Collections.singletonMap("dt", "20250101"), + Collections.singletonMap("dt", "20250102")); catalog.dropDatabase(databaseName, true, true); catalog.createDatabase(databaseName, true); Identifier identifier = Identifier.create(databaseName, "table"); @@ -1051,28 +1054,33 @@ public void testPartitions() throws Exception { .build(), true); - catalog.createPartitions( - identifier, - Arrays.asList( - Collections.singletonMap("dt", "20250101"), - Collections.singletonMap("dt", "20250102"))); + catalog.createPartitions(identifier, partitionSpecs); assertThat(catalog.listPartitions(identifier).stream().map(Partition::spec)) - .containsExactlyInAnyOrder( - Collections.singletonMap("dt", "20250102"), - Collections.singletonMap("dt", "20250101")); + .containsExactlyInAnyOrder(partitionSpecs.get(0), partitionSpecs.get(1)); - assertDoesNotThrow( - () -> - catalog.markDonePartitions( - identifier, - ImmutableList.of(Collections.singletonMap("dt", "20250102")))); + assertDoesNotThrow(() -> catalog.markDonePartitions(identifier, partitionSpecs)); - catalog.dropPartitions( - identifier, - Arrays.asList( - Collections.singletonMap("dt", "20250102"), - Collections.singletonMap("dt", "20250101"))); + catalog.dropPartitions(identifier, partitionSpecs); assertThat(catalog.listPartitions(identifier)).isEmpty(); + + // Test when table does not exist + assertThatExceptionOfType(Catalog.TableNotExistException.class) + .isThrownBy( + () -> + catalog.createPartitions( + Identifier.create(databaseName, "non_existing_table"), + partitionSpecs)); + assertThatExceptionOfType(Catalog.TableNotExistException.class) + .isThrownBy( + () -> + catalog.listPartitions( + Identifier.create(databaseName, "non_existing_table"))); + assertThatExceptionOfType(Catalog.TableNotExistException.class) + .isThrownBy( + () -> + catalog.markDonePartitions( + Identifier.create(databaseName, "non_existing_table"), + partitionSpecs)); } @Test @@ -1108,6 +1116,14 @@ public void testAlterPartitions() throws Exception { catalog.alterPartitions(alterIdentifier, Arrays.asList(partition)); Partition partitionFromServer = catalog.listPartitions(alterIdentifier).get(0); checkPartition(partition, partitionFromServer); + + // Test when table does not exist + assertThatExceptionOfType(Catalog.TableNotExistException.class) + .isThrownBy( + () -> + catalog.alterPartitions( + Identifier.create(databaseName, "non_existing_table"), + Arrays.asList(partition))); } protected boolean supportsAlterDatabase() { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java index a9181ec80f47..a0f820e7ad0b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/TestRESTCatalog.java @@ -69,6 +69,7 @@ public static TestRESTCatalog create(CatalogContext context) { @Override public void createPartitions(Identifier identifier, List> partitions) throws TableNotExistException { + getTable(identifier); tableFullName2Partitions.put( identifier.getFullName(), partitions.stream() @@ -79,6 +80,7 @@ public void createPartitions(Identifier identifier, List> pa @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { + getTable(identifier); List existPartitions = tableFullName2Partitions.get(identifier.getFullName()); partitions.forEach( partition -> { @@ -100,6 +102,7 @@ public void dropPartitions(Identifier identifier, List> part @Override public void alterPartitions(Identifier identifier, List partitions) throws TableNotExistException { + getTable(identifier); List existPartitions = tableFullName2Partitions.get(identifier.getFullName()); partitions.forEach( partition -> { @@ -122,6 +125,7 @@ public void alterPartitions(Identifier identifier, List partitions) @Override public List listPartitions(Identifier identifier) throws TableNotExistException { + getTable(identifier); return tableFullName2Partitions.get(identifier.getFullName()); } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index d314b5c62386..4e2d7b6acb95 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -71,6 +71,7 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.thrift.TException; import org.slf4j.Logger; @@ -492,6 +493,8 @@ public void markDonePartitions(Identifier identifier, List> }); } catch (NoSuchObjectException e) { // do nothing if the partition not exists + } catch (UnknownTableException e) { + throw new TableNotExistException(identifier); } catch (TException | InterruptedException e) { throw new RuntimeException(e); }