diff --git a/docs/content/concepts/system-tables.md b/docs/content/concepts/system-tables.md index c6fc76807133..aea72c000a99 100644 --- a/docs/content/concepts/system-tables.md +++ b/docs/content/concepts/system-tables.md @@ -279,45 +279,45 @@ You can query all manifest files contained in the latest snapshot or the specifi SELECT * FROM my_table$manifests; /* -+--------------------------------+-------------+------------------+-------------------+---------------+ -| file_name | file_size | num_added_files | num_deleted_files | schema_id | -+--------------------------------+-------------+------------------+-------------------+---------------+ -| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | -| manifest-f4dcab43-ef6b-4713... | 1648 | 1 | 0 | 0 | -+--------------------------------+-------------+------------------+-------------------+---------------+ ++--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+ +| file_name | file_size | num_added_files | num_deleted_files | schema_id | min_partition_stats | max_partition_stats | ++--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+ +| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | {20230315, 00} | {20230315, 20} | +| manifest-f4dcab43-ef6b-4713... | 1648 | 1 | 0 | 0 | {20230115, 00} | {20230316, 23} | ++--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+ 2 rows in set */ -- You can also query the manifest with specified snapshot SELECT * FROM my_table$manifests /*+ OPTIONS('scan.snapshot-id'='1') */; /* -+--------------------------------+-------------+------------------+-------------------+---------------+ -| file_name | file_size | num_added_files | num_deleted_files | schema_id | -+--------------------------------+-------------+------------------+-------------------+---------------+ -| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | -+--------------------------------+-------------+------------------+-------------------+---------------+ ++--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+ +| file_name | file_size | num_added_files | num_deleted_files | schema_id | min_partition_stats | max_partition_stats | ++--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+ +| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | {20230315, 00} | {20230315, 20} | ++--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+ 1 rows in set */ - You can also query the manifest with specified tagName SELECT * FROM my_table$manifests /*+ OPTIONS('scan.tag-name'='tag1') */; /* -+--------------------------------+-------------+------------------+-------------------+---------------+ -| file_name | file_size | num_added_files | num_deleted_files | schema_id | -+--------------------------------+-------------+------------------+-------------------+---------------+ -| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | -+--------------------------------+-------------+------------------+-------------------+---------------+ ++--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+ +| file_name | file_size | num_added_files | num_deleted_files | schema_id | min_partition_stats | max_partition_stats | ++--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+ +| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | {20230315, 00} | {20230315, 20} | ++--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+ 1 rows in set */ - You can also query the manifest with specified timestamp in unix milliseconds SELECT * FROM my_table$manifests /*+ OPTIONS('scan.timestamp-millis'='1678883047356') */; /* -+--------------------------------+-------------+------------------+-------------------+---------------+ -| file_name | file_size | num_added_files | num_deleted_files | schema_id | -+--------------------------------+-------------+------------------+-------------------+---------------+ -| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | -+--------------------------------+-------------+------------------+-------------------+---------------+ ++--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+ +| file_name | file_size | num_added_files | num_deleted_files | schema_id | min_partition_stats | max_partition_stats | ++--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+ +| manifest-f4dcab43-ef6b-4713... | 12365| 40 | 0 | 0 | {20230315, 00} | {20230315, 20} | ++--------------------------------+-------------+------------------+-------------------+---------------+---------------------+---------------------+ 1 rows in set */ ``` diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/ManifestsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/ManifestsTable.java index d88636d02aab..4b3ddf30cec4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/ManifestsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/ManifestsTable.java @@ -20,6 +20,8 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.Snapshot; +import org.apache.paimon.casting.CastExecutor; +import org.apache.paimon.casting.CastExecutors; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; @@ -64,7 +66,7 @@ public class ManifestsTable implements ReadonlyTable { private static final Logger LOG = LoggerFactory.getLogger(ManifestsTable.class); - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 2L; public static final String MANIFESTS = "manifests"; @@ -75,7 +77,15 @@ public class ManifestsTable implements ReadonlyTable { new DataField(1, "file_size", new BigIntType(false)), new DataField(2, "num_added_files", new BigIntType(false)), new DataField(3, "num_deleted_files", new BigIntType(false)), - new DataField(4, "schema_id", new BigIntType(false)))); + new DataField(4, "schema_id", new BigIntType(false)), + new DataField( + 5, + "min_partition_stats", + SerializationUtils.newStringType(true)), + new DataField( + 6, + "max_partition_stats", + SerializationUtils.newStringType(true)))); private final FileStoreTable dataTable; @@ -176,8 +186,16 @@ public RecordReader createReader(Split split) { } List manifestFileMetas = allManifests(dataTable); + @SuppressWarnings("unchecked") + CastExecutor partitionCastExecutor = + (CastExecutor) + CastExecutors.resolveToString( + dataTable.schema().logicalPartitionType()); + Iterator rows = - Iterators.transform(manifestFileMetas.iterator(), this::toRow); + Iterators.transform( + manifestFileMetas.iterator(), + meta -> toRow(meta, partitionCastExecutor)); if (readType != null) { rows = Iterators.transform( @@ -189,13 +207,17 @@ public RecordReader createReader(Split split) { return new IteratorRecordReader<>(rows); } - private InternalRow toRow(ManifestFileMeta manifestFileMeta) { + private InternalRow toRow( + ManifestFileMeta manifestFileMeta, + CastExecutor partitionCastExecutor) { return GenericRow.of( BinaryString.fromString(manifestFileMeta.fileName()), manifestFileMeta.fileSize(), manifestFileMeta.numAddedFiles(), manifestFileMeta.numDeletedFiles(), - manifestFileMeta.schemaId()); + manifestFileMeta.schemaId(), + partitionCastExecutor.cast(manifestFileMeta.partitionStats().minValues()), + partitionCastExecutor.cast(manifestFileMeta.partitionStats().maxValues())); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java index a39e6f6fa807..f375dfd2c814 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java @@ -177,7 +177,21 @@ private List getExpectedResult(long snapshotId) { manifestFileMeta.fileSize(), manifestFileMeta.numAddedFiles(), manifestFileMeta.numDeletedFiles(), - manifestFileMeta.schemaId())); + manifestFileMeta.schemaId(), + BinaryString.fromString( + String.format( + "{%d}", + manifestFileMeta + .partitionStats() + .minValues() + .getInt(0))), + BinaryString.fromString( + String.format( + "{%d}", + manifestFileMeta + .partitionStats() + .maxValues() + .getInt(0))))); } return expectedRow; }