Skip to content

[core] Add UT for expire partitions with hms config metastore.partitioned-table #4875

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 6 commits into from
Jan 9, 2025
Merged

[core] Add UT for expire partitions with hms config metastore.partitioned-table #4875

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
dd4816c
add ut
xuzifu666 Jan 9, 2025
8fd7710
fix ut
xuzifu666 Jan 9, 2025
b0993cc
fix conflict
xuzifu666 Jan 9, 2025
08966de
restore ut for hms
xuzifu666 Jan 9, 2025
4f122a1
restore ut for hms
xuzifu666 Jan 9, 2025
1c91400
opt code
xuzifu666 Jan 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions ...imon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,54 @@ abstract class DDLWithHiveCatalogTestBase extends PaimonHiveTestBase {
}
}

test(
"Paimon partition expire test with hive catalog: expire partition for paimon table sparkCatalogName") {
spark.sql(s"USE $paimonHiveCatalogName")
withTempDir {
dBLocation =>
withDatabase("paimon_db") {
spark.sql(s"CREATE DATABASE paimon_db LOCATION '${dBLocation.getCanonicalPath}'")
withTable("paimon_db.paimon_tbl") {
spark.sql(s"""
|CREATE TABLE paimon_db.paimon_tbl (id STRING, name STRING, pt STRING)
|USING PAIMON
|PARTITIONED BY (pt)
|TBLPROPERTIES('metastore.partitioned-table' = 'false')
|""".stripMargin)
spark.sql("insert into paimon_db.paimon_tbl select '1', 'n', '2024-11-01'")
spark.sql("insert into paimon_db.paimon_tbl select '2', 'n', '9999-11-01'")

spark.sql(
"CALL paimon_hive.sys.expire_partitions(table => 'paimon_db.paimon_tbl', expiration_time => '1 d', timestamp_formatter => 'yyyy-MM-dd')")

checkAnswer(
spark.sql("SELECT * FROM paimon_db.paimon_tbl"),
Row("2", "n", "9999-11-01") :: Nil)
}

withTable("paimon_db.paimon_tbl2") {
spark.sql(s"""
|CREATE TABLE paimon_db.paimon_tbl2 (id STRING, name STRING, pt STRING)
|USING PAIMON
|PARTITIONED BY (pt)
|TBLPROPERTIES('metastore.partitioned-table' = 'true')
|""".stripMargin)
spark.sql("insert into paimon_db.paimon_tbl2 select '1', 'n', '2024-11-01'")

spark.sql("insert into paimon_db.paimon_tbl2 select '2', 'n', '9999-11-01'")

spark.sql(
"CALL paimon_hive.sys.expire_partitions(table => 'paimon_db.paimon_tbl2', expiration_time => '1 d', timestamp_formatter => 'yyyy-MM-dd')")

checkAnswer(
spark.sql("SELECT * FROM paimon_db.paimon_tbl2"),
Row("2", "n", "9999-11-01") :: Nil)
}

}
}
}

test("Paimon DDL with hive catalog: create partition for paimon table sparkCatalogName") {
Seq(paimonHiveCatalogName).foreach {
catalogName =>
Expand Down
Loading