Skip to content

Commit a77b70d

Browse files
MaxGekkHyukjinKwon
authored andcommitted
[SPARK-33788][SQL][3.1][3.0][2.4] Throw NoSuchPartitionsException from HiveExternalCatalog.dropPartitions()
### What changes were proposed in this pull request? Throw `NoSuchPartitionsException` from `ALTER TABLE .. DROP TABLE` for not existing partitions of a table in V1 Hive external catalog. ### Why are the changes needed? The behaviour of Hive external catalog deviates from V1/V2 in-memory catalogs that throw `NoSuchPartitionsException`. To improve user experience with Spark SQL, it would be better to throw the same exception. ### Does this PR introduce _any_ user-facing change? Yes, the command throws `NoSuchPartitionsException` instead of the general exception `AnalysisException`. ### How was this patch tested? By running new UT via: ``` $ build/sbt -Phive -Phive-thriftserver "test:testOnly *HiveDDLSuite" ``` Authored-by: Max Gekk <max.gekkgmail.com> Signed-off-by: HyukjinKwon <gurwls223apache.org> (cherry picked from commit 3dfdcf4) Signed-off-by: Max Gekk <max.gekkgmail.com> Closes #30802 from MaxGekk/hive-drop-partition-exception-3.1. Authored-by: Max Gekk <max.gekk@gmail.com> Signed-off-by: HyukjinKwon <gurwls223@apache.org>
1 parent f2c8079 commit a77b70d

File tree

2 files changed

+19
-5
lines changed

2 files changed

+19
-5
lines changed

sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ import org.apache.spark.internal.Logging
4949
import org.apache.spark.metrics.source.HiveCatalogMetrics
5050
import org.apache.spark.sql.AnalysisException
5151
import org.apache.spark.sql.catalyst.TableIdentifier
52-
import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPartitionException, PartitionsAlreadyExistException}
52+
import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPartitionException, NoSuchPartitionsException, PartitionsAlreadyExistException}
5353
import org.apache.spark.sql.catalyst.catalog._
5454
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
5555
import org.apache.spark.sql.catalyst.expressions.Expression
@@ -637,9 +637,7 @@ private[hive] class HiveClientImpl(
637637
// (b='1', c='1') and (b='1', c='2'), a partial spec of (b='1') will match both.
638638
val parts = client.getPartitions(hiveTable, s.asJava).asScala
639639
if (parts.isEmpty && !ignoreIfNotExists) {
640-
throw new AnalysisException(
641-
s"No partition is dropped. One partition spec '$s' does not exist in table '$table' " +
642-
s"database '$db'")
640+
throw new NoSuchPartitionsException(db, table, Seq(s))
643641
}
644642
parts.map(_.getValues)
645643
}.distinct

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import org.scalatest.BeforeAndAfterEach
2828
import org.apache.spark.SparkException
2929
import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode}
3030
import org.apache.spark.sql.catalyst.TableIdentifier
31-
import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, PartitionsAlreadyExistException, TableAlreadyExistsException}
31+
import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, NoSuchPartitionsException, PartitionsAlreadyExistException, TableAlreadyExistsException}
3232
import org.apache.spark.sql.catalyst.catalog._
3333
import org.apache.spark.sql.catalyst.parser.ParseException
3434
import org.apache.spark.sql.connector.catalog.CatalogManager
@@ -2750,4 +2750,20 @@ class HiveDDLSuite
27502750
checkAnswer(sql("SHOW PARTITIONS t"), Seq(Row("id=1"), Row("id=2")))
27512751
}
27522752
}
2753+
2754+
test("SPARK-33788: partition not exists") {
2755+
withTable("t") {
2756+
sql(s"CREATE TABLE t (data string) PARTITIONED BY (id bigint)")
2757+
sql(s"ALTER TABLE t ADD PARTITION (id=1)")
2758+
2759+
val errMsg = intercept[NoSuchPartitionsException] {
2760+
sql(s"ALTER TABLE t DROP PARTITION (id=1), PARTITION (id=2)")
2761+
}.getMessage
2762+
assert(errMsg.contains("partitions not found in table"))
2763+
2764+
checkAnswer(sql("SHOW PARTITIONS t"), Seq(Row("id=1")))
2765+
sql(s"ALTER TABLE t DROP IF EXISTS PARTITION (id=1), PARTITION (id=2)")
2766+
checkAnswer(sql("SHOW PARTITIONS t"), Seq.empty)
2767+
}
2768+
}
27532769
}

0 commit comments

Comments
 (0)