Skip to content

[CORE] Refine the test with specified spark version #9274

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zhztheplayer merged 1 commit into from
Apr 10, 2025
Merged

[CORE] Refine the test with specified spark version #9274

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergMOREqualityDeletionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ class ClickHouseIcebergMOREqualityDeletionSuite extends GlutenClickHouseWholeSta
}

testWithSpecifiedSparkVersion(
"iceberg read mor table with equality deletion", Array("3.3", "3.5")) {
"iceberg read mor table with equality deletion", "3.3", "3.5") {
// The table 'test_upsert_query' was generated by Flink + Iceberg from the iceberg ut,
// the root path must be the '/tmp/junit6640909127060857423/default'
val testTableName = "local.db.test_upsert_query"
Expand Down
17 changes: 8 additions & 9 deletions ...e/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
}

testWithSpecifiedSparkVersion(
"iceberg bucketed join", Array("3.3", "3.5")) {
"iceberg bucketed join", "3.3", "3.5") {
val leftTable = "p_str_tb"
val rightTable = "p_int_tb"
withTable(leftTable, rightTable) {
Expand Down Expand Up @@ -140,7 +140,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
}

testWithSpecifiedSparkVersion(
"iceberg bucketed join with partition", Array("3.3", "3.5")) {
"iceberg bucketed join with partition", "3.3", "3.5") {
val leftTable = "p_str_tb"
val rightTable = "p_int_tb"
withTable(leftTable, rightTable) {
Expand Down Expand Up @@ -222,8 +222,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
}
}

testWithSpecifiedSparkVersion("iceberg bucketed join partition value not exists",
Array("3.4", "3.5")) {
testWithMinSparkVersion("iceberg bucketed join partition value not exists", "3.5") {
val leftTable = "p_str_tb"
val rightTable = "p_int_tb"
withTable(leftTable, rightTable) {
Expand Down Expand Up @@ -302,8 +301,8 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
}
}

testWithSpecifiedSparkVersion(
"iceberg bucketed join partition value not exists partial cluster", Array("3.4", "3.5")) {
testWithMinSparkVersion(
"iceberg bucketed join partition value not exists partial cluster", "3.4") {
val leftTable = "p_str_tb"
val rightTable = "p_int_tb"
withTable(leftTable, rightTable) {
Expand Down Expand Up @@ -383,7 +382,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
}

testWithSpecifiedSparkVersion(
"iceberg bucketed join with partition filter", Array("3.3", "3.5")) {
"iceberg bucketed join with partition filter", "3.3", "3.5") {
val leftTable = "p_str_tb"
val rightTable = "p_int_tb"
withTable(leftTable, rightTable) {
Expand Down Expand Up @@ -466,7 +465,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
}
}

testWithSpecifiedSparkVersion("iceberg: time travel") {
test("iceberg: time travel") {
withTable("iceberg_tm") {
spark.sql(
s"""
Expand Down Expand Up @@ -765,7 +764,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
// Spark configuration spark.sql.iceberg.handle-timestamp-without-timezone is not supported
// in Spark 3.4
// TODO: there is a bug when using timestamp type as the partition column
testWithSpecifiedSparkVersion("iceberg partition type - timestamp", Array("")) {
ignore("iceberg partition type - timestamp") {
Seq("true", "false").foreach {
flag =>
withSQLConf(
Expand Down
4 changes: 1 addition & 3 deletions ...ckhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseExcelFormatSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1485,9 +1485,7 @@ class GlutenClickHouseExcelFormatSuite
}

// TODO: pass spark configuration to FileFormatWriter in Spark 3.3 and 3.2
testWithSpecifiedSparkVersion(
"write succeed even if set wrong snappy compression codec level",
Some("3.5")) {
testWithMinSparkVersion("write succeed even if set wrong snappy compression codec level", "3.5") {
// TODO: remove duplicated test codes
val tablePath = s"$HDFS_URL_ENDPOINT/$SPARK_DIR_NAME/failed_test/"
val format = "parquet"
Expand Down
20 changes: 6 additions & 14 deletions ...nds-velox/src/test/scala/org/apache/gluten/execution/GlutenSQLCollectLimitExecSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,7 @@ class GlutenSQLCollectLimitExecSuite extends WholeStageTransformerSuite {
assert(assertionCondition, assertionMessage)
}

testWithSpecifiedSparkVersion(
"ColumnarCollectLimitExec - basic limit test",
Array("3.2", "3.3")) {
testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - basic limit test", "3.2", "3.3") {
val df = spark.range(0, 1000, 1).toDF("id").limit(5)
val expectedData = Seq(Row(0L), Row(1L), Row(2L), Row(3L), Row(4L))

Expand All @@ -67,7 +65,7 @@ class GlutenSQLCollectLimitExecSuite extends WholeStageTransformerSuite {
assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true)
}

testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - with filter", Array("3.2", "3.3")) {
testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - with filter", "3.2", "3.3") {
val df = spark
.range(0, 20, 1)
.toDF("id")
Expand All @@ -80,9 +78,7 @@ class GlutenSQLCollectLimitExecSuite extends WholeStageTransformerSuite {
assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true)
}

testWithSpecifiedSparkVersion(
"ColumnarCollectLimitExec - range with repartition",
Array("3.2", "3.3")) {
testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - range with repartition", "3.2", "3.3") {

val df = spark
.range(0, 10, 1)
Expand All @@ -96,9 +92,7 @@ class GlutenSQLCollectLimitExecSuite extends WholeStageTransformerSuite {
assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true)
}

testWithSpecifiedSparkVersion(
"ColumnarCollectLimitExec - with distinct values",
Array("3.2", "3.3")) {
testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - with distinct values", "3.2", "3.3") {
val df = spark
.range(0, 10, 1)
.toDF("id")
Expand All @@ -112,7 +106,7 @@ class GlutenSQLCollectLimitExecSuite extends WholeStageTransformerSuite {
assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true)
}

testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - chained limit", Array("3.2", "3.3")) {
testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - chained limit", "3.2", "3.3") {
val df = spark
.range(0, 10, 1)
.toDF("id")
Expand All @@ -125,9 +119,7 @@ class GlutenSQLCollectLimitExecSuite extends WholeStageTransformerSuite {
assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true)
}

testWithSpecifiedSparkVersion(
"ColumnarCollectLimitExec - limit after union",
Array("3.2", "3.3")) {
testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - limit after union", "3.2", "3.3") {
val df1 = spark.range(0, 5).toDF("id")
val df2 = spark.range(5, 10).toDF("id")
val unionDf = df1.union(df2).limit(3)
Expand Down
22 changes: 11 additions & 11 deletions backends-velox/src/test/scala/org/apache/gluten/execution/JsonFunctionsValidateSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("from_json function bool", Some("3.4")) {
testWithMinSparkVersion("from_json function bool", "3.4") {
withTempPath {
path =>
Seq[(String)](
Expand All @@ -101,7 +101,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("from_json function small int", Some("3.4")) {
testWithMinSparkVersion("from_json function small int", "3.4") {
withTempPath {
path =>
Seq[(String)](
Expand All @@ -123,7 +123,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("from_json function int", Some("3.4")) {
testWithMinSparkVersion("from_json function int", "3.4") {
withTempPath {
path =>
Seq[(String)](
Expand All @@ -145,7 +145,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("from_json function big int", Some("3.4")) {
testWithMinSparkVersion("from_json function big int", "3.4") {
withTempPath {
path =>
Seq[(String)](
Expand All @@ -167,7 +167,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("from_json function float", Some("3.4")) {
testWithMinSparkVersion("from_json function float", "3.4") {
withTempPath {
path =>
Seq[(String)](
Expand All @@ -191,7 +191,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("from_json function double", Some("3.4")) {
testWithMinSparkVersion("from_json function double", "3.4") {
withTempPath {
path =>
Seq[(String)](
Expand All @@ -215,7 +215,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("from_json function string", Some("3.4")) {
testWithMinSparkVersion("from_json function string", "3.4") {
withTempPath {
path =>
Seq[(String)](
Expand All @@ -237,7 +237,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("from_json function array", Some("3.4")) {
testWithMinSparkVersion("from_json function array", "3.4") {
withTempPath {
path =>
Seq[(String)](
Expand All @@ -257,7 +257,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("from_json function map", Some("3.4")) {
testWithMinSparkVersion("from_json function map", "3.4") {
withTempPath {
path =>
Seq[(String)](
Expand All @@ -279,7 +279,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("from_json function row", Some("3.4")) {
testWithMinSparkVersion("from_json function row", "3.4") {
withTempPath {
path =>
Seq[(String)](
Expand Down Expand Up @@ -319,7 +319,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("from_json function duplicate key", Some("3.4")) {
testWithMinSparkVersion("from_json function duplicate key", "3.4") {
withTempPath {
path =>
Seq[(String)](
Expand Down
2 changes: 1 addition & 1 deletion backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa
checkLengthAndPlan(df, 5)
}

testWithSpecifiedSparkVersion("coalesce validation", Some("3.4")) {
testWithMinSparkVersion("coalesce validation", "3.4") {
withTempPath {
path =>
val data = "2019-09-09 01:02:03.456789"
Expand Down
26 changes: 13 additions & 13 deletions backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("Test array_append function - INT", Some("3.4")) {
testWithMinSparkVersion("Test array_append function - INT", "3.4") {
withTempPath {
path =>
Seq[(Array[Int], Int)](
Expand All @@ -148,7 +148,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("Test array_append function - STRING", Some("3.4")) {
testWithMinSparkVersion("Test array_append function - STRING", "3.4") {
withTempPath {
path =>
Seq[(Array[String], String)](
Expand Down Expand Up @@ -186,7 +186,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("null input for array_size", Some("3.3")) {
testWithMinSparkVersion("null input for array_size", "3.3") {
withTempPath {
path =>
Seq[(Array[Int])](
Expand Down Expand Up @@ -841,7 +841,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("Test width_bucket function", Some("3.4")) {
testWithMinSparkVersion("Test width_bucket function", "3.4") {
withTempPath {
path =>
Seq[(Integer, Integer, Integer, Integer)](
Expand All @@ -859,7 +859,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("Test url_decode function", Some("3.4")) {
testWithMinSparkVersion("Test url_decode function", "3.4") {
withTempPath {
path =>
Seq("https%3A%2F%2Fspark.apache.org")
Expand All @@ -874,7 +874,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("Test url_encode function", Some("3.4")) {
testWithMinSparkVersion("Test url_encode function", "3.4") {
withTempPath {
path =>
Seq("https://spark.apache.org")
Expand Down Expand Up @@ -1005,7 +1005,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("mask", Some("3.4")) {
testWithMinSparkVersion("mask", "3.4") {
runQueryAndCompare("SELECT mask(c_comment) FROM customer limit 50") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
Expand Down Expand Up @@ -1119,7 +1119,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("try_subtract", Some("3.3")) {
testWithMinSparkVersion("try_subtract", "3.3") {
runQueryAndCompare(
"select try_subtract(2147483647, cast(l_orderkey as int)), " +
"try_subtract(-2147483648, cast(l_orderkey as int)) from lineitem") {
Expand All @@ -1135,7 +1135,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("try_multiply", Some("3.3")) {
testWithMinSparkVersion("try_multiply", "3.3") {
runQueryAndCompare(
"select try_multiply(2147483647, cast(l_orderkey as int)), " +
"try_multiply(-2147483648, cast(l_orderkey as int)) from lineitem") {
Expand Down Expand Up @@ -1301,7 +1301,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("get", Some("3.4")) {
testWithMinSparkVersion("get", "3.4") {
withTempPath {
path =>
Seq[Seq[Integer]](Seq(1, null, 5, 4), Seq(5, -1, 8, 9, -7, 2), Seq.empty, null)
Expand Down Expand Up @@ -1392,7 +1392,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("levenshtein with limit", Some("3.5")) {
testWithMinSparkVersion("levenshtein with limit", "3.5") {
runQueryAndCompare("select levenshtein(c_comment, c_address, 3) from customer limit 50") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
Expand Down Expand Up @@ -1506,7 +1506,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("array insert", Some("3.4")) {
testWithMinSparkVersion("array insert", "3.4") {
withTempPath {
path =>
Seq[Seq[Integer]](Seq(1, null, 5, 4), Seq(5, -1, 8, 9, -7, 2), Seq.empty, null)
Expand Down Expand Up @@ -1560,7 +1560,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
}
}

testWithSpecifiedSparkVersion("Test try_cast", Some("3.4")) {
testWithMinSparkVersion("Test try_cast", "3.4") {
withTempView("try_cast_table") {
withTempPath {
path =>
Expand Down
Loading