Skip to content

chore: [df52] fallback to Spark for width_bucket #3501

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
comphead merged 1 commit into from
Feb 12, 2026
Merged

chore: [df52] fallback to Spark for width_bucket #3501

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import org.apache.comet.CometSparkSessionExtensions.withInfo
import org.apache.comet.expressions.{CometCast, CometEvalMode}
import org.apache.comet.serde.{CommonStringExprs, Compatible, ExprOuterClass, Incompatible}
import org.apache.comet.serde.ExprOuterClass.{BinaryOutputStyle, Expr}
import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithInfo, scalarFunctionExprToProto}
import org.apache.comet.serde.QueryPlanSerde.exprToProtoInternal

/**
* `CometExprShim` acts as a shim for parsing expressions from different Spark versions.
Expand Down Expand Up @@ -82,9 +82,14 @@ trait CometExprShim extends CommonStringExprs {
}

case wb: WidthBucket =>
val childExprs = wb.children.map(exprToProtoInternal(_, inputs, binding))
val optExpr = scalarFunctionExprToProto("width_bucket", childExprs: _*)
optExprWithInfo(optExpr, wb, wb.children: _*)
withInfo(
wb,
"WidthBucket not supported, track https://github.com/apache/datafusion/issues/20320")
None
// https://github.com/apache/datafusion/issues/20320
// val childExprs = wb.children.map(exprToProtoInternal(_, inputs, binding))
// val optExpr = scalarFunctionExprToProto("width_bucket", childExprs: _*)
// optExprWithInfo(optExpr, wb, wb.children: _*)

case _ => None
}
Expand Down
11 changes: 8 additions & 3 deletions spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,14 @@ trait CometExprShim extends CommonStringExprs {
}

case wb: WidthBucket =>
val childExprs = wb.children.map(exprToProtoInternal(_, inputs, binding))
val optExpr = scalarFunctionExprToProto("width_bucket", childExprs: _*)
optExprWithInfo(optExpr, wb, wb.children: _*)
withInfo(
wb,
"WidthBucket not supported, track https://github.com/apache/datafusion/issues/20320")
None
// https://github.com/apache/datafusion/issues/20320
// val childExprs = wb.children.map(exprToProtoInternal(_, inputs, binding))
// val optExpr = scalarFunctionExprToProto("width_bucket", childExprs: _*)
// optExprWithInfo(optExpr, wb, wb.children: _*)

case _ => None
}
Expand Down
12 changes: 8 additions & 4 deletions spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,8 @@ class CometMathExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe
DataGenOptions(generateNegativeZero = generateNegativeZero))
}

test("width_bucket") {
// https://github.com/apache/datafusion/issues/20320
ignore("width_bucket") {
assume(isSpark35Plus, "width_bucket was added in Spark 3.5")
withSQLConf("spark.comet.exec.localTableScan.enabled" -> "true") {
spark
Expand All @@ -105,7 +106,8 @@ class CometMathExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe
}
}

test("width_bucket - edge cases") {
// https://github.com/apache/datafusion/issues/20320
ignore("width_bucket - edge cases") {
assume(isSpark35Plus, "width_bucket was added in Spark 3.5")
withSQLConf("spark.comet.exec.localTableScan.enabled" -> "true") {
spark
Expand All @@ -122,7 +124,8 @@ class CometMathExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe
}
}

test("width_bucket - NaN values") {
// https://github.com/apache/datafusion/issues/20320
ignore("width_bucket - NaN values") {
assume(isSpark35Plus, "width_bucket was added in Spark 3.5")
withSQLConf("spark.comet.exec.localTableScan.enabled" -> "true") {
spark
Expand All @@ -134,7 +137,8 @@ class CometMathExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe
}
}

test("width_bucket - with range data") {
// https://github.com/apache/datafusion/issues/20320
ignore("width_bucket - with range data") {
assume(isSpark35Plus, "width_bucket was added in Spark 3.5")
withSQLConf("spark.comet.exec.localTableScan.enabled" -> "true") {
spark
Expand Down
Loading