feat: CreateArray support

native/Cargo.toml

@@ -42,6 +42,7 @@ parquet = { version = "52.2.0", default-features = false, features = ["experimen
 datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1" }
 datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", features = ["unicode_expressions", "crypto_expressions"] }
 datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", features = ["crypto_expressions"] }
+datafusion-functions-nested = { git = "https://github.com/apache/datafusion.git",  rev = "41.0.0-rc1", default-features = false }
 datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false }
 datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false }
 datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false }

native/core/Cargo.toml

@@ -67,6 +67,7 @@ itertools = "0.11.0"
 paste = "1.0.14"
 datafusion-common = { workspace = true }
 datafusion = {  workspace = true }
+datafusion-functions-nested = { workspace = true }
 datafusion-expr = {  workspace = true }
 datafusion-physical-expr-common = {  workspace = true }
 datafusion-physical-expr = {  workspace = true }

native/core/src/execution/jni_api.rs

@@ -236,10 +236,11 @@ fn prepare_datafusion_session_context(
 
     let runtime = RuntimeEnv::new(rt_config).unwrap();
 
-    Ok(SessionContext::new_with_config_rt(
-        session_config,
-        Arc::new(runtime),
-    ))
+    let mut session_ctx = SessionContext::new_with_config_rt(session_config, Arc::new(runtime));
+
+    datafusion_functions_nested::register_all(&mut session_ctx)?;
+
+    Ok(session_ctx)
 }
 
 fn parse_bool(conf: &HashMap<String, String>, name: &str) -> CometResult<bool> {

spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala

@@ -2361,6 +2361,18 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde with CometExprShim
               .build()
           }
 
+        // datafusion's make_array only supports nullable element types
+        // https://github.com/apache/datafusion/issues/11923
+        case array @ CreateArray(children, _) if array.dataType.containsNull =>
+          val childExprs = children.map(exprToProto(_, inputs, binding))
+
+          if (childExprs.forall(_.isDefined)) {
+            scalarExprToProtoWithReturnType("make_array", array.dataType, childExprs: _*)
+          } else {
+            withInfo(expr, "unsupported arguments for CreateArray", children: _*)
+            None
+          }
+
         case _ =>
           withInfo(expr, s"${expr.prettyName} is not supported", expr.children: _*)
           None

spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala

@@ -1998,4 +1998,16 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper {
       }
     }
   }
+
+  test("CreateArray") {
+    Seq(true, false).foreach { dictionaryEnabled =>
+      withTempDir { dir =>
+        val path = new Path(dir.toURI.toString, "test.parquet")
+        makeParquetFileAllTypes(path, dictionaryEnabled = dictionaryEnabled, 10000)
+        val df = spark.read.parquet(path.toString)
+        checkSparkAnswerAndOperator(df.select(array(col("_2"), col("_3"), col("_4"))))
+        checkSparkAnswerAndOperator(df.select(array(col("_4"), col("_11"), lit(null))))
+      }
+    }
+  }
 }