From 4ab3f1987b0e4d628582c1f9e71592eead065e1c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 29 Jan 2026 07:50:56 -0700 Subject: [PATCH 1/9] Add SQL file-based test framework Introduce a sqllogictest-inspired framework for writing Comet tests as plain .sql files instead of Scala code. Convert 11 tests from CometExpressionSuite to demonstrate the approach. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/pr_build_linux.yml | 1 + .github/workflows/pr_build_macos.yml | 1 + .../sql-tests/expressions/arithmetic.sql | 38 ++++ .../sql-tests/expressions/bitwise.sql | 50 +++++ .../sql-tests/expressions/boolean.sql | 41 +++++ .../sql-tests/expressions/datetime.sql | 28 +++ .../resources/sql-tests/expressions/hash.sql | 28 +++ .../sql-tests/expressions/in_set.sql | 38 ++++ .../expressions/parquet_default_values.sql | 29 +++ .../sql-tests/expressions/string.sql | 49 +++++ .../apache/comet/CometExpressionSuite.scala | 172 ------------------ .../apache/comet/CometSqlFileTestSuite.scala | 110 +++++++++++ .../org/apache/comet/SqlFileTestParser.scala | 138 ++++++++++++++ 13 files changed, 551 insertions(+), 172 deletions(-) create mode 100644 spark/src/test/resources/sql-tests/expressions/arithmetic.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/bitwise.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/boolean.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/datetime.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/hash.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/in_set.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/parquet_default_values.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/string.sql create mode 100644 spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala create mode 100644 spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 7df0aa0697..8376afbc6e 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -250,6 +250,7 @@ jobs: - name: "expressions" value: | org.apache.comet.CometExpressionSuite + org.apache.comet.CometSqlFileTestSuite org.apache.comet.CometExpressionCoverageSuite org.apache.comet.CometHashExpressionSuite org.apache.comet.CometTemporalExpressionSuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index acf052bf13..3a64c0051f 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -193,6 +193,7 @@ jobs: - name: "expressions" value: | org.apache.comet.CometExpressionSuite + org.apache.comet.CometSqlFileTestSuite org.apache.comet.CometExpressionCoverageSuite org.apache.comet.CometHashExpressionSuite org.apache.comet.CometTemporalExpressionSuite diff --git a/spark/src/test/resources/sql-tests/expressions/arithmetic.sql b/spark/src/test/resources/sql-tests/expressions/arithmetic.sql new file mode 100644 index 0000000000..16ac7b2429 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/arithmetic.sql @@ -0,0 +1,38 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- negative +statement +CREATE TABLE test_neg(col1 int) USING parquet + +statement +INSERT INTO test_neg VALUES(1), (2), (3), (3) + +query +SELECT negative(col1), -(col1) FROM test_neg + +-- integral division overflow +statement +CREATE TABLE test_div(c1 long, c2 short) USING parquet + +statement +INSERT INTO test_div VALUES(-9223372036854775808, -1) + +query +SELECT c1 div c2 FROM test_div ORDER BY c1 diff --git a/spark/src/test/resources/sql-tests/expressions/bitwise.sql b/spark/src/test/resources/sql-tests/expressions/bitwise.sql new file mode 100644 index 0000000000..067836a63b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/bitwise.sql @@ -0,0 +1,50 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- Setup +statement +CREATE TABLE test(col1 int, col2 int) USING parquet + +statement +INSERT INTO test VALUES(1111, 2) + +statement +INSERT INTO test VALUES(1111, 2) + +statement +INSERT INTO test VALUES(3333, 4) + +statement +INSERT INTO test VALUES(5555, 6) + +-- Queries +query +SELECT col1 & col2, col1 | col2, col1 ^ col2 FROM test + +query +SELECT col1 & 1234, col1 | 1234, col1 ^ 1234 FROM test + +query +SELECT shiftright(col1, 2), shiftright(col1, col2) FROM test + +query +SELECT shiftleft(col1, 2), shiftleft(col1, col2) FROM test + +query +SELECT ~(11), ~col1, ~col2 FROM test diff --git a/spark/src/test/resources/sql-tests/expressions/boolean.sql b/spark/src/test/resources/sql-tests/expressions/boolean.sql new file mode 100644 index 0000000000..00688f5fa8 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/boolean.sql @@ -0,0 +1,41 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- compare true/false to negative zero +statement +CREATE TABLE test(col1 boolean, col2 float) USING parquet + +statement +INSERT INTO test VALUES(true, -0.0) + +statement +INSERT INTO test VALUES(false, -0.0) + +query +SELECT col1, negative(col2), cast(col1 as float), col1 = negative(col2) FROM test + +-- not +statement +CREATE TABLE test_not(col1 int, col2 boolean) USING parquet + +statement +INSERT INTO test_not VALUES(1, false), (2, true), (3, true), (3, false) + +query +SELECT col1, col2, NOT(col2), !(col2) FROM test_not diff --git a/spark/src/test/resources/sql-tests/expressions/datetime.sql b/spark/src/test/resources/sql-tests/expressions/datetime.sql new file mode 100644 index 0000000000..93c615d5b3 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime.sql @@ -0,0 +1,28 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- DatePart functions +statement +CREATE TABLE test_dt(col timestamp) USING parquet + +statement +INSERT INTO test_dt VALUES (timestamp('2024-06-15 10:30:00')), (timestamp('1900-01-01')), (null) + +query +SELECT col, year(col), month(col), day(col), weekday(col), dayofweek(col), dayofyear(col), weekofyear(col), quarter(col) FROM test_dt diff --git a/spark/src/test/resources/sql-tests/expressions/hash.sql b/spark/src/test/resources/sql-tests/expressions/hash.sql new file mode 100644 index 0000000000..0629bc0a77 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/hash.sql @@ -0,0 +1,28 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- hash functions +statement +CREATE TABLE test(col string, a int, b float) USING parquet + +statement +INSERT INTO test VALUES ('Spark SQL ', 10, 1.2), (NULL, NULL, NULL), ('', 0, 0.0), ('苹果手机', NULL, 3.999999), ('Spark SQL ', 10, 1.2), (NULL, NULL, NULL), ('', 0, 0.0), ('苹果手机', NULL, 3.999999) + +query +SELECT md5(col), md5(cast(a as string)), md5(cast(b as string)), hash(col), hash(col, 1), hash(col, 0), hash(col, a, b), hash(b, a, col), xxhash64(col), xxhash64(col, 1), xxhash64(col, 0), xxhash64(col, a, b), xxhash64(b, a, col), sha2(col, 0), sha2(col, 256), sha2(col, 224), sha2(col, 384), sha2(col, 512), sha2(col, 128), sha2(col, -1), sha1(col), sha1(cast(a as string)), sha1(cast(b as string)) FROM test diff --git a/spark/src/test/resources/sql-tests/expressions/in_set.sql b/spark/src/test/resources/sql-tests/expressions/in_set.sql new file mode 100644 index 0000000000..1cb0f248a6 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/in_set.sql @@ -0,0 +1,38 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: spark.sql.optimizer.inSetConversionThreshold=100,0 +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- test in(set)/not in(set) +statement +CREATE TABLE names(id int, name varchar(20)) USING parquet + +statement +INSERT INTO names VALUES(1, 'James'), (1, 'Jones'), (2, 'Smith'), (3, 'Smith'), (NULL, 'Jones'), (4, NULL) + +query +SELECT * FROM names WHERE id in (1, 2, 4, NULL) + +query +SELECT * FROM names WHERE name in ('Smith', 'Brown', NULL) + +query +SELECT * FROM names WHERE id not in (1) + +query spark_answer_only +SELECT * FROM names WHERE name not in ('Smith', 'Brown', NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/parquet_default_values.sql b/spark/src/test/resources/sql-tests/expressions/parquet_default_values.sql new file mode 100644 index 0000000000..bb1b003c79 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/parquet_default_values.sql @@ -0,0 +1,29 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- parquet default values +statement +CREATE TABLE t1(col1 boolean) USING parquet + +statement +INSERT INTO t1 VALUES(true) + +statement +ALTER TABLE t1 ADD COLUMN col2 string DEFAULT 'hello' + +query +SELECT * FROM t1 diff --git a/spark/src/test/resources/sql-tests/expressions/string.sql b/spark/src/test/resources/sql-tests/expressions/string.sql new file mode 100644 index 0000000000..15c0107fae --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string.sql @@ -0,0 +1,49 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- substring with start < 1 +statement +CREATE TABLE t(col string) USING parquet + +statement +INSERT INTO t VALUES('123456') + +query +SELECT substring(col, 0) FROM t + +query +SELECT substring(col, -1) FROM t + +-- md5 +statement +CREATE TABLE test_md5(col String) USING parquet + +statement +INSERT INTO test_md5 VALUES ('test1'), ('test1'), ('test2'), ('test2'), (NULL), ('') + +query +SELECT md5(col) FROM test_md5 + +-- unhex +statement +CREATE TABLE unhex_table(col string) USING parquet + +statement +INSERT INTO unhex_table VALUES ('537061726B2053514C'), ('737472696E67'), ('\0'), (''), ('###'), ('G123'), ('hello'), ('A1B'), ('0A1B') + +query +SELECT unhex(col) FROM unhex_table diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index fe5ea77a89..576af73745 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -131,31 +131,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("compare true/false to negative zero") { - Seq(false, true).foreach { dictionary => - withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { - val table = "test" - withTable(table) { - sql(s"create table $table(col1 boolean, col2 float) using parquet") - sql(s"insert into $table values(true, -0.0)") - sql(s"insert into $table values(false, -0.0)") - - checkSparkAnswerAndOperator( - s"SELECT col1, negative(col2), cast(col1 as float), col1 = negative(col2) FROM $table") - } - } - } - } - - test("parquet default values") { - withTable("t1") { - sql("create table t1(col1 boolean) using parquet") - sql("insert into t1 values(true)") - sql("alter table t1 add column col2 string default 'hello'") - checkSparkAnswerAndOperator("select * from t1") - } - } - test("decimals divide by zero") { Seq(true, false).foreach { dictionary => withSQLConf( @@ -176,16 +151,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("Integral Division Overflow Handling Matches Spark Behavior") { - withTable("t1") { - val value = Long.MinValue - sql("create table t1(c1 long, c2 short) using parquet") - sql(s"insert into t1 values($value, -1)") - val res = sql("select c1 div c2 from t1 order by c1") - checkSparkAnswerAndOperator(res) - } - } - test("basic data type support") { // this test requires native_comet scan due to unsigned u8/u16 issue withSQLConf(CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_COMET) { @@ -502,17 +467,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("substring with start < 1") { - withTempPath { _ => - withTable("t") { - sql("create table t (col string) using parquet") - sql("insert into t values('123456')") - checkSparkAnswerAndOperator(sql("select substring(col, 0) from t")) - checkSparkAnswerAndOperator(sql("select substring(col, -1) from t")) - } - } - } - test("substring with dictionary") { val data = (0 until 1000) .map(_ % 5) // reduce value space to trigger dictionary encoding @@ -1557,20 +1511,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("md5") { - Seq(false, true).foreach { dictionary => - withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { - val table = "test" - withTable(table) { - sql(s"create table $table(col String) using parquet") - sql( - s"insert into $table values ('test1'), ('test1'), ('test2'), ('test2'), (NULL), ('')") - checkSparkAnswerAndOperator(s"select md5(col) FROM $table") - } - } - } - } - test("hex") { // https://github.com/apache/datafusion-comet/issues/1441 assume(!usingDataSourceExec) @@ -1589,26 +1529,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("unhex") { - val table = "unhex_table" - withTable(table) { - sql(s"create table $table(col string) using parquet") - - sql(s"""INSERT INTO $table VALUES - |('537061726B2053514C'), - |('737472696E67'), - |('\\0'), - |(''), - |('###'), - |('G123'), - |('hello'), - |('A1B'), - |('0A1B')""".stripMargin) - - checkSparkAnswerAndOperator(s"SELECT unhex(col) FROM $table") - } - } - test("EqualNullSafe should preserve comet filter") { Seq("true", "false").foreach(b => withParquetTable( @@ -1633,58 +1553,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { }) } - test("test in(set)/not in(set)") { - Seq("100", "0").foreach { inSetThreshold => - Seq(false, true).foreach { dictionary => - withSQLConf( - SQLConf.OPTIMIZER_INSET_CONVERSION_THRESHOLD.key -> inSetThreshold, - "parquet.enable.dictionary" -> dictionary.toString) { - val table = "names" - withTable(table) { - sql(s"create table $table(id int, name varchar(20)) using parquet") - sql( - s"insert into $table values(1, 'James'), (1, 'Jones'), (2, 'Smith'), (3, 'Smith')," + - "(NULL, 'Jones'), (4, NULL)") - - checkSparkAnswerAndOperator(s"SELECT * FROM $table WHERE id in (1, 2, 4, NULL)") - checkSparkAnswerAndOperator( - s"SELECT * FROM $table WHERE name in ('Smith', 'Brown', NULL)") - - // TODO: why with not in, the plan is only `LocalTableScan`? - checkSparkAnswerAndOperator(s"SELECT * FROM $table WHERE id not in (1)") - checkSparkAnswer(s"SELECT * FROM $table WHERE name not in ('Smith', 'Brown', NULL)") - } - } - } - } - } - - test("not") { - Seq(false, true).foreach { dictionary => - withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { - val table = "test" - withTable(table) { - sql(s"create table $table(col1 int, col2 boolean) using parquet") - sql(s"insert into $table values(1, false), (2, true), (3, true), (3, false)") - checkSparkAnswerAndOperator(s"SELECT col1, col2, NOT(col2), !(col2) FROM $table") - } - } - } - } - - test("negative") { - Seq(false, true).foreach { dictionary => - withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { - val table = "test" - withTable(table) { - sql(s"create table $table(col1 int) using parquet") - sql(s"insert into $table values(1), (2), (3), (3)") - checkSparkAnswerAndOperator(s"SELECT negative(col1), -(col1) FROM $table") - } - } - } - } - test("basic arithmetic") { withSQLConf("parquet.enable.dictionary" -> "false") { withParquetTable((1 until 10).map(i => (i, i + 1)), "tbl", false) { @@ -1719,21 +1587,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("DatePart functions: Year/Month/DayOfMonth/DayOfWeek/DayOfYear/WeekOfYear/Quarter") { - Seq(false, true).foreach { dictionary => - withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { - val table = "test" - withTable(table) { - sql(s"create table $table(col timestamp) using parquet") - sql(s"insert into $table values (now()), (timestamp('1900-01-01')), (null)") - checkSparkAnswerAndOperator( - "SELECT col, year(col), month(col), day(col), weekday(col), " + - s" dayofweek(col), dayofyear(col), weekofyear(col), quarter(col) FROM $table") - } - } - } - } - test("from_unixtime") { Seq(false, true).foreach { dictionary => withSQLConf( @@ -1985,31 +1838,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("hash functions") { - Seq(true, false).foreach { dictionary => - withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { - val table = "test" - withTable(table) { - sql(s"create table $table(col string, a int, b float) using parquet") - sql(s""" - |insert into $table values - |('Spark SQL ', 10, 1.2), (NULL, NULL, NULL), ('', 0, 0.0), ('苹果手机', NULL, 3.999999) - |, ('Spark SQL ', 10, 1.2), (NULL, NULL, NULL), ('', 0, 0.0), ('苹果手机', NULL, 3.999999) - |""".stripMargin) - checkSparkAnswerAndOperator(""" - |select - |md5(col), md5(cast(a as string)), md5(cast(b as string)), - |hash(col), hash(col, 1), hash(col, 0), hash(col, a, b), hash(b, a, col), - |xxhash64(col), xxhash64(col, 1), xxhash64(col, 0), xxhash64(col, a, b), xxhash64(b, a, col), - |sha2(col, 0), sha2(col, 256), sha2(col, 224), sha2(col, 384), sha2(col, 512), sha2(col, 128), sha2(col, -1), - |sha1(col), sha1(cast(a as string)), sha1(cast(b as string)) - |from test - |""".stripMargin) - } - } - } - } - test("remainder function") { def withAnsiMode(enabled: Boolean)(f: => Unit): Unit = { withSQLConf( diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala new file mode 100644 index 0000000000..8d382f5173 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet + +import java.io.File + +import org.scalactic.source.Position +import org.scalatest.Tag + +import org.apache.spark.sql.CometTestBase +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper + +class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { + + override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit + pos: Position): Unit = { + super.test(testName, testTags: _*) { + withSQLConf(CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_AUTO) { + testFun + } + } + } + + private val testResourceDir = { + val url = getClass.getClassLoader.getResource("sql-tests") + assert(url != null, "Could not find sql-tests resource directory") + new File(url.toURI) + } + + private def discoverTestFiles(dir: File): Seq[File] = { + if (!dir.exists()) return Seq.empty + val files = dir.listFiles().toSeq + val sqlFiles = files.filter(f => f.isFile && f.getName.endsWith(".sql")) + val subDirFiles = files.filter(_.isDirectory).flatMap(discoverTestFiles) + sqlFiles ++ subDirFiles + } + + /** Generate all config combinations from a ConfigMatrix specification. */ + private def configCombinations( + matrix: Seq[(String, Seq[String])]): Seq[Seq[(String, String)]] = { + if (matrix.isEmpty) return Seq(Seq.empty) + val (key, values) = matrix.head + val rest = configCombinations(matrix.tail) + for { + value <- values + combo <- rest + } yield (key, value) +: combo + } + + private def runTestFile(file: SqlTestFile): Unit = { + val allConfigs = file.configs + withSQLConf(allConfigs: _*) { + withTable(file.tables: _*) { + file.records.foreach { + case SqlStatement(sql) => + spark.sql(sql) + case SqlQuery(sql, mode) => + mode match { + case CheckOperator => + checkSparkAnswerAndOperator(sql) + case SparkAnswerOnly => + checkSparkAnswer(sql) + case WithTolerance(tol) => + checkSparkAnswerWithTolerance(sql, tol) + } + } + } + } + } + + // Discover and register all .sql test files + discoverTestFiles(testResourceDir).foreach { file => + val relativePath = testResourceDir.toURI.relativize(file.toURI).getPath + val parsed = SqlFileTestParser.parse(file) + val combinations = configCombinations(parsed.configMatrix) + + if (combinations.size <= 1) { + // No matrix or single combination + test(s"sql-file: $relativePath") { + val effectiveConfigs = parsed.configs ++ combinations.headOption.getOrElse(Seq.empty) + runTestFile(parsed.copy(configs = effectiveConfigs)) + } + } else { + // Multiple combinations: generate one test per combination + combinations.foreach { matrixConfigs => + val label = matrixConfigs.map { case (k, v) => s"$k=$v" }.mkString(", ") + test(s"sql-file: $relativePath [$label]") { + runTestFile(parsed.copy(configs = parsed.configs ++ matrixConfigs)) + } + } + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala b/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala new file mode 100644 index 0000000000..f2cbf56544 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet + +import java.io.File + +import scala.io.Source + +/** A record in a SQL test file: either a statement (DDL/DML) or a query (SELECT). */ +sealed trait SqlTestRecord + +/** A SQL statement to execute (CREATE TABLE, INSERT, etc.). */ +case class SqlStatement(sql: String) extends SqlTestRecord + +/** A SQL query whose results are compared between Spark and Comet. */ +case class SqlQuery(sql: String, mode: QueryMode = CheckOperator) extends SqlTestRecord + +sealed trait QueryMode +case object CheckOperator extends QueryMode +case object SparkAnswerOnly extends QueryMode +case class WithTolerance(tol: Double) extends QueryMode + +/** + * Parsed representation of a .sql test file. + * + * @param configs + * Spark SQL configs to set for this test file. + * @param configMatrix + * Map of config key to list of values. The test will run once per combination. + * @param records + * Ordered list of statements and queries. + * @param tables + * Table names extracted from CREATE TABLE statements (for cleanup). + */ +case class SqlTestFile( + configs: Seq[(String, String)], + configMatrix: Seq[(String, Seq[String])], + records: Seq[SqlTestRecord], + tables: Seq[String]) + +object SqlFileTestParser { + + private val ConfigPattern = """--\s*Config:\s*(.+)=(.+)""".r + private val ConfigMatrixPattern = """--\s*ConfigMatrix:\s*(.+)=(.+)""".r + private val CreateTablePattern = """(?i)CREATE\s+TABLE\s+(\w+)""".r.unanchored + + def parse(file: File): SqlTestFile = { + val source = Source.fromFile(file) + try { + parse(source.getLines().toSeq) + } finally { + source.close() + } + } + + def parse(lines: Seq[String]): SqlTestFile = { + var configs = Seq.empty[(String, String)] + var configMatrix = Seq.empty[(String, Seq[String])] + val records = Seq.newBuilder[SqlTestRecord] + val tables = Seq.newBuilder[String] + + var i = 0 + while (i < lines.length) { + val line = lines(i).trim + + line match { + case ConfigPattern(key, value) => + configs :+= (key.trim -> value.trim) + i += 1 + + case ConfigMatrixPattern(key, values) => + configMatrix :+= (key.trim -> values.split(",").map(_.trim).toSeq) + i += 1 + + case "statement" => + i += 1 + val (sql, nextIdx) = collectSql(lines, i) + // Extract table names for cleanup + CreateTablePattern.findFirstMatchIn(sql).foreach(m => tables += m.group(1)) + records += SqlStatement(sql) + i = nextIdx + + case s if s.startsWith("query") => + val mode = parseQueryMode(s) + i += 1 + val (sql, nextIdx) = collectSql(lines, i) + records += SqlQuery(sql, mode) + i = nextIdx + + case _ => + // Skip blank lines and comments + i += 1 + } + } + + SqlTestFile(configs, configMatrix, records.result(), tables.result()) + } + + private def parseQueryMode(directive: String): QueryMode = { + val parts = directive.split("\\s+") + if (parts.length == 1) return CheckOperator + parts(1) match { + case "spark_answer_only" => SparkAnswerOnly + case s if s.startsWith("tolerance=") => + WithTolerance(s.stripPrefix("tolerance=").toDouble) + case _ => CheckOperator + } + } + + /** Collect SQL lines until a blank line or end of file. */ + private def collectSql(lines: Seq[String], start: Int): (String, Int) = { + val sb = new StringBuilder + var i = start + while (i < lines.length && lines(i).trim.nonEmpty) { + if (sb.nonEmpty) sb.append("\n") + sb.append(lines(i)) + i += 1 + } + (sb.toString, i) + } +} From 1c6e4753d6d6009cf766d02d5ad3742645efe4ca Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 29 Jan 2026 09:04:42 -0700 Subject: [PATCH 2/9] Add SQL file tests for all Comet-supported expressions Add ~115 SQL test files covering edge cases for every Comet-supported expression, using the SQL file-based test framework. Tests cover NULL inputs, boundary values, special values (NaN, Infinity, empty strings), and type variety. Adds two new query modes to the test framework: - `query expect_fallback(reason)` - verifies fallback reason AND results - `query ignore(reason)` - skips query (for known bugs with issue links) Includes happy-path tests with configs enabled for incompatible expressions (case conversion, regexp, date_format, from_unix_time, init_cap). Known bugs filed: - https://github.com/apache/datafusion-comet/issues/3326 (space(-1) crash) - https://github.com/apache/datafusion-comet/issues/3327 (map_from_arrays(NULL) crash) Co-Authored-By: Claude Opus 4.5 --- .../resources/sql-tests/expressions/abs.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/acos.sql | 27 ++++++++++++ .../sql-tests/expressions/arithmetic.sql | 29 +++++++++++++ .../sql-tests/expressions/array_append.sql | 27 ++++++++++++ .../sql-tests/expressions/array_compact.sql | 27 ++++++++++++ .../sql-tests/expressions/array_contains.sql | 27 ++++++++++++ .../sql-tests/expressions/array_distinct.sql | 27 ++++++++++++ .../sql-tests/expressions/array_except.sql | 27 ++++++++++++ .../sql-tests/expressions/array_insert.sql | 27 ++++++++++++ .../sql-tests/expressions/array_intersect.sql | 27 ++++++++++++ .../sql-tests/expressions/array_join.sql | 30 +++++++++++++ .../sql-tests/expressions/array_max.sql | 27 ++++++++++++ .../sql-tests/expressions/array_min.sql | 27 ++++++++++++ .../sql-tests/expressions/array_remove.sql | 27 ++++++++++++ .../sql-tests/expressions/array_repeat.sql | 27 ++++++++++++ .../sql-tests/expressions/array_union.sql | 27 ++++++++++++ .../sql-tests/expressions/arrays_overlap.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/ascii.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/asin.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/atan.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/atan2.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/avg.sql | 30 +++++++++++++ .../sql-tests/expressions/bit_agg.sql | 30 +++++++++++++ .../sql-tests/expressions/bit_length.sql | 27 ++++++++++++ .../sql-tests/expressions/bitwise.sql | 20 +++++++++ .../sql-tests/expressions/case_when.sql | 33 +++++++++++++++ .../resources/sql-tests/expressions/cast.sql | 42 +++++++++++++++++++ .../resources/sql-tests/expressions/ceil.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/chr.sql | 27 ++++++++++++ .../sql-tests/expressions/coalesce.sql | 33 +++++++++++++++ .../sql-tests/expressions/concat.sql | 30 +++++++++++++ .../sql-tests/expressions/concat_ws.sql | 33 +++++++++++++++ .../sql-tests/expressions/contains.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/corr.sql | 30 +++++++++++++ .../resources/sql-tests/expressions/cos.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/cosh.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/cot.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/count.sql | 30 +++++++++++++ .../sql-tests/expressions/covariance.sql | 30 +++++++++++++ .../sql-tests/expressions/create_array.sql | 30 +++++++++++++ .../expressions/create_named_struct.sql | 30 +++++++++++++ .../sql-tests/expressions/date_add.sql | 27 ++++++++++++ .../sql-tests/expressions/date_diff.sql | 27 ++++++++++++ .../sql-tests/expressions/date_format.sql | 33 +++++++++++++++ .../expressions/date_format_enabled.sql | 37 ++++++++++++++++ .../sql-tests/expressions/date_sub.sql | 27 ++++++++++++ .../sql-tests/expressions/datetime.sql | 13 ++++++ .../sql-tests/expressions/element_at.sql | 27 ++++++++++++ .../sql-tests/expressions/ends_with.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/exp.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/expm1.sql | 27 ++++++++++++ .../sql-tests/expressions/first_last.sql | 33 +++++++++++++++ .../sql-tests/expressions/flatten.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/floor.sql | 27 ++++++++++++ .../sql-tests/expressions/from_unix_time.sql | 30 +++++++++++++ .../expressions/from_unix_time_enabled.sql | 35 ++++++++++++++++ .../resources/sql-tests/expressions/hex.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/hour.sql | 27 ++++++++++++ .../sql-tests/expressions/if_expr.sql | 30 +++++++++++++ .../sql-tests/expressions/init_cap.sql | 27 ++++++++++++ .../expressions/init_cap_enabled.sql | 29 +++++++++++++ .../sql-tests/expressions/is_not_null.sql | 30 +++++++++++++ .../sql-tests/expressions/is_null.sql | 30 +++++++++++++ .../resources/sql-tests/expressions/isnan.sql | 27 ++++++++++++ .../sql-tests/expressions/json_to_structs.sql | 27 ++++++++++++ .../sql-tests/expressions/last_day.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/left.sql | 27 ++++++++++++ .../sql-tests/expressions/length.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/like.sql | 36 ++++++++++++++++ .../resources/sql-tests/expressions/log.sql | 30 +++++++++++++ .../resources/sql-tests/expressions/log10.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/log2.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/lower.sql | 27 ++++++++++++ .../sql-tests/expressions/lower_enabled.sql | 29 +++++++++++++ .../sql-tests/expressions/map_entries.sql | 27 ++++++++++++ .../sql-tests/expressions/map_from_arrays.sql | 32 ++++++++++++++ .../sql-tests/expressions/map_keys.sql | 27 ++++++++++++ .../sql-tests/expressions/map_values.sql | 27 ++++++++++++ .../sql-tests/expressions/min_max.sql | 30 +++++++++++++ .../sql-tests/expressions/minute.sql | 27 ++++++++++++ .../sql-tests/expressions/octet_length.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/pow.sql | 27 ++++++++++++ .../sql-tests/expressions/predicates.sql | 33 +++++++++++++++ .../sql-tests/expressions/regexp_replace.sql | 30 +++++++++++++ .../expressions/regexp_replace_enabled.sql | 32 ++++++++++++++ .../sql-tests/expressions/reverse.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/rlike.sql | 33 +++++++++++++++ .../sql-tests/expressions/rlike_enabled.sql | 35 ++++++++++++++++ .../resources/sql-tests/expressions/round.sql | 36 ++++++++++++++++ .../sql-tests/expressions/second.sql | 27 ++++++++++++ .../sql-tests/expressions/signum.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/sin.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/sinh.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/size.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/sqrt.sql | 27 ++++++++++++ .../sql-tests/expressions/starts_with.sql | 27 ++++++++++++ .../sql-tests/expressions/stddev.sql | 30 +++++++++++++ .../sql-tests/expressions/string_instr.sql | 27 ++++++++++++ .../sql-tests/expressions/string_lpad.sql | 30 +++++++++++++ .../sql-tests/expressions/string_repeat.sql | 27 ++++++++++++ .../sql-tests/expressions/string_replace.sql | 30 +++++++++++++ .../sql-tests/expressions/string_rpad.sql | 30 +++++++++++++ .../sql-tests/expressions/string_space.sql | 32 ++++++++++++++ .../expressions/string_translate.sql | 27 ++++++++++++ .../sql-tests/expressions/string_trim.sql | 30 +++++++++++++ .../sql-tests/expressions/structs_to_json.sql | 27 ++++++++++++ .../sql-tests/expressions/substring.sql | 42 +++++++++++++++++++ .../resources/sql-tests/expressions/sum.sql | 33 +++++++++++++++ .../resources/sql-tests/expressions/tan.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/tanh.sql | 27 ++++++++++++ .../sql-tests/expressions/trunc_date.sql | 33 +++++++++++++++ .../sql-tests/expressions/trunc_timestamp.sql | 36 ++++++++++++++++ .../resources/sql-tests/expressions/unhex.sql | 30 +++++++++++++ .../sql-tests/expressions/unix_date.sql | 27 ++++++++++++ .../sql-tests/expressions/unix_timestamp.sql | 27 ++++++++++++ .../resources/sql-tests/expressions/upper.sql | 27 ++++++++++++ .../sql-tests/expressions/upper_enabled.sql | 29 +++++++++++++ .../sql-tests/expressions/variance.sql | 30 +++++++++++++ .../apache/comet/CometSqlFileTestSuite.scala | 4 ++ .../org/apache/comet/SqlFileTestParser.scala | 26 ++++++++---- 120 files changed, 3417 insertions(+), 7 deletions(-) create mode 100644 spark/src/test/resources/sql-tests/expressions/abs.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/acos.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_append.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_compact.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_contains.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_distinct.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_except.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_insert.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_intersect.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_join.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_max.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_min.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_remove.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_repeat.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array_union.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/arrays_overlap.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/ascii.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/asin.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/atan.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/atan2.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/avg.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/bit_agg.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/bit_length.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/case_when.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/cast.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/ceil.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/chr.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/coalesce.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/concat.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/concat_ws.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/contains.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/corr.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/cos.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/cosh.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/cot.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/count.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/covariance.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/create_array.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/create_named_struct.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/date_add.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/date_diff.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/date_format.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/date_format_enabled.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/date_sub.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/element_at.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/ends_with.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/exp.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/expm1.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/first_last.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/flatten.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/floor.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/from_unix_time.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/from_unix_time_enabled.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/hex.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/hour.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/if_expr.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/init_cap.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/init_cap_enabled.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/is_not_null.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/is_null.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/isnan.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/json_to_structs.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/last_day.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/left.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/length.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/like.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/log.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/log10.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/log2.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/lower.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/lower_enabled.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/map_entries.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/map_from_arrays.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/map_keys.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/map_values.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/min_max.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/minute.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/octet_length.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/pow.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/predicates.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/regexp_replace.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/regexp_replace_enabled.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/reverse.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/rlike.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/rlike_enabled.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/round.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/second.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/signum.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/sin.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/sinh.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/size.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/sqrt.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/starts_with.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/stddev.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/string_instr.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/string_lpad.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/string_repeat.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/string_replace.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/string_rpad.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/string_space.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/string_translate.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/string_trim.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/structs_to_json.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/substring.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/sum.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/tan.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/tanh.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/trunc_date.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/trunc_timestamp.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/unhex.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/unix_date.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/unix_timestamp.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/upper.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/upper_enabled.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/variance.sql diff --git a/spark/src/test/resources/sql-tests/expressions/abs.sql b/spark/src/test/resources/sql-tests/expressions/abs.sql new file mode 100644 index 0000000000..edb4e71fd9 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/abs.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_abs(i int, l long, f float, d double) USING parquet + +statement +INSERT INTO test_abs VALUES (1, 1, 1.5, 1.5), (-1, -1, -1.5, -1.5), (0, 0, 0.0, 0.0), (NULL, NULL, NULL, NULL), (2147483647, 9223372036854775807, cast('Infinity' as float), cast('NaN' as double)) + +query +SELECT abs(i), abs(l), abs(f), abs(d) FROM test_abs diff --git a/spark/src/test/resources/sql-tests/expressions/acos.sql b/spark/src/test/resources/sql-tests/expressions/acos.sql new file mode 100644 index 0000000000..4f7ead86f8 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/acos.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_acos(d double) USING parquet + +statement +INSERT INTO test_acos VALUES (0.0), (1.0), (-1.0), (0.5), (NULL), (cast('NaN' as double)), (2.0), (-2.0) + +query tolerance=1e-6 +SELECT acos(d) FROM test_acos diff --git a/spark/src/test/resources/sql-tests/expressions/arithmetic.sql b/spark/src/test/resources/sql-tests/expressions/arithmetic.sql index 16ac7b2429..16434b87da 100644 --- a/spark/src/test/resources/sql-tests/expressions/arithmetic.sql +++ b/spark/src/test/resources/sql-tests/expressions/arithmetic.sql @@ -36,3 +36,32 @@ INSERT INTO test_div VALUES(-9223372036854775808, -1) query SELECT c1 div c2 FROM test_div ORDER BY c1 + +-- Add, Subtract, Multiply, Divide, Remainder +statement +CREATE TABLE test_arith(a int, b int, la long, lb long, fa float, fb float, da double, db double) USING parquet + +statement +INSERT INTO test_arith VALUES (10, 3, 10, 3, 10.5, 3.2, 10.5, 3.2), (0, 1, 0, 1, 0.0, 1.0, 0.0, 1.0), (-10, 3, -10, 3, -10.5, 3.2, -10.5, 3.2), (2147483647, 1, 9223372036854775807, 1, cast('Infinity' as float), 1.0, cast('NaN' as double), 1.0), (NULL, 1, NULL, 1, NULL, 1.0, NULL, 1.0) + +query +SELECT a + b, a - b, a * b, a / b, a % b FROM test_arith + +query +SELECT la + lb, la - lb, la * lb, la / lb, la % lb FROM test_arith + +query +SELECT fa + fb, fa - fb, fa * fb, fa / fb FROM test_arith + +query +SELECT da + db, da - db, da * db, da / db FROM test_arith + +-- division by zero +statement +CREATE TABLE test_div_zero(a int, b int, d double) USING parquet + +statement +INSERT INTO test_div_zero VALUES (1, 0, 0.0), (0, 0, 0.0) + +query +SELECT a / b, a % b, d / 0.0 FROM test_div_zero diff --git a/spark/src/test/resources/sql-tests/expressions/array_append.sql b/spark/src/test/resources/sql-tests/expressions/array_append.sql new file mode 100644 index 0000000000..9d3a4e5c7d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_append.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_append(arr array, val int) USING parquet + +statement +INSERT INTO test_array_append VALUES (array(1, 2, 3), 4), (array(), 1), (NULL, 1), (array(1, 2), NULL) + +query spark_answer_only +SELECT array_append(arr, val) FROM test_array_append diff --git a/spark/src/test/resources/sql-tests/expressions/array_compact.sql b/spark/src/test/resources/sql-tests/expressions/array_compact.sql new file mode 100644 index 0000000000..f8baddd14f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_compact.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_compact(arr array) USING parquet + +statement +INSERT INTO test_array_compact VALUES (array(1, NULL, 2, NULL, 3)), (array()), (NULL), (array(NULL, NULL)), (array(1, 2, 3)) + +query spark_answer_only +SELECT array_compact(arr) FROM test_array_compact diff --git a/spark/src/test/resources/sql-tests/expressions/array_contains.sql b/spark/src/test/resources/sql-tests/expressions/array_contains.sql new file mode 100644 index 0000000000..11c72880f0 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_contains.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_contains(arr array, val int) USING parquet + +statement +INSERT INTO test_array_contains VALUES (array(1, 2, 3), 2), (array(1, 2, 3), 4), (array(1, NULL, 3), NULL), (array(), 1), (NULL, 1) + +query spark_answer_only +SELECT array_contains(arr, val) FROM test_array_contains diff --git a/spark/src/test/resources/sql-tests/expressions/array_distinct.sql b/spark/src/test/resources/sql-tests/expressions/array_distinct.sql new file mode 100644 index 0000000000..283eb6da80 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_distinct.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_distinct(arr array) USING parquet + +statement +INSERT INTO test_array_distinct VALUES (array(1, 2, 2, 3, 3)), (array()), (NULL), (array(NULL, 1, NULL, 2)), (array(1)) + +query spark_answer_only +SELECT array_distinct(arr) FROM test_array_distinct diff --git a/spark/src/test/resources/sql-tests/expressions/array_except.sql b/spark/src/test/resources/sql-tests/expressions/array_except.sql new file mode 100644 index 0000000000..7a7e9cfc63 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_except.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_except(a array, b array) USING parquet + +statement +INSERT INTO test_array_except VALUES (array(1, 2, 3), array(2, 3, 4)), (array(1, 2), array()), (array(), array(1)), (NULL, array(1)), (array(1, NULL), array(NULL)) + +query spark_answer_only +SELECT array_except(a, b) FROM test_array_except diff --git a/spark/src/test/resources/sql-tests/expressions/array_insert.sql b/spark/src/test/resources/sql-tests/expressions/array_insert.sql new file mode 100644 index 0000000000..2d5754ea8e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_insert.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_insert(arr array, pos int, val int) USING parquet + +statement +INSERT INTO test_array_insert VALUES (array(1, 2, 3), 2, 10), (array(1, 2, 3), 1, 10), (array(1, 2, 3), 4, 10), (array(), 1, 10), (NULL, 1, 10) + +query spark_answer_only +SELECT array_insert(arr, pos, val) FROM test_array_insert diff --git a/spark/src/test/resources/sql-tests/expressions/array_intersect.sql b/spark/src/test/resources/sql-tests/expressions/array_intersect.sql new file mode 100644 index 0000000000..cb78703e66 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_intersect.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_intersect(a array, b array) USING parquet + +statement +INSERT INTO test_array_intersect VALUES (array(1, 2, 3), array(2, 3, 4)), (array(1, 2), array(3, 4)), (array(), array(1)), (NULL, array(1)), (array(1, NULL), array(NULL, 2)) + +query spark_answer_only +SELECT array_intersect(a, b) FROM test_array_intersect diff --git a/spark/src/test/resources/sql-tests/expressions/array_join.sql b/spark/src/test/resources/sql-tests/expressions/array_join.sql new file mode 100644 index 0000000000..840fc3f2c8 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_join.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_join(arr array) USING parquet + +statement +INSERT INTO test_array_join VALUES (array('a', 'b', 'c')), (array('hello', 'world')), (array()), (NULL), (array('a', NULL, 'c')) + +query spark_answer_only +SELECT array_join(arr, ',') FROM test_array_join + +query spark_answer_only +SELECT array_join(arr, ',', 'NULL') FROM test_array_join diff --git a/spark/src/test/resources/sql-tests/expressions/array_max.sql b/spark/src/test/resources/sql-tests/expressions/array_max.sql new file mode 100644 index 0000000000..f04e1e1935 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_max.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_max(arr array) USING parquet + +statement +INSERT INTO test_array_max VALUES (array(1, 2, 3)), (array(3, 1, 2)), (array()), (NULL), (array(NULL, 1, 2)), (array(-1, -2, -3)) + +query spark_answer_only +SELECT array_max(arr) FROM test_array_max diff --git a/spark/src/test/resources/sql-tests/expressions/array_min.sql b/spark/src/test/resources/sql-tests/expressions/array_min.sql new file mode 100644 index 0000000000..6ab7b8f173 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_min.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_min(arr array) USING parquet + +statement +INSERT INTO test_array_min VALUES (array(1, 2, 3)), (array(3, 1, 2)), (array()), (NULL), (array(NULL, 1, 2)), (array(-1, -2, -3)) + +query spark_answer_only +SELECT array_min(arr) FROM test_array_min diff --git a/spark/src/test/resources/sql-tests/expressions/array_remove.sql b/spark/src/test/resources/sql-tests/expressions/array_remove.sql new file mode 100644 index 0000000000..70d0f254d1 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_remove.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_remove(arr array, val int) USING parquet + +statement +INSERT INTO test_array_remove VALUES (array(1, 2, 3, 2), 2), (array(1, 2, 3), 4), (array(), 1), (NULL, 1), (array(1, NULL, 3), NULL) + +query spark_answer_only +SELECT array_remove(arr, val) FROM test_array_remove diff --git a/spark/src/test/resources/sql-tests/expressions/array_repeat.sql b/spark/src/test/resources/sql-tests/expressions/array_repeat.sql new file mode 100644 index 0000000000..5108e7288e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_repeat.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_repeat(val int, cnt int) USING parquet + +statement +INSERT INTO test_array_repeat VALUES (1, 3), (NULL, 3), (1, 0), (1, -1), (1, NULL) + +query spark_answer_only +SELECT array_repeat(val, cnt) FROM test_array_repeat diff --git a/spark/src/test/resources/sql-tests/expressions/array_union.sql b/spark/src/test/resources/sql-tests/expressions/array_union.sql new file mode 100644 index 0000000000..9333ff7b95 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array_union.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_union(a array, b array) USING parquet + +statement +INSERT INTO test_array_union VALUES (array(1, 2, 3), array(3, 4, 5)), (array(1, 2), array()), (array(), array(1)), (NULL, array(1)), (array(1, NULL), array(NULL, 2)) + +query spark_answer_only +SELECT array_union(a, b) FROM test_array_union diff --git a/spark/src/test/resources/sql-tests/expressions/arrays_overlap.sql b/spark/src/test/resources/sql-tests/expressions/arrays_overlap.sql new file mode 100644 index 0000000000..72561978c0 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/arrays_overlap.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_arrays_overlap(a array, b array) USING parquet + +statement +INSERT INTO test_arrays_overlap VALUES (array(1, 2, 3), array(3, 4, 5)), (array(1, 2), array(3, 4)), (array(), array(1)), (NULL, array(1)), (array(1, NULL), array(NULL, 2)) + +query spark_answer_only +SELECT arrays_overlap(a, b) FROM test_arrays_overlap diff --git a/spark/src/test/resources/sql-tests/expressions/ascii.sql b/spark/src/test/resources/sql-tests/expressions/ascii.sql new file mode 100644 index 0000000000..73aef6881c --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/ascii.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_ascii(s string) USING parquet + +statement +INSERT INTO test_ascii VALUES ('A'), ('z'), ('0'), (''), (NULL), ('hello') + +query +SELECT ascii(s) FROM test_ascii diff --git a/spark/src/test/resources/sql-tests/expressions/asin.sql b/spark/src/test/resources/sql-tests/expressions/asin.sql new file mode 100644 index 0000000000..3e96a7b7d5 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/asin.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_asin(d double) USING parquet + +statement +INSERT INTO test_asin VALUES (0.0), (1.0), (-1.0), (0.5), (NULL), (cast('NaN' as double)), (2.0), (-2.0) + +query tolerance=1e-6 +SELECT asin(d) FROM test_asin diff --git a/spark/src/test/resources/sql-tests/expressions/atan.sql b/spark/src/test/resources/sql-tests/expressions/atan.sql new file mode 100644 index 0000000000..a6bfc2b14e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/atan.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_atan(d double) USING parquet + +statement +INSERT INTO test_atan VALUES (0.0), (1.0), (-1.0), (100.0), (-100.0), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)), (cast('-Infinity' as double)) + +query tolerance=1e-6 +SELECT atan(d) FROM test_atan diff --git a/spark/src/test/resources/sql-tests/expressions/atan2.sql b/spark/src/test/resources/sql-tests/expressions/atan2.sql new file mode 100644 index 0000000000..b9ea886009 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/atan2.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_atan2(y double, x double) USING parquet + +statement +INSERT INTO test_atan2 VALUES (0.0, 1.0), (1.0, 0.0), (1.0, 1.0), (-1.0, -1.0), (0.0, 0.0), (NULL, 1.0), (1.0, NULL), (cast('NaN' as double), 1.0), (cast('Infinity' as double), 1.0) + +query tolerance=1e-6 +SELECT atan2(y, x) FROM test_atan2 diff --git a/spark/src/test/resources/sql-tests/expressions/avg.sql b/spark/src/test/resources/sql-tests/expressions/avg.sql new file mode 100644 index 0000000000..c46a14c617 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/avg.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_avg(i int, l long, f float, d double, grp string) USING parquet + +statement +INSERT INTO test_avg VALUES (1, 10, 1.5, 1.5, 'a'), (2, 20, 2.5, 2.5, 'a'), (3, 30, 3.5, 3.5, 'b'), (NULL, NULL, NULL, NULL, 'b'), (0, 0, 0.0, 0.0, 'a') + +query tolerance=1e-6 +SELECT avg(i), avg(l), avg(f), avg(d) FROM test_avg + +query tolerance=1e-6 +SELECT grp, avg(d) FROM test_avg GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/bit_agg.sql b/spark/src/test/resources/sql-tests/expressions/bit_agg.sql new file mode 100644 index 0000000000..32a09febc2 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/bit_agg.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_bit_agg(i int, grp string) USING parquet + +statement +INSERT INTO test_bit_agg VALUES (1, 'a'), (2, 'a'), (3, 'a'), (4, 'b'), (5, 'b'), (NULL, 'b') + +query +SELECT bit_and(i), bit_or(i), bit_xor(i) FROM test_bit_agg + +query +SELECT grp, bit_and(i), bit_or(i), bit_xor(i) FROM test_bit_agg GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/bit_length.sql b/spark/src/test/resources/sql-tests/expressions/bit_length.sql new file mode 100644 index 0000000000..b74d1365cf --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/bit_length.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_bit_length(s string) USING parquet + +statement +INSERT INTO test_bit_length VALUES (''), ('a'), ('hello'), (NULL), ('café') + +query +SELECT bit_length(s) FROM test_bit_length diff --git a/spark/src/test/resources/sql-tests/expressions/bitwise.sql b/spark/src/test/resources/sql-tests/expressions/bitwise.sql index 067836a63b..88b70930da 100644 --- a/spark/src/test/resources/sql-tests/expressions/bitwise.sql +++ b/spark/src/test/resources/sql-tests/expressions/bitwise.sql @@ -48,3 +48,23 @@ SELECT shiftleft(col1, 2), shiftleft(col1, col2) FROM test query SELECT ~(11), ~col1, ~col2 FROM test + +-- BitwiseCount +statement +CREATE TABLE test_bit_count(i int, l long) USING parquet + +statement +INSERT INTO test_bit_count VALUES (0, 0), (1, 1), (7, 7), (-1, -1), (2147483647, 9223372036854775807), (NULL, NULL) + +query +SELECT bit_count(i), bit_count(l) FROM test_bit_count + +-- BitwiseGet +statement +CREATE TABLE test_bit_get(i int, pos int) USING parquet + +statement +INSERT INTO test_bit_get VALUES (11, 0), (11, 1), (11, 2), (11, 3), (0, 0), (NULL, 0), (11, NULL) + +query spark_answer_only +SELECT bit_get(i, pos) FROM test_bit_get diff --git a/spark/src/test/resources/sql-tests/expressions/case_when.sql b/spark/src/test/resources/sql-tests/expressions/case_when.sql new file mode 100644 index 0000000000..ee0d651c42 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/case_when.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_case_when(i int, s string) USING parquet + +statement +INSERT INTO test_case_when VALUES (1, 'a'), (2, 'b'), (3, 'c'), (NULL, 'd'), (1, NULL), (NULL, NULL) + +query +SELECT CASE WHEN i = 1 THEN 'one' WHEN i = 2 THEN 'two' ELSE 'other' END FROM test_case_when + +query +SELECT CASE i WHEN 1 THEN 'one' WHEN 2 THEN 'two' END FROM test_case_when + +query +SELECT CASE WHEN s IS NULL THEN 'null_val' ELSE s END FROM test_case_when diff --git a/spark/src/test/resources/sql-tests/expressions/cast.sql b/spark/src/test/resources/sql-tests/expressions/cast.sql new file mode 100644 index 0000000000..18375bc0b8 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/cast.sql @@ -0,0 +1,42 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_cast(i int, l long, f float, d double, s string, b boolean) USING parquet + +statement +INSERT INTO test_cast VALUES (1, 1, 1.5, 1.5, '123', true), (0, 0, 0.0, 0.0, '0', false), (NULL, NULL, NULL, NULL, NULL, NULL), (-1, -1, -1.5, -1.5, '-1', true), (2147483647, 9223372036854775807, cast('NaN' as float), cast('Infinity' as double), 'abc', false) + +query +SELECT cast(i as long), cast(i as double), cast(i as string) FROM test_cast + +query +SELECT cast(l as int), cast(l as double), cast(l as string) FROM test_cast + +query +SELECT cast(f as double), cast(f as int), cast(f as string) FROM test_cast + +query +SELECT cast(d as float), cast(d as int), cast(d as string) FROM test_cast + +query +SELECT cast(s as int), cast(s as double) FROM test_cast + +query +SELECT cast(b as int), cast(b as string), cast(i as boolean) FROM test_cast diff --git a/spark/src/test/resources/sql-tests/expressions/ceil.sql b/spark/src/test/resources/sql-tests/expressions/ceil.sql new file mode 100644 index 0000000000..2155d4b17e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/ceil.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_ceil(f float, d double) USING parquet + +statement +INSERT INTO test_ceil VALUES (1.1, 1.1), (-1.1, -1.1), (0.0, 0.0), (1.0, 1.0), (NULL, NULL), (cast('NaN' as float), cast('NaN' as double)), (cast('Infinity' as float), cast('Infinity' as double)) + +query +SELECT ceil(f), ceil(d) FROM test_ceil diff --git a/spark/src/test/resources/sql-tests/expressions/chr.sql b/spark/src/test/resources/sql-tests/expressions/chr.sql new file mode 100644 index 0000000000..96dba6e009 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/chr.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_chr(i long) USING parquet + +statement +INSERT INTO test_chr VALUES (65), (97), (0), (NULL), (128522), (256), (48) + +query +SELECT chr(i) FROM test_chr diff --git a/spark/src/test/resources/sql-tests/expressions/coalesce.sql b/spark/src/test/resources/sql-tests/expressions/coalesce.sql new file mode 100644 index 0000000000..c683a1b3a9 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/coalesce.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_coalesce(a int, b int, c int) USING parquet + +statement +INSERT INTO test_coalesce VALUES (1, 2, 3), (NULL, 2, 3), (NULL, NULL, 3), (NULL, NULL, NULL), (1, NULL, NULL) + +query +SELECT coalesce(a, b, c) FROM test_coalesce + +query +SELECT coalesce(a) FROM test_coalesce + +query +SELECT coalesce(a, 99) FROM test_coalesce diff --git a/spark/src/test/resources/sql-tests/expressions/concat.sql b/spark/src/test/resources/sql-tests/expressions/concat.sql new file mode 100644 index 0000000000..9b2dce2b1e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/concat.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_concat(a string, b string, c string) USING parquet + +statement +INSERT INTO test_concat VALUES ('hello', ' ', 'world'), ('', '', ''), (NULL, 'b', 'c'), ('a', NULL, 'c'), (NULL, NULL, NULL) + +query +SELECT concat(a, b, c) FROM test_concat + +query +SELECT a || b || c FROM test_concat diff --git a/spark/src/test/resources/sql-tests/expressions/concat_ws.sql b/spark/src/test/resources/sql-tests/expressions/concat_ws.sql new file mode 100644 index 0000000000..422e97225e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/concat_ws.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_concat_ws(a string, b string, c string) USING parquet + +statement +INSERT INTO test_concat_ws VALUES ('hello', 'beautiful', 'world'), ('', '', ''), (NULL, 'b', 'c'), ('a', NULL, 'c'), (NULL, NULL, NULL) + +query +SELECT concat_ws(',', a, b, c) FROM test_concat_ws + +query +SELECT concat_ws('', a, b, c) FROM test_concat_ws + +query +SELECT concat_ws(NULL, a, b, c) FROM test_concat_ws diff --git a/spark/src/test/resources/sql-tests/expressions/contains.sql b/spark/src/test/resources/sql-tests/expressions/contains.sql new file mode 100644 index 0000000000..79658b038d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/contains.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_contains(s string, sub string) USING parquet + +statement +INSERT INTO test_contains VALUES ('hello world', 'world'), ('hello', ''), ('', ''), ('hello', 'xyz'), (NULL, 'a'), ('hello', NULL) + +query +SELECT contains(s, sub) FROM test_contains diff --git a/spark/src/test/resources/sql-tests/expressions/corr.sql b/spark/src/test/resources/sql-tests/expressions/corr.sql new file mode 100644 index 0000000000..9d11ba0ca3 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/corr.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_corr(x double, y double, grp string) USING parquet + +statement +INSERT INTO test_corr VALUES (1.0, 2.0, 'a'), (2.0, 4.0, 'a'), (3.0, 6.0, 'a'), (1.0, 1.0, 'b'), (2.0, 3.0, 'b'), (NULL, 1.0, 'b') + +query tolerance=1e-6 +SELECT corr(x, y) FROM test_corr + +query tolerance=1e-6 +SELECT grp, corr(x, y) FROM test_corr GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/cos.sql b/spark/src/test/resources/sql-tests/expressions/cos.sql new file mode 100644 index 0000000000..97bdaec31e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/cos.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_cos(d double) USING parquet + +statement +INSERT INTO test_cos VALUES (0.0), (3.141592653589793), (1.5707963267948966), (-3.141592653589793), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT cos(d) FROM test_cos diff --git a/spark/src/test/resources/sql-tests/expressions/cosh.sql b/spark/src/test/resources/sql-tests/expressions/cosh.sql new file mode 100644 index 0000000000..3c482959dc --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/cosh.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_cosh(d double) USING parquet + +statement +INSERT INTO test_cosh VALUES (0.0), (1.0), (-1.0), (100.0), (NULL), (cast('NaN' as double)) + +query tolerance=1e-6 +SELECT cosh(d) FROM test_cosh diff --git a/spark/src/test/resources/sql-tests/expressions/cot.sql b/spark/src/test/resources/sql-tests/expressions/cot.sql new file mode 100644 index 0000000000..5acd14c19c --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/cot.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_cot(d double) USING parquet + +statement +INSERT INTO test_cot VALUES (0.7853981633974483), (1.0), (-1.0), (0.1), (NULL), (cast('NaN' as double)) + +query tolerance=1e-6 +SELECT cot(d) FROM test_cot diff --git a/spark/src/test/resources/sql-tests/expressions/count.sql b/spark/src/test/resources/sql-tests/expressions/count.sql new file mode 100644 index 0000000000..b7fec97557 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/count.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_count(i int, s string, grp string) USING parquet + +statement +INSERT INTO test_count VALUES (1, 'a', 'x'), (2, 'b', 'x'), (NULL, NULL, 'y'), (3, 'c', 'y'), (NULL, 'd', 'y') + +query +SELECT count(*), count(i), count(s) FROM test_count + +query +SELECT grp, count(*), count(i) FROM test_count GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/covariance.sql b/spark/src/test/resources/sql-tests/expressions/covariance.sql new file mode 100644 index 0000000000..eca4b5ff1b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/covariance.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_covar(x double, y double, grp string) USING parquet + +statement +INSERT INTO test_covar VALUES (1.0, 2.0, 'a'), (2.0, 4.0, 'a'), (3.0, 6.0, 'a'), (1.0, 1.0, 'b'), (2.0, 3.0, 'b'), (NULL, 1.0, 'b') + +query tolerance=1e-6 +SELECT covar_samp(x, y), covar_pop(x, y) FROM test_covar + +query tolerance=1e-6 +SELECT grp, covar_samp(x, y), covar_pop(x, y) FROM test_covar GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/create_array.sql b/spark/src/test/resources/sql-tests/expressions/create_array.sql new file mode 100644 index 0000000000..0b41f0220c --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/create_array.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_create_array(a int, b int, c int) USING parquet + +statement +INSERT INTO test_create_array VALUES (1, 2, 3), (NULL, 2, 3), (NULL, NULL, NULL) + +query +SELECT array(a, b, c) FROM test_create_array + +query +SELECT array(1, 2, 3, NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/create_named_struct.sql b/spark/src/test/resources/sql-tests/expressions/create_named_struct.sql new file mode 100644 index 0000000000..4b2461be8e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/create_named_struct.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_named_struct(a int, b string, c double) USING parquet + +statement +INSERT INTO test_named_struct VALUES (1, 'hello', 1.5), (NULL, NULL, NULL), (0, '', 0.0) + +query +SELECT named_struct('x', a, 'y', b, 'z', c) FROM test_named_struct + +query +SELECT struct(a, b, c) FROM test_named_struct diff --git a/spark/src/test/resources/sql-tests/expressions/date_add.sql b/spark/src/test/resources/sql-tests/expressions/date_add.sql new file mode 100644 index 0000000000..3391cb3b78 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/date_add.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_date_add(d date, n int) USING parquet + +statement +INSERT INTO test_date_add VALUES (date('2024-01-15'), 1), (date('2024-01-15'), -1), (date('2024-01-15'), 0), (date('2024-12-31'), 1), (NULL, 1), (date('2024-01-15'), NULL) + +query +SELECT date_add(d, n) FROM test_date_add diff --git a/spark/src/test/resources/sql-tests/expressions/date_diff.sql b/spark/src/test/resources/sql-tests/expressions/date_diff.sql new file mode 100644 index 0000000000..c956074223 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/date_diff.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_datediff(d1 date, d2 date) USING parquet + +statement +INSERT INTO test_datediff VALUES (date('2024-01-15'), date('2024-01-10')), (date('2024-01-10'), date('2024-01-15')), (date('2024-01-15'), date('2024-01-15')), (NULL, date('2024-01-15')), (date('2024-01-15'), NULL) + +query +SELECT datediff(d1, d2) FROM test_datediff diff --git a/spark/src/test/resources/sql-tests/expressions/date_format.sql b/spark/src/test/resources/sql-tests/expressions/date_format.sql new file mode 100644 index 0000000000..92f8228f63 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/date_format.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_date_format(ts timestamp) USING parquet + +statement +INSERT INTO test_date_format VALUES (timestamp('2024-06-15 10:30:45')), (timestamp('1970-01-01 00:00:00')), (NULL) + +query expect_fallback(Non-UTC timezone) +SELECT date_format(ts, 'yyyy-MM-dd') FROM test_date_format + +query expect_fallback(Non-UTC timezone) +SELECT date_format(ts, 'HH:mm:ss') FROM test_date_format + +query expect_fallback(Non-UTC timezone) +SELECT date_format(ts, 'yyyy-MM-dd HH:mm:ss') FROM test_date_format diff --git a/spark/src/test/resources/sql-tests/expressions/date_format_enabled.sql b/spark/src/test/resources/sql-tests/expressions/date_format_enabled.sql new file mode 100644 index 0000000000..ee88382814 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/date_format_enabled.sql @@ -0,0 +1,37 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test date_format() with allowIncompatible enabled (happy path) +-- Uses UTC timezone to ensure results match Spark +-- Config: spark.comet.expression.DateFormatClass.allowIncompatible=true +-- Config: spark.sql.session.timeZone=UTC +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_date_format_enabled(ts timestamp) USING parquet + +statement +INSERT INTO test_date_format_enabled VALUES (timestamp('2024-06-15 10:30:45')), (timestamp('1970-01-01 00:00:00')), (NULL) + +query +SELECT date_format(ts, 'yyyy-MM-dd') FROM test_date_format_enabled + +query +SELECT date_format(ts, 'HH:mm:ss') FROM test_date_format_enabled + +query +SELECT date_format(ts, 'yyyy-MM-dd HH:mm:ss') FROM test_date_format_enabled diff --git a/spark/src/test/resources/sql-tests/expressions/date_sub.sql b/spark/src/test/resources/sql-tests/expressions/date_sub.sql new file mode 100644 index 0000000000..fcee705e20 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/date_sub.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_date_sub(d date, n int) USING parquet + +statement +INSERT INTO test_date_sub VALUES (date('2024-01-15'), 1), (date('2024-01-15'), -1), (date('2024-01-15'), 0), (date('2024-01-01'), 1), (NULL, 1), (date('2024-01-15'), NULL) + +query +SELECT date_sub(d, n) FROM test_date_sub diff --git a/spark/src/test/resources/sql-tests/expressions/datetime.sql b/spark/src/test/resources/sql-tests/expressions/datetime.sql index 93c615d5b3..a00516cdb3 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime.sql @@ -26,3 +26,16 @@ INSERT INTO test_dt VALUES (timestamp('2024-06-15 10:30:00')), (timestamp('1900- query SELECT col, year(col), month(col), day(col), weekday(col), dayofweek(col), dayofyear(col), weekofyear(col), quarter(col) FROM test_dt + +query +SELECT hour(col), minute(col), second(col) FROM test_dt + +-- Midnight and end-of-day +statement +CREATE TABLE test_dt_hms(ts timestamp) USING parquet + +statement +INSERT INTO test_dt_hms VALUES (timestamp('2024-01-01 00:00:00')), (timestamp('2024-01-01 23:59:59')), (timestamp('2024-06-15 12:30:45')), (NULL) + +query +SELECT hour(ts), minute(ts), second(ts) FROM test_dt_hms diff --git a/spark/src/test/resources/sql-tests/expressions/element_at.sql b/spark/src/test/resources/sql-tests/expressions/element_at.sql new file mode 100644 index 0000000000..fabbe6f298 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/element_at.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_element_at(arr array) USING parquet + +statement +INSERT INTO test_element_at VALUES (array(1, 2, 3)), (array(10)), (NULL) + +query spark_answer_only +SELECT element_at(arr, 1), element_at(arr, -1) FROM test_element_at diff --git a/spark/src/test/resources/sql-tests/expressions/ends_with.sql b/spark/src/test/resources/sql-tests/expressions/ends_with.sql new file mode 100644 index 0000000000..cd77743d2f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/ends_with.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_ends_with(s string, suffix string) USING parquet + +statement +INSERT INTO test_ends_with VALUES ('hello world', 'world'), ('hello', ''), ('', ''), ('hello', 'xyz'), (NULL, 'a'), ('hello', NULL) + +query +SELECT endswith(s, suffix) FROM test_ends_with diff --git a/spark/src/test/resources/sql-tests/expressions/exp.sql b/spark/src/test/resources/sql-tests/expressions/exp.sql new file mode 100644 index 0000000000..23fa613bb0 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/exp.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_exp(d double) USING parquet + +statement +INSERT INTO test_exp VALUES (0.0), (1.0), (-1.0), (100.0), (-100.0), (NULL), (cast('NaN' as double)), (cast('-Infinity' as double)) + +query tolerance=1e-6 +SELECT exp(d) FROM test_exp diff --git a/spark/src/test/resources/sql-tests/expressions/expm1.sql b/spark/src/test/resources/sql-tests/expressions/expm1.sql new file mode 100644 index 0000000000..f2ef06f6e7 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/expm1.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_expm1(d double) USING parquet + +statement +INSERT INTO test_expm1 VALUES (0.0), (1.0), (-1.0), (0.001), (NULL), (cast('NaN' as double)), (cast('-Infinity' as double)) + +query tolerance=1e-6 +SELECT expm1(d) FROM test_expm1 diff --git a/spark/src/test/resources/sql-tests/expressions/first_last.sql b/spark/src/test/resources/sql-tests/expressions/first_last.sql new file mode 100644 index 0000000000..7bc617bffd --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/first_last.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_first_last(i int, grp string) USING parquet + +statement +INSERT INTO test_first_last VALUES (1, 'a'), (2, 'a'), (3, 'a'), (NULL, 'b'), (4, 'b') + +query spark_answer_only +SELECT first(i), last(i) FROM test_first_last + +query spark_answer_only +SELECT first(i, true), last(i, true) FROM test_first_last + +query spark_answer_only +SELECT grp, first(i), last(i) FROM test_first_last GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/flatten.sql b/spark/src/test/resources/sql-tests/expressions/flatten.sql new file mode 100644 index 0000000000..2ff67a8da7 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/flatten.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_flatten(arr array>) USING parquet + +statement +INSERT INTO test_flatten VALUES (array(array(1, 2), array(3, 4))), (array(array(), array(1))), (array()), (NULL), (array(array(1, NULL), array(NULL))) + +query spark_answer_only +SELECT flatten(arr) FROM test_flatten diff --git a/spark/src/test/resources/sql-tests/expressions/floor.sql b/spark/src/test/resources/sql-tests/expressions/floor.sql new file mode 100644 index 0000000000..f365459b03 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/floor.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_floor(f float, d double) USING parquet + +statement +INSERT INTO test_floor VALUES (1.9, 1.9), (-1.1, -1.1), (0.0, 0.0), (1.0, 1.0), (NULL, NULL), (cast('NaN' as float), cast('NaN' as double)), (cast('Infinity' as float), cast('Infinity' as double)) + +query +SELECT floor(f), floor(d) FROM test_floor diff --git a/spark/src/test/resources/sql-tests/expressions/from_unix_time.sql b/spark/src/test/resources/sql-tests/expressions/from_unix_time.sql new file mode 100644 index 0000000000..2d0aaba4ed --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/from_unix_time.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_from_unix_time(t long) USING parquet + +statement +INSERT INTO test_from_unix_time VALUES (0), (1718451045), (-1), (NULL), (2147483647) + +query expect_fallback(not fully compatible with Spark) +SELECT from_unixtime(t) FROM test_from_unix_time + +query expect_fallback(not fully compatible with Spark) +SELECT from_unixtime(t, 'yyyy-MM-dd') FROM test_from_unix_time diff --git a/spark/src/test/resources/sql-tests/expressions/from_unix_time_enabled.sql b/spark/src/test/resources/sql-tests/expressions/from_unix_time_enabled.sql new file mode 100644 index 0000000000..8d31e98e16 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/from_unix_time_enabled.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test from_unixtime() with allowIncompatible enabled (happy path) +-- Uses UTC timezone to ensure results match Spark +-- Config: spark.comet.expression.FromUnixTime.allowIncompatible=true +-- Config: spark.sql.session.timeZone=UTC +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_from_unix_time_enabled(t long) USING parquet + +statement +INSERT INTO test_from_unix_time_enabled VALUES (0), (1718451045), (-1), (NULL), (2147483647) + +-- Even with allowIncompatible=true, the default datetime pattern is unsupported natively +query spark_answer_only +SELECT from_unixtime(t) FROM test_from_unix_time_enabled + +query spark_answer_only +SELECT from_unixtime(t, 'yyyy-MM-dd') FROM test_from_unix_time_enabled diff --git a/spark/src/test/resources/sql-tests/expressions/hex.sql b/spark/src/test/resources/sql-tests/expressions/hex.sql new file mode 100644 index 0000000000..e99df6127d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/hex.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_hex(i int, l long, s string) USING parquet + +statement +INSERT INTO test_hex VALUES (0, 0, ''), (255, 255, 'Spark'), (-1, -1, NULL), (NULL, NULL, 'ABC'), (2147483647, 9223372036854775807, 'a') + +query +SELECT hex(i), hex(l), hex(s) FROM test_hex diff --git a/spark/src/test/resources/sql-tests/expressions/hour.sql b/spark/src/test/resources/sql-tests/expressions/hour.sql new file mode 100644 index 0000000000..009c1876e3 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/hour.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_hour(ts timestamp) USING parquet + +statement +INSERT INTO test_hour VALUES (timestamp('2024-01-15 00:00:00')), (timestamp('2024-01-15 12:30:45')), (timestamp('2024-01-15 23:59:59')), (NULL) + +query +SELECT hour(ts) FROM test_hour diff --git a/spark/src/test/resources/sql-tests/expressions/if_expr.sql b/spark/src/test/resources/sql-tests/expressions/if_expr.sql new file mode 100644 index 0000000000..8dd8c8b090 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/if_expr.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_if(cond boolean, a int, b int) USING parquet + +statement +INSERT INTO test_if VALUES (true, 1, 2), (false, 1, 2), (NULL, 1, 2), (true, NULL, 2), (false, 1, NULL) + +query +SELECT IF(cond, a, b) FROM test_if + +query +SELECT IF(a > 0, 'positive', 'non-positive') FROM test_if diff --git a/spark/src/test/resources/sql-tests/expressions/init_cap.sql b/spark/src/test/resources/sql-tests/expressions/init_cap.sql new file mode 100644 index 0000000000..90f5afc8f4 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/init_cap.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_initcap(s string) USING parquet + +statement +INSERT INTO test_initcap VALUES ('hello world'), ('HELLO WORLD'), (''), (NULL), ('hello-world'), ('123abc'), (' spaces ') + +query expect_fallback(not fully compatible with Spark) +SELECT initcap(s) FROM test_initcap diff --git a/spark/src/test/resources/sql-tests/expressions/init_cap_enabled.sql b/spark/src/test/resources/sql-tests/expressions/init_cap_enabled.sql new file mode 100644 index 0000000000..f26694ff27 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/init_cap_enabled.sql @@ -0,0 +1,29 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test initcap() with allowIncompatible enabled (happy path) +-- Config: spark.comet.expression.InitCap.allowIncompatible=true +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_initcap_enabled(s string) USING parquet + +statement +INSERT INTO test_initcap_enabled VALUES ('hello world'), ('HELLO WORLD'), (''), (NULL), ('123abc'), (' spaces ') + +query +SELECT initcap(s) FROM test_initcap_enabled diff --git a/spark/src/test/resources/sql-tests/expressions/is_not_null.sql b/spark/src/test/resources/sql-tests/expressions/is_not_null.sql new file mode 100644 index 0000000000..d27a0a8b99 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/is_not_null.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_is_not_null(i int, s string, d double) USING parquet + +statement +INSERT INTO test_is_not_null VALUES (1, 'a', 1.0), (NULL, NULL, NULL), (0, '', 0.0), (NULL, 'b', cast('NaN' as double)) + +query +SELECT i IS NOT NULL, s IS NOT NULL, d IS NOT NULL FROM test_is_not_null + +query +SELECT isnotnull(i), isnotnull(s), isnotnull(d) FROM test_is_not_null diff --git a/spark/src/test/resources/sql-tests/expressions/is_null.sql b/spark/src/test/resources/sql-tests/expressions/is_null.sql new file mode 100644 index 0000000000..c3a1c5a741 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/is_null.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_is_null(i int, s string, d double) USING parquet + +statement +INSERT INTO test_is_null VALUES (1, 'a', 1.0), (NULL, NULL, NULL), (0, '', 0.0), (NULL, 'b', cast('NaN' as double)) + +query +SELECT i IS NULL, s IS NULL, d IS NULL FROM test_is_null + +query +SELECT isnull(i), isnull(s), isnull(d) FROM test_is_null diff --git a/spark/src/test/resources/sql-tests/expressions/isnan.sql b/spark/src/test/resources/sql-tests/expressions/isnan.sql new file mode 100644 index 0000000000..c9d0521a9e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/isnan.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_isnan(f float, d double) USING parquet + +statement +INSERT INTO test_isnan VALUES (1.0, 1.0), (cast('NaN' as float), cast('NaN' as double)), (NULL, NULL), (cast('Infinity' as float), cast('Infinity' as double)), (0.0, 0.0) + +query +SELECT isnan(f), isnan(d) FROM test_isnan diff --git a/spark/src/test/resources/sql-tests/expressions/json_to_structs.sql b/spark/src/test/resources/sql-tests/expressions/json_to_structs.sql new file mode 100644 index 0000000000..798ff80698 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/json_to_structs.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_from_json(j string) USING parquet + +statement +INSERT INTO test_from_json VALUES ('{"a": 1, "b": "hello"}'), ('{}'), (NULL), ('{"a": null}'), ('invalid json') + +query spark_answer_only +SELECT from_json(j, 'a INT, b STRING') FROM test_from_json diff --git a/spark/src/test/resources/sql-tests/expressions/last_day.sql b/spark/src/test/resources/sql-tests/expressions/last_day.sql new file mode 100644 index 0000000000..e7d60cc069 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/last_day.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_last_day(d date) USING parquet + +statement +INSERT INTO test_last_day VALUES (date('2024-01-15')), (date('2024-02-15')), (date('2023-02-15')), (date('2024-12-01')), (NULL) + +query +SELECT last_day(d) FROM test_last_day diff --git a/spark/src/test/resources/sql-tests/expressions/left.sql b/spark/src/test/resources/sql-tests/expressions/left.sql new file mode 100644 index 0000000000..1c2c393335 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/left.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_str_left(s string, n int) USING parquet + +statement +INSERT INTO test_str_left VALUES ('hello', 3), ('hello', 0), ('hello', -1), ('hello', 10), ('', 3), (NULL, 3), ('hello', NULL) + +query expect_fallback(Substring pos and len must be literals) +SELECT left(s, n) FROM test_str_left diff --git a/spark/src/test/resources/sql-tests/expressions/length.sql b/spark/src/test/resources/sql-tests/expressions/length.sql new file mode 100644 index 0000000000..db8ead6883 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/length.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_length(s string) USING parquet + +statement +INSERT INTO test_length VALUES (''), ('a'), ('hello'), (NULL), ('café') + +query +SELECT length(s), char_length(s) FROM test_length diff --git a/spark/src/test/resources/sql-tests/expressions/like.sql b/spark/src/test/resources/sql-tests/expressions/like.sql new file mode 100644 index 0000000000..aa4bbff4eb --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/like.sql @@ -0,0 +1,36 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_like(s string) USING parquet + +statement +INSERT INTO test_like VALUES ('hello'), ('world'), (''), (NULL), ('Hello'), ('h%llo'), ('h_llo') + +query +SELECT s LIKE 'h%' FROM test_like + +query +SELECT s LIKE '%llo' FROM test_like + +query +SELECT s LIKE 'h_llo' FROM test_like + +query +SELECT s LIKE '' FROM test_like diff --git a/spark/src/test/resources/sql-tests/expressions/log.sql b/spark/src/test/resources/sql-tests/expressions/log.sql new file mode 100644 index 0000000000..025c47d6df --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/log.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_log(d double) USING parquet + +statement +INSERT INTO test_log VALUES (1.0), (2.718281828459045), (10.0), (0.5), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT ln(d) FROM test_log + +query tolerance=1e-6 +SELECT log(10.0, d) FROM test_log diff --git a/spark/src/test/resources/sql-tests/expressions/log10.sql b/spark/src/test/resources/sql-tests/expressions/log10.sql new file mode 100644 index 0000000000..8f14574eb1 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/log10.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_log10(d double) USING parquet + +statement +INSERT INTO test_log10 VALUES (1.0), (10.0), (100.0), (0.1), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT log10(d) FROM test_log10 diff --git a/spark/src/test/resources/sql-tests/expressions/log2.sql b/spark/src/test/resources/sql-tests/expressions/log2.sql new file mode 100644 index 0000000000..f157c8c069 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/log2.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_log2(d double) USING parquet + +statement +INSERT INTO test_log2 VALUES (1.0), (2.0), (4.0), (8.0), (0.5), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT log2(d) FROM test_log2 diff --git a/spark/src/test/resources/sql-tests/expressions/lower.sql b/spark/src/test/resources/sql-tests/expressions/lower.sql new file mode 100644 index 0000000000..91676738c8 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/lower.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_lower(s string) USING parquet + +statement +INSERT INTO test_lower VALUES ('HELLO'), ('hello'), ('Hello World'), (''), (NULL), ('123ABC') + +query expect_fallback(case conversion) +SELECT lower(s) FROM test_lower diff --git a/spark/src/test/resources/sql-tests/expressions/lower_enabled.sql b/spark/src/test/resources/sql-tests/expressions/lower_enabled.sql new file mode 100644 index 0000000000..b521d87f91 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/lower_enabled.sql @@ -0,0 +1,29 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test lower() with case conversion enabled (happy path) +-- Config: spark.comet.caseConversion.enabled=true +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_lower_enabled(s string) USING parquet + +statement +INSERT INTO test_lower_enabled VALUES ('HELLO'), ('hello'), ('Hello World'), (''), (NULL), ('123ABC') + +query +SELECT lower(s) FROM test_lower_enabled diff --git a/spark/src/test/resources/sql-tests/expressions/map_entries.sql b/spark/src/test/resources/sql-tests/expressions/map_entries.sql new file mode 100644 index 0000000000..b89fdec060 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map_entries.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_map_entries(m map) USING parquet + +statement +INSERT INTO test_map_entries VALUES (map('a', 1, 'b', 2)), (map()), (NULL) + +query spark_answer_only +SELECT map_entries(m) FROM test_map_entries diff --git a/spark/src/test/resources/sql-tests/expressions/map_from_arrays.sql b/spark/src/test/resources/sql-tests/expressions/map_from_arrays.sql new file mode 100644 index 0000000000..7869021066 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map_from_arrays.sql @@ -0,0 +1,32 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_map_from_arrays(k array, v array) USING parquet + +statement +INSERT INTO test_map_from_arrays VALUES (array('a', 'b', 'c'), array(1, 2, 3)), (array(), array()), (NULL, NULL) + +query spark_answer_only +SELECT map_from_arrays(k, v) FROM test_map_from_arrays WHERE k IS NOT NULL + +-- Comet bug: map_from_arrays(NULL, NULL) causes native crash "map key cannot be null" +-- https://github.com/apache/datafusion-comet/issues/3327 +query ignore(https://github.com/apache/datafusion-comet/issues/3327) +SELECT map_from_arrays(k, v) FROM test_map_from_arrays WHERE k IS NULL diff --git a/spark/src/test/resources/sql-tests/expressions/map_keys.sql b/spark/src/test/resources/sql-tests/expressions/map_keys.sql new file mode 100644 index 0000000000..a6591ce800 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map_keys.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_map_keys(m map) USING parquet + +statement +INSERT INTO test_map_keys VALUES (map('a', 1, 'b', 2, 'c', 3)), (map()), (NULL) + +query spark_answer_only +SELECT map_keys(m) FROM test_map_keys diff --git a/spark/src/test/resources/sql-tests/expressions/map_values.sql b/spark/src/test/resources/sql-tests/expressions/map_values.sql new file mode 100644 index 0000000000..717e614e0d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map_values.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_map_values(m map) USING parquet + +statement +INSERT INTO test_map_values VALUES (map('a', 1, 'b', 2, 'c', 3)), (map()), (NULL) + +query spark_answer_only +SELECT map_values(m) FROM test_map_values diff --git a/spark/src/test/resources/sql-tests/expressions/min_max.sql b/spark/src/test/resources/sql-tests/expressions/min_max.sql new file mode 100644 index 0000000000..50a13d5896 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/min_max.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_min_max(i int, d double, s string, grp string) USING parquet + +statement +INSERT INTO test_min_max VALUES (1, 1.5, 'b', 'x'), (3, 3.5, 'a', 'x'), (2, 2.5, 'c', 'y'), (NULL, NULL, NULL, 'y'), (-1, -1.5, 'z', 'x') + +query expect_fallback(SortAggregate is not supported) +SELECT min(i), max(i), min(d), max(d), min(s), max(s) FROM test_min_max + +query +SELECT grp, min(i), max(i) FROM test_min_max GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/minute.sql b/spark/src/test/resources/sql-tests/expressions/minute.sql new file mode 100644 index 0000000000..fbd159c7a7 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/minute.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_minute(ts timestamp) USING parquet + +statement +INSERT INTO test_minute VALUES (timestamp('2024-01-15 10:00:00')), (timestamp('2024-01-15 10:30:00')), (timestamp('2024-01-15 10:59:59')), (NULL) + +query +SELECT minute(ts) FROM test_minute diff --git a/spark/src/test/resources/sql-tests/expressions/octet_length.sql b/spark/src/test/resources/sql-tests/expressions/octet_length.sql new file mode 100644 index 0000000000..7dd336af83 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/octet_length.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_octet_length(s string) USING parquet + +statement +INSERT INTO test_octet_length VALUES (''), ('a'), ('hello'), (NULL), ('café') + +query +SELECT octet_length(s) FROM test_octet_length diff --git a/spark/src/test/resources/sql-tests/expressions/pow.sql b/spark/src/test/resources/sql-tests/expressions/pow.sql new file mode 100644 index 0000000000..9c6f8914e0 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/pow.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_pow(base double, exp double) USING parquet + +statement +INSERT INTO test_pow VALUES (2.0, 3.0), (0.0, 0.0), (-1.0, 2.0), (-1.0, 0.5), (2.0, -1.0), (NULL, 2.0), (2.0, NULL), (cast('NaN' as double), 2.0), (cast('Infinity' as double), 2.0), (2.0, cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT pow(base, exp) FROM test_pow diff --git a/spark/src/test/resources/sql-tests/expressions/predicates.sql b/spark/src/test/resources/sql-tests/expressions/predicates.sql new file mode 100644 index 0000000000..35498d0f32 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/predicates.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_pred(a int, b int) USING parquet + +statement +INSERT INTO test_pred VALUES (1, 2), (2, 2), (3, 1), (NULL, 1), (1, NULL), (NULL, NULL) + +query +SELECT a = b, a <=> b, a < b, a > b, a <= b, a >= b FROM test_pred + +query +SELECT a != b, NOT (a = b) FROM test_pred + +query +SELECT (a > 1) AND (b > 1), (a > 1) OR (b > 1), NOT (a > 1) FROM test_pred diff --git a/spark/src/test/resources/sql-tests/expressions/regexp_replace.sql b/spark/src/test/resources/sql-tests/expressions/regexp_replace.sql new file mode 100644 index 0000000000..acaa238c00 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/regexp_replace.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_regexp_replace(s string) USING parquet + +statement +INSERT INTO test_regexp_replace VALUES ('100-200'), ('abc'), (''), (NULL), ('phone 123-456-7890') + +query expect_fallback(Regexp pattern) +SELECT regexp_replace(s, '(\\d+)', 'X') FROM test_regexp_replace + +query expect_fallback(Regexp pattern) +SELECT regexp_replace(s, '(\\d+)', 'X', 1) FROM test_regexp_replace diff --git a/spark/src/test/resources/sql-tests/expressions/regexp_replace_enabled.sql b/spark/src/test/resources/sql-tests/expressions/regexp_replace_enabled.sql new file mode 100644 index 0000000000..25624d24de --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/regexp_replace_enabled.sql @@ -0,0 +1,32 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test regexp_replace() with regexp allowIncompatible enabled (happy path) +-- Config: spark.comet.regexp.allowIncompatible=true +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_regexp_replace_enabled(s string) USING parquet + +statement +INSERT INTO test_regexp_replace_enabled VALUES ('100-200'), ('abc'), (''), (NULL), ('phone 123-456-7890') + +query +SELECT regexp_replace(s, '(\d+)', 'X') FROM test_regexp_replace_enabled + +query +SELECT regexp_replace(s, '(\d+)', 'X', 1) FROM test_regexp_replace_enabled diff --git a/spark/src/test/resources/sql-tests/expressions/reverse.sql b/spark/src/test/resources/sql-tests/expressions/reverse.sql new file mode 100644 index 0000000000..5ec46ad060 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/reverse.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_reverse(s string) USING parquet + +statement +INSERT INTO test_reverse VALUES ('hello'), (''), (NULL), ('a'), ('abcde'), ('café') + +query +SELECT reverse(s) FROM test_reverse diff --git a/spark/src/test/resources/sql-tests/expressions/rlike.sql b/spark/src/test/resources/sql-tests/expressions/rlike.sql new file mode 100644 index 0000000000..7c95211df9 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/rlike.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_rlike(s string) USING parquet + +statement +INSERT INTO test_rlike VALUES ('hello'), ('12345'), (''), (NULL), ('Hello World'), ('abc123') + +query expect_fallback(Regexp pattern) +SELECT s RLIKE '^[0-9]+$' FROM test_rlike + +query expect_fallback(Regexp pattern) +SELECT s RLIKE '^[a-z]+$' FROM test_rlike + +query spark_answer_only +SELECT s RLIKE '' FROM test_rlike diff --git a/spark/src/test/resources/sql-tests/expressions/rlike_enabled.sql b/spark/src/test/resources/sql-tests/expressions/rlike_enabled.sql new file mode 100644 index 0000000000..2f129397a6 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/rlike_enabled.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test RLIKE with regexp allowIncompatible enabled (happy path) +-- Config: spark.comet.regexp.allowIncompatible=true +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_rlike_enabled(s string) USING parquet + +statement +INSERT INTO test_rlike_enabled VALUES ('hello'), ('12345'), (''), (NULL), ('Hello World'), ('abc123') + +query +SELECT s RLIKE '^[0-9]+$' FROM test_rlike_enabled + +query +SELECT s RLIKE '^[a-z]+$' FROM test_rlike_enabled + +query +SELECT s RLIKE '' FROM test_rlike_enabled diff --git a/spark/src/test/resources/sql-tests/expressions/round.sql b/spark/src/test/resources/sql-tests/expressions/round.sql new file mode 100644 index 0000000000..4647d5f397 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/round.sql @@ -0,0 +1,36 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_round(d double, i int) USING parquet + +statement +INSERT INTO test_round VALUES (2.5, 0), (3.5, 0), (-2.5, 0), (123.456, 2), (123.456, -1), (NULL, 0), (cast('NaN' as double), 0), (cast('Infinity' as double), 0), (0.0, 0) + +query expect_fallback(BigDecimal rounding) +SELECT round(d, 0) FROM test_round WHERE i = 0 + +query expect_fallback(BigDecimal rounding) +SELECT round(d, 2) FROM test_round WHERE i = 2 + +query expect_fallback(BigDecimal rounding) +SELECT round(d, -1) FROM test_round WHERE i = -1 + +query expect_fallback(BigDecimal rounding) +SELECT round(d) FROM test_round diff --git a/spark/src/test/resources/sql-tests/expressions/second.sql b/spark/src/test/resources/sql-tests/expressions/second.sql new file mode 100644 index 0000000000..ec306fc8e4 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/second.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_second(ts timestamp) USING parquet + +statement +INSERT INTO test_second VALUES (timestamp('2024-01-15 10:30:00')), (timestamp('2024-01-15 10:30:30')), (timestamp('2024-01-15 10:30:59')), (NULL) + +query +SELECT second(ts) FROM test_second diff --git a/spark/src/test/resources/sql-tests/expressions/signum.sql b/spark/src/test/resources/sql-tests/expressions/signum.sql new file mode 100644 index 0000000000..dfdebd864d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/signum.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_signum(d double) USING parquet + +statement +INSERT INTO test_signum VALUES (5.0), (-5.0), (0.0), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)), (cast('-Infinity' as double)) + +query +SELECT signum(d) FROM test_signum diff --git a/spark/src/test/resources/sql-tests/expressions/sin.sql b/spark/src/test/resources/sql-tests/expressions/sin.sql new file mode 100644 index 0000000000..85db4c8864 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/sin.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_sin(d double) USING parquet + +statement +INSERT INTO test_sin VALUES (0.0), (3.141592653589793), (1.5707963267948966), (-1.5707963267948966), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT sin(d) FROM test_sin diff --git a/spark/src/test/resources/sql-tests/expressions/sinh.sql b/spark/src/test/resources/sql-tests/expressions/sinh.sql new file mode 100644 index 0000000000..d5f25d6e7f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/sinh.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_sinh(d double) USING parquet + +statement +INSERT INTO test_sinh VALUES (0.0), (1.0), (-1.0), (NULL), (cast('NaN' as double)) + +query tolerance=1e-6 +SELECT sinh(d) FROM test_sinh diff --git a/spark/src/test/resources/sql-tests/expressions/size.sql b/spark/src/test/resources/sql-tests/expressions/size.sql new file mode 100644 index 0000000000..9215747af7 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/size.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_size(arr array, m map) USING parquet + +statement +INSERT INTO test_size VALUES (array(1, 2, 3), map('a', 1, 'b', 2)), (array(), map()), (NULL, NULL) + +query spark_answer_only +SELECT size(arr), size(m) FROM test_size diff --git a/spark/src/test/resources/sql-tests/expressions/sqrt.sql b/spark/src/test/resources/sql-tests/expressions/sqrt.sql new file mode 100644 index 0000000000..a776ab28e8 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/sqrt.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_sqrt(d double) USING parquet + +statement +INSERT INTO test_sqrt VALUES (0.0), (1.0), (4.0), (2.0), (-1.0), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT sqrt(d) FROM test_sqrt diff --git a/spark/src/test/resources/sql-tests/expressions/starts_with.sql b/spark/src/test/resources/sql-tests/expressions/starts_with.sql new file mode 100644 index 0000000000..12fb1e8de3 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/starts_with.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_starts_with(s string, prefix string) USING parquet + +statement +INSERT INTO test_starts_with VALUES ('hello world', 'hello'), ('hello', ''), ('', ''), ('hello', 'xyz'), (NULL, 'a'), ('hello', NULL) + +query +SELECT startswith(s, prefix) FROM test_starts_with diff --git a/spark/src/test/resources/sql-tests/expressions/stddev.sql b/spark/src/test/resources/sql-tests/expressions/stddev.sql new file mode 100644 index 0000000000..cd70852daf --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/stddev.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_stddev(d double, grp string) USING parquet + +statement +INSERT INTO test_stddev VALUES (1.0, 'a'), (2.0, 'a'), (3.0, 'a'), (4.0, 'b'), (5.0, 'b'), (NULL, 'b') + +query tolerance=1e-6 +SELECT stddev(d), stddev_samp(d), stddev_pop(d) FROM test_stddev + +query tolerance=1e-6 +SELECT grp, stddev(d) FROM test_stddev GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/string_instr.sql b/spark/src/test/resources/sql-tests/expressions/string_instr.sql new file mode 100644 index 0000000000..982a371f80 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string_instr.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_instr(s string, sub string) USING parquet + +statement +INSERT INTO test_instr VALUES ('hello world', 'world'), ('hello', 'xyz'), ('hello', ''), ('', ''), (NULL, 'a'), ('hello', NULL), ('abcabc', 'bc') + +query +SELECT instr(s, sub) FROM test_instr diff --git a/spark/src/test/resources/sql-tests/expressions/string_lpad.sql b/spark/src/test/resources/sql-tests/expressions/string_lpad.sql new file mode 100644 index 0000000000..e2a40c1543 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string_lpad.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_lpad(s string, len int, pad string) USING parquet + +statement +INSERT INTO test_lpad VALUES ('hi', 5, 'x'), ('hello', 3, 'x'), ('hi', 5, 'xy'), ('', 3, 'a'), (NULL, 5, 'x'), ('hi', 0, 'x'), ('hi', -1, 'x') + +query expect_fallback(Only scalar values are supported for the pad argument) +SELECT lpad(s, len, pad) FROM test_lpad + +query +SELECT lpad(s, len) FROM test_lpad diff --git a/spark/src/test/resources/sql-tests/expressions/string_repeat.sql b/spark/src/test/resources/sql-tests/expressions/string_repeat.sql new file mode 100644 index 0000000000..0f537e9b48 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string_repeat.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_repeat(s string, n int) USING parquet + +statement +INSERT INTO test_repeat VALUES ('hi', 3), ('', 5), ('a', 0), ('a', -1), (NULL, 3), ('hi', NULL) + +query +SELECT repeat(s, n) FROM test_repeat diff --git a/spark/src/test/resources/sql-tests/expressions/string_replace.sql b/spark/src/test/resources/sql-tests/expressions/string_replace.sql new file mode 100644 index 0000000000..0cfaa4983f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string_replace.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_str_replace(s string, search string, replace string) USING parquet + +statement +INSERT INTO test_str_replace VALUES ('hello world', 'world', 'there'), ('aaa', 'a', 'bb'), ('hello', 'xyz', 'abc'), ('', 'a', 'b'), (NULL, 'a', 'b') + +query +SELECT replace(s, search, replace) FROM test_str_replace + +query spark_answer_only +SELECT replace('hello', '', 'x') diff --git a/spark/src/test/resources/sql-tests/expressions/string_rpad.sql b/spark/src/test/resources/sql-tests/expressions/string_rpad.sql new file mode 100644 index 0000000000..ba5e9311b6 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string_rpad.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_rpad(s string, len int, pad string) USING parquet + +statement +INSERT INTO test_rpad VALUES ('hi', 5, 'x'), ('hello', 3, 'x'), ('hi', 5, 'xy'), ('', 3, 'a'), (NULL, 5, 'x'), ('hi', 0, 'x'), ('hi', -1, 'x') + +query expect_fallback(Only scalar values are supported for the pad argument) +SELECT rpad(s, len, pad) FROM test_rpad + +query +SELECT rpad(s, len) FROM test_rpad diff --git a/spark/src/test/resources/sql-tests/expressions/string_space.sql b/spark/src/test/resources/sql-tests/expressions/string_space.sql new file mode 100644 index 0000000000..fe231e45b0 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string_space.sql @@ -0,0 +1,32 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_space(n int) USING parquet + +statement +INSERT INTO test_space VALUES (0), (1), (5), (NULL), (-1) + +query +SELECT concat('[', space(n), ']') FROM test_space WHERE n >= 0 OR n IS NULL + +-- Comet bug: space(-1) causes native crash "failed to round upto multiple of 64" +-- https://github.com/apache/datafusion-comet/issues/3326 +query ignore(https://github.com/apache/datafusion-comet/issues/3326) +SELECT concat('[', space(n), ']') FROM test_space WHERE n < 0 diff --git a/spark/src/test/resources/sql-tests/expressions/string_translate.sql b/spark/src/test/resources/sql-tests/expressions/string_translate.sql new file mode 100644 index 0000000000..1795a6cef1 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string_translate.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_translate(s string, from_str string, to_str string) USING parquet + +statement +INSERT INTO test_translate VALUES ('hello', 'el', 'ip'), ('hello', 'aeiou', '12345'), ('', 'a', 'b'), (NULL, 'a', 'b'), ('hello', '', ''), ('abc', 'abc', 'x') + +query +SELECT translate(s, from_str, to_str) FROM test_translate diff --git a/spark/src/test/resources/sql-tests/expressions/string_trim.sql b/spark/src/test/resources/sql-tests/expressions/string_trim.sql new file mode 100644 index 0000000000..7edc8c7fbe --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string_trim.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_trim(s string) USING parquet + +statement +INSERT INTO test_trim VALUES (' hello '), ('hello'), (''), (NULL), (' '), (' hello world ') + +query +SELECT trim(s), ltrim(s), rtrim(s) FROM test_trim + +query +SELECT trim(BOTH 'h' FROM s) FROM test_trim diff --git a/spark/src/test/resources/sql-tests/expressions/structs_to_json.sql b/spark/src/test/resources/sql-tests/expressions/structs_to_json.sql new file mode 100644 index 0000000000..e2498faffe --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/structs_to_json.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_to_json(a int, b string) USING parquet + +statement +INSERT INTO test_to_json VALUES (1, 'hello'), (NULL, NULL), (0, '') + +query spark_answer_only +SELECT to_json(named_struct('a', a, 'b', b)) FROM test_to_json diff --git a/spark/src/test/resources/sql-tests/expressions/substring.sql b/spark/src/test/resources/sql-tests/expressions/substring.sql new file mode 100644 index 0000000000..4ed982d7ab --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/substring.sql @@ -0,0 +1,42 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_substring(s string) USING parquet + +statement +INSERT INTO test_substring VALUES ('hello world'), (''), (NULL), ('abc') + +query +SELECT substring(s, 1, 5) FROM test_substring + +query +SELECT substring(s, -3) FROM test_substring + +query +SELECT substring(s, 0, 3) FROM test_substring + +query +SELECT substring(s, 1, 0) FROM test_substring + +query +SELECT substring(s, 1, -1) FROM test_substring + +query +SELECT substring(s, 100) FROM test_substring diff --git a/spark/src/test/resources/sql-tests/expressions/sum.sql b/spark/src/test/resources/sql-tests/expressions/sum.sql new file mode 100644 index 0000000000..306bbe9128 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/sum.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_sum(i int, l long, f float, d double, grp string) USING parquet + +statement +INSERT INTO test_sum VALUES (1, 10, 1.5, 1.5, 'a'), (2, 20, 2.5, 2.5, 'a'), (3, 30, 3.5, 3.5, 'b'), (NULL, NULL, NULL, NULL, 'b'), (2147483647, 9223372036854775807, cast('Infinity' as float), cast('Infinity' as double), 'c') + +query +SELECT sum(i), sum(l) FROM test_sum + +query tolerance=1e-6 +SELECT sum(f), sum(d) FROM test_sum + +query +SELECT grp, sum(i) FROM test_sum GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/tan.sql b/spark/src/test/resources/sql-tests/expressions/tan.sql new file mode 100644 index 0000000000..00dfc64131 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/tan.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_tan(d double) USING parquet + +statement +INSERT INTO test_tan VALUES (0.0), (0.7853981633974483), (-0.7853981633974483), (1.0), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT tan(d) FROM test_tan diff --git a/spark/src/test/resources/sql-tests/expressions/tanh.sql b/spark/src/test/resources/sql-tests/expressions/tanh.sql new file mode 100644 index 0000000000..85643dc935 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/tanh.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_tanh(d double) USING parquet + +statement +INSERT INTO test_tanh VALUES (0.0), (1.0), (-1.0), (100.0), (-100.0), (NULL), (cast('NaN' as double)) + +query tolerance=1e-6 +SELECT tanh(d) FROM test_tanh diff --git a/spark/src/test/resources/sql-tests/expressions/trunc_date.sql b/spark/src/test/resources/sql-tests/expressions/trunc_date.sql new file mode 100644 index 0000000000..e35a4537d2 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/trunc_date.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_trunc_date(d date) USING parquet + +statement +INSERT INTO test_trunc_date VALUES (date('2024-06-15')), (date('2024-01-01')), (date('2024-12-31')), (NULL) + +query +SELECT trunc(d, 'year') FROM test_trunc_date + +query +SELECT trunc(d, 'month') FROM test_trunc_date + +query +SELECT trunc(d, 'quarter') FROM test_trunc_date diff --git a/spark/src/test/resources/sql-tests/expressions/trunc_timestamp.sql b/spark/src/test/resources/sql-tests/expressions/trunc_timestamp.sql new file mode 100644 index 0000000000..6db602944a --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/trunc_timestamp.sql @@ -0,0 +1,36 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_trunc_ts(ts timestamp) USING parquet + +statement +INSERT INTO test_trunc_ts VALUES (timestamp('2024-06-15 10:30:45')), (timestamp('2024-01-01 00:00:00')), (NULL) + +query +SELECT date_trunc('year', ts) FROM test_trunc_ts + +query +SELECT date_trunc('month', ts) FROM test_trunc_ts + +query +SELECT date_trunc('day', ts) FROM test_trunc_ts + +query +SELECT date_trunc('hour', ts) FROM test_trunc_ts diff --git a/spark/src/test/resources/sql-tests/expressions/unhex.sql b/spark/src/test/resources/sql-tests/expressions/unhex.sql new file mode 100644 index 0000000000..8a9a3153d7 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/unhex.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_unhex(s string) USING parquet + +statement +INSERT INTO test_unhex VALUES ('537061726B2053514C'), ('41'), ('0A1B'), (''), (NULL), ('GG'), ('hello'), ('A1B') + +query +SELECT hex(unhex(s)) FROM test_unhex + +query +SELECT unhex(s) IS NULL FROM test_unhex diff --git a/spark/src/test/resources/sql-tests/expressions/unix_date.sql b/spark/src/test/resources/sql-tests/expressions/unix_date.sql new file mode 100644 index 0000000000..028f0ddea2 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/unix_date.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_unix_date(d date) USING parquet + +statement +INSERT INTO test_unix_date VALUES (date('1970-01-01')), (date('2024-01-15')), (date('1969-12-31')), (NULL) + +query spark_answer_only +SELECT unix_date(d) FROM test_unix_date diff --git a/spark/src/test/resources/sql-tests/expressions/unix_timestamp.sql b/spark/src/test/resources/sql-tests/expressions/unix_timestamp.sql new file mode 100644 index 0000000000..01a7194aad --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/unix_timestamp.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_unix_ts(ts timestamp) USING parquet + +statement +INSERT INTO test_unix_ts VALUES (timestamp('1970-01-01 00:00:00')), (timestamp('2024-06-15 10:30:45')), (NULL) + +query +SELECT unix_timestamp(ts) FROM test_unix_ts diff --git a/spark/src/test/resources/sql-tests/expressions/upper.sql b/spark/src/test/resources/sql-tests/expressions/upper.sql new file mode 100644 index 0000000000..b9ac2c5b75 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/upper.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_upper(s string) USING parquet + +statement +INSERT INTO test_upper VALUES ('hello'), ('HELLO'), ('Hello World'), (''), (NULL), ('123abc') + +query expect_fallback(case conversion) +SELECT upper(s) FROM test_upper diff --git a/spark/src/test/resources/sql-tests/expressions/upper_enabled.sql b/spark/src/test/resources/sql-tests/expressions/upper_enabled.sql new file mode 100644 index 0000000000..b8e4dc128d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/upper_enabled.sql @@ -0,0 +1,29 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test upper() with case conversion enabled (happy path) +-- Config: spark.comet.caseConversion.enabled=true +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_upper_enabled(s string) USING parquet + +statement +INSERT INTO test_upper_enabled VALUES ('hello'), ('HELLO'), ('Hello World'), (''), (NULL), ('123abc') + +query +SELECT upper(s) FROM test_upper_enabled diff --git a/spark/src/test/resources/sql-tests/expressions/variance.sql b/spark/src/test/resources/sql-tests/expressions/variance.sql new file mode 100644 index 0000000000..be4905d5fa --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/variance.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_variance(d double, grp string) USING parquet + +statement +INSERT INTO test_variance VALUES (1.0, 'a'), (2.0, 'a'), (3.0, 'a'), (4.0, 'b'), (5.0, 'b'), (NULL, 'b') + +query tolerance=1e-6 +SELECT variance(d), var_samp(d), var_pop(d) FROM test_variance + +query tolerance=1e-6 +SELECT grp, variance(d) FROM test_variance GROUP BY grp ORDER BY grp diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala index 8d382f5173..e3cbf89333 100644 --- a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -79,6 +79,10 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { checkSparkAnswer(sql) case WithTolerance(tol) => checkSparkAnswerWithTolerance(sql, tol) + case ExpectFallback(reason) => + checkSparkAnswerAndFallbackReason(sql, reason) + case Ignore(reason) => + logInfo(s"IGNORED query (${reason}): $sql") } } } diff --git a/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala b/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala index f2cbf56544..ae6a33f419 100644 --- a/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala +++ b/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala @@ -36,6 +36,8 @@ sealed trait QueryMode case object CheckOperator extends QueryMode case object SparkAnswerOnly extends QueryMode case class WithTolerance(tol: Double) extends QueryMode +case class ExpectFallback(reason: String) extends QueryMode +case class Ignore(reason: String) extends QueryMode /** * Parsed representation of a .sql test file. @@ -113,14 +115,24 @@ object SqlFileTestParser { SqlTestFile(configs, configMatrix, records.result(), tables.result()) } + private val FallbackPattern = """query\s+expect_fallback\((.+)\)""".r + private val IgnorePattern = """query\s+ignore\((.+)\)""".r + private def parseQueryMode(directive: String): QueryMode = { - val parts = directive.split("\\s+") - if (parts.length == 1) return CheckOperator - parts(1) match { - case "spark_answer_only" => SparkAnswerOnly - case s if s.startsWith("tolerance=") => - WithTolerance(s.stripPrefix("tolerance=").toDouble) - case _ => CheckOperator + directive match { + case FallbackPattern(reason) => + ExpectFallback(reason.trim) + case IgnorePattern(reason) => + Ignore(reason.trim) + case _ => + val parts = directive.split("\\s+") + if (parts.length == 1) return CheckOperator + parts(1) match { + case "spark_answer_only" => SparkAnswerOnly + case s if s.startsWith("tolerance=") => + WithTolerance(s.stripPrefix("tolerance=").toDouble) + case _ => CheckOperator + } } } From 39d2ba77a14ea4c3da9ac1f69b9c0127500304b7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 29 Jan 2026 09:24:24 -0700 Subject: [PATCH 3/9] Specify UTF-8 encoding when reading SQL test files Co-Authored-By: Claude Opus 4.5 --- spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala b/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala index ae6a33f419..1d18f9d64a 100644 --- a/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala +++ b/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala @@ -64,7 +64,7 @@ object SqlFileTestParser { private val CreateTablePattern = """(?i)CREATE\s+TABLE\s+(\w+)""".r.unanchored def parse(file: File): SqlTestFile = { - val source = Source.fromFile(file) + val source = Source.fromFile(file, "UTF-8") try { parse(source.getLines().toSeq) } finally { From 5dab0f0a02a29adb29feaf04fb28e0ed944fe2db Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 29 Jan 2026 09:30:07 -0700 Subject: [PATCH 4/9] revert --- .../apache/comet/CometExpressionSuite.scala | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 576af73745..fe5ea77a89 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -131,6 +131,31 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("compare true/false to negative zero") { + Seq(false, true).foreach { dictionary => + withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { + val table = "test" + withTable(table) { + sql(s"create table $table(col1 boolean, col2 float) using parquet") + sql(s"insert into $table values(true, -0.0)") + sql(s"insert into $table values(false, -0.0)") + + checkSparkAnswerAndOperator( + s"SELECT col1, negative(col2), cast(col1 as float), col1 = negative(col2) FROM $table") + } + } + } + } + + test("parquet default values") { + withTable("t1") { + sql("create table t1(col1 boolean) using parquet") + sql("insert into t1 values(true)") + sql("alter table t1 add column col2 string default 'hello'") + checkSparkAnswerAndOperator("select * from t1") + } + } + test("decimals divide by zero") { Seq(true, false).foreach { dictionary => withSQLConf( @@ -151,6 +176,16 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("Integral Division Overflow Handling Matches Spark Behavior") { + withTable("t1") { + val value = Long.MinValue + sql("create table t1(c1 long, c2 short) using parquet") + sql(s"insert into t1 values($value, -1)") + val res = sql("select c1 div c2 from t1 order by c1") + checkSparkAnswerAndOperator(res) + } + } + test("basic data type support") { // this test requires native_comet scan due to unsigned u8/u16 issue withSQLConf(CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_COMET) { @@ -467,6 +502,17 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("substring with start < 1") { + withTempPath { _ => + withTable("t") { + sql("create table t (col string) using parquet") + sql("insert into t values('123456')") + checkSparkAnswerAndOperator(sql("select substring(col, 0) from t")) + checkSparkAnswerAndOperator(sql("select substring(col, -1) from t")) + } + } + } + test("substring with dictionary") { val data = (0 until 1000) .map(_ % 5) // reduce value space to trigger dictionary encoding @@ -1511,6 +1557,20 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("md5") { + Seq(false, true).foreach { dictionary => + withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { + val table = "test" + withTable(table) { + sql(s"create table $table(col String) using parquet") + sql( + s"insert into $table values ('test1'), ('test1'), ('test2'), ('test2'), (NULL), ('')") + checkSparkAnswerAndOperator(s"select md5(col) FROM $table") + } + } + } + } + test("hex") { // https://github.com/apache/datafusion-comet/issues/1441 assume(!usingDataSourceExec) @@ -1529,6 +1589,26 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("unhex") { + val table = "unhex_table" + withTable(table) { + sql(s"create table $table(col string) using parquet") + + sql(s"""INSERT INTO $table VALUES + |('537061726B2053514C'), + |('737472696E67'), + |('\\0'), + |(''), + |('###'), + |('G123'), + |('hello'), + |('A1B'), + |('0A1B')""".stripMargin) + + checkSparkAnswerAndOperator(s"SELECT unhex(col) FROM $table") + } + } + test("EqualNullSafe should preserve comet filter") { Seq("true", "false").foreach(b => withParquetTable( @@ -1553,6 +1633,58 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { }) } + test("test in(set)/not in(set)") { + Seq("100", "0").foreach { inSetThreshold => + Seq(false, true).foreach { dictionary => + withSQLConf( + SQLConf.OPTIMIZER_INSET_CONVERSION_THRESHOLD.key -> inSetThreshold, + "parquet.enable.dictionary" -> dictionary.toString) { + val table = "names" + withTable(table) { + sql(s"create table $table(id int, name varchar(20)) using parquet") + sql( + s"insert into $table values(1, 'James'), (1, 'Jones'), (2, 'Smith'), (3, 'Smith')," + + "(NULL, 'Jones'), (4, NULL)") + + checkSparkAnswerAndOperator(s"SELECT * FROM $table WHERE id in (1, 2, 4, NULL)") + checkSparkAnswerAndOperator( + s"SELECT * FROM $table WHERE name in ('Smith', 'Brown', NULL)") + + // TODO: why with not in, the plan is only `LocalTableScan`? + checkSparkAnswerAndOperator(s"SELECT * FROM $table WHERE id not in (1)") + checkSparkAnswer(s"SELECT * FROM $table WHERE name not in ('Smith', 'Brown', NULL)") + } + } + } + } + } + + test("not") { + Seq(false, true).foreach { dictionary => + withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { + val table = "test" + withTable(table) { + sql(s"create table $table(col1 int, col2 boolean) using parquet") + sql(s"insert into $table values(1, false), (2, true), (3, true), (3, false)") + checkSparkAnswerAndOperator(s"SELECT col1, col2, NOT(col2), !(col2) FROM $table") + } + } + } + } + + test("negative") { + Seq(false, true).foreach { dictionary => + withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { + val table = "test" + withTable(table) { + sql(s"create table $table(col1 int) using parquet") + sql(s"insert into $table values(1), (2), (3), (3)") + checkSparkAnswerAndOperator(s"SELECT negative(col1), -(col1) FROM $table") + } + } + } + } + test("basic arithmetic") { withSQLConf("parquet.enable.dictionary" -> "false") { withParquetTable((1 until 10).map(i => (i, i + 1)), "tbl", false) { @@ -1587,6 +1719,21 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("DatePart functions: Year/Month/DayOfMonth/DayOfWeek/DayOfYear/WeekOfYear/Quarter") { + Seq(false, true).foreach { dictionary => + withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { + val table = "test" + withTable(table) { + sql(s"create table $table(col timestamp) using parquet") + sql(s"insert into $table values (now()), (timestamp('1900-01-01')), (null)") + checkSparkAnswerAndOperator( + "SELECT col, year(col), month(col), day(col), weekday(col), " + + s" dayofweek(col), dayofyear(col), weekofyear(col), quarter(col) FROM $table") + } + } + } + } + test("from_unixtime") { Seq(false, true).foreach { dictionary => withSQLConf( @@ -1838,6 +1985,31 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("hash functions") { + Seq(true, false).foreach { dictionary => + withSQLConf("parquet.enable.dictionary" -> dictionary.toString) { + val table = "test" + withTable(table) { + sql(s"create table $table(col string, a int, b float) using parquet") + sql(s""" + |insert into $table values + |('Spark SQL ', 10, 1.2), (NULL, NULL, NULL), ('', 0, 0.0), ('苹果手机', NULL, 3.999999) + |, ('Spark SQL ', 10, 1.2), (NULL, NULL, NULL), ('', 0, 0.0), ('苹果手机', NULL, 3.999999) + |""".stripMargin) + checkSparkAnswerAndOperator(""" + |select + |md5(col), md5(cast(a as string)), md5(cast(b as string)), + |hash(col), hash(col, 1), hash(col, 0), hash(col, a, b), hash(b, a, col), + |xxhash64(col), xxhash64(col, 1), xxhash64(col, 0), xxhash64(col, a, b), xxhash64(b, a, col), + |sha2(col, 0), sha2(col, 256), sha2(col, 224), sha2(col, 384), sha2(col, 512), sha2(col, 128), sha2(col, -1), + |sha1(col), sha1(cast(a as string)), sha1(cast(b as string)) + |from test + |""".stripMargin) + } + } + } + } + test("remainder function") { def withAnsiMode(enabled: Boolean)(f: => Unit): Unit = { withSQLConf( From 81941e24c5b3e91cac27d052bb63541bd6314889 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 29 Jan 2026 09:41:02 -0700 Subject: [PATCH 5/9] Organize SQL test files into category subdirectories and add missing expression tests Add 8 new SQL test files for previously untested expressions (GetArrayItem, GetStructField, GetMapValue, GetArrayStructFields, ArrayFilter, ScalarSubquery, decimal ops, WidthBucket). Reorganize all 131 SQL test files into 13 category subdirectories (math, string, array, map, struct, aggregate, datetime, conditional, bitwise, hash, cast, decimal, misc). Add MinSparkVersion directive support to the test framework for version-gated tests. Co-Authored-By: Claude Opus 4.5 --- .../expressions/{ => aggregate}/avg.sql | 0 .../expressions/{ => aggregate}/bit_agg.sql | 0 .../expressions/{ => aggregate}/corr.sql | 0 .../expressions/{ => aggregate}/count.sql | 0 .../{ => aggregate}/covariance.sql | 0 .../{ => aggregate}/first_last.sql | 0 .../expressions/{ => aggregate}/min_max.sql | 0 .../expressions/{ => aggregate}/stddev.sql | 0 .../expressions/{ => aggregate}/sum.sql | 0 .../expressions/{ => aggregate}/variance.sql | 0 .../expressions/{ => array}/array_append.sql | 0 .../expressions/{ => array}/array_compact.sql | 0 .../{ => array}/array_contains.sql | 0 .../{ => array}/array_distinct.sql | 0 .../expressions/{ => array}/array_except.sql | 0 .../expressions/array/array_filter.sql | 33 ++++++++++++ .../expressions/{ => array}/array_insert.sql | 0 .../{ => array}/array_intersect.sql | 0 .../expressions/{ => array}/array_join.sql | 0 .../expressions/{ => array}/array_max.sql | 0 .../expressions/{ => array}/array_min.sql | 0 .../expressions/{ => array}/array_remove.sql | 0 .../expressions/{ => array}/array_repeat.sql | 0 .../expressions/{ => array}/array_union.sql | 0 .../{ => array}/arrays_overlap.sql | 0 .../expressions/{ => array}/create_array.sql | 0 .../expressions/{ => array}/element_at.sql | 0 .../expressions/{ => array}/flatten.sql | 0 .../expressions/array/get_array_item.sql | 30 +++++++++++ .../array/get_array_struct_fields.sql | 27 ++++++++++ .../expressions/{ => array}/size.sql | 0 .../expressions/{ => bitwise}/bitwise.sql | 0 .../sql-tests/expressions/{ => cast}/cast.sql | 0 .../expressions/{ => conditional}/boolean.sql | 0 .../{ => conditional}/case_when.sql | 0 .../{ => conditional}/coalesce.sql | 0 .../expressions/{ => conditional}/if_expr.sql | 0 .../expressions/{ => conditional}/in_set.sql | 0 .../{ => conditional}/is_not_null.sql | 0 .../expressions/{ => conditional}/is_null.sql | 0 .../{ => conditional}/predicates.sql | 0 .../expressions/{ => datetime}/date_add.sql | 0 .../expressions/{ => datetime}/date_diff.sql | 0 .../{ => datetime}/date_format.sql | 0 .../{ => datetime}/date_format_enabled.sql | 0 .../expressions/{ => datetime}/date_sub.sql | 0 .../expressions/{ => datetime}/datetime.sql | 0 .../{ => datetime}/from_unix_time.sql | 0 .../{ => datetime}/from_unix_time_enabled.sql | 0 .../expressions/{ => datetime}/hour.sql | 0 .../expressions/{ => datetime}/last_day.sql | 0 .../expressions/{ => datetime}/minute.sql | 0 .../expressions/{ => datetime}/second.sql | 0 .../expressions/{ => datetime}/trunc_date.sql | 0 .../{ => datetime}/trunc_timestamp.sql | 0 .../expressions/{ => datetime}/unix_date.sql | 0 .../{ => datetime}/unix_timestamp.sql | 0 .../expressions/decimal/decimal_ops.sql | 54 +++++++++++++++++++ .../sql-tests/expressions/{ => hash}/hash.sql | 0 .../expressions/map/get_map_value.sql | 30 +++++++++++ .../expressions/{ => map}/map_entries.sql | 0 .../expressions/{ => map}/map_from_arrays.sql | 0 .../expressions/{ => map}/map_keys.sql | 0 .../expressions/{ => map}/map_values.sql | 0 .../sql-tests/expressions/{ => math}/abs.sql | 0 .../sql-tests/expressions/{ => math}/acos.sql | 0 .../expressions/{ => math}/arithmetic.sql | 0 .../sql-tests/expressions/{ => math}/asin.sql | 0 .../sql-tests/expressions/{ => math}/atan.sql | 0 .../expressions/{ => math}/atan2.sql | 0 .../sql-tests/expressions/{ => math}/ceil.sql | 0 .../sql-tests/expressions/{ => math}/cos.sql | 0 .../sql-tests/expressions/{ => math}/cosh.sql | 0 .../sql-tests/expressions/{ => math}/cot.sql | 0 .../sql-tests/expressions/{ => math}/exp.sql | 0 .../expressions/{ => math}/expm1.sql | 0 .../expressions/{ => math}/floor.sql | 0 .../expressions/{ => math}/isnan.sql | 0 .../sql-tests/expressions/{ => math}/log.sql | 0 .../expressions/{ => math}/log10.sql | 0 .../sql-tests/expressions/{ => math}/log2.sql | 0 .../sql-tests/expressions/{ => math}/pow.sql | 0 .../expressions/{ => math}/round.sql | 0 .../expressions/{ => math}/signum.sql | 0 .../sql-tests/expressions/{ => math}/sin.sql | 0 .../sql-tests/expressions/{ => math}/sinh.sql | 0 .../sql-tests/expressions/{ => math}/sqrt.sql | 0 .../sql-tests/expressions/{ => math}/tan.sql | 0 .../sql-tests/expressions/{ => math}/tanh.sql | 0 .../{ => misc}/parquet_default_values.sql | 0 .../expressions/misc/scalar_subquery.sql | 31 +++++++++++ .../expressions/misc/width_bucket.sql | 33 ++++++++++++ .../expressions/{ => string}/ascii.sql | 0 .../expressions/{ => string}/bit_length.sql | 0 .../expressions/{ => string}/chr.sql | 0 .../expressions/{ => string}/concat.sql | 0 .../expressions/{ => string}/concat_ws.sql | 0 .../expressions/{ => string}/contains.sql | 0 .../expressions/{ => string}/ends_with.sql | 0 .../expressions/{ => string}/hex.sql | 0 .../expressions/{ => string}/init_cap.sql | 0 .../{ => string}/init_cap_enabled.sql | 0 .../expressions/{ => string}/left.sql | 0 .../expressions/{ => string}/length.sql | 0 .../expressions/{ => string}/like.sql | 0 .../expressions/{ => string}/lower.sql | 0 .../{ => string}/lower_enabled.sql | 0 .../expressions/{ => string}/octet_length.sql | 0 .../{ => string}/regexp_replace.sql | 0 .../{ => string}/regexp_replace_enabled.sql | 0 .../expressions/{ => string}/reverse.sql | 0 .../expressions/{ => string}/rlike.sql | 0 .../{ => string}/rlike_enabled.sql | 0 .../expressions/{ => string}/starts_with.sql | 0 .../expressions/{ => string}/string.sql | 0 .../expressions/{ => string}/string_instr.sql | 0 .../expressions/{ => string}/string_lpad.sql | 0 .../{ => string}/string_repeat.sql | 0 .../{ => string}/string_replace.sql | 0 .../expressions/{ => string}/string_rpad.sql | 0 .../expressions/{ => string}/string_space.sql | 0 .../{ => string}/string_translate.sql | 0 .../expressions/{ => string}/string_trim.sql | 0 .../expressions/{ => string}/substring.sql | 0 .../expressions/{ => string}/unhex.sql | 0 .../expressions/{ => string}/upper.sql | 0 .../{ => string}/upper_enabled.sql | 0 .../{ => struct}/create_named_struct.sql | 0 .../expressions/struct/get_struct_field.sql | 30 +++++++++++ .../{ => struct}/json_to_structs.sql | 0 .../{ => struct}/structs_to_json.sql | 0 .../apache/comet/CometSqlFileTestSuite.scala | 25 +++++++-- .../org/apache/comet/SqlFileTestParser.scala | 13 ++++- 133 files changed, 301 insertions(+), 5 deletions(-) rename spark/src/test/resources/sql-tests/expressions/{ => aggregate}/avg.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => aggregate}/bit_agg.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => aggregate}/corr.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => aggregate}/count.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => aggregate}/covariance.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => aggregate}/first_last.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => aggregate}/min_max.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => aggregate}/stddev.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => aggregate}/sum.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => aggregate}/variance.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_append.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_compact.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_contains.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_distinct.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_except.sql (100%) create mode 100644 spark/src/test/resources/sql-tests/expressions/array/array_filter.sql rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_insert.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_intersect.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_join.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_max.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_min.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_remove.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_repeat.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/array_union.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/arrays_overlap.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/create_array.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/element_at.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => array}/flatten.sql (100%) create mode 100644 spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/array/get_array_struct_fields.sql rename spark/src/test/resources/sql-tests/expressions/{ => array}/size.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => bitwise}/bitwise.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => cast}/cast.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => conditional}/boolean.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => conditional}/case_when.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => conditional}/coalesce.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => conditional}/if_expr.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => conditional}/in_set.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => conditional}/is_not_null.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => conditional}/is_null.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => conditional}/predicates.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/date_add.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/date_diff.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/date_format.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/date_format_enabled.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/date_sub.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/datetime.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/from_unix_time.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/from_unix_time_enabled.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/hour.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/last_day.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/minute.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/second.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/trunc_date.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/trunc_timestamp.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/unix_date.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => datetime}/unix_timestamp.sql (100%) create mode 100644 spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql rename spark/src/test/resources/sql-tests/expressions/{ => hash}/hash.sql (100%) create mode 100644 spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql rename spark/src/test/resources/sql-tests/expressions/{ => map}/map_entries.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => map}/map_from_arrays.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => map}/map_keys.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => map}/map_values.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/abs.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/acos.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/arithmetic.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/asin.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/atan.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/atan2.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/ceil.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/cos.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/cosh.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/cot.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/exp.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/expm1.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/floor.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/isnan.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/log.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/log10.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/log2.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/pow.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/round.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/signum.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/sin.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/sinh.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/sqrt.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/tan.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => math}/tanh.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => misc}/parquet_default_values.sql (100%) create mode 100644 spark/src/test/resources/sql-tests/expressions/misc/scalar_subquery.sql create mode 100644 spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql rename spark/src/test/resources/sql-tests/expressions/{ => string}/ascii.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/bit_length.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/chr.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/concat.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/concat_ws.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/contains.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/ends_with.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/hex.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/init_cap.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/init_cap_enabled.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/left.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/length.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/like.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/lower.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/lower_enabled.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/octet_length.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/regexp_replace.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/regexp_replace_enabled.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/reverse.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/rlike.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/rlike_enabled.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/starts_with.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/string.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/string_instr.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/string_lpad.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/string_repeat.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/string_replace.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/string_rpad.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/string_space.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/string_translate.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/string_trim.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/substring.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/unhex.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/upper.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => string}/upper_enabled.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => struct}/create_named_struct.sql (100%) create mode 100644 spark/src/test/resources/sql-tests/expressions/struct/get_struct_field.sql rename spark/src/test/resources/sql-tests/expressions/{ => struct}/json_to_structs.sql (100%) rename spark/src/test/resources/sql-tests/expressions/{ => struct}/structs_to_json.sql (100%) diff --git a/spark/src/test/resources/sql-tests/expressions/avg.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/avg.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/avg.sql rename to spark/src/test/resources/sql-tests/expressions/aggregate/avg.sql diff --git a/spark/src/test/resources/sql-tests/expressions/bit_agg.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/bit_agg.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/bit_agg.sql rename to spark/src/test/resources/sql-tests/expressions/aggregate/bit_agg.sql diff --git a/spark/src/test/resources/sql-tests/expressions/corr.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/corr.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/corr.sql rename to spark/src/test/resources/sql-tests/expressions/aggregate/corr.sql diff --git a/spark/src/test/resources/sql-tests/expressions/count.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/count.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/count.sql rename to spark/src/test/resources/sql-tests/expressions/aggregate/count.sql diff --git a/spark/src/test/resources/sql-tests/expressions/covariance.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/covariance.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/covariance.sql rename to spark/src/test/resources/sql-tests/expressions/aggregate/covariance.sql diff --git a/spark/src/test/resources/sql-tests/expressions/first_last.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/first_last.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/first_last.sql rename to spark/src/test/resources/sql-tests/expressions/aggregate/first_last.sql diff --git a/spark/src/test/resources/sql-tests/expressions/min_max.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/min_max.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/min_max.sql rename to spark/src/test/resources/sql-tests/expressions/aggregate/min_max.sql diff --git a/spark/src/test/resources/sql-tests/expressions/stddev.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/stddev.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/stddev.sql rename to spark/src/test/resources/sql-tests/expressions/aggregate/stddev.sql diff --git a/spark/src/test/resources/sql-tests/expressions/sum.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/sum.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/sum.sql rename to spark/src/test/resources/sql-tests/expressions/aggregate/sum.sql diff --git a/spark/src/test/resources/sql-tests/expressions/variance.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/variance.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/variance.sql rename to spark/src/test/resources/sql-tests/expressions/aggregate/variance.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array_append.sql b/spark/src/test/resources/sql-tests/expressions/array/array_append.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_append.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_append.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array_compact.sql b/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_compact.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_compact.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array_contains.sql b/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_contains.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_contains.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array_distinct.sql b/spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_distinct.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array_except.sql b/spark/src/test/resources/sql-tests/expressions/array/array_except.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_except.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_except.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_filter.sql b/spark/src/test/resources/sql-tests/expressions/array/array_filter.sql new file mode 100644 index 0000000000..a5cc165815 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_filter.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_filter(arr array) USING parquet + +statement +INSERT INTO test_array_filter VALUES (array(1, 2, 3, 4, 5)), (array(-1, 0, 1)), (array(10)), (NULL) + +query spark_answer_only +SELECT filter(arr, x -> x > 2) FROM test_array_filter + +query spark_answer_only +SELECT filter(arr, x -> x >= 0) FROM test_array_filter + +query spark_answer_only +SELECT filter(arr, (x, i) -> i > 0) FROM test_array_filter diff --git a/spark/src/test/resources/sql-tests/expressions/array_insert.sql b/spark/src/test/resources/sql-tests/expressions/array/array_insert.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_insert.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_insert.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array_intersect.sql b/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_intersect.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array_join.sql b/spark/src/test/resources/sql-tests/expressions/array/array_join.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_join.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_join.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array_max.sql b/spark/src/test/resources/sql-tests/expressions/array/array_max.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_max.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_max.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array_min.sql b/spark/src/test/resources/sql-tests/expressions/array/array_min.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_min.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_min.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array_remove.sql b/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_remove.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_remove.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array_repeat.sql b/spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_repeat.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array_union.sql b/spark/src/test/resources/sql-tests/expressions/array/array_union.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/array_union.sql rename to spark/src/test/resources/sql-tests/expressions/array/array_union.sql diff --git a/spark/src/test/resources/sql-tests/expressions/arrays_overlap.sql b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/arrays_overlap.sql rename to spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql diff --git a/spark/src/test/resources/sql-tests/expressions/create_array.sql b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/create_array.sql rename to spark/src/test/resources/sql-tests/expressions/array/create_array.sql diff --git a/spark/src/test/resources/sql-tests/expressions/element_at.sql b/spark/src/test/resources/sql-tests/expressions/array/element_at.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/element_at.sql rename to spark/src/test/resources/sql-tests/expressions/array/element_at.sql diff --git a/spark/src/test/resources/sql-tests/expressions/flatten.sql b/spark/src/test/resources/sql-tests/expressions/array/flatten.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/flatten.sql rename to spark/src/test/resources/sql-tests/expressions/array/flatten.sql diff --git a/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql b/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql new file mode 100644 index 0000000000..f51e62da3b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_get_array_item(arr array, idx int) USING parquet + +statement +INSERT INTO test_get_array_item VALUES (array(10, 20, 30), 0), (array(10, 20, 30), 1), (array(10, 20, 30), 2), (array(1), 0), (NULL, 0), (array(10, 20), NULL) + +query spark_answer_only +SELECT arr[0], arr[1], arr[2] FROM test_get_array_item + +query spark_answer_only +SELECT arr[idx] FROM test_get_array_item diff --git a/spark/src/test/resources/sql-tests/expressions/array/get_array_struct_fields.sql b/spark/src/test/resources/sql-tests/expressions/array/get_array_struct_fields.sql new file mode 100644 index 0000000000..e65bd297dd --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/get_array_struct_fields.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_arr_struct(arr array>) USING parquet + +statement +INSERT INTO test_arr_struct VALUES (array(named_struct('name', 'a', 'value', 1), named_struct('name', 'b', 'value', 2))), (array(named_struct('name', 'x', 'value', 10))), (NULL) + +query spark_answer_only +SELECT arr.name, arr.value FROM test_arr_struct diff --git a/spark/src/test/resources/sql-tests/expressions/size.sql b/spark/src/test/resources/sql-tests/expressions/array/size.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/size.sql rename to spark/src/test/resources/sql-tests/expressions/array/size.sql diff --git a/spark/src/test/resources/sql-tests/expressions/bitwise.sql b/spark/src/test/resources/sql-tests/expressions/bitwise/bitwise.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/bitwise.sql rename to spark/src/test/resources/sql-tests/expressions/bitwise/bitwise.sql diff --git a/spark/src/test/resources/sql-tests/expressions/cast.sql b/spark/src/test/resources/sql-tests/expressions/cast/cast.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/cast.sql rename to spark/src/test/resources/sql-tests/expressions/cast/cast.sql diff --git a/spark/src/test/resources/sql-tests/expressions/boolean.sql b/spark/src/test/resources/sql-tests/expressions/conditional/boolean.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/boolean.sql rename to spark/src/test/resources/sql-tests/expressions/conditional/boolean.sql diff --git a/spark/src/test/resources/sql-tests/expressions/case_when.sql b/spark/src/test/resources/sql-tests/expressions/conditional/case_when.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/case_when.sql rename to spark/src/test/resources/sql-tests/expressions/conditional/case_when.sql diff --git a/spark/src/test/resources/sql-tests/expressions/coalesce.sql b/spark/src/test/resources/sql-tests/expressions/conditional/coalesce.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/coalesce.sql rename to spark/src/test/resources/sql-tests/expressions/conditional/coalesce.sql diff --git a/spark/src/test/resources/sql-tests/expressions/if_expr.sql b/spark/src/test/resources/sql-tests/expressions/conditional/if_expr.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/if_expr.sql rename to spark/src/test/resources/sql-tests/expressions/conditional/if_expr.sql diff --git a/spark/src/test/resources/sql-tests/expressions/in_set.sql b/spark/src/test/resources/sql-tests/expressions/conditional/in_set.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/in_set.sql rename to spark/src/test/resources/sql-tests/expressions/conditional/in_set.sql diff --git a/spark/src/test/resources/sql-tests/expressions/is_not_null.sql b/spark/src/test/resources/sql-tests/expressions/conditional/is_not_null.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/is_not_null.sql rename to spark/src/test/resources/sql-tests/expressions/conditional/is_not_null.sql diff --git a/spark/src/test/resources/sql-tests/expressions/is_null.sql b/spark/src/test/resources/sql-tests/expressions/conditional/is_null.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/is_null.sql rename to spark/src/test/resources/sql-tests/expressions/conditional/is_null.sql diff --git a/spark/src/test/resources/sql-tests/expressions/predicates.sql b/spark/src/test/resources/sql-tests/expressions/conditional/predicates.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/predicates.sql rename to spark/src/test/resources/sql-tests/expressions/conditional/predicates.sql diff --git a/spark/src/test/resources/sql-tests/expressions/date_add.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_add.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/date_add.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/date_add.sql diff --git a/spark/src/test/resources/sql-tests/expressions/date_diff.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_diff.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/date_diff.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/date_diff.sql diff --git a/spark/src/test/resources/sql-tests/expressions/date_format.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_format.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/date_format.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/date_format.sql diff --git a/spark/src/test/resources/sql-tests/expressions/date_format_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_format_enabled.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/date_format_enabled.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/date_format_enabled.sql diff --git a/spark/src/test/resources/sql-tests/expressions/date_sub.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_sub.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/date_sub.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/date_sub.sql diff --git a/spark/src/test/resources/sql-tests/expressions/datetime.sql b/spark/src/test/resources/sql-tests/expressions/datetime/datetime.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/datetime.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/datetime.sql diff --git a/spark/src/test/resources/sql-tests/expressions/from_unix_time.sql b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/from_unix_time.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql diff --git a/spark/src/test/resources/sql-tests/expressions/from_unix_time_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/from_unix_time_enabled.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql diff --git a/spark/src/test/resources/sql-tests/expressions/hour.sql b/spark/src/test/resources/sql-tests/expressions/datetime/hour.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/hour.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/hour.sql diff --git a/spark/src/test/resources/sql-tests/expressions/last_day.sql b/spark/src/test/resources/sql-tests/expressions/datetime/last_day.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/last_day.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/last_day.sql diff --git a/spark/src/test/resources/sql-tests/expressions/minute.sql b/spark/src/test/resources/sql-tests/expressions/datetime/minute.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/minute.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/minute.sql diff --git a/spark/src/test/resources/sql-tests/expressions/second.sql b/spark/src/test/resources/sql-tests/expressions/datetime/second.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/second.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/second.sql diff --git a/spark/src/test/resources/sql-tests/expressions/trunc_date.sql b/spark/src/test/resources/sql-tests/expressions/datetime/trunc_date.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/trunc_date.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/trunc_date.sql diff --git a/spark/src/test/resources/sql-tests/expressions/trunc_timestamp.sql b/spark/src/test/resources/sql-tests/expressions/datetime/trunc_timestamp.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/trunc_timestamp.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/trunc_timestamp.sql diff --git a/spark/src/test/resources/sql-tests/expressions/unix_date.sql b/spark/src/test/resources/sql-tests/expressions/datetime/unix_date.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/unix_date.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/unix_date.sql diff --git a/spark/src/test/resources/sql-tests/expressions/unix_timestamp.sql b/spark/src/test/resources/sql-tests/expressions/datetime/unix_timestamp.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/unix_timestamp.sql rename to spark/src/test/resources/sql-tests/expressions/datetime/unix_timestamp.sql diff --git a/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql b/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql new file mode 100644 index 0000000000..1fbc362b72 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql @@ -0,0 +1,54 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- Decimal arithmetic exercises CheckOverflow, MakeDecimal, UnscaledValue +statement +CREATE TABLE test_decimal(a decimal(10,2), b decimal(10,2)) USING parquet + +statement +INSERT INTO test_decimal VALUES (10.50, 3.20), (0.00, 1.00), (-10.50, 3.20), (99999999.99, 0.01), (NULL, 1.00) + +query +SELECT a + b, a - b, a * b, a / b FROM test_decimal + +query +SELECT a % b FROM test_decimal + +-- Mixed precision +statement +CREATE TABLE test_decimal_mix(a decimal(18,6), b decimal(10,2)) USING parquet + +statement +INSERT INTO test_decimal_mix VALUES (123456.789012, 99.99), (0.000001, 1.00), (-123456.789012, 0.01), (NULL, NULL) + +query +SELECT a + b, a - b, a * b FROM test_decimal_mix + +query spark_answer_only +SELECT a / b FROM test_decimal_mix + +-- Cast to decimal +statement +CREATE TABLE test_dec_cast(i int, l long, d double, s string) USING parquet + +statement +INSERT INTO test_dec_cast VALUES (42, 123456789, 3.14159, '99.99'), (0, 0, 0.0, '0.00'), (NULL, NULL, NULL, NULL) + +query +SELECT cast(i as decimal(10,2)), cast(l as decimal(18,2)), cast(d as decimal(10,5)), cast(s as decimal(10,2)) FROM test_dec_cast diff --git a/spark/src/test/resources/sql-tests/expressions/hash.sql b/spark/src/test/resources/sql-tests/expressions/hash/hash.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/hash.sql rename to spark/src/test/resources/sql-tests/expressions/hash/hash.sql diff --git a/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql b/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql new file mode 100644 index 0000000000..6a500f6ce2 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_map(m map) USING parquet + +statement +INSERT INTO test_map VALUES (map('a', 1, 'b', 2, 'c', 3)), (map('x', 10)), (NULL) + +query spark_answer_only +SELECT m['a'], m['b'], m['c'] FROM test_map + +query spark_answer_only +SELECT m['x'], m['missing'] FROM test_map diff --git a/spark/src/test/resources/sql-tests/expressions/map_entries.sql b/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/map_entries.sql rename to spark/src/test/resources/sql-tests/expressions/map/map_entries.sql diff --git a/spark/src/test/resources/sql-tests/expressions/map_from_arrays.sql b/spark/src/test/resources/sql-tests/expressions/map/map_from_arrays.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/map_from_arrays.sql rename to spark/src/test/resources/sql-tests/expressions/map/map_from_arrays.sql diff --git a/spark/src/test/resources/sql-tests/expressions/map_keys.sql b/spark/src/test/resources/sql-tests/expressions/map/map_keys.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/map_keys.sql rename to spark/src/test/resources/sql-tests/expressions/map/map_keys.sql diff --git a/spark/src/test/resources/sql-tests/expressions/map_values.sql b/spark/src/test/resources/sql-tests/expressions/map/map_values.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/map_values.sql rename to spark/src/test/resources/sql-tests/expressions/map/map_values.sql diff --git a/spark/src/test/resources/sql-tests/expressions/abs.sql b/spark/src/test/resources/sql-tests/expressions/math/abs.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/abs.sql rename to spark/src/test/resources/sql-tests/expressions/math/abs.sql diff --git a/spark/src/test/resources/sql-tests/expressions/acos.sql b/spark/src/test/resources/sql-tests/expressions/math/acos.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/acos.sql rename to spark/src/test/resources/sql-tests/expressions/math/acos.sql diff --git a/spark/src/test/resources/sql-tests/expressions/arithmetic.sql b/spark/src/test/resources/sql-tests/expressions/math/arithmetic.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/arithmetic.sql rename to spark/src/test/resources/sql-tests/expressions/math/arithmetic.sql diff --git a/spark/src/test/resources/sql-tests/expressions/asin.sql b/spark/src/test/resources/sql-tests/expressions/math/asin.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/asin.sql rename to spark/src/test/resources/sql-tests/expressions/math/asin.sql diff --git a/spark/src/test/resources/sql-tests/expressions/atan.sql b/spark/src/test/resources/sql-tests/expressions/math/atan.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/atan.sql rename to spark/src/test/resources/sql-tests/expressions/math/atan.sql diff --git a/spark/src/test/resources/sql-tests/expressions/atan2.sql b/spark/src/test/resources/sql-tests/expressions/math/atan2.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/atan2.sql rename to spark/src/test/resources/sql-tests/expressions/math/atan2.sql diff --git a/spark/src/test/resources/sql-tests/expressions/ceil.sql b/spark/src/test/resources/sql-tests/expressions/math/ceil.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/ceil.sql rename to spark/src/test/resources/sql-tests/expressions/math/ceil.sql diff --git a/spark/src/test/resources/sql-tests/expressions/cos.sql b/spark/src/test/resources/sql-tests/expressions/math/cos.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/cos.sql rename to spark/src/test/resources/sql-tests/expressions/math/cos.sql diff --git a/spark/src/test/resources/sql-tests/expressions/cosh.sql b/spark/src/test/resources/sql-tests/expressions/math/cosh.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/cosh.sql rename to spark/src/test/resources/sql-tests/expressions/math/cosh.sql diff --git a/spark/src/test/resources/sql-tests/expressions/cot.sql b/spark/src/test/resources/sql-tests/expressions/math/cot.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/cot.sql rename to spark/src/test/resources/sql-tests/expressions/math/cot.sql diff --git a/spark/src/test/resources/sql-tests/expressions/exp.sql b/spark/src/test/resources/sql-tests/expressions/math/exp.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/exp.sql rename to spark/src/test/resources/sql-tests/expressions/math/exp.sql diff --git a/spark/src/test/resources/sql-tests/expressions/expm1.sql b/spark/src/test/resources/sql-tests/expressions/math/expm1.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/expm1.sql rename to spark/src/test/resources/sql-tests/expressions/math/expm1.sql diff --git a/spark/src/test/resources/sql-tests/expressions/floor.sql b/spark/src/test/resources/sql-tests/expressions/math/floor.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/floor.sql rename to spark/src/test/resources/sql-tests/expressions/math/floor.sql diff --git a/spark/src/test/resources/sql-tests/expressions/isnan.sql b/spark/src/test/resources/sql-tests/expressions/math/isnan.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/isnan.sql rename to spark/src/test/resources/sql-tests/expressions/math/isnan.sql diff --git a/spark/src/test/resources/sql-tests/expressions/log.sql b/spark/src/test/resources/sql-tests/expressions/math/log.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/log.sql rename to spark/src/test/resources/sql-tests/expressions/math/log.sql diff --git a/spark/src/test/resources/sql-tests/expressions/log10.sql b/spark/src/test/resources/sql-tests/expressions/math/log10.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/log10.sql rename to spark/src/test/resources/sql-tests/expressions/math/log10.sql diff --git a/spark/src/test/resources/sql-tests/expressions/log2.sql b/spark/src/test/resources/sql-tests/expressions/math/log2.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/log2.sql rename to spark/src/test/resources/sql-tests/expressions/math/log2.sql diff --git a/spark/src/test/resources/sql-tests/expressions/pow.sql b/spark/src/test/resources/sql-tests/expressions/math/pow.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/pow.sql rename to spark/src/test/resources/sql-tests/expressions/math/pow.sql diff --git a/spark/src/test/resources/sql-tests/expressions/round.sql b/spark/src/test/resources/sql-tests/expressions/math/round.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/round.sql rename to spark/src/test/resources/sql-tests/expressions/math/round.sql diff --git a/spark/src/test/resources/sql-tests/expressions/signum.sql b/spark/src/test/resources/sql-tests/expressions/math/signum.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/signum.sql rename to spark/src/test/resources/sql-tests/expressions/math/signum.sql diff --git a/spark/src/test/resources/sql-tests/expressions/sin.sql b/spark/src/test/resources/sql-tests/expressions/math/sin.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/sin.sql rename to spark/src/test/resources/sql-tests/expressions/math/sin.sql diff --git a/spark/src/test/resources/sql-tests/expressions/sinh.sql b/spark/src/test/resources/sql-tests/expressions/math/sinh.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/sinh.sql rename to spark/src/test/resources/sql-tests/expressions/math/sinh.sql diff --git a/spark/src/test/resources/sql-tests/expressions/sqrt.sql b/spark/src/test/resources/sql-tests/expressions/math/sqrt.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/sqrt.sql rename to spark/src/test/resources/sql-tests/expressions/math/sqrt.sql diff --git a/spark/src/test/resources/sql-tests/expressions/tan.sql b/spark/src/test/resources/sql-tests/expressions/math/tan.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/tan.sql rename to spark/src/test/resources/sql-tests/expressions/math/tan.sql diff --git a/spark/src/test/resources/sql-tests/expressions/tanh.sql b/spark/src/test/resources/sql-tests/expressions/math/tanh.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/tanh.sql rename to spark/src/test/resources/sql-tests/expressions/math/tanh.sql diff --git a/spark/src/test/resources/sql-tests/expressions/parquet_default_values.sql b/spark/src/test/resources/sql-tests/expressions/misc/parquet_default_values.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/parquet_default_values.sql rename to spark/src/test/resources/sql-tests/expressions/misc/parquet_default_values.sql diff --git a/spark/src/test/resources/sql-tests/expressions/misc/scalar_subquery.sql b/spark/src/test/resources/sql-tests/expressions/misc/scalar_subquery.sql new file mode 100644 index 0000000000..8b2cad053d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/misc/scalar_subquery.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +statement +CREATE TABLE test_subq(a int, b int) USING parquet + +statement +INSERT INTO test_subq VALUES (1, 10), (2, 20), (3, 30), (NULL, 40) + +query spark_answer_only +SELECT a, (SELECT max(b) FROM test_subq) FROM test_subq + +query spark_answer_only +SELECT a, (SELECT min(b) FROM test_subq) + a FROM test_subq + +query spark_answer_only +SELECT a, b, b > (SELECT avg(b) FROM test_subq) FROM test_subq diff --git a/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql b/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql new file mode 100644 index 0000000000..965405d575 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MinSparkVersion: 3.5 + +statement +CREATE TABLE test_wb(v double) USING parquet + +statement +INSERT INTO test_wb VALUES (0.0), (2.5), (5.0), (7.5), (10.0), (-1.0), (11.0), (NULL) + +query spark_answer_only +SELECT v, width_bucket(v, 0, 10, 4) FROM test_wb + +query spark_answer_only +SELECT v, width_bucket(v, 10, 0, 4) FROM test_wb + +query spark_answer_only +SELECT v, width_bucket(v, 0, 10, 1) FROM test_wb diff --git a/spark/src/test/resources/sql-tests/expressions/ascii.sql b/spark/src/test/resources/sql-tests/expressions/string/ascii.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/ascii.sql rename to spark/src/test/resources/sql-tests/expressions/string/ascii.sql diff --git a/spark/src/test/resources/sql-tests/expressions/bit_length.sql b/spark/src/test/resources/sql-tests/expressions/string/bit_length.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/bit_length.sql rename to spark/src/test/resources/sql-tests/expressions/string/bit_length.sql diff --git a/spark/src/test/resources/sql-tests/expressions/chr.sql b/spark/src/test/resources/sql-tests/expressions/string/chr.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/chr.sql rename to spark/src/test/resources/sql-tests/expressions/string/chr.sql diff --git a/spark/src/test/resources/sql-tests/expressions/concat.sql b/spark/src/test/resources/sql-tests/expressions/string/concat.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/concat.sql rename to spark/src/test/resources/sql-tests/expressions/string/concat.sql diff --git a/spark/src/test/resources/sql-tests/expressions/concat_ws.sql b/spark/src/test/resources/sql-tests/expressions/string/concat_ws.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/concat_ws.sql rename to spark/src/test/resources/sql-tests/expressions/string/concat_ws.sql diff --git a/spark/src/test/resources/sql-tests/expressions/contains.sql b/spark/src/test/resources/sql-tests/expressions/string/contains.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/contains.sql rename to spark/src/test/resources/sql-tests/expressions/string/contains.sql diff --git a/spark/src/test/resources/sql-tests/expressions/ends_with.sql b/spark/src/test/resources/sql-tests/expressions/string/ends_with.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/ends_with.sql rename to spark/src/test/resources/sql-tests/expressions/string/ends_with.sql diff --git a/spark/src/test/resources/sql-tests/expressions/hex.sql b/spark/src/test/resources/sql-tests/expressions/string/hex.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/hex.sql rename to spark/src/test/resources/sql-tests/expressions/string/hex.sql diff --git a/spark/src/test/resources/sql-tests/expressions/init_cap.sql b/spark/src/test/resources/sql-tests/expressions/string/init_cap.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/init_cap.sql rename to spark/src/test/resources/sql-tests/expressions/string/init_cap.sql diff --git a/spark/src/test/resources/sql-tests/expressions/init_cap_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/init_cap_enabled.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/init_cap_enabled.sql rename to spark/src/test/resources/sql-tests/expressions/string/init_cap_enabled.sql diff --git a/spark/src/test/resources/sql-tests/expressions/left.sql b/spark/src/test/resources/sql-tests/expressions/string/left.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/left.sql rename to spark/src/test/resources/sql-tests/expressions/string/left.sql diff --git a/spark/src/test/resources/sql-tests/expressions/length.sql b/spark/src/test/resources/sql-tests/expressions/string/length.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/length.sql rename to spark/src/test/resources/sql-tests/expressions/string/length.sql diff --git a/spark/src/test/resources/sql-tests/expressions/like.sql b/spark/src/test/resources/sql-tests/expressions/string/like.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/like.sql rename to spark/src/test/resources/sql-tests/expressions/string/like.sql diff --git a/spark/src/test/resources/sql-tests/expressions/lower.sql b/spark/src/test/resources/sql-tests/expressions/string/lower.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/lower.sql rename to spark/src/test/resources/sql-tests/expressions/string/lower.sql diff --git a/spark/src/test/resources/sql-tests/expressions/lower_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/lower_enabled.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/lower_enabled.sql rename to spark/src/test/resources/sql-tests/expressions/string/lower_enabled.sql diff --git a/spark/src/test/resources/sql-tests/expressions/octet_length.sql b/spark/src/test/resources/sql-tests/expressions/string/octet_length.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/octet_length.sql rename to spark/src/test/resources/sql-tests/expressions/string/octet_length.sql diff --git a/spark/src/test/resources/sql-tests/expressions/regexp_replace.sql b/spark/src/test/resources/sql-tests/expressions/string/regexp_replace.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/regexp_replace.sql rename to spark/src/test/resources/sql-tests/expressions/string/regexp_replace.sql diff --git a/spark/src/test/resources/sql-tests/expressions/regexp_replace_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/regexp_replace_enabled.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/regexp_replace_enabled.sql rename to spark/src/test/resources/sql-tests/expressions/string/regexp_replace_enabled.sql diff --git a/spark/src/test/resources/sql-tests/expressions/reverse.sql b/spark/src/test/resources/sql-tests/expressions/string/reverse.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/reverse.sql rename to spark/src/test/resources/sql-tests/expressions/string/reverse.sql diff --git a/spark/src/test/resources/sql-tests/expressions/rlike.sql b/spark/src/test/resources/sql-tests/expressions/string/rlike.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/rlike.sql rename to spark/src/test/resources/sql-tests/expressions/string/rlike.sql diff --git a/spark/src/test/resources/sql-tests/expressions/rlike_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/rlike_enabled.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/rlike_enabled.sql rename to spark/src/test/resources/sql-tests/expressions/string/rlike_enabled.sql diff --git a/spark/src/test/resources/sql-tests/expressions/starts_with.sql b/spark/src/test/resources/sql-tests/expressions/string/starts_with.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/starts_with.sql rename to spark/src/test/resources/sql-tests/expressions/string/starts_with.sql diff --git a/spark/src/test/resources/sql-tests/expressions/string.sql b/spark/src/test/resources/sql-tests/expressions/string/string.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/string.sql rename to spark/src/test/resources/sql-tests/expressions/string/string.sql diff --git a/spark/src/test/resources/sql-tests/expressions/string_instr.sql b/spark/src/test/resources/sql-tests/expressions/string/string_instr.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/string_instr.sql rename to spark/src/test/resources/sql-tests/expressions/string/string_instr.sql diff --git a/spark/src/test/resources/sql-tests/expressions/string_lpad.sql b/spark/src/test/resources/sql-tests/expressions/string/string_lpad.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/string_lpad.sql rename to spark/src/test/resources/sql-tests/expressions/string/string_lpad.sql diff --git a/spark/src/test/resources/sql-tests/expressions/string_repeat.sql b/spark/src/test/resources/sql-tests/expressions/string/string_repeat.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/string_repeat.sql rename to spark/src/test/resources/sql-tests/expressions/string/string_repeat.sql diff --git a/spark/src/test/resources/sql-tests/expressions/string_replace.sql b/spark/src/test/resources/sql-tests/expressions/string/string_replace.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/string_replace.sql rename to spark/src/test/resources/sql-tests/expressions/string/string_replace.sql diff --git a/spark/src/test/resources/sql-tests/expressions/string_rpad.sql b/spark/src/test/resources/sql-tests/expressions/string/string_rpad.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/string_rpad.sql rename to spark/src/test/resources/sql-tests/expressions/string/string_rpad.sql diff --git a/spark/src/test/resources/sql-tests/expressions/string_space.sql b/spark/src/test/resources/sql-tests/expressions/string/string_space.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/string_space.sql rename to spark/src/test/resources/sql-tests/expressions/string/string_space.sql diff --git a/spark/src/test/resources/sql-tests/expressions/string_translate.sql b/spark/src/test/resources/sql-tests/expressions/string/string_translate.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/string_translate.sql rename to spark/src/test/resources/sql-tests/expressions/string/string_translate.sql diff --git a/spark/src/test/resources/sql-tests/expressions/string_trim.sql b/spark/src/test/resources/sql-tests/expressions/string/string_trim.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/string_trim.sql rename to spark/src/test/resources/sql-tests/expressions/string/string_trim.sql diff --git a/spark/src/test/resources/sql-tests/expressions/substring.sql b/spark/src/test/resources/sql-tests/expressions/string/substring.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/substring.sql rename to spark/src/test/resources/sql-tests/expressions/string/substring.sql diff --git a/spark/src/test/resources/sql-tests/expressions/unhex.sql b/spark/src/test/resources/sql-tests/expressions/string/unhex.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/unhex.sql rename to spark/src/test/resources/sql-tests/expressions/string/unhex.sql diff --git a/spark/src/test/resources/sql-tests/expressions/upper.sql b/spark/src/test/resources/sql-tests/expressions/string/upper.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/upper.sql rename to spark/src/test/resources/sql-tests/expressions/string/upper.sql diff --git a/spark/src/test/resources/sql-tests/expressions/upper_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/upper_enabled.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/upper_enabled.sql rename to spark/src/test/resources/sql-tests/expressions/string/upper_enabled.sql diff --git a/spark/src/test/resources/sql-tests/expressions/create_named_struct.sql b/spark/src/test/resources/sql-tests/expressions/struct/create_named_struct.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/create_named_struct.sql rename to spark/src/test/resources/sql-tests/expressions/struct/create_named_struct.sql diff --git a/spark/src/test/resources/sql-tests/expressions/struct/get_struct_field.sql b/spark/src/test/resources/sql-tests/expressions/struct/get_struct_field.sql new file mode 100644 index 0000000000..cf569ae121 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/struct/get_struct_field.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_struct(s struct) USING parquet + +statement +INSERT INTO test_struct VALUES (named_struct('name', 'Alice', 'age', 30, 'score', 95.5)), (named_struct('name', 'Bob', 'age', 25, 'score', 88.0)), (NULL) + +query +SELECT s.name, s.age, s.score FROM test_struct + +query +SELECT s.name, s.age + 1, s.score * 2 FROM test_struct diff --git a/spark/src/test/resources/sql-tests/expressions/json_to_structs.sql b/spark/src/test/resources/sql-tests/expressions/struct/json_to_structs.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/json_to_structs.sql rename to spark/src/test/resources/sql-tests/expressions/struct/json_to_structs.sql diff --git a/spark/src/test/resources/sql-tests/expressions/structs_to_json.sql b/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql similarity index 100% rename from spark/src/test/resources/sql-tests/expressions/structs_to_json.sql rename to spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala index e3cbf89333..06ae482a59 100644 --- a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -38,6 +38,14 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + /** Check if the current Spark version meets a minimum version requirement. */ + private def meetsMinSparkVersion(minVersion: String): Boolean = { + val current = org.apache.spark.SPARK_VERSION.split("[.-]").take(2).map(_.toInt) + val required = minVersion.split("[.-]").take(2).map(_.toInt) + (current(0) > required(0)) || + (current(0) == required(0) && current(1) >= required(1)) + } + private val testResourceDir = { val url = getClass.getClassLoader.getResource("sql-tests") assert(url != null, "Could not find sql-tests resource directory") @@ -95,18 +103,29 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { val parsed = SqlFileTestParser.parse(file) val combinations = configCombinations(parsed.configMatrix) + // Skip tests that require a newer Spark version + val skip = parsed.minSparkVersion.exists(!meetsMinSparkVersion(_)) + if (combinations.size <= 1) { // No matrix or single combination test(s"sql-file: $relativePath") { - val effectiveConfigs = parsed.configs ++ combinations.headOption.getOrElse(Seq.empty) - runTestFile(parsed.copy(configs = effectiveConfigs)) + if (skip) { + logInfo(s"SKIPPED (requires Spark ${parsed.minSparkVersion.get}): $relativePath") + } else { + val effectiveConfigs = parsed.configs ++ combinations.headOption.getOrElse(Seq.empty) + runTestFile(parsed.copy(configs = effectiveConfigs)) + } } } else { // Multiple combinations: generate one test per combination combinations.foreach { matrixConfigs => val label = matrixConfigs.map { case (k, v) => s"$k=$v" }.mkString(", ") test(s"sql-file: $relativePath [$label]") { - runTestFile(parsed.copy(configs = parsed.configs ++ matrixConfigs)) + if (skip) { + logInfo(s"SKIPPED (requires Spark ${parsed.minSparkVersion.get}): $relativePath") + } else { + runTestFile(parsed.copy(configs = parsed.configs ++ matrixConfigs)) + } } } } diff --git a/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala b/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala index 1d18f9d64a..1d68972411 100644 --- a/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala +++ b/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala @@ -50,17 +50,21 @@ case class Ignore(reason: String) extends QueryMode * Ordered list of statements and queries. * @param tables * Table names extracted from CREATE TABLE statements (for cleanup). + * @param minSparkVersion + * Optional minimum Spark version required to run this test (e.g. "3.5"). */ case class SqlTestFile( configs: Seq[(String, String)], configMatrix: Seq[(String, Seq[String])], records: Seq[SqlTestRecord], - tables: Seq[String]) + tables: Seq[String], + minSparkVersion: Option[String] = None) object SqlFileTestParser { private val ConfigPattern = """--\s*Config:\s*(.+)=(.+)""".r private val ConfigMatrixPattern = """--\s*ConfigMatrix:\s*(.+)=(.+)""".r + private val MinSparkVersionPattern = """--\s*MinSparkVersion:\s*(.+)""".r private val CreateTablePattern = """(?i)CREATE\s+TABLE\s+(\w+)""".r.unanchored def parse(file: File): SqlTestFile = { @@ -75,6 +79,7 @@ object SqlFileTestParser { def parse(lines: Seq[String]): SqlTestFile = { var configs = Seq.empty[(String, String)] var configMatrix = Seq.empty[(String, Seq[String])] + var minSparkVersion: Option[String] = None val records = Seq.newBuilder[SqlTestRecord] val tables = Seq.newBuilder[String] @@ -91,6 +96,10 @@ object SqlFileTestParser { configMatrix :+= (key.trim -> values.split(",").map(_.trim).toSeq) i += 1 + case MinSparkVersionPattern(version) => + minSparkVersion = Some(version.trim) + i += 1 + case "statement" => i += 1 val (sql, nextIdx) = collectSql(lines, i) @@ -112,7 +121,7 @@ object SqlFileTestParser { } } - SqlTestFile(configs, configMatrix, records.result(), tables.result()) + SqlTestFile(configs, configMatrix, records.result(), tables.result(), minSparkVersion) } private val FallbackPattern = """query\s+expect_fallback\((.+)\)""".r From 53fc89756a560f29429e639b040da56a0b612111 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 29 Jan 2026 10:40:13 -0700 Subject: [PATCH 6/9] Enable allowIncompatible for decimal cast test to run natively in Comet Co-Authored-By: Claude Opus 4.5 --- .../test/resources/sql-tests/expressions/decimal/decimal_ops.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql b/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql index 1fbc362b72..4cffb1b6d9 100644 --- a/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql +++ b/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql @@ -15,6 +15,7 @@ -- specific language governing permissions and limitations -- under the License. +-- Config: spark.comet.expression.Cast.allowIncompatible=true -- ConfigMatrix: parquet.enable.dictionary=false,true -- Decimal arithmetic exercises CheckOverflow, MakeDecimal, UnscaledValue From 9a711b4f791e4369978ca94e29bc2177770f9ee1 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 29 Jan 2026 10:59:03 -0700 Subject: [PATCH 7/9] Ignore failing SQL file tests and rename configCombinations to configMatrix Ignore get_array_item dynamic index test (#3332) and width_bucket tests (#3331) with links to tracking issues. Rename configCombinations method to configMatrix for clarity. Co-Authored-By: Claude Opus 4.5 --- .../sql-tests/expressions/array/get_array_item.sql | 2 +- .../resources/sql-tests/expressions/misc/width_bucket.sql | 6 +++--- .../scala/org/apache/comet/CometSqlFileTestSuite.scala | 7 +++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql b/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql index f51e62da3b..4fcb57f09a 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql @@ -26,5 +26,5 @@ INSERT INTO test_get_array_item VALUES (array(10, 20, 30), 0), (array(10, 20, 30 query spark_answer_only SELECT arr[0], arr[1], arr[2] FROM test_get_array_item -query spark_answer_only +query ignore(https://github.com/apache/datafusion-comet/issues/3332) SELECT arr[idx] FROM test_get_array_item diff --git a/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql b/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql index 965405d575..6497ff3812 100644 --- a/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql +++ b/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql @@ -23,11 +23,11 @@ CREATE TABLE test_wb(v double) USING parquet statement INSERT INTO test_wb VALUES (0.0), (2.5), (5.0), (7.5), (10.0), (-1.0), (11.0), (NULL) -query spark_answer_only +query ignore(https://github.com/apache/datafusion-comet/issues/3331) SELECT v, width_bucket(v, 0, 10, 4) FROM test_wb -query spark_answer_only +query ignore(https://github.com/apache/datafusion-comet/issues/3331) SELECT v, width_bucket(v, 10, 0, 4) FROM test_wb -query spark_answer_only +query ignore(https://github.com/apache/datafusion-comet/issues/3331) SELECT v, width_bucket(v, 0, 10, 1) FROM test_wb diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala index 06ae482a59..1cab6d4269 100644 --- a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -61,11 +61,10 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { } /** Generate all config combinations from a ConfigMatrix specification. */ - private def configCombinations( - matrix: Seq[(String, Seq[String])]): Seq[Seq[(String, String)]] = { + private def configMatrix(matrix: Seq[(String, Seq[String])]): Seq[Seq[(String, String)]] = { if (matrix.isEmpty) return Seq(Seq.empty) val (key, values) = matrix.head - val rest = configCombinations(matrix.tail) + val rest = configMatrix(matrix.tail) for { value <- values combo <- rest @@ -101,7 +100,7 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { discoverTestFiles(testResourceDir).foreach { file => val relativePath = testResourceDir.toURI.relativize(file.toURI).getPath val parsed = SqlFileTestParser.parse(file) - val combinations = configCombinations(parsed.configMatrix) + val combinations = configMatrix(parsed.configMatrix) // Skip tests that require a newer Spark version val skip = parsed.minSparkVersion.exists(!meetsMinSparkVersion(_)) From 70d072e22962eac1808dd9271db5d1a366bb6316 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 29 Jan 2026 11:07:46 -0700 Subject: [PATCH 8/9] Address PR feedback: rename QueryMode, CheckOperator, and i variable - Rename QueryMode to QueryAssertionMode - Rename CheckOperator to CheckCoverageAndAnswer - Rename loop variable i to lineIdx in SqlFileTestParser Co-Authored-By: Claude Opus 4.5 --- .../apache/comet/CometSqlFileTestSuite.scala | 2 +- .../org/apache/comet/SqlFileTestParser.scala | 59 ++++++++++--------- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala index 1cab6d4269..4fbeba349e 100644 --- a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -80,7 +80,7 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { spark.sql(sql) case SqlQuery(sql, mode) => mode match { - case CheckOperator => + case CheckCoverageAndAnswer => checkSparkAnswerAndOperator(sql) case SparkAnswerOnly => checkSparkAnswer(sql) diff --git a/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala b/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala index 1d68972411..62a349cdea 100644 --- a/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala +++ b/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala @@ -30,14 +30,15 @@ sealed trait SqlTestRecord case class SqlStatement(sql: String) extends SqlTestRecord /** A SQL query whose results are compared between Spark and Comet. */ -case class SqlQuery(sql: String, mode: QueryMode = CheckOperator) extends SqlTestRecord +case class SqlQuery(sql: String, mode: QueryAssertionMode = CheckCoverageAndAnswer) + extends SqlTestRecord -sealed trait QueryMode -case object CheckOperator extends QueryMode -case object SparkAnswerOnly extends QueryMode -case class WithTolerance(tol: Double) extends QueryMode -case class ExpectFallback(reason: String) extends QueryMode -case class Ignore(reason: String) extends QueryMode +sealed trait QueryAssertionMode +case object CheckCoverageAndAnswer extends QueryAssertionMode +case object SparkAnswerOnly extends QueryAssertionMode +case class WithTolerance(tol: Double) extends QueryAssertionMode +case class ExpectFallback(reason: String) extends QueryAssertionMode +case class Ignore(reason: String) extends QueryAssertionMode /** * Parsed representation of a .sql test file. @@ -83,41 +84,41 @@ object SqlFileTestParser { val records = Seq.newBuilder[SqlTestRecord] val tables = Seq.newBuilder[String] - var i = 0 - while (i < lines.length) { - val line = lines(i).trim + var lineIdx = 0 + while (lineIdx < lines.length) { + val line = lines(lineIdx).trim line match { case ConfigPattern(key, value) => configs :+= (key.trim -> value.trim) - i += 1 + lineIdx += 1 case ConfigMatrixPattern(key, values) => configMatrix :+= (key.trim -> values.split(",").map(_.trim).toSeq) - i += 1 + lineIdx += 1 case MinSparkVersionPattern(version) => minSparkVersion = Some(version.trim) - i += 1 + lineIdx += 1 case "statement" => - i += 1 - val (sql, nextIdx) = collectSql(lines, i) + lineIdx += 1 + val (sql, nextIdx) = collectSql(lines, lineIdx) // Extract table names for cleanup CreateTablePattern.findFirstMatchIn(sql).foreach(m => tables += m.group(1)) records += SqlStatement(sql) - i = nextIdx + lineIdx = nextIdx case s if s.startsWith("query") => - val mode = parseQueryMode(s) - i += 1 - val (sql, nextIdx) = collectSql(lines, i) + val mode = parseQueryAssertionMode(s) + lineIdx += 1 + val (sql, nextIdx) = collectSql(lines, lineIdx) records += SqlQuery(sql, mode) - i = nextIdx + lineIdx = nextIdx case _ => // Skip blank lines and comments - i += 1 + lineIdx += 1 } } @@ -127,7 +128,7 @@ object SqlFileTestParser { private val FallbackPattern = """query\s+expect_fallback\((.+)\)""".r private val IgnorePattern = """query\s+ignore\((.+)\)""".r - private def parseQueryMode(directive: String): QueryMode = { + private def parseQueryAssertionMode(directive: String): QueryAssertionMode = { directive match { case FallbackPattern(reason) => ExpectFallback(reason.trim) @@ -135,12 +136,12 @@ object SqlFileTestParser { Ignore(reason.trim) case _ => val parts = directive.split("\\s+") - if (parts.length == 1) return CheckOperator + if (parts.length == 1) return CheckCoverageAndAnswer parts(1) match { case "spark_answer_only" => SparkAnswerOnly case s if s.startsWith("tolerance=") => WithTolerance(s.stripPrefix("tolerance=").toDouble) - case _ => CheckOperator + case _ => CheckCoverageAndAnswer } } } @@ -148,12 +149,12 @@ object SqlFileTestParser { /** Collect SQL lines until a blank line or end of file. */ private def collectSql(lines: Seq[String], start: Int): (String, Int) = { val sb = new StringBuilder - var i = start - while (i < lines.length && lines(i).trim.nonEmpty) { + var lineIdx = start + while (lineIdx < lines.length && lines(lineIdx).trim.nonEmpty) { if (sb.nonEmpty) sb.append("\n") - sb.append(lines(i)) - i += 1 + sb.append(lines(lineIdx)) + lineIdx += 1 } - (sb.toString, i) + (sb.toString, lineIdx) } } From b79459f42fb960902156794565f43890e90e11c9 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 29 Jan 2026 15:49:47 -0700 Subject: [PATCH 9/9] Add literal argument tests for all SQL file expression tests Add literal value test coverage for all expression test files to ensure Comet's native engine handles literal arguments correctly, not just column references. For N-argument functions, test all combinations of literal/column arguments (col+lit, lit+col, lit+lit, etc.). Disable Spark's ConstantFolding optimizer rule in the test runner so literal-only expressions reach Comet's native engine rather than being folded away at plan time. Mark queries that expose native engine bugs with ignore directives linked to filed GitHub issues: - #3336: datetime scalar panics (hour, minute, second, unix_timestamp) - #3337: Substring/StringSpace scalar panics - #3338: array literal index out of bounds - #3339: concat_ws NULL separator crash - #3340: sha2 literal crash - #3341: bit_count scalar panic - #3342: DateTrunc/TimestampTrunc literal crash - #3343: all-literal RLIKE crash - #3344: replace() empty-string search wrong result - #3345: array_contains literal result mismatch - #3346: array_contains empty array returns null instead of false Mark expected fallback cases with expect_fallback directives: - lpad/rpad: scalar str argument not supported - initcap: moved literal tests to init_cap_enabled.sql Co-Authored-By: Claude Opus 4.5 --- .../sql-tests/expressions/array/array_append.sql | 12 ++++++++++++ .../expressions/array/array_compact.sql | 4 ++++ .../expressions/array/array_contains.sql | 12 ++++++++++++ .../expressions/array/array_distinct.sql | 4 ++++ .../sql-tests/expressions/array/array_except.sql | 12 ++++++++++++ .../expressions/array/array_intersect.sql | 12 ++++++++++++ .../sql-tests/expressions/array/array_max.sql | 4 ++++ .../sql-tests/expressions/array/array_min.sql | 4 ++++ .../sql-tests/expressions/array/array_remove.sql | 12 ++++++++++++ .../sql-tests/expressions/array/array_repeat.sql | 12 ++++++++++++ .../sql-tests/expressions/array/array_union.sql | 12 ++++++++++++ .../expressions/array/arrays_overlap.sql | 12 ++++++++++++ .../sql-tests/expressions/array/element_at.sql | 4 ++++ .../sql-tests/expressions/array/flatten.sql | 4 ++++ .../expressions/array/get_array_item.sql | 4 ++++ .../sql-tests/expressions/array/size.sql | 4 ++++ .../sql-tests/expressions/bitwise/bitwise.sql | 13 +++++++++++++ .../sql-tests/expressions/cast/cast.sql | 4 ++++ .../expressions/conditional/case_when.sql | 4 ++++ .../expressions/conditional/coalesce.sql | 4 ++++ .../expressions/conditional/if_expr.sql | 4 ++++ .../expressions/conditional/predicates.sql | 12 ++++++++++++ .../sql-tests/expressions/datetime/date_add.sql | 12 ++++++++++++ .../sql-tests/expressions/datetime/date_diff.sql | 12 ++++++++++++ .../expressions/datetime/date_format.sql | 4 ++++ .../expressions/datetime/date_format_enabled.sql | 4 ++++ .../sql-tests/expressions/datetime/date_sub.sql | 12 ++++++++++++ .../expressions/datetime/from_unix_time.sql | 7 +++++++ .../datetime/from_unix_time_enabled.sql | 7 +++++++ .../sql-tests/expressions/datetime/hour.sql | 4 ++++ .../sql-tests/expressions/datetime/last_day.sql | 4 ++++ .../sql-tests/expressions/datetime/minute.sql | 4 ++++ .../sql-tests/expressions/datetime/second.sql | 4 ++++ .../expressions/datetime/trunc_date.sql | 4 ++++ .../expressions/datetime/trunc_timestamp.sql | 4 ++++ .../sql-tests/expressions/datetime/unix_date.sql | 4 ++++ .../expressions/datetime/unix_timestamp.sql | 4 ++++ .../expressions/decimal/decimal_ops.sql | 16 ++++++++++++++++ .../sql-tests/expressions/hash/hash.sql | 4 ++++ .../sql-tests/expressions/map/get_map_value.sql | 4 ++++ .../expressions/map/map_from_arrays.sql | 4 ++++ .../resources/sql-tests/expressions/math/abs.sql | 4 ++++ .../sql-tests/expressions/math/acos.sql | 4 ++++ .../sql-tests/expressions/math/arithmetic.sql | 16 ++++++++++++++++ .../sql-tests/expressions/math/asin.sql | 4 ++++ .../sql-tests/expressions/math/atan.sql | 4 ++++ .../sql-tests/expressions/math/atan2.sql | 12 ++++++++++++ .../sql-tests/expressions/math/ceil.sql | 4 ++++ .../resources/sql-tests/expressions/math/cos.sql | 4 ++++ .../sql-tests/expressions/math/cosh.sql | 4 ++++ .../resources/sql-tests/expressions/math/cot.sql | 4 ++++ .../resources/sql-tests/expressions/math/exp.sql | 4 ++++ .../sql-tests/expressions/math/expm1.sql | 4 ++++ .../sql-tests/expressions/math/floor.sql | 4 ++++ .../sql-tests/expressions/math/isnan.sql | 4 ++++ .../resources/sql-tests/expressions/math/log.sql | 12 ++++++++++++ .../sql-tests/expressions/math/log10.sql | 4 ++++ .../sql-tests/expressions/math/log2.sql | 4 ++++ .../resources/sql-tests/expressions/math/pow.sql | 12 ++++++++++++ .../sql-tests/expressions/math/round.sql | 4 ++++ .../sql-tests/expressions/math/signum.sql | 4 ++++ .../resources/sql-tests/expressions/math/sin.sql | 4 ++++ .../sql-tests/expressions/math/sinh.sql | 4 ++++ .../sql-tests/expressions/math/sqrt.sql | 4 ++++ .../resources/sql-tests/expressions/math/tan.sql | 4 ++++ .../sql-tests/expressions/math/tanh.sql | 4 ++++ .../sql-tests/expressions/misc/width_bucket.sql | 4 ++++ .../sql-tests/expressions/string/ascii.sql | 4 ++++ .../sql-tests/expressions/string/bit_length.sql | 4 ++++ .../sql-tests/expressions/string/chr.sql | 4 ++++ .../sql-tests/expressions/string/concat.sql | 8 ++++++++ .../sql-tests/expressions/string/concat_ws.sql | 4 ++++ .../sql-tests/expressions/string/contains.sql | 12 ++++++++++++ .../sql-tests/expressions/string/ends_with.sql | 12 ++++++++++++ .../sql-tests/expressions/string/hex.sql | 4 ++++ .../expressions/string/init_cap_enabled.sql | 4 ++++ .../sql-tests/expressions/string/left.sql | 12 ++++++++++++ .../sql-tests/expressions/string/length.sql | 4 ++++ .../sql-tests/expressions/string/like.sql | 4 ++++ .../sql-tests/expressions/string/lower.sql | 4 ++++ .../expressions/string/octet_length.sql | 4 ++++ .../string/regexp_replace_enabled.sql | 4 ++++ .../sql-tests/expressions/string/reverse.sql | 4 ++++ .../expressions/string/rlike_enabled.sql | 4 ++++ .../sql-tests/expressions/string/starts_with.sql | 12 ++++++++++++ .../expressions/string/string_instr.sql | 12 ++++++++++++ .../sql-tests/expressions/string/string_lpad.sql | 8 ++++++++ .../expressions/string/string_repeat.sql | 12 ++++++++++++ .../expressions/string/string_replace.sql | 14 +++++++++++++- .../sql-tests/expressions/string/string_rpad.sql | 8 ++++++++ .../expressions/string/string_space.sql | 4 ++++ .../expressions/string/string_translate.sql | 12 ++++++++++++ .../sql-tests/expressions/string/string_trim.sql | 7 +++++++ .../sql-tests/expressions/string/substring.sql | 4 ++++ .../sql-tests/expressions/string/unhex.sql | 4 ++++ .../sql-tests/expressions/string/upper.sql | 4 ++++ .../expressions/struct/create_named_struct.sql | 7 +++++++ .../expressions/struct/json_to_structs.sql | 4 ++++ .../expressions/struct/structs_to_json.sql | 4 ++++ .../org/apache/comet/CometSqlFileTestSuite.scala | 8 +++++++- 100 files changed, 645 insertions(+), 2 deletions(-) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_append.sql b/spark/src/test/resources/sql-tests/expressions/array/array_append.sql index 9d3a4e5c7d..2efd13f1c2 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_append.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_append.sql @@ -25,3 +25,15 @@ INSERT INTO test_array_append VALUES (array(1, 2, 3), 4), (array(), 1), (NULL, 1 query spark_answer_only SELECT array_append(arr, val) FROM test_array_append + +-- column + literal +query spark_answer_only +SELECT array_append(arr, 99) FROM test_array_append + +-- literal + column +query spark_answer_only +SELECT array_append(array(1, 2, 3), val) FROM test_array_append + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_append(array(1, 2, 3), 4), array_append(array(), 1), array_append(cast(NULL as array), 1) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql b/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql index f8baddd14f..9b834a4dbd 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql @@ -25,3 +25,7 @@ INSERT INTO test_array_compact VALUES (array(1, NULL, 2, NULL, 3)), (array()), ( query spark_answer_only SELECT array_compact(arr) FROM test_array_compact + +-- literal arguments +query spark_answer_only +SELECT array_compact(array(1, NULL, 2, NULL, 3)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql b/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql index 11c72880f0..86ad0cc488 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql @@ -25,3 +25,15 @@ INSERT INTO test_array_contains VALUES (array(1, 2, 3), 2), (array(1, 2, 3), 4), query spark_answer_only SELECT array_contains(arr, val) FROM test_array_contains + +-- column + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3346) +SELECT array_contains(arr, 2) FROM test_array_contains + +-- literal + column +query spark_answer_only +SELECT array_contains(array(1, 2, 3), val) FROM test_array_contains + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3345) +SELECT array_contains(array(1, 2, 3), 2), array_contains(array(1, 2, 3), 4), array_contains(array(), 1), array_contains(cast(NULL as array), 1) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql b/spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql index 283eb6da80..43c18e9889 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql @@ -25,3 +25,7 @@ INSERT INTO test_array_distinct VALUES (array(1, 2, 2, 3, 3)), (array()), (NULL) query spark_answer_only SELECT array_distinct(arr) FROM test_array_distinct + +-- literal arguments +query spark_answer_only +SELECT array_distinct(array(1, 2, 2, 3, 3)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_except.sql b/spark/src/test/resources/sql-tests/expressions/array/array_except.sql index 7a7e9cfc63..110f798f97 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_except.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_except.sql @@ -25,3 +25,15 @@ INSERT INTO test_array_except VALUES (array(1, 2, 3), array(2, 3, 4)), (array(1, query spark_answer_only SELECT array_except(a, b) FROM test_array_except + +-- column + literal +query spark_answer_only +SELECT array_except(a, array(2, 3)) FROM test_array_except + +-- literal + column +query spark_answer_only +SELECT array_except(array(1, 2, 3), b) FROM test_array_except + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_except(array(1, 2, 3), array(2, 3, 4)), array_except(array(1, 2), array()), array_except(array(), array(1)), array_except(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql b/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql index cb78703e66..379f16a642 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql @@ -25,3 +25,15 @@ INSERT INTO test_array_intersect VALUES (array(1, 2, 3), array(2, 3, 4)), (array query spark_answer_only SELECT array_intersect(a, b) FROM test_array_intersect + +-- column + literal +query spark_answer_only +SELECT array_intersect(a, array(2, 3)) FROM test_array_intersect + +-- literal + column +query spark_answer_only +SELECT array_intersect(array(1, 2, 3), b) FROM test_array_intersect + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_intersect(array(1, 2, 3), array(2, 3, 4)), array_intersect(array(1, 2), array(3, 4)), array_intersect(array(), array(1)), array_intersect(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_max.sql b/spark/src/test/resources/sql-tests/expressions/array/array_max.sql index f04e1e1935..17c572b952 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_max.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_max.sql @@ -25,3 +25,7 @@ INSERT INTO test_array_max VALUES (array(1, 2, 3)), (array(3, 1, 2)), (array()), query spark_answer_only SELECT array_max(arr) FROM test_array_max + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_max(array(1, 2, 3)), array_max(array()), array_max(cast(NULL as array)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_min.sql b/spark/src/test/resources/sql-tests/expressions/array/array_min.sql index 6ab7b8f173..eb86d6c15f 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_min.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_min.sql @@ -25,3 +25,7 @@ INSERT INTO test_array_min VALUES (array(1, 2, 3)), (array(3, 1, 2)), (array()), query spark_answer_only SELECT array_min(arr) FROM test_array_min + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_min(array(1, 2, 3)), array_min(array()), array_min(cast(NULL as array)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql b/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql index 70d0f254d1..bd1af4bc24 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql @@ -25,3 +25,15 @@ INSERT INTO test_array_remove VALUES (array(1, 2, 3, 2), 2), (array(1, 2, 3), 4) query spark_answer_only SELECT array_remove(arr, val) FROM test_array_remove + +-- column + literal +query spark_answer_only +SELECT array_remove(arr, 2) FROM test_array_remove + +-- literal + column +query spark_answer_only +SELECT array_remove(array(1, 2, 3, 2), val) FROM test_array_remove + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_remove(array(1, 2, 3, 2), 2), array_remove(array(1, 2, 3), 4), array_remove(array(), 1), array_remove(cast(NULL as array), 1) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql b/spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql index 5108e7288e..926b1141ef 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql @@ -25,3 +25,15 @@ INSERT INTO test_array_repeat VALUES (1, 3), (NULL, 3), (1, 0), (1, -1), (1, NUL query spark_answer_only SELECT array_repeat(val, cnt) FROM test_array_repeat + +-- column + literal +query spark_answer_only +SELECT array_repeat(val, 3) FROM test_array_repeat + +-- literal + column +query spark_answer_only +SELECT array_repeat(1, cnt) FROM test_array_repeat + +-- literal + literal +query spark_answer_only +SELECT array_repeat(1, 3), array_repeat(NULL, 3), array_repeat(1, 0), array_repeat(1, -1) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_union.sql b/spark/src/test/resources/sql-tests/expressions/array/array_union.sql index 9333ff7b95..4c699ae546 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_union.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_union.sql @@ -25,3 +25,15 @@ INSERT INTO test_array_union VALUES (array(1, 2, 3), array(3, 4, 5)), (array(1, query spark_answer_only SELECT array_union(a, b) FROM test_array_union + +-- column + literal +query spark_answer_only +SELECT array_union(a, array(3, 4, 5)) FROM test_array_union + +-- literal + column +query spark_answer_only +SELECT array_union(array(1, 2, 3), b) FROM test_array_union + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_union(array(1, 2, 3), array(3, 4, 5)), array_union(array(1, 2), array()), array_union(array(), array(1)), array_union(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql index 72561978c0..6c28224ed6 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql @@ -25,3 +25,15 @@ INSERT INTO test_arrays_overlap VALUES (array(1, 2, 3), array(3, 4, 5)), (array( query spark_answer_only SELECT arrays_overlap(a, b) FROM test_arrays_overlap + +-- column + literal +query spark_answer_only +SELECT arrays_overlap(a, array(3, 4, 5)) FROM test_arrays_overlap + +-- literal + column +query spark_answer_only +SELECT arrays_overlap(array(1, 2, 3), b) FROM test_arrays_overlap + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)), arrays_overlap(array(1, 2), array(3, 4)), arrays_overlap(array(), array(1)), arrays_overlap(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/element_at.sql b/spark/src/test/resources/sql-tests/expressions/array/element_at.sql index fabbe6f298..a2e98849fc 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/element_at.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/element_at.sql @@ -25,3 +25,7 @@ INSERT INTO test_element_at VALUES (array(1, 2, 3)), (array(10)), (NULL) query spark_answer_only SELECT element_at(arr, 1), element_at(arr, -1) FROM test_element_at + +-- literal arguments +query ignore(Spark codegen bug with literal element_at when constant folding is disabled) +SELECT element_at(array(1, 2, 3), 1), element_at(array(1, 2, 3), -1) diff --git a/spark/src/test/resources/sql-tests/expressions/array/flatten.sql b/spark/src/test/resources/sql-tests/expressions/array/flatten.sql index 2ff67a8da7..effc123605 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/flatten.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/flatten.sql @@ -25,3 +25,7 @@ INSERT INTO test_flatten VALUES (array(array(1, 2), array(3, 4))), (array(array( query spark_answer_only SELECT flatten(arr) FROM test_flatten + +-- literal arguments +query spark_answer_only +SELECT flatten(array(array(1, 2), array(3, 4))) diff --git a/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql b/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql index 4fcb57f09a..99e6d68f9d 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql @@ -28,3 +28,7 @@ SELECT arr[0], arr[1], arr[2] FROM test_get_array_item query ignore(https://github.com/apache/datafusion-comet/issues/3332) SELECT arr[idx] FROM test_get_array_item + +-- literal arguments +query spark_answer_only +SELECT array(10, 20, 30)[0], array(10, 20, 30)[2], array()[0] diff --git a/spark/src/test/resources/sql-tests/expressions/array/size.sql b/spark/src/test/resources/sql-tests/expressions/array/size.sql index 9215747af7..a096502bc5 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/size.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/size.sql @@ -25,3 +25,7 @@ INSERT INTO test_size VALUES (array(1, 2, 3), map('a', 1, 'b', 2)), (array(), ma query spark_answer_only SELECT size(arr), size(m) FROM test_size + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT size(array(1, 2, 3)), size(array()), size(cast(NULL as array)) diff --git a/spark/src/test/resources/sql-tests/expressions/bitwise/bitwise.sql b/spark/src/test/resources/sql-tests/expressions/bitwise/bitwise.sql index 88b70930da..640aa1e990 100644 --- a/spark/src/test/resources/sql-tests/expressions/bitwise/bitwise.sql +++ b/spark/src/test/resources/sql-tests/expressions/bitwise/bitwise.sql @@ -68,3 +68,16 @@ INSERT INTO test_bit_get VALUES (11, 0), (11, 1), (11, 2), (11, 3), (0, 0), (NUL query spark_answer_only SELECT bit_get(i, pos) FROM test_bit_get + +-- literal arguments +query +SELECT 1111 & 2, 1111 | 2, 1111 ^ 2 + +query ignore(https://github.com/apache/datafusion-comet/issues/3341) +SELECT bit_count(0), bit_count(7), bit_count(-1) + +query spark_answer_only +SELECT bit_get(11, 0), bit_get(11, 1), bit_get(11, 2), bit_get(11, 3) + +query +SELECT shiftright(1111, 2), shiftleft(1111, 2) diff --git a/spark/src/test/resources/sql-tests/expressions/cast/cast.sql b/spark/src/test/resources/sql-tests/expressions/cast/cast.sql index 18375bc0b8..e848e0e636 100644 --- a/spark/src/test/resources/sql-tests/expressions/cast/cast.sql +++ b/spark/src/test/resources/sql-tests/expressions/cast/cast.sql @@ -40,3 +40,7 @@ SELECT cast(s as int), cast(s as double) FROM test_cast query SELECT cast(b as int), cast(b as string), cast(i as boolean) FROM test_cast + +-- literal arguments +query +SELECT cast(1 as long), cast(1 as double), cast(1 as string), cast('123' as int), cast('3.14' as double), cast(true as int), cast(NULL as int) diff --git a/spark/src/test/resources/sql-tests/expressions/conditional/case_when.sql b/spark/src/test/resources/sql-tests/expressions/conditional/case_when.sql index ee0d651c42..eff3123743 100644 --- a/spark/src/test/resources/sql-tests/expressions/conditional/case_when.sql +++ b/spark/src/test/resources/sql-tests/expressions/conditional/case_when.sql @@ -31,3 +31,7 @@ SELECT CASE i WHEN 1 THEN 'one' WHEN 2 THEN 'two' END FROM test_case_when query SELECT CASE WHEN s IS NULL THEN 'null_val' ELSE s END FROM test_case_when + +-- literal arguments +query +SELECT CASE WHEN i = 1 THEN s WHEN i = 2 THEN 'fixed' ELSE s END FROM test_case_when diff --git a/spark/src/test/resources/sql-tests/expressions/conditional/coalesce.sql b/spark/src/test/resources/sql-tests/expressions/conditional/coalesce.sql index c683a1b3a9..26fa6a2553 100644 --- a/spark/src/test/resources/sql-tests/expressions/conditional/coalesce.sql +++ b/spark/src/test/resources/sql-tests/expressions/conditional/coalesce.sql @@ -31,3 +31,7 @@ SELECT coalesce(a) FROM test_coalesce query SELECT coalesce(a, 99) FROM test_coalesce + +-- literal arguments +query +SELECT coalesce(NULL, NULL, 99), coalesce(1, NULL, 99), coalesce(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/conditional/if_expr.sql b/spark/src/test/resources/sql-tests/expressions/conditional/if_expr.sql index 8dd8c8b090..ecc17f6b55 100644 --- a/spark/src/test/resources/sql-tests/expressions/conditional/if_expr.sql +++ b/spark/src/test/resources/sql-tests/expressions/conditional/if_expr.sql @@ -28,3 +28,7 @@ SELECT IF(cond, a, b) FROM test_if query SELECT IF(a > 0, 'positive', 'non-positive') FROM test_if + +-- literal arguments +query +SELECT IF(true, 1, 2), IF(false, 1, 2), IF(NULL, 1, 2) diff --git a/spark/src/test/resources/sql-tests/expressions/conditional/predicates.sql b/spark/src/test/resources/sql-tests/expressions/conditional/predicates.sql index 35498d0f32..089bcc30fc 100644 --- a/spark/src/test/resources/sql-tests/expressions/conditional/predicates.sql +++ b/spark/src/test/resources/sql-tests/expressions/conditional/predicates.sql @@ -31,3 +31,15 @@ SELECT a != b, NOT (a = b) FROM test_pred query SELECT (a > 1) AND (b > 1), (a > 1) OR (b > 1), NOT (a > 1) FROM test_pred + +-- column op literal +query +SELECT a = 1, a <=> 1, a < 2, a > 0, a <= 1, a >= 2 FROM test_pred + +-- literal op column +query +SELECT 2 = a, 2 < a, 0 > a, 1 <= a, 3 >= a FROM test_pred + +-- literal op literal +query +SELECT 1 = 1, 1 = 2, 1 <=> NULL, NULL <=> NULL, NULL = NULL, 1 < 2, 2 > 1 diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_add.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_add.sql index 3391cb3b78..c516cff07a 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/date_add.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_add.sql @@ -25,3 +25,15 @@ INSERT INTO test_date_add VALUES (date('2024-01-15'), 1), (date('2024-01-15'), - query SELECT date_add(d, n) FROM test_date_add + +-- column + literal +query +SELECT date_add(d, 5) FROM test_date_add + +-- literal + column +query +SELECT date_add(date('2024-01-15'), n) FROM test_date_add + +-- literal + literal +query +SELECT date_add(date('2024-01-15'), 5), date_add(date('2024-12-31'), 1), date_add(NULL, 1) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_diff.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_diff.sql index c956074223..50ba8923c3 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/date_diff.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_diff.sql @@ -25,3 +25,15 @@ INSERT INTO test_datediff VALUES (date('2024-01-15'), date('2024-01-10')), (date query SELECT datediff(d1, d2) FROM test_datediff + +-- column + literal +query +SELECT datediff(d1, date('2024-01-10')) FROM test_datediff + +-- literal + column +query +SELECT datediff(date('2024-01-20'), d2) FROM test_datediff + +-- literal + literal +query +SELECT datediff(date('2024-01-15'), date('2024-01-10')), datediff(date('2024-01-10'), date('2024-01-15')), datediff(NULL, date('2024-01-15')) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_format.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_format.sql index 92f8228f63..7f31e8f811 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/date_format.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_format.sql @@ -31,3 +31,7 @@ SELECT date_format(ts, 'HH:mm:ss') FROM test_date_format query expect_fallback(Non-UTC timezone) SELECT date_format(ts, 'yyyy-MM-dd HH:mm:ss') FROM test_date_format + +-- literal arguments +query expect_fallback(Non-UTC timezone) +SELECT date_format(timestamp('2024-06-15 10:30:45'), 'yyyy-MM-dd') diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_format_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_format_enabled.sql index ee88382814..b225f9f1b6 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/date_format_enabled.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_format_enabled.sql @@ -35,3 +35,7 @@ SELECT date_format(ts, 'HH:mm:ss') FROM test_date_format_enabled query SELECT date_format(ts, 'yyyy-MM-dd HH:mm:ss') FROM test_date_format_enabled + +-- literal arguments +query +SELECT date_format(timestamp('2024-06-15 10:30:45'), 'yyyy-MM-dd') diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_sub.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_sub.sql index fcee705e20..2dafdee89b 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/date_sub.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_sub.sql @@ -25,3 +25,15 @@ INSERT INTO test_date_sub VALUES (date('2024-01-15'), 1), (date('2024-01-15'), - query SELECT date_sub(d, n) FROM test_date_sub + +-- column + literal +query +SELECT date_sub(d, 5) FROM test_date_sub + +-- literal + column +query +SELECT date_sub(date('2024-01-15'), n) FROM test_date_sub + +-- literal + literal +query +SELECT date_sub(date('2024-01-15'), 5), date_sub(date('2024-01-01'), 1), date_sub(NULL, 1) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql index 2d0aaba4ed..5af2864593 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql @@ -28,3 +28,10 @@ SELECT from_unixtime(t) FROM test_from_unix_time query expect_fallback(not fully compatible with Spark) SELECT from_unixtime(t, 'yyyy-MM-dd') FROM test_from_unix_time + +-- literal arguments +query expect_fallback(not fully compatible with Spark) +SELECT from_unixtime(0) + +query expect_fallback(not fully compatible with Spark) +SELECT from_unixtime(1718451045, 'yyyy-MM-dd') diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql index 8d31e98e16..293a03fb5f 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql @@ -33,3 +33,10 @@ SELECT from_unixtime(t) FROM test_from_unix_time_enabled query spark_answer_only SELECT from_unixtime(t, 'yyyy-MM-dd') FROM test_from_unix_time_enabled + +-- literal arguments +query spark_answer_only +SELECT from_unixtime(0) + +query spark_answer_only +SELECT from_unixtime(1718451045, 'yyyy-MM-dd') diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/hour.sql b/spark/src/test/resources/sql-tests/expressions/datetime/hour.sql index 009c1876e3..551dd60c06 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/hour.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/hour.sql @@ -25,3 +25,7 @@ INSERT INTO test_hour VALUES (timestamp('2024-01-15 00:00:00')), (timestamp('202 query SELECT hour(ts) FROM test_hour + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3336) +SELECT hour(timestamp('2024-01-15 00:00:00')), hour(timestamp('2024-01-15 12:30:45')), hour(timestamp('2024-01-15 23:59:59')) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/last_day.sql b/spark/src/test/resources/sql-tests/expressions/datetime/last_day.sql index e7d60cc069..b1482f13a5 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/last_day.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/last_day.sql @@ -25,3 +25,7 @@ INSERT INTO test_last_day VALUES (date('2024-01-15')), (date('2024-02-15')), (da query SELECT last_day(d) FROM test_last_day + +-- literal arguments +query +SELECT last_day(date('2024-01-15')), last_day(date('2024-02-15')), last_day(date('2023-02-15')), last_day(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/minute.sql b/spark/src/test/resources/sql-tests/expressions/datetime/minute.sql index fbd159c7a7..0b75084352 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/minute.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/minute.sql @@ -25,3 +25,7 @@ INSERT INTO test_minute VALUES (timestamp('2024-01-15 10:00:00')), (timestamp('2 query SELECT minute(ts) FROM test_minute + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3336) +SELECT minute(timestamp('2024-01-15 10:00:00')), minute(timestamp('2024-01-15 10:30:00')), minute(timestamp('2024-01-15 10:59:59')) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/second.sql b/spark/src/test/resources/sql-tests/expressions/datetime/second.sql index ec306fc8e4..5db4e9e743 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/second.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/second.sql @@ -25,3 +25,7 @@ INSERT INTO test_second VALUES (timestamp('2024-01-15 10:30:00')), (timestamp('2 query SELECT second(ts) FROM test_second + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3336) +SELECT second(timestamp('2024-01-15 10:30:00')), second(timestamp('2024-01-15 10:30:30')), second(timestamp('2024-01-15 10:30:59')) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/trunc_date.sql b/spark/src/test/resources/sql-tests/expressions/datetime/trunc_date.sql index e35a4537d2..899a06f4bd 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/trunc_date.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/trunc_date.sql @@ -31,3 +31,7 @@ SELECT trunc(d, 'month') FROM test_trunc_date query SELECT trunc(d, 'quarter') FROM test_trunc_date + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3342) +SELECT trunc(date('2024-06-15'), 'year'), trunc(date('2024-06-15'), 'month'), trunc(date('2024-06-15'), 'quarter') diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/trunc_timestamp.sql b/spark/src/test/resources/sql-tests/expressions/datetime/trunc_timestamp.sql index 6db602944a..1105d014f3 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/trunc_timestamp.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/trunc_timestamp.sql @@ -34,3 +34,7 @@ SELECT date_trunc('day', ts) FROM test_trunc_ts query SELECT date_trunc('hour', ts) FROM test_trunc_ts + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3342) +SELECT date_trunc('year', timestamp('2024-06-15 10:30:45')), date_trunc('month', timestamp('2024-06-15 10:30:45')), date_trunc('day', timestamp('2024-06-15 10:30:45')) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/unix_date.sql b/spark/src/test/resources/sql-tests/expressions/datetime/unix_date.sql index 028f0ddea2..3e067dfd00 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/unix_date.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/unix_date.sql @@ -25,3 +25,7 @@ INSERT INTO test_unix_date VALUES (date('1970-01-01')), (date('2024-01-15')), (d query spark_answer_only SELECT unix_date(d) FROM test_unix_date + +-- literal arguments +query spark_answer_only +SELECT unix_date(date('1970-01-01')), unix_date(date('2024-01-15')), unix_date(date('1969-12-31')), unix_date(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/unix_timestamp.sql b/spark/src/test/resources/sql-tests/expressions/datetime/unix_timestamp.sql index 01a7194aad..fa4d9df07f 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/unix_timestamp.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/unix_timestamp.sql @@ -25,3 +25,7 @@ INSERT INTO test_unix_ts VALUES (timestamp('1970-01-01 00:00:00')), (timestamp(' query SELECT unix_timestamp(ts) FROM test_unix_ts + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3336) +SELECT unix_timestamp(timestamp('1970-01-01 00:00:00')), unix_timestamp(timestamp('2024-06-15 10:30:45')) diff --git a/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql b/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql index 4cffb1b6d9..3a856eb565 100644 --- a/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql +++ b/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql @@ -31,6 +31,18 @@ SELECT a + b, a - b, a * b, a / b FROM test_decimal query SELECT a % b FROM test_decimal +-- column + literal +query +SELECT a + cast(1.00 as decimal(10,2)), a - cast(1.00 as decimal(10,2)), a * cast(2.00 as decimal(10,2)) FROM test_decimal + +-- literal + column +query +SELECT cast(100.00 as decimal(10,2)) + b, cast(100.00 as decimal(10,2)) - b, cast(100.00 as decimal(10,2)) * b FROM test_decimal + +-- literal + literal +query +SELECT cast(10.50 as decimal(10,2)) + cast(3.20 as decimal(10,2)), cast(10.50 as decimal(10,2)) - cast(3.20 as decimal(10,2)), cast(10.50 as decimal(10,2)) * cast(3.20 as decimal(10,2)), cast(10.50 as decimal(10,2)) / cast(3.20 as decimal(10,2)) + -- Mixed precision statement CREATE TABLE test_decimal_mix(a decimal(18,6), b decimal(10,2)) USING parquet @@ -53,3 +65,7 @@ INSERT INTO test_dec_cast VALUES (42, 123456789, 3.14159, '99.99'), (0, 0, 0.0, query SELECT cast(i as decimal(10,2)), cast(l as decimal(18,2)), cast(d as decimal(10,5)), cast(s as decimal(10,2)) FROM test_dec_cast + +-- literal arguments +query +SELECT cast(42 as decimal(10,2)), cast(3.14 as decimal(10,5)), cast(NULL as decimal(10,2)) diff --git a/spark/src/test/resources/sql-tests/expressions/hash/hash.sql b/spark/src/test/resources/sql-tests/expressions/hash/hash.sql index 0629bc0a77..35031ea7e4 100644 --- a/spark/src/test/resources/sql-tests/expressions/hash/hash.sql +++ b/spark/src/test/resources/sql-tests/expressions/hash/hash.sql @@ -26,3 +26,7 @@ INSERT INTO test VALUES ('Spark SQL ', 10, 1.2), (NULL, NULL, NULL), ('', 0, 0. query SELECT md5(col), md5(cast(a as string)), md5(cast(b as string)), hash(col), hash(col, 1), hash(col, 0), hash(col, a, b), hash(b, a, col), xxhash64(col), xxhash64(col, 1), xxhash64(col, 0), xxhash64(col, a, b), xxhash64(b, a, col), sha2(col, 0), sha2(col, 256), sha2(col, 224), sha2(col, 384), sha2(col, 512), sha2(col, 128), sha2(col, -1), sha1(col), sha1(cast(a as string)), sha1(cast(b as string)) FROM test + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3340) +SELECT md5('Spark SQL'), sha1('test'), sha2('test', 256), hash('test'), xxhash64('test') diff --git a/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql b/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql index 6a500f6ce2..f97ffe8729 100644 --- a/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql +++ b/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql @@ -28,3 +28,7 @@ SELECT m['a'], m['b'], m['c'] FROM test_map query spark_answer_only SELECT m['x'], m['missing'] FROM test_map + +-- literal arguments +query spark_answer_only +SELECT map('a', 1, 'b', 2)['a'], map('a', 1, 'b', 2)['missing'], map('a', 1, 'b', 2)[NULL] diff --git a/spark/src/test/resources/sql-tests/expressions/map/map_from_arrays.sql b/spark/src/test/resources/sql-tests/expressions/map/map_from_arrays.sql index 7869021066..5d6ac3d550 100644 --- a/spark/src/test/resources/sql-tests/expressions/map/map_from_arrays.sql +++ b/spark/src/test/resources/sql-tests/expressions/map/map_from_arrays.sql @@ -30,3 +30,7 @@ SELECT map_from_arrays(k, v) FROM test_map_from_arrays WHERE k IS NOT NULL -- https://github.com/apache/datafusion-comet/issues/3327 query ignore(https://github.com/apache/datafusion-comet/issues/3327) SELECT map_from_arrays(k, v) FROM test_map_from_arrays WHERE k IS NULL + +-- literal arguments +query spark_answer_only +SELECT map_from_arrays(array('a', 'b'), array(1, 2)) diff --git a/spark/src/test/resources/sql-tests/expressions/math/abs.sql b/spark/src/test/resources/sql-tests/expressions/math/abs.sql index edb4e71fd9..c51270ae35 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/abs.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/abs.sql @@ -25,3 +25,7 @@ INSERT INTO test_abs VALUES (1, 1, 1.5, 1.5), (-1, -1, -1.5, -1.5), (0, 0, 0.0, query SELECT abs(i), abs(l), abs(f), abs(d) FROM test_abs + +-- literal arguments +query +SELECT abs(-5), abs(-1.5), abs(0), abs(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/acos.sql b/spark/src/test/resources/sql-tests/expressions/math/acos.sql index 4f7ead86f8..8b8e3c2565 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/acos.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/acos.sql @@ -25,3 +25,7 @@ INSERT INTO test_acos VALUES (0.0), (1.0), (-1.0), (0.5), (NULL), (cast('NaN' as query tolerance=1e-6 SELECT acos(d) FROM test_acos + +-- literal arguments +query tolerance=1e-6 +SELECT acos(0.5), acos(1.0), acos(-1.0), acos(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/arithmetic.sql b/spark/src/test/resources/sql-tests/expressions/math/arithmetic.sql index 16434b87da..de611ddfc7 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/arithmetic.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/arithmetic.sql @@ -56,6 +56,22 @@ SELECT fa + fb, fa - fb, fa * fb, fa / fb FROM test_arith query SELECT da + db, da - db, da * db, da / db FROM test_arith +-- column + literal +query +SELECT a + 3, a - 3, a * 3, a / 3, a % 3 FROM test_arith + +-- literal + column +query +SELECT 10 + b, 10 - b, 10 * b, 10 / b, 10 % b FROM test_arith + +-- literal + literal +query +SELECT 10 + 3, 10 - 3, 10 * 3, 10 / 3, 10 % 3 + +-- unary negative with literal +query +SELECT negative(5), negative(-5), negative(0), -(5) + -- division by zero statement CREATE TABLE test_div_zero(a int, b int, d double) USING parquet diff --git a/spark/src/test/resources/sql-tests/expressions/math/asin.sql b/spark/src/test/resources/sql-tests/expressions/math/asin.sql index 3e96a7b7d5..5a59ed944d 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/asin.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/asin.sql @@ -25,3 +25,7 @@ INSERT INTO test_asin VALUES (0.0), (1.0), (-1.0), (0.5), (NULL), (cast('NaN' as query tolerance=1e-6 SELECT asin(d) FROM test_asin + +-- literal arguments +query tolerance=1e-6 +SELECT asin(0.5), asin(1.0), asin(-1.0), asin(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/atan.sql b/spark/src/test/resources/sql-tests/expressions/math/atan.sql index a6bfc2b14e..b9037c7775 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/atan.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/atan.sql @@ -25,3 +25,7 @@ INSERT INTO test_atan VALUES (0.0), (1.0), (-1.0), (100.0), (-100.0), (NULL), (c query tolerance=1e-6 SELECT atan(d) FROM test_atan + +-- literal arguments +query tolerance=1e-6 +SELECT atan(1.0), atan(0.0), atan(-1.0), atan(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/atan2.sql b/spark/src/test/resources/sql-tests/expressions/math/atan2.sql index b9ea886009..7a912930b8 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/atan2.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/atan2.sql @@ -25,3 +25,15 @@ INSERT INTO test_atan2 VALUES (0.0, 1.0), (1.0, 0.0), (1.0, 1.0), (-1.0, -1.0), query tolerance=1e-6 SELECT atan2(y, x) FROM test_atan2 + +-- column + literal +query tolerance=1e-6 +SELECT atan2(y, 1.0) FROM test_atan2 + +-- literal + column +query tolerance=1e-6 +SELECT atan2(1.0, x) FROM test_atan2 + +-- literal + literal +query tolerance=1e-6 +SELECT atan2(1.0, 1.0), atan2(0.0, 0.0), atan2(-1.0, -1.0), atan2(NULL, 1.0) diff --git a/spark/src/test/resources/sql-tests/expressions/math/ceil.sql b/spark/src/test/resources/sql-tests/expressions/math/ceil.sql index 2155d4b17e..fade75d28a 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/ceil.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/ceil.sql @@ -25,3 +25,7 @@ INSERT INTO test_ceil VALUES (1.1, 1.1), (-1.1, -1.1), (0.0, 0.0), (1.0, 1.0), ( query SELECT ceil(f), ceil(d) FROM test_ceil + +-- literal arguments +query +SELECT ceil(1.1), ceil(-1.1), ceil(0.0), ceil(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/cos.sql b/spark/src/test/resources/sql-tests/expressions/math/cos.sql index 97bdaec31e..7ebc24bb99 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/cos.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/cos.sql @@ -25,3 +25,7 @@ INSERT INTO test_cos VALUES (0.0), (3.141592653589793), (1.5707963267948966), (- query tolerance=1e-6 SELECT cos(d) FROM test_cos + +-- literal arguments +query tolerance=1e-6 +SELECT cos(0.0), cos(3.141592653589793), cos(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/cosh.sql b/spark/src/test/resources/sql-tests/expressions/math/cosh.sql index 3c482959dc..a751960362 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/cosh.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/cosh.sql @@ -25,3 +25,7 @@ INSERT INTO test_cosh VALUES (0.0), (1.0), (-1.0), (100.0), (NULL), (cast('NaN' query tolerance=1e-6 SELECT cosh(d) FROM test_cosh + +-- literal arguments +query tolerance=1e-6 +SELECT cosh(0.0), cosh(1.0), cosh(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/cot.sql b/spark/src/test/resources/sql-tests/expressions/math/cot.sql index 5acd14c19c..fc6193f8f5 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/cot.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/cot.sql @@ -25,3 +25,7 @@ INSERT INTO test_cot VALUES (0.7853981633974483), (1.0), (-1.0), (0.1), (NULL), query tolerance=1e-6 SELECT cot(d) FROM test_cot + +-- literal arguments +query tolerance=1e-6 +SELECT cot(1.0), cot(0.5), cot(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/exp.sql b/spark/src/test/resources/sql-tests/expressions/math/exp.sql index 23fa613bb0..243aefba96 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/exp.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/exp.sql @@ -25,3 +25,7 @@ INSERT INTO test_exp VALUES (0.0), (1.0), (-1.0), (100.0), (-100.0), (NULL), (ca query tolerance=1e-6 SELECT exp(d) FROM test_exp + +-- literal arguments +query tolerance=1e-6 +SELECT exp(0.0), exp(1.0), exp(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/expm1.sql b/spark/src/test/resources/sql-tests/expressions/math/expm1.sql index f2ef06f6e7..2eaba2b739 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/expm1.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/expm1.sql @@ -25,3 +25,7 @@ INSERT INTO test_expm1 VALUES (0.0), (1.0), (-1.0), (0.001), (NULL), (cast('NaN' query tolerance=1e-6 SELECT expm1(d) FROM test_expm1 + +-- literal arguments +query tolerance=1e-6 +SELECT expm1(0.0), expm1(1.0), expm1(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/floor.sql b/spark/src/test/resources/sql-tests/expressions/math/floor.sql index f365459b03..3960002846 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/floor.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/floor.sql @@ -25,3 +25,7 @@ INSERT INTO test_floor VALUES (1.9, 1.9), (-1.1, -1.1), (0.0, 0.0), (1.0, 1.0), query SELECT floor(f), floor(d) FROM test_floor + +-- literal arguments +query +SELECT floor(1.9), floor(-1.1), floor(0.0), floor(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/isnan.sql b/spark/src/test/resources/sql-tests/expressions/math/isnan.sql index c9d0521a9e..7b7914c3cb 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/isnan.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/isnan.sql @@ -25,3 +25,7 @@ INSERT INTO test_isnan VALUES (1.0, 1.0), (cast('NaN' as float), cast('NaN' as d query SELECT isnan(f), isnan(d) FROM test_isnan + +-- literal arguments +query +SELECT isnan(cast('NaN' as double)), isnan(1.0), isnan(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/log.sql b/spark/src/test/resources/sql-tests/expressions/math/log.sql index 025c47d6df..e7420954cf 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/log.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/log.sql @@ -28,3 +28,15 @@ SELECT ln(d) FROM test_log query tolerance=1e-6 SELECT log(10.0, d) FROM test_log + +-- column + literal (log base from column) +query tolerance=1e-6 +SELECT log(d, 10.0) FROM test_log + +-- literal (1-arg form) +query tolerance=1e-6 +SELECT ln(1.0), ln(2.718281828459045), ln(10.0), ln(NULL) + +-- literal + literal (2-arg form) +query tolerance=1e-6 +SELECT log(10.0, 100.0), log(2.0, 8.0), log(10.0, 1.0), log(NULL, 10.0) diff --git a/spark/src/test/resources/sql-tests/expressions/math/log10.sql b/spark/src/test/resources/sql-tests/expressions/math/log10.sql index 8f14574eb1..1b3c9417f1 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/log10.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/log10.sql @@ -25,3 +25,7 @@ INSERT INTO test_log10 VALUES (1.0), (10.0), (100.0), (0.1), (NULL), (cast('NaN' query tolerance=1e-6 SELECT log10(d) FROM test_log10 + +-- literal arguments +query tolerance=1e-6 +SELECT log10(100.0), log10(1.0), log10(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/log2.sql b/spark/src/test/resources/sql-tests/expressions/math/log2.sql index f157c8c069..5db0ca484b 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/log2.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/log2.sql @@ -25,3 +25,7 @@ INSERT INTO test_log2 VALUES (1.0), (2.0), (4.0), (8.0), (0.5), (NULL), (cast('N query tolerance=1e-6 SELECT log2(d) FROM test_log2 + +-- literal arguments +query tolerance=1e-6 +SELECT log2(8.0), log2(1.0), log2(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/pow.sql b/spark/src/test/resources/sql-tests/expressions/math/pow.sql index 9c6f8914e0..82b1155303 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/pow.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/pow.sql @@ -25,3 +25,15 @@ INSERT INTO test_pow VALUES (2.0, 3.0), (0.0, 0.0), (-1.0, 2.0), (-1.0, 0.5), (2 query tolerance=1e-6 SELECT pow(base, exp) FROM test_pow + +-- column + literal +query tolerance=1e-6 +SELECT pow(base, 2.0) FROM test_pow + +-- literal + column +query tolerance=1e-6 +SELECT pow(2.0, exp) FROM test_pow + +-- literal + literal +query tolerance=1e-6 +SELECT pow(2.0, 3.0), pow(0.0, 0.0), pow(-1.0, 2.0), pow(NULL, 2.0) diff --git a/spark/src/test/resources/sql-tests/expressions/math/round.sql b/spark/src/test/resources/sql-tests/expressions/math/round.sql index 4647d5f397..617968eb35 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/round.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/round.sql @@ -34,3 +34,7 @@ SELECT round(d, -1) FROM test_round WHERE i = -1 query expect_fallback(BigDecimal rounding) SELECT round(d) FROM test_round + +-- literal + literal +query expect_fallback(BigDecimal rounding) +SELECT round(123.456, 2), round(2.5, 0), round(3.5, 0), round(-2.5, 0), round(NULL, 0) diff --git a/spark/src/test/resources/sql-tests/expressions/math/signum.sql b/spark/src/test/resources/sql-tests/expressions/math/signum.sql index dfdebd864d..6dd2f2209a 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/signum.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/signum.sql @@ -25,3 +25,7 @@ INSERT INTO test_signum VALUES (5.0), (-5.0), (0.0), (NULL), (cast('NaN' as doub query SELECT signum(d) FROM test_signum + +-- literal arguments +query +SELECT signum(-5.0), signum(5.0), signum(0.0), signum(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/sin.sql b/spark/src/test/resources/sql-tests/expressions/math/sin.sql index 85db4c8864..dca3ad77b4 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/sin.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/sin.sql @@ -25,3 +25,7 @@ INSERT INTO test_sin VALUES (0.0), (3.141592653589793), (1.5707963267948966), (- query tolerance=1e-6 SELECT sin(d) FROM test_sin + +-- literal arguments +query tolerance=1e-6 +SELECT sin(0.0), sin(1.5707963267948966), sin(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/sinh.sql b/spark/src/test/resources/sql-tests/expressions/math/sinh.sql index d5f25d6e7f..432c58c583 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/sinh.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/sinh.sql @@ -25,3 +25,7 @@ INSERT INTO test_sinh VALUES (0.0), (1.0), (-1.0), (NULL), (cast('NaN' as double query tolerance=1e-6 SELECT sinh(d) FROM test_sinh + +-- literal arguments +query tolerance=1e-6 +SELECT sinh(0.0), sinh(1.0), sinh(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/sqrt.sql b/spark/src/test/resources/sql-tests/expressions/math/sqrt.sql index a776ab28e8..ec3c066eb9 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/sqrt.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/sqrt.sql @@ -25,3 +25,7 @@ INSERT INTO test_sqrt VALUES (0.0), (1.0), (4.0), (2.0), (-1.0), (NULL), (cast(' query tolerance=1e-6 SELECT sqrt(d) FROM test_sqrt + +-- literal arguments +query tolerance=1e-6 +SELECT sqrt(4.0), sqrt(2.0), sqrt(0.0), sqrt(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/tan.sql b/spark/src/test/resources/sql-tests/expressions/math/tan.sql index 00dfc64131..21bd44f907 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/tan.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/tan.sql @@ -25,3 +25,7 @@ INSERT INTO test_tan VALUES (0.0), (0.7853981633974483), (-0.7853981633974483), query tolerance=1e-6 SELECT tan(d) FROM test_tan + +-- literal arguments +query tolerance=1e-6 +SELECT tan(0.0), tan(0.7853981633974483), tan(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/tanh.sql b/spark/src/test/resources/sql-tests/expressions/math/tanh.sql index 85643dc935..9d70202776 100644 --- a/spark/src/test/resources/sql-tests/expressions/math/tanh.sql +++ b/spark/src/test/resources/sql-tests/expressions/math/tanh.sql @@ -25,3 +25,7 @@ INSERT INTO test_tanh VALUES (0.0), (1.0), (-1.0), (100.0), (-100.0), (NULL), (c query tolerance=1e-6 SELECT tanh(d) FROM test_tanh + +-- literal arguments +query tolerance=1e-6 +SELECT tanh(0.0), tanh(1.0), tanh(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql b/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql index 6497ff3812..443864abe7 100644 --- a/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql +++ b/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql @@ -31,3 +31,7 @@ SELECT v, width_bucket(v, 10, 0, 4) FROM test_wb query ignore(https://github.com/apache/datafusion-comet/issues/3331) SELECT v, width_bucket(v, 0, 10, 1) FROM test_wb + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3331) +SELECT width_bucket(5.0, 0, 10, 4), width_bucket(0.0, 0, 10, 4), width_bucket(NULL, 0, 10, 4) diff --git a/spark/src/test/resources/sql-tests/expressions/string/ascii.sql b/spark/src/test/resources/sql-tests/expressions/string/ascii.sql index 73aef6881c..bca7b77932 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/ascii.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/ascii.sql @@ -25,3 +25,7 @@ INSERT INTO test_ascii VALUES ('A'), ('z'), ('0'), (''), (NULL), ('hello') query SELECT ascii(s) FROM test_ascii + +-- literal arguments +query +SELECT ascii('A'), ascii(''), ascii(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/bit_length.sql b/spark/src/test/resources/sql-tests/expressions/string/bit_length.sql index b74d1365cf..83044267de 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/bit_length.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/bit_length.sql @@ -25,3 +25,7 @@ INSERT INTO test_bit_length VALUES (''), ('a'), ('hello'), (NULL), ('café') query SELECT bit_length(s) FROM test_bit_length + +-- literal arguments +query +SELECT bit_length('hello'), bit_length(''), bit_length(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/chr.sql b/spark/src/test/resources/sql-tests/expressions/string/chr.sql index 96dba6e009..bce93d75dd 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/chr.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/chr.sql @@ -25,3 +25,7 @@ INSERT INTO test_chr VALUES (65), (97), (0), (NULL), (128522), (256), (48) query SELECT chr(i) FROM test_chr + +-- literal arguments +query +SELECT chr(65), chr(0), chr(-1), chr(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/concat.sql b/spark/src/test/resources/sql-tests/expressions/string/concat.sql index 9b2dce2b1e..fcf2416bc2 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/concat.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/concat.sql @@ -28,3 +28,11 @@ SELECT concat(a, b, c) FROM test_concat query SELECT a || b || c FROM test_concat + +-- mixed: column + literal + column +query +SELECT concat(a, ' ', c) FROM test_concat + +-- literal + literal + literal +query +SELECT concat('hello', ' ', 'world'), concat('', '', ''), concat(NULL, 'b', 'c') diff --git a/spark/src/test/resources/sql-tests/expressions/string/concat_ws.sql b/spark/src/test/resources/sql-tests/expressions/string/concat_ws.sql index 422e97225e..fd277fd7c4 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/concat_ws.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/concat_ws.sql @@ -31,3 +31,7 @@ SELECT concat_ws('', a, b, c) FROM test_concat_ws query SELECT concat_ws(NULL, a, b, c) FROM test_concat_ws + +-- literal + literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3339) +SELECT concat_ws(',', 'hello', 'world'), concat_ws(',', '', ''), concat_ws(',', NULL, 'b', 'c'), concat_ws(NULL, 'a', 'b') diff --git a/spark/src/test/resources/sql-tests/expressions/string/contains.sql b/spark/src/test/resources/sql-tests/expressions/string/contains.sql index 79658b038d..32a9bca01e 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/contains.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/contains.sql @@ -25,3 +25,15 @@ INSERT INTO test_contains VALUES ('hello world', 'world'), ('hello', ''), ('', ' query SELECT contains(s, sub) FROM test_contains + +-- column + literal +query +SELECT contains(s, 'world') FROM test_contains + +-- literal + column +query +SELECT contains('hello world', sub) FROM test_contains + +-- literal + literal +query +SELECT contains('hello world', 'world'), contains('hello', 'xyz'), contains('', ''), contains(NULL, 'a') diff --git a/spark/src/test/resources/sql-tests/expressions/string/ends_with.sql b/spark/src/test/resources/sql-tests/expressions/string/ends_with.sql index cd77743d2f..957cb93c12 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/ends_with.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/ends_with.sql @@ -25,3 +25,15 @@ INSERT INTO test_ends_with VALUES ('hello world', 'world'), ('hello', ''), ('', query SELECT endswith(s, suffix) FROM test_ends_with + +-- column + literal +query +SELECT endswith(s, 'world') FROM test_ends_with + +-- literal + column +query +SELECT endswith('hello world', suffix) FROM test_ends_with + +-- literal + literal +query +SELECT endswith('hello world', 'world'), endswith('hello', 'xyz'), endswith('', ''), endswith(NULL, 'a') diff --git a/spark/src/test/resources/sql-tests/expressions/string/hex.sql b/spark/src/test/resources/sql-tests/expressions/string/hex.sql index e99df6127d..d0be03372d 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/hex.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/hex.sql @@ -25,3 +25,7 @@ INSERT INTO test_hex VALUES (0, 0, ''), (255, 255, 'Spark'), (-1, -1, NULL), (NU query SELECT hex(i), hex(l), hex(s) FROM test_hex + +-- literal arguments +query +SELECT hex(255), hex('Spark'), hex(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/init_cap_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/init_cap_enabled.sql index f26694ff27..03c9de8817 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/init_cap_enabled.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/init_cap_enabled.sql @@ -27,3 +27,7 @@ INSERT INTO test_initcap_enabled VALUES ('hello world'), ('HELLO WORLD'), (''), query SELECT initcap(s) FROM test_initcap_enabled + +-- literal arguments +query +SELECT initcap('hello world'), initcap(''), initcap(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/left.sql b/spark/src/test/resources/sql-tests/expressions/string/left.sql index 1c2c393335..4605622e8b 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/left.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/left.sql @@ -25,3 +25,15 @@ INSERT INTO test_str_left VALUES ('hello', 3), ('hello', 0), ('hello', -1), ('he query expect_fallback(Substring pos and len must be literals) SELECT left(s, n) FROM test_str_left + +-- column + literal +query +SELECT left(s, 3) FROM test_str_left + +-- literal + column +query expect_fallback(Substring pos and len must be literals) +SELECT left('hello', n) FROM test_str_left + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3337) +SELECT left('hello', 3), left('hello', 0), left('hello', -1), left('', 3), left(NULL, 3) diff --git a/spark/src/test/resources/sql-tests/expressions/string/length.sql b/spark/src/test/resources/sql-tests/expressions/string/length.sql index db8ead6883..c09fbe80c1 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/length.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/length.sql @@ -25,3 +25,7 @@ INSERT INTO test_length VALUES (''), ('a'), ('hello'), (NULL), ('café') query SELECT length(s), char_length(s) FROM test_length + +-- literal arguments +query +SELECT length('hello'), length(''), length(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/like.sql b/spark/src/test/resources/sql-tests/expressions/string/like.sql index aa4bbff4eb..5f56881d3c 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/like.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/like.sql @@ -34,3 +34,7 @@ SELECT s LIKE 'h_llo' FROM test_like query SELECT s LIKE '' FROM test_like + +-- literal arguments +query +SELECT 'hello' LIKE 'h%', 'hello' LIKE 'xyz%', '' LIKE '', NULL LIKE 'a' diff --git a/spark/src/test/resources/sql-tests/expressions/string/lower.sql b/spark/src/test/resources/sql-tests/expressions/string/lower.sql index 91676738c8..f2f8173c39 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/lower.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/lower.sql @@ -25,3 +25,7 @@ INSERT INTO test_lower VALUES ('HELLO'), ('hello'), ('Hello World'), (''), (NULL query expect_fallback(case conversion) SELECT lower(s) FROM test_lower + +-- literal arguments +query expect_fallback(case conversion) +SELECT lower('HELLO'), lower(''), lower(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/octet_length.sql b/spark/src/test/resources/sql-tests/expressions/string/octet_length.sql index 7dd336af83..2e006c688f 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/octet_length.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/octet_length.sql @@ -25,3 +25,7 @@ INSERT INTO test_octet_length VALUES (''), ('a'), ('hello'), (NULL), ('café') query SELECT octet_length(s) FROM test_octet_length + +-- literal arguments +query +SELECT octet_length('hello'), octet_length(''), octet_length(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/regexp_replace_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/regexp_replace_enabled.sql index 25624d24de..7ea13a18a2 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/regexp_replace_enabled.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/regexp_replace_enabled.sql @@ -30,3 +30,7 @@ SELECT regexp_replace(s, '(\d+)', 'X') FROM test_regexp_replace_enabled query SELECT regexp_replace(s, '(\d+)', 'X', 1) FROM test_regexp_replace_enabled + +-- literal + literal + literal +query +SELECT regexp_replace('100-200', '(\d+)', 'X'), regexp_replace('abc', '(\d+)', 'X'), regexp_replace(NULL, '(\d+)', 'X') diff --git a/spark/src/test/resources/sql-tests/expressions/string/reverse.sql b/spark/src/test/resources/sql-tests/expressions/string/reverse.sql index 5ec46ad060..766b097932 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/reverse.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/reverse.sql @@ -25,3 +25,7 @@ INSERT INTO test_reverse VALUES ('hello'), (''), (NULL), ('a'), ('abcde'), ('caf query SELECT reverse(s) FROM test_reverse + +-- literal arguments +query +SELECT reverse('hello'), reverse(''), reverse(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/rlike_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/rlike_enabled.sql index 2f129397a6..968a2f22f7 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/rlike_enabled.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/rlike_enabled.sql @@ -33,3 +33,7 @@ SELECT s RLIKE '^[a-z]+$' FROM test_rlike_enabled query SELECT s RLIKE '' FROM test_rlike_enabled + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3343) +SELECT 'hello' RLIKE '^[a-z]+$', '12345' RLIKE '^[a-z]+$', '' RLIKE '', NULL RLIKE 'a' diff --git a/spark/src/test/resources/sql-tests/expressions/string/starts_with.sql b/spark/src/test/resources/sql-tests/expressions/string/starts_with.sql index 12fb1e8de3..eeea2b856d 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/starts_with.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/starts_with.sql @@ -25,3 +25,15 @@ INSERT INTO test_starts_with VALUES ('hello world', 'hello'), ('hello', ''), ('' query SELECT startswith(s, prefix) FROM test_starts_with + +-- column + literal +query +SELECT startswith(s, 'hello') FROM test_starts_with + +-- literal + column +query +SELECT startswith('hello world', prefix) FROM test_starts_with + +-- literal + literal +query +SELECT startswith('hello world', 'hello'), startswith('hello', 'xyz'), startswith('', ''), startswith(NULL, 'a') diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_instr.sql b/spark/src/test/resources/sql-tests/expressions/string/string_instr.sql index 982a371f80..92298a143b 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/string_instr.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/string_instr.sql @@ -25,3 +25,15 @@ INSERT INTO test_instr VALUES ('hello world', 'world'), ('hello', 'xyz'), ('hell query SELECT instr(s, sub) FROM test_instr + +-- column + literal +query +SELECT instr(s, 'world') FROM test_instr + +-- literal + column +query +SELECT instr('hello world', sub) FROM test_instr + +-- literal + literal +query +SELECT instr('hello world', 'world'), instr('hello', 'xyz'), instr('', ''), instr(NULL, 'a') diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_lpad.sql b/spark/src/test/resources/sql-tests/expressions/string/string_lpad.sql index e2a40c1543..358be456b1 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/string_lpad.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/string_lpad.sql @@ -28,3 +28,11 @@ SELECT lpad(s, len, pad) FROM test_lpad query SELECT lpad(s, len) FROM test_lpad + +-- column + literal + literal +query +SELECT lpad(s, 5, 'x') FROM test_lpad + +-- literal + literal + literal +query expect_fallback(Scalar values are not supported for the str argument) +SELECT lpad('hi', 5, 'x'), lpad('hello', 3, 'x'), lpad('', 3, 'a'), lpad(NULL, 5, 'x') diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_repeat.sql b/spark/src/test/resources/sql-tests/expressions/string/string_repeat.sql index 0f537e9b48..32f08db101 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/string_repeat.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/string_repeat.sql @@ -25,3 +25,15 @@ INSERT INTO test_repeat VALUES ('hi', 3), ('', 5), ('a', 0), ('a', -1), (NULL, 3 query SELECT repeat(s, n) FROM test_repeat + +-- column + literal +query +SELECT repeat(s, 3) FROM test_repeat + +-- literal + column +query +SELECT repeat('hi', n) FROM test_repeat + +-- literal + literal +query +SELECT repeat('hi', 3), repeat('', 5), repeat('a', 0), repeat(NULL, 3) diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_replace.sql b/spark/src/test/resources/sql-tests/expressions/string/string_replace.sql index 0cfaa4983f..d1c03d18e6 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/string_replace.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/string_replace.sql @@ -26,5 +26,17 @@ INSERT INTO test_str_replace VALUES ('hello world', 'world', 'there'), ('aaa', ' query SELECT replace(s, search, replace) FROM test_str_replace -query spark_answer_only +query ignore(https://github.com/apache/datafusion-comet/issues/3344) SELECT replace('hello', '', 'x') + +-- column + literal + literal +query +SELECT replace(s, 'world', 'there') FROM test_str_replace + +-- literal + column + column +query +SELECT replace('hello world', search, replace) FROM test_str_replace + +-- literal + literal + literal +query +SELECT replace('hello world', 'world', 'there'), replace('aaa', 'a', 'bb'), replace('hello', 'xyz', 'abc'), replace(NULL, 'a', 'b') diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_rpad.sql b/spark/src/test/resources/sql-tests/expressions/string/string_rpad.sql index ba5e9311b6..c2f526a88f 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/string_rpad.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/string_rpad.sql @@ -28,3 +28,11 @@ SELECT rpad(s, len, pad) FROM test_rpad query SELECT rpad(s, len) FROM test_rpad + +-- column + literal + literal +query +SELECT rpad(s, 5, 'x') FROM test_rpad + +-- literal + literal + literal +query expect_fallback(Scalar values are not supported for the str argument) +SELECT rpad('hi', 5, 'x'), rpad('hello', 3, 'x'), rpad('', 3, 'a'), rpad(NULL, 5, 'x') diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_space.sql b/spark/src/test/resources/sql-tests/expressions/string/string_space.sql index fe231e45b0..ec24bfb97e 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/string_space.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/string_space.sql @@ -30,3 +30,7 @@ SELECT concat('[', space(n), ']') FROM test_space WHERE n >= 0 OR n IS NULL -- https://github.com/apache/datafusion-comet/issues/3326 query ignore(https://github.com/apache/datafusion-comet/issues/3326) SELECT concat('[', space(n), ']') FROM test_space WHERE n < 0 + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3337) +SELECT concat('[', space(5), ']'), concat('[', space(0), ']'), space(-1), space(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_translate.sql b/spark/src/test/resources/sql-tests/expressions/string/string_translate.sql index 1795a6cef1..01f5426dd7 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/string_translate.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/string_translate.sql @@ -25,3 +25,15 @@ INSERT INTO test_translate VALUES ('hello', 'el', 'ip'), ('hello', 'aeiou', '123 query SELECT translate(s, from_str, to_str) FROM test_translate + +-- column + literal + literal +query +SELECT translate(s, 'el', 'ip') FROM test_translate + +-- literal + column + column +query +SELECT translate('hello', from_str, to_str) FROM test_translate + +-- literal + literal + literal +query +SELECT translate('hello', 'el', 'ip'), translate('hello', 'aeiou', '12345'), translate('', 'a', 'b'), translate(NULL, 'a', 'b') diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_trim.sql b/spark/src/test/resources/sql-tests/expressions/string/string_trim.sql index 7edc8c7fbe..9822643ef1 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/string_trim.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/string_trim.sql @@ -28,3 +28,10 @@ SELECT trim(s), ltrim(s), rtrim(s) FROM test_trim query SELECT trim(BOTH 'h' FROM s) FROM test_trim + +-- literal arguments +query +SELECT trim(' hello '), ltrim(' hello '), rtrim(' hello ') + +query +SELECT trim(BOTH 'h' FROM 'hello'), trim(LEADING ' ' FROM ' hello '), trim(TRAILING ' ' FROM ' hello ') diff --git a/spark/src/test/resources/sql-tests/expressions/string/substring.sql b/spark/src/test/resources/sql-tests/expressions/string/substring.sql index 4ed982d7ab..42b99f1900 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/substring.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/substring.sql @@ -40,3 +40,7 @@ SELECT substring(s, 1, -1) FROM test_substring query SELECT substring(s, 100) FROM test_substring + +-- literal + literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3337) +SELECT substring('hello world', 1, 5), substring('hello world', -3), substring('', 1, 5), substring(NULL, 1, 5) diff --git a/spark/src/test/resources/sql-tests/expressions/string/unhex.sql b/spark/src/test/resources/sql-tests/expressions/string/unhex.sql index 8a9a3153d7..2b20cf489f 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/unhex.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/unhex.sql @@ -28,3 +28,7 @@ SELECT hex(unhex(s)) FROM test_unhex query SELECT unhex(s) IS NULL FROM test_unhex + +-- literal arguments +query +SELECT unhex('41'), unhex('GG'), unhex(''), unhex(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/upper.sql b/spark/src/test/resources/sql-tests/expressions/string/upper.sql index b9ac2c5b75..6582a299ef 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/upper.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/upper.sql @@ -25,3 +25,7 @@ INSERT INTO test_upper VALUES ('hello'), ('HELLO'), ('Hello World'), (''), (NULL query expect_fallback(case conversion) SELECT upper(s) FROM test_upper + +-- literal arguments +query expect_fallback(case conversion) +SELECT upper('hello'), upper(''), upper(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/struct/create_named_struct.sql b/spark/src/test/resources/sql-tests/expressions/struct/create_named_struct.sql index 4b2461be8e..d188aed7bb 100644 --- a/spark/src/test/resources/sql-tests/expressions/struct/create_named_struct.sql +++ b/spark/src/test/resources/sql-tests/expressions/struct/create_named_struct.sql @@ -28,3 +28,10 @@ SELECT named_struct('x', a, 'y', b, 'z', c) FROM test_named_struct query SELECT struct(a, b, c) FROM test_named_struct + +-- literal arguments +query +SELECT named_struct('x', 1, 'y', 'hello', 'z', 3.14) + +query +SELECT named_struct('x', a, 'y', 'fixed_val', 'z', c) FROM test_named_struct diff --git a/spark/src/test/resources/sql-tests/expressions/struct/json_to_structs.sql b/spark/src/test/resources/sql-tests/expressions/struct/json_to_structs.sql index 798ff80698..a0ae20b655 100644 --- a/spark/src/test/resources/sql-tests/expressions/struct/json_to_structs.sql +++ b/spark/src/test/resources/sql-tests/expressions/struct/json_to_structs.sql @@ -25,3 +25,7 @@ INSERT INTO test_from_json VALUES ('{"a": 1, "b": "hello"}'), ('{}'), (NULL), (' query spark_answer_only SELECT from_json(j, 'a INT, b STRING') FROM test_from_json + +-- literal arguments +query spark_answer_only +SELECT from_json('{"a": 1, "b": "hello"}', 'a INT, b STRING') diff --git a/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql b/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql index e2498faffe..7f2310f147 100644 --- a/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql +++ b/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql @@ -25,3 +25,7 @@ INSERT INTO test_to_json VALUES (1, 'hello'), (NULL, NULL), (0, '') query spark_answer_only SELECT to_json(named_struct('a', a, 'b', b)) FROM test_to_json + +-- literal arguments +query spark_answer_only +SELECT to_json(named_struct('a', 1, 'b', 'hello')) diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala index 4fbeba349e..80ccf92557 100644 --- a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -71,8 +71,14 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { } yield (key, value) +: combo } + // Disable constant folding so that literal expressions are evaluated by Comet's + // native engine rather than being folded away by Spark's optimizer at plan time. + private val constantFoldingExcluded = Seq( + "spark.sql.optimizer.excludedRules" -> + "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") + private def runTestFile(file: SqlTestFile): Unit = { - val allConfigs = file.configs + val allConfigs = file.configs ++ constantFoldingExcluded withSQLConf(allConfigs: _*) { withTable(file.tables: _*) { file.records.foreach {