diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 7df0aa0697..8376afbc6e 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -250,6 +250,7 @@ jobs: - name: "expressions" value: | org.apache.comet.CometExpressionSuite + org.apache.comet.CometSqlFileTestSuite org.apache.comet.CometExpressionCoverageSuite org.apache.comet.CometHashExpressionSuite org.apache.comet.CometTemporalExpressionSuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index acf052bf13..3a64c0051f 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -193,6 +193,7 @@ jobs: - name: "expressions" value: | org.apache.comet.CometExpressionSuite + org.apache.comet.CometSqlFileTestSuite org.apache.comet.CometExpressionCoverageSuite org.apache.comet.CometHashExpressionSuite org.apache.comet.CometTemporalExpressionSuite diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/avg.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/avg.sql new file mode 100644 index 0000000000..c46a14c617 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/avg.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_avg(i int, l long, f float, d double, grp string) USING parquet + +statement +INSERT INTO test_avg VALUES (1, 10, 1.5, 1.5, 'a'), (2, 20, 2.5, 2.5, 'a'), (3, 30, 3.5, 3.5, 'b'), (NULL, NULL, NULL, NULL, 'b'), (0, 0, 0.0, 0.0, 'a') + +query tolerance=1e-6 +SELECT avg(i), avg(l), avg(f), avg(d) FROM test_avg + +query tolerance=1e-6 +SELECT grp, avg(d) FROM test_avg GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/bit_agg.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/bit_agg.sql new file mode 100644 index 0000000000..32a09febc2 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/bit_agg.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_bit_agg(i int, grp string) USING parquet + +statement +INSERT INTO test_bit_agg VALUES (1, 'a'), (2, 'a'), (3, 'a'), (4, 'b'), (5, 'b'), (NULL, 'b') + +query +SELECT bit_and(i), bit_or(i), bit_xor(i) FROM test_bit_agg + +query +SELECT grp, bit_and(i), bit_or(i), bit_xor(i) FROM test_bit_agg GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/corr.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/corr.sql new file mode 100644 index 0000000000..9d11ba0ca3 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/corr.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_corr(x double, y double, grp string) USING parquet + +statement +INSERT INTO test_corr VALUES (1.0, 2.0, 'a'), (2.0, 4.0, 'a'), (3.0, 6.0, 'a'), (1.0, 1.0, 'b'), (2.0, 3.0, 'b'), (NULL, 1.0, 'b') + +query tolerance=1e-6 +SELECT corr(x, y) FROM test_corr + +query tolerance=1e-6 +SELECT grp, corr(x, y) FROM test_corr GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/count.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/count.sql new file mode 100644 index 0000000000..b7fec97557 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/count.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_count(i int, s string, grp string) USING parquet + +statement +INSERT INTO test_count VALUES (1, 'a', 'x'), (2, 'b', 'x'), (NULL, NULL, 'y'), (3, 'c', 'y'), (NULL, 'd', 'y') + +query +SELECT count(*), count(i), count(s) FROM test_count + +query +SELECT grp, count(*), count(i) FROM test_count GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/covariance.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/covariance.sql new file mode 100644 index 0000000000..eca4b5ff1b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/covariance.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_covar(x double, y double, grp string) USING parquet + +statement +INSERT INTO test_covar VALUES (1.0, 2.0, 'a'), (2.0, 4.0, 'a'), (3.0, 6.0, 'a'), (1.0, 1.0, 'b'), (2.0, 3.0, 'b'), (NULL, 1.0, 'b') + +query tolerance=1e-6 +SELECT covar_samp(x, y), covar_pop(x, y) FROM test_covar + +query tolerance=1e-6 +SELECT grp, covar_samp(x, y), covar_pop(x, y) FROM test_covar GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/first_last.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/first_last.sql new file mode 100644 index 0000000000..7bc617bffd --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/first_last.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_first_last(i int, grp string) USING parquet + +statement +INSERT INTO test_first_last VALUES (1, 'a'), (2, 'a'), (3, 'a'), (NULL, 'b'), (4, 'b') + +query spark_answer_only +SELECT first(i), last(i) FROM test_first_last + +query spark_answer_only +SELECT first(i, true), last(i, true) FROM test_first_last + +query spark_answer_only +SELECT grp, first(i), last(i) FROM test_first_last GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/min_max.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/min_max.sql new file mode 100644 index 0000000000..50a13d5896 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/min_max.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_min_max(i int, d double, s string, grp string) USING parquet + +statement +INSERT INTO test_min_max VALUES (1, 1.5, 'b', 'x'), (3, 3.5, 'a', 'x'), (2, 2.5, 'c', 'y'), (NULL, NULL, NULL, 'y'), (-1, -1.5, 'z', 'x') + +query expect_fallback(SortAggregate is not supported) +SELECT min(i), max(i), min(d), max(d), min(s), max(s) FROM test_min_max + +query +SELECT grp, min(i), max(i) FROM test_min_max GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/stddev.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/stddev.sql new file mode 100644 index 0000000000..cd70852daf --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/stddev.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_stddev(d double, grp string) USING parquet + +statement +INSERT INTO test_stddev VALUES (1.0, 'a'), (2.0, 'a'), (3.0, 'a'), (4.0, 'b'), (5.0, 'b'), (NULL, 'b') + +query tolerance=1e-6 +SELECT stddev(d), stddev_samp(d), stddev_pop(d) FROM test_stddev + +query tolerance=1e-6 +SELECT grp, stddev(d) FROM test_stddev GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/sum.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/sum.sql new file mode 100644 index 0000000000..306bbe9128 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/sum.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_sum(i int, l long, f float, d double, grp string) USING parquet + +statement +INSERT INTO test_sum VALUES (1, 10, 1.5, 1.5, 'a'), (2, 20, 2.5, 2.5, 'a'), (3, 30, 3.5, 3.5, 'b'), (NULL, NULL, NULL, NULL, 'b'), (2147483647, 9223372036854775807, cast('Infinity' as float), cast('Infinity' as double), 'c') + +query +SELECT sum(i), sum(l) FROM test_sum + +query tolerance=1e-6 +SELECT sum(f), sum(d) FROM test_sum + +query +SELECT grp, sum(i) FROM test_sum GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/variance.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/variance.sql new file mode 100644 index 0000000000..be4905d5fa --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/variance.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_variance(d double, grp string) USING parquet + +statement +INSERT INTO test_variance VALUES (1.0, 'a'), (2.0, 'a'), (3.0, 'a'), (4.0, 'b'), (5.0, 'b'), (NULL, 'b') + +query tolerance=1e-6 +SELECT variance(d), var_samp(d), var_pop(d) FROM test_variance + +query tolerance=1e-6 +SELECT grp, variance(d) FROM test_variance GROUP BY grp ORDER BY grp diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_append.sql b/spark/src/test/resources/sql-tests/expressions/array/array_append.sql new file mode 100644 index 0000000000..2efd13f1c2 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_append.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_append(arr array, val int) USING parquet + +statement +INSERT INTO test_array_append VALUES (array(1, 2, 3), 4), (array(), 1), (NULL, 1), (array(1, 2), NULL) + +query spark_answer_only +SELECT array_append(arr, val) FROM test_array_append + +-- column + literal +query spark_answer_only +SELECT array_append(arr, 99) FROM test_array_append + +-- literal + column +query spark_answer_only +SELECT array_append(array(1, 2, 3), val) FROM test_array_append + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_append(array(1, 2, 3), 4), array_append(array(), 1), array_append(cast(NULL as array), 1) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql b/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql new file mode 100644 index 0000000000..9b834a4dbd --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_compact(arr array) USING parquet + +statement +INSERT INTO test_array_compact VALUES (array(1, NULL, 2, NULL, 3)), (array()), (NULL), (array(NULL, NULL)), (array(1, 2, 3)) + +query spark_answer_only +SELECT array_compact(arr) FROM test_array_compact + +-- literal arguments +query spark_answer_only +SELECT array_compact(array(1, NULL, 2, NULL, 3)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql b/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql new file mode 100644 index 0000000000..86ad0cc488 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_contains(arr array, val int) USING parquet + +statement +INSERT INTO test_array_contains VALUES (array(1, 2, 3), 2), (array(1, 2, 3), 4), (array(1, NULL, 3), NULL), (array(), 1), (NULL, 1) + +query spark_answer_only +SELECT array_contains(arr, val) FROM test_array_contains + +-- column + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3346) +SELECT array_contains(arr, 2) FROM test_array_contains + +-- literal + column +query spark_answer_only +SELECT array_contains(array(1, 2, 3), val) FROM test_array_contains + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3345) +SELECT array_contains(array(1, 2, 3), 2), array_contains(array(1, 2, 3), 4), array_contains(array(), 1), array_contains(cast(NULL as array), 1) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql b/spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql new file mode 100644 index 0000000000..43c18e9889 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_distinct(arr array) USING parquet + +statement +INSERT INTO test_array_distinct VALUES (array(1, 2, 2, 3, 3)), (array()), (NULL), (array(NULL, 1, NULL, 2)), (array(1)) + +query spark_answer_only +SELECT array_distinct(arr) FROM test_array_distinct + +-- literal arguments +query spark_answer_only +SELECT array_distinct(array(1, 2, 2, 3, 3)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_except.sql b/spark/src/test/resources/sql-tests/expressions/array/array_except.sql new file mode 100644 index 0000000000..110f798f97 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_except.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_except(a array, b array) USING parquet + +statement +INSERT INTO test_array_except VALUES (array(1, 2, 3), array(2, 3, 4)), (array(1, 2), array()), (array(), array(1)), (NULL, array(1)), (array(1, NULL), array(NULL)) + +query spark_answer_only +SELECT array_except(a, b) FROM test_array_except + +-- column + literal +query spark_answer_only +SELECT array_except(a, array(2, 3)) FROM test_array_except + +-- literal + column +query spark_answer_only +SELECT array_except(array(1, 2, 3), b) FROM test_array_except + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_except(array(1, 2, 3), array(2, 3, 4)), array_except(array(1, 2), array()), array_except(array(), array(1)), array_except(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_filter.sql b/spark/src/test/resources/sql-tests/expressions/array/array_filter.sql new file mode 100644 index 0000000000..a5cc165815 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_filter.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_filter(arr array) USING parquet + +statement +INSERT INTO test_array_filter VALUES (array(1, 2, 3, 4, 5)), (array(-1, 0, 1)), (array(10)), (NULL) + +query spark_answer_only +SELECT filter(arr, x -> x > 2) FROM test_array_filter + +query spark_answer_only +SELECT filter(arr, x -> x >= 0) FROM test_array_filter + +query spark_answer_only +SELECT filter(arr, (x, i) -> i > 0) FROM test_array_filter diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_insert.sql b/spark/src/test/resources/sql-tests/expressions/array/array_insert.sql new file mode 100644 index 0000000000..2d5754ea8e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_insert.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_insert(arr array, pos int, val int) USING parquet + +statement +INSERT INTO test_array_insert VALUES (array(1, 2, 3), 2, 10), (array(1, 2, 3), 1, 10), (array(1, 2, 3), 4, 10), (array(), 1, 10), (NULL, 1, 10) + +query spark_answer_only +SELECT array_insert(arr, pos, val) FROM test_array_insert diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql b/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql new file mode 100644 index 0000000000..379f16a642 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_intersect(a array, b array) USING parquet + +statement +INSERT INTO test_array_intersect VALUES (array(1, 2, 3), array(2, 3, 4)), (array(1, 2), array(3, 4)), (array(), array(1)), (NULL, array(1)), (array(1, NULL), array(NULL, 2)) + +query spark_answer_only +SELECT array_intersect(a, b) FROM test_array_intersect + +-- column + literal +query spark_answer_only +SELECT array_intersect(a, array(2, 3)) FROM test_array_intersect + +-- literal + column +query spark_answer_only +SELECT array_intersect(array(1, 2, 3), b) FROM test_array_intersect + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_intersect(array(1, 2, 3), array(2, 3, 4)), array_intersect(array(1, 2), array(3, 4)), array_intersect(array(), array(1)), array_intersect(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_join.sql b/spark/src/test/resources/sql-tests/expressions/array/array_join.sql new file mode 100644 index 0000000000..840fc3f2c8 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_join.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_join(arr array) USING parquet + +statement +INSERT INTO test_array_join VALUES (array('a', 'b', 'c')), (array('hello', 'world')), (array()), (NULL), (array('a', NULL, 'c')) + +query spark_answer_only +SELECT array_join(arr, ',') FROM test_array_join + +query spark_answer_only +SELECT array_join(arr, ',', 'NULL') FROM test_array_join diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_max.sql b/spark/src/test/resources/sql-tests/expressions/array/array_max.sql new file mode 100644 index 0000000000..17c572b952 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_max.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_max(arr array) USING parquet + +statement +INSERT INTO test_array_max VALUES (array(1, 2, 3)), (array(3, 1, 2)), (array()), (NULL), (array(NULL, 1, 2)), (array(-1, -2, -3)) + +query spark_answer_only +SELECT array_max(arr) FROM test_array_max + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_max(array(1, 2, 3)), array_max(array()), array_max(cast(NULL as array)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_min.sql b/spark/src/test/resources/sql-tests/expressions/array/array_min.sql new file mode 100644 index 0000000000..eb86d6c15f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_min.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_min(arr array) USING parquet + +statement +INSERT INTO test_array_min VALUES (array(1, 2, 3)), (array(3, 1, 2)), (array()), (NULL), (array(NULL, 1, 2)), (array(-1, -2, -3)) + +query spark_answer_only +SELECT array_min(arr) FROM test_array_min + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_min(array(1, 2, 3)), array_min(array()), array_min(cast(NULL as array)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql b/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql new file mode 100644 index 0000000000..bd1af4bc24 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_remove.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_remove(arr array, val int) USING parquet + +statement +INSERT INTO test_array_remove VALUES (array(1, 2, 3, 2), 2), (array(1, 2, 3), 4), (array(), 1), (NULL, 1), (array(1, NULL, 3), NULL) + +query spark_answer_only +SELECT array_remove(arr, val) FROM test_array_remove + +-- column + literal +query spark_answer_only +SELECT array_remove(arr, 2) FROM test_array_remove + +-- literal + column +query spark_answer_only +SELECT array_remove(array(1, 2, 3, 2), val) FROM test_array_remove + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_remove(array(1, 2, 3, 2), 2), array_remove(array(1, 2, 3), 4), array_remove(array(), 1), array_remove(cast(NULL as array), 1) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql b/spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql new file mode 100644 index 0000000000..926b1141ef --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_repeat(val int, cnt int) USING parquet + +statement +INSERT INTO test_array_repeat VALUES (1, 3), (NULL, 3), (1, 0), (1, -1), (1, NULL) + +query spark_answer_only +SELECT array_repeat(val, cnt) FROM test_array_repeat + +-- column + literal +query spark_answer_only +SELECT array_repeat(val, 3) FROM test_array_repeat + +-- literal + column +query spark_answer_only +SELECT array_repeat(1, cnt) FROM test_array_repeat + +-- literal + literal +query spark_answer_only +SELECT array_repeat(1, 3), array_repeat(NULL, 3), array_repeat(1, 0), array_repeat(1, -1) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_union.sql b/spark/src/test/resources/sql-tests/expressions/array/array_union.sql new file mode 100644 index 0000000000..4c699ae546 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/array_union.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_array_union(a array, b array) USING parquet + +statement +INSERT INTO test_array_union VALUES (array(1, 2, 3), array(3, 4, 5)), (array(1, 2), array()), (array(), array(1)), (NULL, array(1)), (array(1, NULL), array(NULL, 2)) + +query spark_answer_only +SELECT array_union(a, b) FROM test_array_union + +-- column + literal +query spark_answer_only +SELECT array_union(a, array(3, 4, 5)) FROM test_array_union + +-- literal + column +query spark_answer_only +SELECT array_union(array(1, 2, 3), b) FROM test_array_union + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT array_union(array(1, 2, 3), array(3, 4, 5)), array_union(array(1, 2), array()), array_union(array(), array(1)), array_union(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql new file mode 100644 index 0000000000..6c28224ed6 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_arrays_overlap(a array, b array) USING parquet + +statement +INSERT INTO test_arrays_overlap VALUES (array(1, 2, 3), array(3, 4, 5)), (array(1, 2), array(3, 4)), (array(), array(1)), (NULL, array(1)), (array(1, NULL), array(NULL, 2)) + +query spark_answer_only +SELECT arrays_overlap(a, b) FROM test_arrays_overlap + +-- column + literal +query spark_answer_only +SELECT arrays_overlap(a, array(3, 4, 5)) FROM test_arrays_overlap + +-- literal + column +query spark_answer_only +SELECT arrays_overlap(array(1, 2, 3), b) FROM test_arrays_overlap + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)), arrays_overlap(array(1, 2), array(3, 4)), arrays_overlap(array(), array(1)), arrays_overlap(cast(NULL as array), array(1)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql new file mode 100644 index 0000000000..0b41f0220c --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_create_array(a int, b int, c int) USING parquet + +statement +INSERT INTO test_create_array VALUES (1, 2, 3), (NULL, 2, 3), (NULL, NULL, NULL) + +query +SELECT array(a, b, c) FROM test_create_array + +query +SELECT array(1, 2, 3, NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/array/element_at.sql b/spark/src/test/resources/sql-tests/expressions/array/element_at.sql new file mode 100644 index 0000000000..a2e98849fc --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/element_at.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_element_at(arr array) USING parquet + +statement +INSERT INTO test_element_at VALUES (array(1, 2, 3)), (array(10)), (NULL) + +query spark_answer_only +SELECT element_at(arr, 1), element_at(arr, -1) FROM test_element_at + +-- literal arguments +query ignore(Spark codegen bug with literal element_at when constant folding is disabled) +SELECT element_at(array(1, 2, 3), 1), element_at(array(1, 2, 3), -1) diff --git a/spark/src/test/resources/sql-tests/expressions/array/flatten.sql b/spark/src/test/resources/sql-tests/expressions/array/flatten.sql new file mode 100644 index 0000000000..effc123605 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/flatten.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_flatten(arr array>) USING parquet + +statement +INSERT INTO test_flatten VALUES (array(array(1, 2), array(3, 4))), (array(array(), array(1))), (array()), (NULL), (array(array(1, NULL), array(NULL))) + +query spark_answer_only +SELECT flatten(arr) FROM test_flatten + +-- literal arguments +query spark_answer_only +SELECT flatten(array(array(1, 2), array(3, 4))) diff --git a/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql b/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql new file mode 100644 index 0000000000..99e6d68f9d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_get_array_item(arr array, idx int) USING parquet + +statement +INSERT INTO test_get_array_item VALUES (array(10, 20, 30), 0), (array(10, 20, 30), 1), (array(10, 20, 30), 2), (array(1), 0), (NULL, 0), (array(10, 20), NULL) + +query spark_answer_only +SELECT arr[0], arr[1], arr[2] FROM test_get_array_item + +query ignore(https://github.com/apache/datafusion-comet/issues/3332) +SELECT arr[idx] FROM test_get_array_item + +-- literal arguments +query spark_answer_only +SELECT array(10, 20, 30)[0], array(10, 20, 30)[2], array()[0] diff --git a/spark/src/test/resources/sql-tests/expressions/array/get_array_struct_fields.sql b/spark/src/test/resources/sql-tests/expressions/array/get_array_struct_fields.sql new file mode 100644 index 0000000000..e65bd297dd --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/get_array_struct_fields.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_arr_struct(arr array>) USING parquet + +statement +INSERT INTO test_arr_struct VALUES (array(named_struct('name', 'a', 'value', 1), named_struct('name', 'b', 'value', 2))), (array(named_struct('name', 'x', 'value', 10))), (NULL) + +query spark_answer_only +SELECT arr.name, arr.value FROM test_arr_struct diff --git a/spark/src/test/resources/sql-tests/expressions/array/size.sql b/spark/src/test/resources/sql-tests/expressions/array/size.sql new file mode 100644 index 0000000000..a096502bc5 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/array/size.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_size(arr array, m map) USING parquet + +statement +INSERT INTO test_size VALUES (array(1, 2, 3), map('a', 1, 'b', 2)), (array(), map()), (NULL, NULL) + +query spark_answer_only +SELECT size(arr), size(m) FROM test_size + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3338) +SELECT size(array(1, 2, 3)), size(array()), size(cast(NULL as array)) diff --git a/spark/src/test/resources/sql-tests/expressions/bitwise/bitwise.sql b/spark/src/test/resources/sql-tests/expressions/bitwise/bitwise.sql new file mode 100644 index 0000000000..640aa1e990 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/bitwise/bitwise.sql @@ -0,0 +1,83 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- Setup +statement +CREATE TABLE test(col1 int, col2 int) USING parquet + +statement +INSERT INTO test VALUES(1111, 2) + +statement +INSERT INTO test VALUES(1111, 2) + +statement +INSERT INTO test VALUES(3333, 4) + +statement +INSERT INTO test VALUES(5555, 6) + +-- Queries +query +SELECT col1 & col2, col1 | col2, col1 ^ col2 FROM test + +query +SELECT col1 & 1234, col1 | 1234, col1 ^ 1234 FROM test + +query +SELECT shiftright(col1, 2), shiftright(col1, col2) FROM test + +query +SELECT shiftleft(col1, 2), shiftleft(col1, col2) FROM test + +query +SELECT ~(11), ~col1, ~col2 FROM test + +-- BitwiseCount +statement +CREATE TABLE test_bit_count(i int, l long) USING parquet + +statement +INSERT INTO test_bit_count VALUES (0, 0), (1, 1), (7, 7), (-1, -1), (2147483647, 9223372036854775807), (NULL, NULL) + +query +SELECT bit_count(i), bit_count(l) FROM test_bit_count + +-- BitwiseGet +statement +CREATE TABLE test_bit_get(i int, pos int) USING parquet + +statement +INSERT INTO test_bit_get VALUES (11, 0), (11, 1), (11, 2), (11, 3), (0, 0), (NULL, 0), (11, NULL) + +query spark_answer_only +SELECT bit_get(i, pos) FROM test_bit_get + +-- literal arguments +query +SELECT 1111 & 2, 1111 | 2, 1111 ^ 2 + +query ignore(https://github.com/apache/datafusion-comet/issues/3341) +SELECT bit_count(0), bit_count(7), bit_count(-1) + +query spark_answer_only +SELECT bit_get(11, 0), bit_get(11, 1), bit_get(11, 2), bit_get(11, 3) + +query +SELECT shiftright(1111, 2), shiftleft(1111, 2) diff --git a/spark/src/test/resources/sql-tests/expressions/cast/cast.sql b/spark/src/test/resources/sql-tests/expressions/cast/cast.sql new file mode 100644 index 0000000000..e848e0e636 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/cast/cast.sql @@ -0,0 +1,46 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_cast(i int, l long, f float, d double, s string, b boolean) USING parquet + +statement +INSERT INTO test_cast VALUES (1, 1, 1.5, 1.5, '123', true), (0, 0, 0.0, 0.0, '0', false), (NULL, NULL, NULL, NULL, NULL, NULL), (-1, -1, -1.5, -1.5, '-1', true), (2147483647, 9223372036854775807, cast('NaN' as float), cast('Infinity' as double), 'abc', false) + +query +SELECT cast(i as long), cast(i as double), cast(i as string) FROM test_cast + +query +SELECT cast(l as int), cast(l as double), cast(l as string) FROM test_cast + +query +SELECT cast(f as double), cast(f as int), cast(f as string) FROM test_cast + +query +SELECT cast(d as float), cast(d as int), cast(d as string) FROM test_cast + +query +SELECT cast(s as int), cast(s as double) FROM test_cast + +query +SELECT cast(b as int), cast(b as string), cast(i as boolean) FROM test_cast + +-- literal arguments +query +SELECT cast(1 as long), cast(1 as double), cast(1 as string), cast('123' as int), cast('3.14' as double), cast(true as int), cast(NULL as int) diff --git a/spark/src/test/resources/sql-tests/expressions/conditional/boolean.sql b/spark/src/test/resources/sql-tests/expressions/conditional/boolean.sql new file mode 100644 index 0000000000..00688f5fa8 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/conditional/boolean.sql @@ -0,0 +1,41 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- compare true/false to negative zero +statement +CREATE TABLE test(col1 boolean, col2 float) USING parquet + +statement +INSERT INTO test VALUES(true, -0.0) + +statement +INSERT INTO test VALUES(false, -0.0) + +query +SELECT col1, negative(col2), cast(col1 as float), col1 = negative(col2) FROM test + +-- not +statement +CREATE TABLE test_not(col1 int, col2 boolean) USING parquet + +statement +INSERT INTO test_not VALUES(1, false), (2, true), (3, true), (3, false) + +query +SELECT col1, col2, NOT(col2), !(col2) FROM test_not diff --git a/spark/src/test/resources/sql-tests/expressions/conditional/case_when.sql b/spark/src/test/resources/sql-tests/expressions/conditional/case_when.sql new file mode 100644 index 0000000000..eff3123743 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/conditional/case_when.sql @@ -0,0 +1,37 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_case_when(i int, s string) USING parquet + +statement +INSERT INTO test_case_when VALUES (1, 'a'), (2, 'b'), (3, 'c'), (NULL, 'd'), (1, NULL), (NULL, NULL) + +query +SELECT CASE WHEN i = 1 THEN 'one' WHEN i = 2 THEN 'two' ELSE 'other' END FROM test_case_when + +query +SELECT CASE i WHEN 1 THEN 'one' WHEN 2 THEN 'two' END FROM test_case_when + +query +SELECT CASE WHEN s IS NULL THEN 'null_val' ELSE s END FROM test_case_when + +-- literal arguments +query +SELECT CASE WHEN i = 1 THEN s WHEN i = 2 THEN 'fixed' ELSE s END FROM test_case_when diff --git a/spark/src/test/resources/sql-tests/expressions/conditional/coalesce.sql b/spark/src/test/resources/sql-tests/expressions/conditional/coalesce.sql new file mode 100644 index 0000000000..26fa6a2553 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/conditional/coalesce.sql @@ -0,0 +1,37 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_coalesce(a int, b int, c int) USING parquet + +statement +INSERT INTO test_coalesce VALUES (1, 2, 3), (NULL, 2, 3), (NULL, NULL, 3), (NULL, NULL, NULL), (1, NULL, NULL) + +query +SELECT coalesce(a, b, c) FROM test_coalesce + +query +SELECT coalesce(a) FROM test_coalesce + +query +SELECT coalesce(a, 99) FROM test_coalesce + +-- literal arguments +query +SELECT coalesce(NULL, NULL, 99), coalesce(1, NULL, 99), coalesce(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/conditional/if_expr.sql b/spark/src/test/resources/sql-tests/expressions/conditional/if_expr.sql new file mode 100644 index 0000000000..ecc17f6b55 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/conditional/if_expr.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_if(cond boolean, a int, b int) USING parquet + +statement +INSERT INTO test_if VALUES (true, 1, 2), (false, 1, 2), (NULL, 1, 2), (true, NULL, 2), (false, 1, NULL) + +query +SELECT IF(cond, a, b) FROM test_if + +query +SELECT IF(a > 0, 'positive', 'non-positive') FROM test_if + +-- literal arguments +query +SELECT IF(true, 1, 2), IF(false, 1, 2), IF(NULL, 1, 2) diff --git a/spark/src/test/resources/sql-tests/expressions/conditional/in_set.sql b/spark/src/test/resources/sql-tests/expressions/conditional/in_set.sql new file mode 100644 index 0000000000..1cb0f248a6 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/conditional/in_set.sql @@ -0,0 +1,38 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: spark.sql.optimizer.inSetConversionThreshold=100,0 +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- test in(set)/not in(set) +statement +CREATE TABLE names(id int, name varchar(20)) USING parquet + +statement +INSERT INTO names VALUES(1, 'James'), (1, 'Jones'), (2, 'Smith'), (3, 'Smith'), (NULL, 'Jones'), (4, NULL) + +query +SELECT * FROM names WHERE id in (1, 2, 4, NULL) + +query +SELECT * FROM names WHERE name in ('Smith', 'Brown', NULL) + +query +SELECT * FROM names WHERE id not in (1) + +query spark_answer_only +SELECT * FROM names WHERE name not in ('Smith', 'Brown', NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/conditional/is_not_null.sql b/spark/src/test/resources/sql-tests/expressions/conditional/is_not_null.sql new file mode 100644 index 0000000000..d27a0a8b99 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/conditional/is_not_null.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_is_not_null(i int, s string, d double) USING parquet + +statement +INSERT INTO test_is_not_null VALUES (1, 'a', 1.0), (NULL, NULL, NULL), (0, '', 0.0), (NULL, 'b', cast('NaN' as double)) + +query +SELECT i IS NOT NULL, s IS NOT NULL, d IS NOT NULL FROM test_is_not_null + +query +SELECT isnotnull(i), isnotnull(s), isnotnull(d) FROM test_is_not_null diff --git a/spark/src/test/resources/sql-tests/expressions/conditional/is_null.sql b/spark/src/test/resources/sql-tests/expressions/conditional/is_null.sql new file mode 100644 index 0000000000..c3a1c5a741 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/conditional/is_null.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_is_null(i int, s string, d double) USING parquet + +statement +INSERT INTO test_is_null VALUES (1, 'a', 1.0), (NULL, NULL, NULL), (0, '', 0.0), (NULL, 'b', cast('NaN' as double)) + +query +SELECT i IS NULL, s IS NULL, d IS NULL FROM test_is_null + +query +SELECT isnull(i), isnull(s), isnull(d) FROM test_is_null diff --git a/spark/src/test/resources/sql-tests/expressions/conditional/predicates.sql b/spark/src/test/resources/sql-tests/expressions/conditional/predicates.sql new file mode 100644 index 0000000000..089bcc30fc --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/conditional/predicates.sql @@ -0,0 +1,45 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_pred(a int, b int) USING parquet + +statement +INSERT INTO test_pred VALUES (1, 2), (2, 2), (3, 1), (NULL, 1), (1, NULL), (NULL, NULL) + +query +SELECT a = b, a <=> b, a < b, a > b, a <= b, a >= b FROM test_pred + +query +SELECT a != b, NOT (a = b) FROM test_pred + +query +SELECT (a > 1) AND (b > 1), (a > 1) OR (b > 1), NOT (a > 1) FROM test_pred + +-- column op literal +query +SELECT a = 1, a <=> 1, a < 2, a > 0, a <= 1, a >= 2 FROM test_pred + +-- literal op column +query +SELECT 2 = a, 2 < a, 0 > a, 1 <= a, 3 >= a FROM test_pred + +-- literal op literal +query +SELECT 1 = 1, 1 = 2, 1 <=> NULL, NULL <=> NULL, NULL = NULL, 1 < 2, 2 > 1 diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_add.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_add.sql new file mode 100644 index 0000000000..c516cff07a --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_add.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_date_add(d date, n int) USING parquet + +statement +INSERT INTO test_date_add VALUES (date('2024-01-15'), 1), (date('2024-01-15'), -1), (date('2024-01-15'), 0), (date('2024-12-31'), 1), (NULL, 1), (date('2024-01-15'), NULL) + +query +SELECT date_add(d, n) FROM test_date_add + +-- column + literal +query +SELECT date_add(d, 5) FROM test_date_add + +-- literal + column +query +SELECT date_add(date('2024-01-15'), n) FROM test_date_add + +-- literal + literal +query +SELECT date_add(date('2024-01-15'), 5), date_add(date('2024-12-31'), 1), date_add(NULL, 1) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_diff.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_diff.sql new file mode 100644 index 0000000000..50ba8923c3 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_diff.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_datediff(d1 date, d2 date) USING parquet + +statement +INSERT INTO test_datediff VALUES (date('2024-01-15'), date('2024-01-10')), (date('2024-01-10'), date('2024-01-15')), (date('2024-01-15'), date('2024-01-15')), (NULL, date('2024-01-15')), (date('2024-01-15'), NULL) + +query +SELECT datediff(d1, d2) FROM test_datediff + +-- column + literal +query +SELECT datediff(d1, date('2024-01-10')) FROM test_datediff + +-- literal + column +query +SELECT datediff(date('2024-01-20'), d2) FROM test_datediff + +-- literal + literal +query +SELECT datediff(date('2024-01-15'), date('2024-01-10')), datediff(date('2024-01-10'), date('2024-01-15')), datediff(NULL, date('2024-01-15')) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_format.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_format.sql new file mode 100644 index 0000000000..7f31e8f811 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_format.sql @@ -0,0 +1,37 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_date_format(ts timestamp) USING parquet + +statement +INSERT INTO test_date_format VALUES (timestamp('2024-06-15 10:30:45')), (timestamp('1970-01-01 00:00:00')), (NULL) + +query expect_fallback(Non-UTC timezone) +SELECT date_format(ts, 'yyyy-MM-dd') FROM test_date_format + +query expect_fallback(Non-UTC timezone) +SELECT date_format(ts, 'HH:mm:ss') FROM test_date_format + +query expect_fallback(Non-UTC timezone) +SELECT date_format(ts, 'yyyy-MM-dd HH:mm:ss') FROM test_date_format + +-- literal arguments +query expect_fallback(Non-UTC timezone) +SELECT date_format(timestamp('2024-06-15 10:30:45'), 'yyyy-MM-dd') diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_format_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_format_enabled.sql new file mode 100644 index 0000000000..b225f9f1b6 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_format_enabled.sql @@ -0,0 +1,41 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test date_format() with allowIncompatible enabled (happy path) +-- Uses UTC timezone to ensure results match Spark +-- Config: spark.comet.expression.DateFormatClass.allowIncompatible=true +-- Config: spark.sql.session.timeZone=UTC +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_date_format_enabled(ts timestamp) USING parquet + +statement +INSERT INTO test_date_format_enabled VALUES (timestamp('2024-06-15 10:30:45')), (timestamp('1970-01-01 00:00:00')), (NULL) + +query +SELECT date_format(ts, 'yyyy-MM-dd') FROM test_date_format_enabled + +query +SELECT date_format(ts, 'HH:mm:ss') FROM test_date_format_enabled + +query +SELECT date_format(ts, 'yyyy-MM-dd HH:mm:ss') FROM test_date_format_enabled + +-- literal arguments +query +SELECT date_format(timestamp('2024-06-15 10:30:45'), 'yyyy-MM-dd') diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/date_sub.sql b/spark/src/test/resources/sql-tests/expressions/datetime/date_sub.sql new file mode 100644 index 0000000000..2dafdee89b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/date_sub.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_date_sub(d date, n int) USING parquet + +statement +INSERT INTO test_date_sub VALUES (date('2024-01-15'), 1), (date('2024-01-15'), -1), (date('2024-01-15'), 0), (date('2024-01-01'), 1), (NULL, 1), (date('2024-01-15'), NULL) + +query +SELECT date_sub(d, n) FROM test_date_sub + +-- column + literal +query +SELECT date_sub(d, 5) FROM test_date_sub + +-- literal + column +query +SELECT date_sub(date('2024-01-15'), n) FROM test_date_sub + +-- literal + literal +query +SELECT date_sub(date('2024-01-15'), 5), date_sub(date('2024-01-01'), 1), date_sub(NULL, 1) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/datetime.sql b/spark/src/test/resources/sql-tests/expressions/datetime/datetime.sql new file mode 100644 index 0000000000..a00516cdb3 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/datetime.sql @@ -0,0 +1,41 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- DatePart functions +statement +CREATE TABLE test_dt(col timestamp) USING parquet + +statement +INSERT INTO test_dt VALUES (timestamp('2024-06-15 10:30:00')), (timestamp('1900-01-01')), (null) + +query +SELECT col, year(col), month(col), day(col), weekday(col), dayofweek(col), dayofyear(col), weekofyear(col), quarter(col) FROM test_dt + +query +SELECT hour(col), minute(col), second(col) FROM test_dt + +-- Midnight and end-of-day +statement +CREATE TABLE test_dt_hms(ts timestamp) USING parquet + +statement +INSERT INTO test_dt_hms VALUES (timestamp('2024-01-01 00:00:00')), (timestamp('2024-01-01 23:59:59')), (timestamp('2024-06-15 12:30:45')), (NULL) + +query +SELECT hour(ts), minute(ts), second(ts) FROM test_dt_hms diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql new file mode 100644 index 0000000000..5af2864593 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql @@ -0,0 +1,37 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_from_unix_time(t long) USING parquet + +statement +INSERT INTO test_from_unix_time VALUES (0), (1718451045), (-1), (NULL), (2147483647) + +query expect_fallback(not fully compatible with Spark) +SELECT from_unixtime(t) FROM test_from_unix_time + +query expect_fallback(not fully compatible with Spark) +SELECT from_unixtime(t, 'yyyy-MM-dd') FROM test_from_unix_time + +-- literal arguments +query expect_fallback(not fully compatible with Spark) +SELECT from_unixtime(0) + +query expect_fallback(not fully compatible with Spark) +SELECT from_unixtime(1718451045, 'yyyy-MM-dd') diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql new file mode 100644 index 0000000000..293a03fb5f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql @@ -0,0 +1,42 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test from_unixtime() with allowIncompatible enabled (happy path) +-- Uses UTC timezone to ensure results match Spark +-- Config: spark.comet.expression.FromUnixTime.allowIncompatible=true +-- Config: spark.sql.session.timeZone=UTC +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_from_unix_time_enabled(t long) USING parquet + +statement +INSERT INTO test_from_unix_time_enabled VALUES (0), (1718451045), (-1), (NULL), (2147483647) + +-- Even with allowIncompatible=true, the default datetime pattern is unsupported natively +query spark_answer_only +SELECT from_unixtime(t) FROM test_from_unix_time_enabled + +query spark_answer_only +SELECT from_unixtime(t, 'yyyy-MM-dd') FROM test_from_unix_time_enabled + +-- literal arguments +query spark_answer_only +SELECT from_unixtime(0) + +query spark_answer_only +SELECT from_unixtime(1718451045, 'yyyy-MM-dd') diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/hour.sql b/spark/src/test/resources/sql-tests/expressions/datetime/hour.sql new file mode 100644 index 0000000000..551dd60c06 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/hour.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_hour(ts timestamp) USING parquet + +statement +INSERT INTO test_hour VALUES (timestamp('2024-01-15 00:00:00')), (timestamp('2024-01-15 12:30:45')), (timestamp('2024-01-15 23:59:59')), (NULL) + +query +SELECT hour(ts) FROM test_hour + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3336) +SELECT hour(timestamp('2024-01-15 00:00:00')), hour(timestamp('2024-01-15 12:30:45')), hour(timestamp('2024-01-15 23:59:59')) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/last_day.sql b/spark/src/test/resources/sql-tests/expressions/datetime/last_day.sql new file mode 100644 index 0000000000..b1482f13a5 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/last_day.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_last_day(d date) USING parquet + +statement +INSERT INTO test_last_day VALUES (date('2024-01-15')), (date('2024-02-15')), (date('2023-02-15')), (date('2024-12-01')), (NULL) + +query +SELECT last_day(d) FROM test_last_day + +-- literal arguments +query +SELECT last_day(date('2024-01-15')), last_day(date('2024-02-15')), last_day(date('2023-02-15')), last_day(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/minute.sql b/spark/src/test/resources/sql-tests/expressions/datetime/minute.sql new file mode 100644 index 0000000000..0b75084352 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/minute.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_minute(ts timestamp) USING parquet + +statement +INSERT INTO test_minute VALUES (timestamp('2024-01-15 10:00:00')), (timestamp('2024-01-15 10:30:00')), (timestamp('2024-01-15 10:59:59')), (NULL) + +query +SELECT minute(ts) FROM test_minute + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3336) +SELECT minute(timestamp('2024-01-15 10:00:00')), minute(timestamp('2024-01-15 10:30:00')), minute(timestamp('2024-01-15 10:59:59')) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/second.sql b/spark/src/test/resources/sql-tests/expressions/datetime/second.sql new file mode 100644 index 0000000000..5db4e9e743 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/second.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_second(ts timestamp) USING parquet + +statement +INSERT INTO test_second VALUES (timestamp('2024-01-15 10:30:00')), (timestamp('2024-01-15 10:30:30')), (timestamp('2024-01-15 10:30:59')), (NULL) + +query +SELECT second(ts) FROM test_second + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3336) +SELECT second(timestamp('2024-01-15 10:30:00')), second(timestamp('2024-01-15 10:30:30')), second(timestamp('2024-01-15 10:30:59')) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/trunc_date.sql b/spark/src/test/resources/sql-tests/expressions/datetime/trunc_date.sql new file mode 100644 index 0000000000..899a06f4bd --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/trunc_date.sql @@ -0,0 +1,37 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_trunc_date(d date) USING parquet + +statement +INSERT INTO test_trunc_date VALUES (date('2024-06-15')), (date('2024-01-01')), (date('2024-12-31')), (NULL) + +query +SELECT trunc(d, 'year') FROM test_trunc_date + +query +SELECT trunc(d, 'month') FROM test_trunc_date + +query +SELECT trunc(d, 'quarter') FROM test_trunc_date + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3342) +SELECT trunc(date('2024-06-15'), 'year'), trunc(date('2024-06-15'), 'month'), trunc(date('2024-06-15'), 'quarter') diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/trunc_timestamp.sql b/spark/src/test/resources/sql-tests/expressions/datetime/trunc_timestamp.sql new file mode 100644 index 0000000000..1105d014f3 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/trunc_timestamp.sql @@ -0,0 +1,40 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_trunc_ts(ts timestamp) USING parquet + +statement +INSERT INTO test_trunc_ts VALUES (timestamp('2024-06-15 10:30:45')), (timestamp('2024-01-01 00:00:00')), (NULL) + +query +SELECT date_trunc('year', ts) FROM test_trunc_ts + +query +SELECT date_trunc('month', ts) FROM test_trunc_ts + +query +SELECT date_trunc('day', ts) FROM test_trunc_ts + +query +SELECT date_trunc('hour', ts) FROM test_trunc_ts + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3342) +SELECT date_trunc('year', timestamp('2024-06-15 10:30:45')), date_trunc('month', timestamp('2024-06-15 10:30:45')), date_trunc('day', timestamp('2024-06-15 10:30:45')) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/unix_date.sql b/spark/src/test/resources/sql-tests/expressions/datetime/unix_date.sql new file mode 100644 index 0000000000..3e067dfd00 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/unix_date.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_unix_date(d date) USING parquet + +statement +INSERT INTO test_unix_date VALUES (date('1970-01-01')), (date('2024-01-15')), (date('1969-12-31')), (NULL) + +query spark_answer_only +SELECT unix_date(d) FROM test_unix_date + +-- literal arguments +query spark_answer_only +SELECT unix_date(date('1970-01-01')), unix_date(date('2024-01-15')), unix_date(date('1969-12-31')), unix_date(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/unix_timestamp.sql b/spark/src/test/resources/sql-tests/expressions/datetime/unix_timestamp.sql new file mode 100644 index 0000000000..fa4d9df07f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/unix_timestamp.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_unix_ts(ts timestamp) USING parquet + +statement +INSERT INTO test_unix_ts VALUES (timestamp('1970-01-01 00:00:00')), (timestamp('2024-06-15 10:30:45')), (NULL) + +query +SELECT unix_timestamp(ts) FROM test_unix_ts + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3336) +SELECT unix_timestamp(timestamp('1970-01-01 00:00:00')), unix_timestamp(timestamp('2024-06-15 10:30:45')) diff --git a/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql b/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql new file mode 100644 index 0000000000..3a856eb565 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/decimal/decimal_ops.sql @@ -0,0 +1,71 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.expression.Cast.allowIncompatible=true +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- Decimal arithmetic exercises CheckOverflow, MakeDecimal, UnscaledValue +statement +CREATE TABLE test_decimal(a decimal(10,2), b decimal(10,2)) USING parquet + +statement +INSERT INTO test_decimal VALUES (10.50, 3.20), (0.00, 1.00), (-10.50, 3.20), (99999999.99, 0.01), (NULL, 1.00) + +query +SELECT a + b, a - b, a * b, a / b FROM test_decimal + +query +SELECT a % b FROM test_decimal + +-- column + literal +query +SELECT a + cast(1.00 as decimal(10,2)), a - cast(1.00 as decimal(10,2)), a * cast(2.00 as decimal(10,2)) FROM test_decimal + +-- literal + column +query +SELECT cast(100.00 as decimal(10,2)) + b, cast(100.00 as decimal(10,2)) - b, cast(100.00 as decimal(10,2)) * b FROM test_decimal + +-- literal + literal +query +SELECT cast(10.50 as decimal(10,2)) + cast(3.20 as decimal(10,2)), cast(10.50 as decimal(10,2)) - cast(3.20 as decimal(10,2)), cast(10.50 as decimal(10,2)) * cast(3.20 as decimal(10,2)), cast(10.50 as decimal(10,2)) / cast(3.20 as decimal(10,2)) + +-- Mixed precision +statement +CREATE TABLE test_decimal_mix(a decimal(18,6), b decimal(10,2)) USING parquet + +statement +INSERT INTO test_decimal_mix VALUES (123456.789012, 99.99), (0.000001, 1.00), (-123456.789012, 0.01), (NULL, NULL) + +query +SELECT a + b, a - b, a * b FROM test_decimal_mix + +query spark_answer_only +SELECT a / b FROM test_decimal_mix + +-- Cast to decimal +statement +CREATE TABLE test_dec_cast(i int, l long, d double, s string) USING parquet + +statement +INSERT INTO test_dec_cast VALUES (42, 123456789, 3.14159, '99.99'), (0, 0, 0.0, '0.00'), (NULL, NULL, NULL, NULL) + +query +SELECT cast(i as decimal(10,2)), cast(l as decimal(18,2)), cast(d as decimal(10,5)), cast(s as decimal(10,2)) FROM test_dec_cast + +-- literal arguments +query +SELECT cast(42 as decimal(10,2)), cast(3.14 as decimal(10,5)), cast(NULL as decimal(10,2)) diff --git a/spark/src/test/resources/sql-tests/expressions/hash/hash.sql b/spark/src/test/resources/sql-tests/expressions/hash/hash.sql new file mode 100644 index 0000000000..35031ea7e4 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/hash/hash.sql @@ -0,0 +1,32 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- hash functions +statement +CREATE TABLE test(col string, a int, b float) USING parquet + +statement +INSERT INTO test VALUES ('Spark SQL ', 10, 1.2), (NULL, NULL, NULL), ('', 0, 0.0), ('苹果手机', NULL, 3.999999), ('Spark SQL ', 10, 1.2), (NULL, NULL, NULL), ('', 0, 0.0), ('苹果手机', NULL, 3.999999) + +query +SELECT md5(col), md5(cast(a as string)), md5(cast(b as string)), hash(col), hash(col, 1), hash(col, 0), hash(col, a, b), hash(b, a, col), xxhash64(col), xxhash64(col, 1), xxhash64(col, 0), xxhash64(col, a, b), xxhash64(b, a, col), sha2(col, 0), sha2(col, 256), sha2(col, 224), sha2(col, 384), sha2(col, 512), sha2(col, 128), sha2(col, -1), sha1(col), sha1(cast(a as string)), sha1(cast(b as string)) FROM test + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3340) +SELECT md5('Spark SQL'), sha1('test'), sha2('test', 256), hash('test'), xxhash64('test') diff --git a/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql b/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql new file mode 100644 index 0000000000..f97ffe8729 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_map(m map) USING parquet + +statement +INSERT INTO test_map VALUES (map('a', 1, 'b', 2, 'c', 3)), (map('x', 10)), (NULL) + +query spark_answer_only +SELECT m['a'], m['b'], m['c'] FROM test_map + +query spark_answer_only +SELECT m['x'], m['missing'] FROM test_map + +-- literal arguments +query spark_answer_only +SELECT map('a', 1, 'b', 2)['a'], map('a', 1, 'b', 2)['missing'], map('a', 1, 'b', 2)[NULL] diff --git a/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql b/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql new file mode 100644 index 0000000000..b89fdec060 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_map_entries(m map) USING parquet + +statement +INSERT INTO test_map_entries VALUES (map('a', 1, 'b', 2)), (map()), (NULL) + +query spark_answer_only +SELECT map_entries(m) FROM test_map_entries diff --git a/spark/src/test/resources/sql-tests/expressions/map/map_from_arrays.sql b/spark/src/test/resources/sql-tests/expressions/map/map_from_arrays.sql new file mode 100644 index 0000000000..5d6ac3d550 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/map_from_arrays.sql @@ -0,0 +1,36 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_map_from_arrays(k array, v array) USING parquet + +statement +INSERT INTO test_map_from_arrays VALUES (array('a', 'b', 'c'), array(1, 2, 3)), (array(), array()), (NULL, NULL) + +query spark_answer_only +SELECT map_from_arrays(k, v) FROM test_map_from_arrays WHERE k IS NOT NULL + +-- Comet bug: map_from_arrays(NULL, NULL) causes native crash "map key cannot be null" +-- https://github.com/apache/datafusion-comet/issues/3327 +query ignore(https://github.com/apache/datafusion-comet/issues/3327) +SELECT map_from_arrays(k, v) FROM test_map_from_arrays WHERE k IS NULL + +-- literal arguments +query spark_answer_only +SELECT map_from_arrays(array('a', 'b'), array(1, 2)) diff --git a/spark/src/test/resources/sql-tests/expressions/map/map_keys.sql b/spark/src/test/resources/sql-tests/expressions/map/map_keys.sql new file mode 100644 index 0000000000..a6591ce800 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/map_keys.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_map_keys(m map) USING parquet + +statement +INSERT INTO test_map_keys VALUES (map('a', 1, 'b', 2, 'c', 3)), (map()), (NULL) + +query spark_answer_only +SELECT map_keys(m) FROM test_map_keys diff --git a/spark/src/test/resources/sql-tests/expressions/map/map_values.sql b/spark/src/test/resources/sql-tests/expressions/map/map_values.sql new file mode 100644 index 0000000000..717e614e0d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/map_values.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_map_values(m map) USING parquet + +statement +INSERT INTO test_map_values VALUES (map('a', 1, 'b', 2, 'c', 3)), (map()), (NULL) + +query spark_answer_only +SELECT map_values(m) FROM test_map_values diff --git a/spark/src/test/resources/sql-tests/expressions/math/abs.sql b/spark/src/test/resources/sql-tests/expressions/math/abs.sql new file mode 100644 index 0000000000..c51270ae35 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/abs.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_abs(i int, l long, f float, d double) USING parquet + +statement +INSERT INTO test_abs VALUES (1, 1, 1.5, 1.5), (-1, -1, -1.5, -1.5), (0, 0, 0.0, 0.0), (NULL, NULL, NULL, NULL), (2147483647, 9223372036854775807, cast('Infinity' as float), cast('NaN' as double)) + +query +SELECT abs(i), abs(l), abs(f), abs(d) FROM test_abs + +-- literal arguments +query +SELECT abs(-5), abs(-1.5), abs(0), abs(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/acos.sql b/spark/src/test/resources/sql-tests/expressions/math/acos.sql new file mode 100644 index 0000000000..8b8e3c2565 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/acos.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_acos(d double) USING parquet + +statement +INSERT INTO test_acos VALUES (0.0), (1.0), (-1.0), (0.5), (NULL), (cast('NaN' as double)), (2.0), (-2.0) + +query tolerance=1e-6 +SELECT acos(d) FROM test_acos + +-- literal arguments +query tolerance=1e-6 +SELECT acos(0.5), acos(1.0), acos(-1.0), acos(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/arithmetic.sql b/spark/src/test/resources/sql-tests/expressions/math/arithmetic.sql new file mode 100644 index 0000000000..de611ddfc7 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/arithmetic.sql @@ -0,0 +1,83 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +-- negative +statement +CREATE TABLE test_neg(col1 int) USING parquet + +statement +INSERT INTO test_neg VALUES(1), (2), (3), (3) + +query +SELECT negative(col1), -(col1) FROM test_neg + +-- integral division overflow +statement +CREATE TABLE test_div(c1 long, c2 short) USING parquet + +statement +INSERT INTO test_div VALUES(-9223372036854775808, -1) + +query +SELECT c1 div c2 FROM test_div ORDER BY c1 + +-- Add, Subtract, Multiply, Divide, Remainder +statement +CREATE TABLE test_arith(a int, b int, la long, lb long, fa float, fb float, da double, db double) USING parquet + +statement +INSERT INTO test_arith VALUES (10, 3, 10, 3, 10.5, 3.2, 10.5, 3.2), (0, 1, 0, 1, 0.0, 1.0, 0.0, 1.0), (-10, 3, -10, 3, -10.5, 3.2, -10.5, 3.2), (2147483647, 1, 9223372036854775807, 1, cast('Infinity' as float), 1.0, cast('NaN' as double), 1.0), (NULL, 1, NULL, 1, NULL, 1.0, NULL, 1.0) + +query +SELECT a + b, a - b, a * b, a / b, a % b FROM test_arith + +query +SELECT la + lb, la - lb, la * lb, la / lb, la % lb FROM test_arith + +query +SELECT fa + fb, fa - fb, fa * fb, fa / fb FROM test_arith + +query +SELECT da + db, da - db, da * db, da / db FROM test_arith + +-- column + literal +query +SELECT a + 3, a - 3, a * 3, a / 3, a % 3 FROM test_arith + +-- literal + column +query +SELECT 10 + b, 10 - b, 10 * b, 10 / b, 10 % b FROM test_arith + +-- literal + literal +query +SELECT 10 + 3, 10 - 3, 10 * 3, 10 / 3, 10 % 3 + +-- unary negative with literal +query +SELECT negative(5), negative(-5), negative(0), -(5) + +-- division by zero +statement +CREATE TABLE test_div_zero(a int, b int, d double) USING parquet + +statement +INSERT INTO test_div_zero VALUES (1, 0, 0.0), (0, 0, 0.0) + +query +SELECT a / b, a % b, d / 0.0 FROM test_div_zero diff --git a/spark/src/test/resources/sql-tests/expressions/math/asin.sql b/spark/src/test/resources/sql-tests/expressions/math/asin.sql new file mode 100644 index 0000000000..5a59ed944d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/asin.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_asin(d double) USING parquet + +statement +INSERT INTO test_asin VALUES (0.0), (1.0), (-1.0), (0.5), (NULL), (cast('NaN' as double)), (2.0), (-2.0) + +query tolerance=1e-6 +SELECT asin(d) FROM test_asin + +-- literal arguments +query tolerance=1e-6 +SELECT asin(0.5), asin(1.0), asin(-1.0), asin(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/atan.sql b/spark/src/test/resources/sql-tests/expressions/math/atan.sql new file mode 100644 index 0000000000..b9037c7775 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/atan.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_atan(d double) USING parquet + +statement +INSERT INTO test_atan VALUES (0.0), (1.0), (-1.0), (100.0), (-100.0), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)), (cast('-Infinity' as double)) + +query tolerance=1e-6 +SELECT atan(d) FROM test_atan + +-- literal arguments +query tolerance=1e-6 +SELECT atan(1.0), atan(0.0), atan(-1.0), atan(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/atan2.sql b/spark/src/test/resources/sql-tests/expressions/math/atan2.sql new file mode 100644 index 0000000000..7a912930b8 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/atan2.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_atan2(y double, x double) USING parquet + +statement +INSERT INTO test_atan2 VALUES (0.0, 1.0), (1.0, 0.0), (1.0, 1.0), (-1.0, -1.0), (0.0, 0.0), (NULL, 1.0), (1.0, NULL), (cast('NaN' as double), 1.0), (cast('Infinity' as double), 1.0) + +query tolerance=1e-6 +SELECT atan2(y, x) FROM test_atan2 + +-- column + literal +query tolerance=1e-6 +SELECT atan2(y, 1.0) FROM test_atan2 + +-- literal + column +query tolerance=1e-6 +SELECT atan2(1.0, x) FROM test_atan2 + +-- literal + literal +query tolerance=1e-6 +SELECT atan2(1.0, 1.0), atan2(0.0, 0.0), atan2(-1.0, -1.0), atan2(NULL, 1.0) diff --git a/spark/src/test/resources/sql-tests/expressions/math/ceil.sql b/spark/src/test/resources/sql-tests/expressions/math/ceil.sql new file mode 100644 index 0000000000..fade75d28a --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/ceil.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_ceil(f float, d double) USING parquet + +statement +INSERT INTO test_ceil VALUES (1.1, 1.1), (-1.1, -1.1), (0.0, 0.0), (1.0, 1.0), (NULL, NULL), (cast('NaN' as float), cast('NaN' as double)), (cast('Infinity' as float), cast('Infinity' as double)) + +query +SELECT ceil(f), ceil(d) FROM test_ceil + +-- literal arguments +query +SELECT ceil(1.1), ceil(-1.1), ceil(0.0), ceil(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/cos.sql b/spark/src/test/resources/sql-tests/expressions/math/cos.sql new file mode 100644 index 0000000000..7ebc24bb99 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/cos.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_cos(d double) USING parquet + +statement +INSERT INTO test_cos VALUES (0.0), (3.141592653589793), (1.5707963267948966), (-3.141592653589793), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT cos(d) FROM test_cos + +-- literal arguments +query tolerance=1e-6 +SELECT cos(0.0), cos(3.141592653589793), cos(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/cosh.sql b/spark/src/test/resources/sql-tests/expressions/math/cosh.sql new file mode 100644 index 0000000000..a751960362 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/cosh.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_cosh(d double) USING parquet + +statement +INSERT INTO test_cosh VALUES (0.0), (1.0), (-1.0), (100.0), (NULL), (cast('NaN' as double)) + +query tolerance=1e-6 +SELECT cosh(d) FROM test_cosh + +-- literal arguments +query tolerance=1e-6 +SELECT cosh(0.0), cosh(1.0), cosh(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/cot.sql b/spark/src/test/resources/sql-tests/expressions/math/cot.sql new file mode 100644 index 0000000000..fc6193f8f5 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/cot.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_cot(d double) USING parquet + +statement +INSERT INTO test_cot VALUES (0.7853981633974483), (1.0), (-1.0), (0.1), (NULL), (cast('NaN' as double)) + +query tolerance=1e-6 +SELECT cot(d) FROM test_cot + +-- literal arguments +query tolerance=1e-6 +SELECT cot(1.0), cot(0.5), cot(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/exp.sql b/spark/src/test/resources/sql-tests/expressions/math/exp.sql new file mode 100644 index 0000000000..243aefba96 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/exp.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_exp(d double) USING parquet + +statement +INSERT INTO test_exp VALUES (0.0), (1.0), (-1.0), (100.0), (-100.0), (NULL), (cast('NaN' as double)), (cast('-Infinity' as double)) + +query tolerance=1e-6 +SELECT exp(d) FROM test_exp + +-- literal arguments +query tolerance=1e-6 +SELECT exp(0.0), exp(1.0), exp(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/expm1.sql b/spark/src/test/resources/sql-tests/expressions/math/expm1.sql new file mode 100644 index 0000000000..2eaba2b739 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/expm1.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_expm1(d double) USING parquet + +statement +INSERT INTO test_expm1 VALUES (0.0), (1.0), (-1.0), (0.001), (NULL), (cast('NaN' as double)), (cast('-Infinity' as double)) + +query tolerance=1e-6 +SELECT expm1(d) FROM test_expm1 + +-- literal arguments +query tolerance=1e-6 +SELECT expm1(0.0), expm1(1.0), expm1(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/floor.sql b/spark/src/test/resources/sql-tests/expressions/math/floor.sql new file mode 100644 index 0000000000..3960002846 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/floor.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_floor(f float, d double) USING parquet + +statement +INSERT INTO test_floor VALUES (1.9, 1.9), (-1.1, -1.1), (0.0, 0.0), (1.0, 1.0), (NULL, NULL), (cast('NaN' as float), cast('NaN' as double)), (cast('Infinity' as float), cast('Infinity' as double)) + +query +SELECT floor(f), floor(d) FROM test_floor + +-- literal arguments +query +SELECT floor(1.9), floor(-1.1), floor(0.0), floor(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/isnan.sql b/spark/src/test/resources/sql-tests/expressions/math/isnan.sql new file mode 100644 index 0000000000..7b7914c3cb --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/isnan.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_isnan(f float, d double) USING parquet + +statement +INSERT INTO test_isnan VALUES (1.0, 1.0), (cast('NaN' as float), cast('NaN' as double)), (NULL, NULL), (cast('Infinity' as float), cast('Infinity' as double)), (0.0, 0.0) + +query +SELECT isnan(f), isnan(d) FROM test_isnan + +-- literal arguments +query +SELECT isnan(cast('NaN' as double)), isnan(1.0), isnan(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/log.sql b/spark/src/test/resources/sql-tests/expressions/math/log.sql new file mode 100644 index 0000000000..e7420954cf --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/log.sql @@ -0,0 +1,42 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_log(d double) USING parquet + +statement +INSERT INTO test_log VALUES (1.0), (2.718281828459045), (10.0), (0.5), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT ln(d) FROM test_log + +query tolerance=1e-6 +SELECT log(10.0, d) FROM test_log + +-- column + literal (log base from column) +query tolerance=1e-6 +SELECT log(d, 10.0) FROM test_log + +-- literal (1-arg form) +query tolerance=1e-6 +SELECT ln(1.0), ln(2.718281828459045), ln(10.0), ln(NULL) + +-- literal + literal (2-arg form) +query tolerance=1e-6 +SELECT log(10.0, 100.0), log(2.0, 8.0), log(10.0, 1.0), log(NULL, 10.0) diff --git a/spark/src/test/resources/sql-tests/expressions/math/log10.sql b/spark/src/test/resources/sql-tests/expressions/math/log10.sql new file mode 100644 index 0000000000..1b3c9417f1 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/log10.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_log10(d double) USING parquet + +statement +INSERT INTO test_log10 VALUES (1.0), (10.0), (100.0), (0.1), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT log10(d) FROM test_log10 + +-- literal arguments +query tolerance=1e-6 +SELECT log10(100.0), log10(1.0), log10(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/log2.sql b/spark/src/test/resources/sql-tests/expressions/math/log2.sql new file mode 100644 index 0000000000..5db0ca484b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/log2.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_log2(d double) USING parquet + +statement +INSERT INTO test_log2 VALUES (1.0), (2.0), (4.0), (8.0), (0.5), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT log2(d) FROM test_log2 + +-- literal arguments +query tolerance=1e-6 +SELECT log2(8.0), log2(1.0), log2(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/pow.sql b/spark/src/test/resources/sql-tests/expressions/math/pow.sql new file mode 100644 index 0000000000..82b1155303 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/pow.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_pow(base double, exp double) USING parquet + +statement +INSERT INTO test_pow VALUES (2.0, 3.0), (0.0, 0.0), (-1.0, 2.0), (-1.0, 0.5), (2.0, -1.0), (NULL, 2.0), (2.0, NULL), (cast('NaN' as double), 2.0), (cast('Infinity' as double), 2.0), (2.0, cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT pow(base, exp) FROM test_pow + +-- column + literal +query tolerance=1e-6 +SELECT pow(base, 2.0) FROM test_pow + +-- literal + column +query tolerance=1e-6 +SELECT pow(2.0, exp) FROM test_pow + +-- literal + literal +query tolerance=1e-6 +SELECT pow(2.0, 3.0), pow(0.0, 0.0), pow(-1.0, 2.0), pow(NULL, 2.0) diff --git a/spark/src/test/resources/sql-tests/expressions/math/round.sql b/spark/src/test/resources/sql-tests/expressions/math/round.sql new file mode 100644 index 0000000000..617968eb35 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/round.sql @@ -0,0 +1,40 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_round(d double, i int) USING parquet + +statement +INSERT INTO test_round VALUES (2.5, 0), (3.5, 0), (-2.5, 0), (123.456, 2), (123.456, -1), (NULL, 0), (cast('NaN' as double), 0), (cast('Infinity' as double), 0), (0.0, 0) + +query expect_fallback(BigDecimal rounding) +SELECT round(d, 0) FROM test_round WHERE i = 0 + +query expect_fallback(BigDecimal rounding) +SELECT round(d, 2) FROM test_round WHERE i = 2 + +query expect_fallback(BigDecimal rounding) +SELECT round(d, -1) FROM test_round WHERE i = -1 + +query expect_fallback(BigDecimal rounding) +SELECT round(d) FROM test_round + +-- literal + literal +query expect_fallback(BigDecimal rounding) +SELECT round(123.456, 2), round(2.5, 0), round(3.5, 0), round(-2.5, 0), round(NULL, 0) diff --git a/spark/src/test/resources/sql-tests/expressions/math/signum.sql b/spark/src/test/resources/sql-tests/expressions/math/signum.sql new file mode 100644 index 0000000000..6dd2f2209a --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/signum.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_signum(d double) USING parquet + +statement +INSERT INTO test_signum VALUES (5.0), (-5.0), (0.0), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)), (cast('-Infinity' as double)) + +query +SELECT signum(d) FROM test_signum + +-- literal arguments +query +SELECT signum(-5.0), signum(5.0), signum(0.0), signum(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/sin.sql b/spark/src/test/resources/sql-tests/expressions/math/sin.sql new file mode 100644 index 0000000000..dca3ad77b4 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/sin.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_sin(d double) USING parquet + +statement +INSERT INTO test_sin VALUES (0.0), (3.141592653589793), (1.5707963267948966), (-1.5707963267948966), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT sin(d) FROM test_sin + +-- literal arguments +query tolerance=1e-6 +SELECT sin(0.0), sin(1.5707963267948966), sin(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/sinh.sql b/spark/src/test/resources/sql-tests/expressions/math/sinh.sql new file mode 100644 index 0000000000..432c58c583 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/sinh.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_sinh(d double) USING parquet + +statement +INSERT INTO test_sinh VALUES (0.0), (1.0), (-1.0), (NULL), (cast('NaN' as double)) + +query tolerance=1e-6 +SELECT sinh(d) FROM test_sinh + +-- literal arguments +query tolerance=1e-6 +SELECT sinh(0.0), sinh(1.0), sinh(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/sqrt.sql b/spark/src/test/resources/sql-tests/expressions/math/sqrt.sql new file mode 100644 index 0000000000..ec3c066eb9 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/sqrt.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_sqrt(d double) USING parquet + +statement +INSERT INTO test_sqrt VALUES (0.0), (1.0), (4.0), (2.0), (-1.0), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT sqrt(d) FROM test_sqrt + +-- literal arguments +query tolerance=1e-6 +SELECT sqrt(4.0), sqrt(2.0), sqrt(0.0), sqrt(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/tan.sql b/spark/src/test/resources/sql-tests/expressions/math/tan.sql new file mode 100644 index 0000000000..21bd44f907 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/tan.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_tan(d double) USING parquet + +statement +INSERT INTO test_tan VALUES (0.0), (0.7853981633974483), (-0.7853981633974483), (1.0), (NULL), (cast('NaN' as double)), (cast('Infinity' as double)) + +query tolerance=1e-6 +SELECT tan(d) FROM test_tan + +-- literal arguments +query tolerance=1e-6 +SELECT tan(0.0), tan(0.7853981633974483), tan(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/math/tanh.sql b/spark/src/test/resources/sql-tests/expressions/math/tanh.sql new file mode 100644 index 0000000000..9d70202776 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/math/tanh.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_tanh(d double) USING parquet + +statement +INSERT INTO test_tanh VALUES (0.0), (1.0), (-1.0), (100.0), (-100.0), (NULL), (cast('NaN' as double)) + +query tolerance=1e-6 +SELECT tanh(d) FROM test_tanh + +-- literal arguments +query tolerance=1e-6 +SELECT tanh(0.0), tanh(1.0), tanh(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/misc/parquet_default_values.sql b/spark/src/test/resources/sql-tests/expressions/misc/parquet_default_values.sql new file mode 100644 index 0000000000..bb1b003c79 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/misc/parquet_default_values.sql @@ -0,0 +1,29 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- parquet default values +statement +CREATE TABLE t1(col1 boolean) USING parquet + +statement +INSERT INTO t1 VALUES(true) + +statement +ALTER TABLE t1 ADD COLUMN col2 string DEFAULT 'hello' + +query +SELECT * FROM t1 diff --git a/spark/src/test/resources/sql-tests/expressions/misc/scalar_subquery.sql b/spark/src/test/resources/sql-tests/expressions/misc/scalar_subquery.sql new file mode 100644 index 0000000000..8b2cad053d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/misc/scalar_subquery.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +statement +CREATE TABLE test_subq(a int, b int) USING parquet + +statement +INSERT INTO test_subq VALUES (1, 10), (2, 20), (3, 30), (NULL, 40) + +query spark_answer_only +SELECT a, (SELECT max(b) FROM test_subq) FROM test_subq + +query spark_answer_only +SELECT a, (SELECT min(b) FROM test_subq) + a FROM test_subq + +query spark_answer_only +SELECT a, b, b > (SELECT avg(b) FROM test_subq) FROM test_subq diff --git a/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql b/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql new file mode 100644 index 0000000000..443864abe7 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/misc/width_bucket.sql @@ -0,0 +1,37 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MinSparkVersion: 3.5 + +statement +CREATE TABLE test_wb(v double) USING parquet + +statement +INSERT INTO test_wb VALUES (0.0), (2.5), (5.0), (7.5), (10.0), (-1.0), (11.0), (NULL) + +query ignore(https://github.com/apache/datafusion-comet/issues/3331) +SELECT v, width_bucket(v, 0, 10, 4) FROM test_wb + +query ignore(https://github.com/apache/datafusion-comet/issues/3331) +SELECT v, width_bucket(v, 10, 0, 4) FROM test_wb + +query ignore(https://github.com/apache/datafusion-comet/issues/3331) +SELECT v, width_bucket(v, 0, 10, 1) FROM test_wb + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3331) +SELECT width_bucket(5.0, 0, 10, 4), width_bucket(0.0, 0, 10, 4), width_bucket(NULL, 0, 10, 4) diff --git a/spark/src/test/resources/sql-tests/expressions/string/ascii.sql b/spark/src/test/resources/sql-tests/expressions/string/ascii.sql new file mode 100644 index 0000000000..bca7b77932 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/ascii.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_ascii(s string) USING parquet + +statement +INSERT INTO test_ascii VALUES ('A'), ('z'), ('0'), (''), (NULL), ('hello') + +query +SELECT ascii(s) FROM test_ascii + +-- literal arguments +query +SELECT ascii('A'), ascii(''), ascii(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/bit_length.sql b/spark/src/test/resources/sql-tests/expressions/string/bit_length.sql new file mode 100644 index 0000000000..83044267de --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/bit_length.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_bit_length(s string) USING parquet + +statement +INSERT INTO test_bit_length VALUES (''), ('a'), ('hello'), (NULL), ('café') + +query +SELECT bit_length(s) FROM test_bit_length + +-- literal arguments +query +SELECT bit_length('hello'), bit_length(''), bit_length(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/chr.sql b/spark/src/test/resources/sql-tests/expressions/string/chr.sql new file mode 100644 index 0000000000..bce93d75dd --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/chr.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_chr(i long) USING parquet + +statement +INSERT INTO test_chr VALUES (65), (97), (0), (NULL), (128522), (256), (48) + +query +SELECT chr(i) FROM test_chr + +-- literal arguments +query +SELECT chr(65), chr(0), chr(-1), chr(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/concat.sql b/spark/src/test/resources/sql-tests/expressions/string/concat.sql new file mode 100644 index 0000000000..fcf2416bc2 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/concat.sql @@ -0,0 +1,38 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_concat(a string, b string, c string) USING parquet + +statement +INSERT INTO test_concat VALUES ('hello', ' ', 'world'), ('', '', ''), (NULL, 'b', 'c'), ('a', NULL, 'c'), (NULL, NULL, NULL) + +query +SELECT concat(a, b, c) FROM test_concat + +query +SELECT a || b || c FROM test_concat + +-- mixed: column + literal + column +query +SELECT concat(a, ' ', c) FROM test_concat + +-- literal + literal + literal +query +SELECT concat('hello', ' ', 'world'), concat('', '', ''), concat(NULL, 'b', 'c') diff --git a/spark/src/test/resources/sql-tests/expressions/string/concat_ws.sql b/spark/src/test/resources/sql-tests/expressions/string/concat_ws.sql new file mode 100644 index 0000000000..fd277fd7c4 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/concat_ws.sql @@ -0,0 +1,37 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_concat_ws(a string, b string, c string) USING parquet + +statement +INSERT INTO test_concat_ws VALUES ('hello', 'beautiful', 'world'), ('', '', ''), (NULL, 'b', 'c'), ('a', NULL, 'c'), (NULL, NULL, NULL) + +query +SELECT concat_ws(',', a, b, c) FROM test_concat_ws + +query +SELECT concat_ws('', a, b, c) FROM test_concat_ws + +query +SELECT concat_ws(NULL, a, b, c) FROM test_concat_ws + +-- literal + literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3339) +SELECT concat_ws(',', 'hello', 'world'), concat_ws(',', '', ''), concat_ws(',', NULL, 'b', 'c'), concat_ws(NULL, 'a', 'b') diff --git a/spark/src/test/resources/sql-tests/expressions/string/contains.sql b/spark/src/test/resources/sql-tests/expressions/string/contains.sql new file mode 100644 index 0000000000..32a9bca01e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/contains.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_contains(s string, sub string) USING parquet + +statement +INSERT INTO test_contains VALUES ('hello world', 'world'), ('hello', ''), ('', ''), ('hello', 'xyz'), (NULL, 'a'), ('hello', NULL) + +query +SELECT contains(s, sub) FROM test_contains + +-- column + literal +query +SELECT contains(s, 'world') FROM test_contains + +-- literal + column +query +SELECT contains('hello world', sub) FROM test_contains + +-- literal + literal +query +SELECT contains('hello world', 'world'), contains('hello', 'xyz'), contains('', ''), contains(NULL, 'a') diff --git a/spark/src/test/resources/sql-tests/expressions/string/ends_with.sql b/spark/src/test/resources/sql-tests/expressions/string/ends_with.sql new file mode 100644 index 0000000000..957cb93c12 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/ends_with.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_ends_with(s string, suffix string) USING parquet + +statement +INSERT INTO test_ends_with VALUES ('hello world', 'world'), ('hello', ''), ('', ''), ('hello', 'xyz'), (NULL, 'a'), ('hello', NULL) + +query +SELECT endswith(s, suffix) FROM test_ends_with + +-- column + literal +query +SELECT endswith(s, 'world') FROM test_ends_with + +-- literal + column +query +SELECT endswith('hello world', suffix) FROM test_ends_with + +-- literal + literal +query +SELECT endswith('hello world', 'world'), endswith('hello', 'xyz'), endswith('', ''), endswith(NULL, 'a') diff --git a/spark/src/test/resources/sql-tests/expressions/string/hex.sql b/spark/src/test/resources/sql-tests/expressions/string/hex.sql new file mode 100644 index 0000000000..d0be03372d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/hex.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_hex(i int, l long, s string) USING parquet + +statement +INSERT INTO test_hex VALUES (0, 0, ''), (255, 255, 'Spark'), (-1, -1, NULL), (NULL, NULL, 'ABC'), (2147483647, 9223372036854775807, 'a') + +query +SELECT hex(i), hex(l), hex(s) FROM test_hex + +-- literal arguments +query +SELECT hex(255), hex('Spark'), hex(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/init_cap.sql b/spark/src/test/resources/sql-tests/expressions/string/init_cap.sql new file mode 100644 index 0000000000..90f5afc8f4 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/init_cap.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_initcap(s string) USING parquet + +statement +INSERT INTO test_initcap VALUES ('hello world'), ('HELLO WORLD'), (''), (NULL), ('hello-world'), ('123abc'), (' spaces ') + +query expect_fallback(not fully compatible with Spark) +SELECT initcap(s) FROM test_initcap diff --git a/spark/src/test/resources/sql-tests/expressions/string/init_cap_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/init_cap_enabled.sql new file mode 100644 index 0000000000..03c9de8817 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/init_cap_enabled.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test initcap() with allowIncompatible enabled (happy path) +-- Config: spark.comet.expression.InitCap.allowIncompatible=true +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_initcap_enabled(s string) USING parquet + +statement +INSERT INTO test_initcap_enabled VALUES ('hello world'), ('HELLO WORLD'), (''), (NULL), ('123abc'), (' spaces ') + +query +SELECT initcap(s) FROM test_initcap_enabled + +-- literal arguments +query +SELECT initcap('hello world'), initcap(''), initcap(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/left.sql b/spark/src/test/resources/sql-tests/expressions/string/left.sql new file mode 100644 index 0000000000..4605622e8b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/left.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_str_left(s string, n int) USING parquet + +statement +INSERT INTO test_str_left VALUES ('hello', 3), ('hello', 0), ('hello', -1), ('hello', 10), ('', 3), (NULL, 3), ('hello', NULL) + +query expect_fallback(Substring pos and len must be literals) +SELECT left(s, n) FROM test_str_left + +-- column + literal +query +SELECT left(s, 3) FROM test_str_left + +-- literal + column +query expect_fallback(Substring pos and len must be literals) +SELECT left('hello', n) FROM test_str_left + +-- literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3337) +SELECT left('hello', 3), left('hello', 0), left('hello', -1), left('', 3), left(NULL, 3) diff --git a/spark/src/test/resources/sql-tests/expressions/string/length.sql b/spark/src/test/resources/sql-tests/expressions/string/length.sql new file mode 100644 index 0000000000..c09fbe80c1 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/length.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_length(s string) USING parquet + +statement +INSERT INTO test_length VALUES (''), ('a'), ('hello'), (NULL), ('café') + +query +SELECT length(s), char_length(s) FROM test_length + +-- literal arguments +query +SELECT length('hello'), length(''), length(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/like.sql b/spark/src/test/resources/sql-tests/expressions/string/like.sql new file mode 100644 index 0000000000..5f56881d3c --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/like.sql @@ -0,0 +1,40 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_like(s string) USING parquet + +statement +INSERT INTO test_like VALUES ('hello'), ('world'), (''), (NULL), ('Hello'), ('h%llo'), ('h_llo') + +query +SELECT s LIKE 'h%' FROM test_like + +query +SELECT s LIKE '%llo' FROM test_like + +query +SELECT s LIKE 'h_llo' FROM test_like + +query +SELECT s LIKE '' FROM test_like + +-- literal arguments +query +SELECT 'hello' LIKE 'h%', 'hello' LIKE 'xyz%', '' LIKE '', NULL LIKE 'a' diff --git a/spark/src/test/resources/sql-tests/expressions/string/lower.sql b/spark/src/test/resources/sql-tests/expressions/string/lower.sql new file mode 100644 index 0000000000..f2f8173c39 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/lower.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_lower(s string) USING parquet + +statement +INSERT INTO test_lower VALUES ('HELLO'), ('hello'), ('Hello World'), (''), (NULL), ('123ABC') + +query expect_fallback(case conversion) +SELECT lower(s) FROM test_lower + +-- literal arguments +query expect_fallback(case conversion) +SELECT lower('HELLO'), lower(''), lower(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/lower_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/lower_enabled.sql new file mode 100644 index 0000000000..b521d87f91 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/lower_enabled.sql @@ -0,0 +1,29 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test lower() with case conversion enabled (happy path) +-- Config: spark.comet.caseConversion.enabled=true +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_lower_enabled(s string) USING parquet + +statement +INSERT INTO test_lower_enabled VALUES ('HELLO'), ('hello'), ('Hello World'), (''), (NULL), ('123ABC') + +query +SELECT lower(s) FROM test_lower_enabled diff --git a/spark/src/test/resources/sql-tests/expressions/string/octet_length.sql b/spark/src/test/resources/sql-tests/expressions/string/octet_length.sql new file mode 100644 index 0000000000..2e006c688f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/octet_length.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_octet_length(s string) USING parquet + +statement +INSERT INTO test_octet_length VALUES (''), ('a'), ('hello'), (NULL), ('café') + +query +SELECT octet_length(s) FROM test_octet_length + +-- literal arguments +query +SELECT octet_length('hello'), octet_length(''), octet_length(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/regexp_replace.sql b/spark/src/test/resources/sql-tests/expressions/string/regexp_replace.sql new file mode 100644 index 0000000000..acaa238c00 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/regexp_replace.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_regexp_replace(s string) USING parquet + +statement +INSERT INTO test_regexp_replace VALUES ('100-200'), ('abc'), (''), (NULL), ('phone 123-456-7890') + +query expect_fallback(Regexp pattern) +SELECT regexp_replace(s, '(\\d+)', 'X') FROM test_regexp_replace + +query expect_fallback(Regexp pattern) +SELECT regexp_replace(s, '(\\d+)', 'X', 1) FROM test_regexp_replace diff --git a/spark/src/test/resources/sql-tests/expressions/string/regexp_replace_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/regexp_replace_enabled.sql new file mode 100644 index 0000000000..7ea13a18a2 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/regexp_replace_enabled.sql @@ -0,0 +1,36 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test regexp_replace() with regexp allowIncompatible enabled (happy path) +-- Config: spark.comet.regexp.allowIncompatible=true +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_regexp_replace_enabled(s string) USING parquet + +statement +INSERT INTO test_regexp_replace_enabled VALUES ('100-200'), ('abc'), (''), (NULL), ('phone 123-456-7890') + +query +SELECT regexp_replace(s, '(\d+)', 'X') FROM test_regexp_replace_enabled + +query +SELECT regexp_replace(s, '(\d+)', 'X', 1) FROM test_regexp_replace_enabled + +-- literal + literal + literal +query +SELECT regexp_replace('100-200', '(\d+)', 'X'), regexp_replace('abc', '(\d+)', 'X'), regexp_replace(NULL, '(\d+)', 'X') diff --git a/spark/src/test/resources/sql-tests/expressions/string/reverse.sql b/spark/src/test/resources/sql-tests/expressions/string/reverse.sql new file mode 100644 index 0000000000..766b097932 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/reverse.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_reverse(s string) USING parquet + +statement +INSERT INTO test_reverse VALUES ('hello'), (''), (NULL), ('a'), ('abcde'), ('café') + +query +SELECT reverse(s) FROM test_reverse + +-- literal arguments +query +SELECT reverse('hello'), reverse(''), reverse(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/rlike.sql b/spark/src/test/resources/sql-tests/expressions/string/rlike.sql new file mode 100644 index 0000000000..7c95211df9 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/rlike.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_rlike(s string) USING parquet + +statement +INSERT INTO test_rlike VALUES ('hello'), ('12345'), (''), (NULL), ('Hello World'), ('abc123') + +query expect_fallback(Regexp pattern) +SELECT s RLIKE '^[0-9]+$' FROM test_rlike + +query expect_fallback(Regexp pattern) +SELECT s RLIKE '^[a-z]+$' FROM test_rlike + +query spark_answer_only +SELECT s RLIKE '' FROM test_rlike diff --git a/spark/src/test/resources/sql-tests/expressions/string/rlike_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/rlike_enabled.sql new file mode 100644 index 0000000000..968a2f22f7 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/rlike_enabled.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test RLIKE with regexp allowIncompatible enabled (happy path) +-- Config: spark.comet.regexp.allowIncompatible=true +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_rlike_enabled(s string) USING parquet + +statement +INSERT INTO test_rlike_enabled VALUES ('hello'), ('12345'), (''), (NULL), ('Hello World'), ('abc123') + +query +SELECT s RLIKE '^[0-9]+$' FROM test_rlike_enabled + +query +SELECT s RLIKE '^[a-z]+$' FROM test_rlike_enabled + +query +SELECT s RLIKE '' FROM test_rlike_enabled + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3343) +SELECT 'hello' RLIKE '^[a-z]+$', '12345' RLIKE '^[a-z]+$', '' RLIKE '', NULL RLIKE 'a' diff --git a/spark/src/test/resources/sql-tests/expressions/string/starts_with.sql b/spark/src/test/resources/sql-tests/expressions/string/starts_with.sql new file mode 100644 index 0000000000..eeea2b856d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/starts_with.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_starts_with(s string, prefix string) USING parquet + +statement +INSERT INTO test_starts_with VALUES ('hello world', 'hello'), ('hello', ''), ('', ''), ('hello', 'xyz'), (NULL, 'a'), ('hello', NULL) + +query +SELECT startswith(s, prefix) FROM test_starts_with + +-- column + literal +query +SELECT startswith(s, 'hello') FROM test_starts_with + +-- literal + column +query +SELECT startswith('hello world', prefix) FROM test_starts_with + +-- literal + literal +query +SELECT startswith('hello world', 'hello'), startswith('hello', 'xyz'), startswith('', ''), startswith(NULL, 'a') diff --git a/spark/src/test/resources/sql-tests/expressions/string/string.sql b/spark/src/test/resources/sql-tests/expressions/string/string.sql new file mode 100644 index 0000000000..15c0107fae --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/string.sql @@ -0,0 +1,49 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- substring with start < 1 +statement +CREATE TABLE t(col string) USING parquet + +statement +INSERT INTO t VALUES('123456') + +query +SELECT substring(col, 0) FROM t + +query +SELECT substring(col, -1) FROM t + +-- md5 +statement +CREATE TABLE test_md5(col String) USING parquet + +statement +INSERT INTO test_md5 VALUES ('test1'), ('test1'), ('test2'), ('test2'), (NULL), ('') + +query +SELECT md5(col) FROM test_md5 + +-- unhex +statement +CREATE TABLE unhex_table(col string) USING parquet + +statement +INSERT INTO unhex_table VALUES ('537061726B2053514C'), ('737472696E67'), ('\0'), (''), ('###'), ('G123'), ('hello'), ('A1B'), ('0A1B') + +query +SELECT unhex(col) FROM unhex_table diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_instr.sql b/spark/src/test/resources/sql-tests/expressions/string/string_instr.sql new file mode 100644 index 0000000000..92298a143b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/string_instr.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_instr(s string, sub string) USING parquet + +statement +INSERT INTO test_instr VALUES ('hello world', 'world'), ('hello', 'xyz'), ('hello', ''), ('', ''), (NULL, 'a'), ('hello', NULL), ('abcabc', 'bc') + +query +SELECT instr(s, sub) FROM test_instr + +-- column + literal +query +SELECT instr(s, 'world') FROM test_instr + +-- literal + column +query +SELECT instr('hello world', sub) FROM test_instr + +-- literal + literal +query +SELECT instr('hello world', 'world'), instr('hello', 'xyz'), instr('', ''), instr(NULL, 'a') diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_lpad.sql b/spark/src/test/resources/sql-tests/expressions/string/string_lpad.sql new file mode 100644 index 0000000000..358be456b1 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/string_lpad.sql @@ -0,0 +1,38 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_lpad(s string, len int, pad string) USING parquet + +statement +INSERT INTO test_lpad VALUES ('hi', 5, 'x'), ('hello', 3, 'x'), ('hi', 5, 'xy'), ('', 3, 'a'), (NULL, 5, 'x'), ('hi', 0, 'x'), ('hi', -1, 'x') + +query expect_fallback(Only scalar values are supported for the pad argument) +SELECT lpad(s, len, pad) FROM test_lpad + +query +SELECT lpad(s, len) FROM test_lpad + +-- column + literal + literal +query +SELECT lpad(s, 5, 'x') FROM test_lpad + +-- literal + literal + literal +query expect_fallback(Scalar values are not supported for the str argument) +SELECT lpad('hi', 5, 'x'), lpad('hello', 3, 'x'), lpad('', 3, 'a'), lpad(NULL, 5, 'x') diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_repeat.sql b/spark/src/test/resources/sql-tests/expressions/string/string_repeat.sql new file mode 100644 index 0000000000..32f08db101 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/string_repeat.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_repeat(s string, n int) USING parquet + +statement +INSERT INTO test_repeat VALUES ('hi', 3), ('', 5), ('a', 0), ('a', -1), (NULL, 3), ('hi', NULL) + +query +SELECT repeat(s, n) FROM test_repeat + +-- column + literal +query +SELECT repeat(s, 3) FROM test_repeat + +-- literal + column +query +SELECT repeat('hi', n) FROM test_repeat + +-- literal + literal +query +SELECT repeat('hi', 3), repeat('', 5), repeat('a', 0), repeat(NULL, 3) diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_replace.sql b/spark/src/test/resources/sql-tests/expressions/string/string_replace.sql new file mode 100644 index 0000000000..d1c03d18e6 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/string_replace.sql @@ -0,0 +1,42 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_str_replace(s string, search string, replace string) USING parquet + +statement +INSERT INTO test_str_replace VALUES ('hello world', 'world', 'there'), ('aaa', 'a', 'bb'), ('hello', 'xyz', 'abc'), ('', 'a', 'b'), (NULL, 'a', 'b') + +query +SELECT replace(s, search, replace) FROM test_str_replace + +query ignore(https://github.com/apache/datafusion-comet/issues/3344) +SELECT replace('hello', '', 'x') + +-- column + literal + literal +query +SELECT replace(s, 'world', 'there') FROM test_str_replace + +-- literal + column + column +query +SELECT replace('hello world', search, replace) FROM test_str_replace + +-- literal + literal + literal +query +SELECT replace('hello world', 'world', 'there'), replace('aaa', 'a', 'bb'), replace('hello', 'xyz', 'abc'), replace(NULL, 'a', 'b') diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_rpad.sql b/spark/src/test/resources/sql-tests/expressions/string/string_rpad.sql new file mode 100644 index 0000000000..c2f526a88f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/string_rpad.sql @@ -0,0 +1,38 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_rpad(s string, len int, pad string) USING parquet + +statement +INSERT INTO test_rpad VALUES ('hi', 5, 'x'), ('hello', 3, 'x'), ('hi', 5, 'xy'), ('', 3, 'a'), (NULL, 5, 'x'), ('hi', 0, 'x'), ('hi', -1, 'x') + +query expect_fallback(Only scalar values are supported for the pad argument) +SELECT rpad(s, len, pad) FROM test_rpad + +query +SELECT rpad(s, len) FROM test_rpad + +-- column + literal + literal +query +SELECT rpad(s, 5, 'x') FROM test_rpad + +-- literal + literal + literal +query expect_fallback(Scalar values are not supported for the str argument) +SELECT rpad('hi', 5, 'x'), rpad('hello', 3, 'x'), rpad('', 3, 'a'), rpad(NULL, 5, 'x') diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_space.sql b/spark/src/test/resources/sql-tests/expressions/string/string_space.sql new file mode 100644 index 0000000000..ec24bfb97e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/string_space.sql @@ -0,0 +1,36 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_space(n int) USING parquet + +statement +INSERT INTO test_space VALUES (0), (1), (5), (NULL), (-1) + +query +SELECT concat('[', space(n), ']') FROM test_space WHERE n >= 0 OR n IS NULL + +-- Comet bug: space(-1) causes native crash "failed to round upto multiple of 64" +-- https://github.com/apache/datafusion-comet/issues/3326 +query ignore(https://github.com/apache/datafusion-comet/issues/3326) +SELECT concat('[', space(n), ']') FROM test_space WHERE n < 0 + +-- literal arguments +query ignore(https://github.com/apache/datafusion-comet/issues/3337) +SELECT concat('[', space(5), ']'), concat('[', space(0), ']'), space(-1), space(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_translate.sql b/spark/src/test/resources/sql-tests/expressions/string/string_translate.sql new file mode 100644 index 0000000000..01f5426dd7 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/string_translate.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_translate(s string, from_str string, to_str string) USING parquet + +statement +INSERT INTO test_translate VALUES ('hello', 'el', 'ip'), ('hello', 'aeiou', '12345'), ('', 'a', 'b'), (NULL, 'a', 'b'), ('hello', '', ''), ('abc', 'abc', 'x') + +query +SELECT translate(s, from_str, to_str) FROM test_translate + +-- column + literal + literal +query +SELECT translate(s, 'el', 'ip') FROM test_translate + +-- literal + column + column +query +SELECT translate('hello', from_str, to_str) FROM test_translate + +-- literal + literal + literal +query +SELECT translate('hello', 'el', 'ip'), translate('hello', 'aeiou', '12345'), translate('', 'a', 'b'), translate(NULL, 'a', 'b') diff --git a/spark/src/test/resources/sql-tests/expressions/string/string_trim.sql b/spark/src/test/resources/sql-tests/expressions/string/string_trim.sql new file mode 100644 index 0000000000..9822643ef1 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/string_trim.sql @@ -0,0 +1,37 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_trim(s string) USING parquet + +statement +INSERT INTO test_trim VALUES (' hello '), ('hello'), (''), (NULL), (' '), (' hello world ') + +query +SELECT trim(s), ltrim(s), rtrim(s) FROM test_trim + +query +SELECT trim(BOTH 'h' FROM s) FROM test_trim + +-- literal arguments +query +SELECT trim(' hello '), ltrim(' hello '), rtrim(' hello ') + +query +SELECT trim(BOTH 'h' FROM 'hello'), trim(LEADING ' ' FROM ' hello '), trim(TRAILING ' ' FROM ' hello ') diff --git a/spark/src/test/resources/sql-tests/expressions/string/substring.sql b/spark/src/test/resources/sql-tests/expressions/string/substring.sql new file mode 100644 index 0000000000..42b99f1900 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/substring.sql @@ -0,0 +1,46 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_substring(s string) USING parquet + +statement +INSERT INTO test_substring VALUES ('hello world'), (''), (NULL), ('abc') + +query +SELECT substring(s, 1, 5) FROM test_substring + +query +SELECT substring(s, -3) FROM test_substring + +query +SELECT substring(s, 0, 3) FROM test_substring + +query +SELECT substring(s, 1, 0) FROM test_substring + +query +SELECT substring(s, 1, -1) FROM test_substring + +query +SELECT substring(s, 100) FROM test_substring + +-- literal + literal + literal +query ignore(https://github.com/apache/datafusion-comet/issues/3337) +SELECT substring('hello world', 1, 5), substring('hello world', -3), substring('', 1, 5), substring(NULL, 1, 5) diff --git a/spark/src/test/resources/sql-tests/expressions/string/unhex.sql b/spark/src/test/resources/sql-tests/expressions/string/unhex.sql new file mode 100644 index 0000000000..2b20cf489f --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/unhex.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_unhex(s string) USING parquet + +statement +INSERT INTO test_unhex VALUES ('537061726B2053514C'), ('41'), ('0A1B'), (''), (NULL), ('GG'), ('hello'), ('A1B') + +query +SELECT hex(unhex(s)) FROM test_unhex + +query +SELECT unhex(s) IS NULL FROM test_unhex + +-- literal arguments +query +SELECT unhex('41'), unhex('GG'), unhex(''), unhex(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/upper.sql b/spark/src/test/resources/sql-tests/expressions/string/upper.sql new file mode 100644 index 0000000000..6582a299ef --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/upper.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_upper(s string) USING parquet + +statement +INSERT INTO test_upper VALUES ('hello'), ('HELLO'), ('Hello World'), (''), (NULL), ('123abc') + +query expect_fallback(case conversion) +SELECT upper(s) FROM test_upper + +-- literal arguments +query expect_fallback(case conversion) +SELECT upper('hello'), upper(''), upper(NULL) diff --git a/spark/src/test/resources/sql-tests/expressions/string/upper_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/upper_enabled.sql new file mode 100644 index 0000000000..b8e4dc128d --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/string/upper_enabled.sql @@ -0,0 +1,29 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Test upper() with case conversion enabled (happy path) +-- Config: spark.comet.caseConversion.enabled=true +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_upper_enabled(s string) USING parquet + +statement +INSERT INTO test_upper_enabled VALUES ('hello'), ('HELLO'), ('Hello World'), (''), (NULL), ('123abc') + +query +SELECT upper(s) FROM test_upper_enabled diff --git a/spark/src/test/resources/sql-tests/expressions/struct/create_named_struct.sql b/spark/src/test/resources/sql-tests/expressions/struct/create_named_struct.sql new file mode 100644 index 0000000000..d188aed7bb --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/struct/create_named_struct.sql @@ -0,0 +1,37 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_named_struct(a int, b string, c double) USING parquet + +statement +INSERT INTO test_named_struct VALUES (1, 'hello', 1.5), (NULL, NULL, NULL), (0, '', 0.0) + +query +SELECT named_struct('x', a, 'y', b, 'z', c) FROM test_named_struct + +query +SELECT struct(a, b, c) FROM test_named_struct + +-- literal arguments +query +SELECT named_struct('x', 1, 'y', 'hello', 'z', 3.14) + +query +SELECT named_struct('x', a, 'y', 'fixed_val', 'z', c) FROM test_named_struct diff --git a/spark/src/test/resources/sql-tests/expressions/struct/get_struct_field.sql b/spark/src/test/resources/sql-tests/expressions/struct/get_struct_field.sql new file mode 100644 index 0000000000..cf569ae121 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/struct/get_struct_field.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_struct(s struct) USING parquet + +statement +INSERT INTO test_struct VALUES (named_struct('name', 'Alice', 'age', 30, 'score', 95.5)), (named_struct('name', 'Bob', 'age', 25, 'score', 88.0)), (NULL) + +query +SELECT s.name, s.age, s.score FROM test_struct + +query +SELECT s.name, s.age + 1, s.score * 2 FROM test_struct diff --git a/spark/src/test/resources/sql-tests/expressions/struct/json_to_structs.sql b/spark/src/test/resources/sql-tests/expressions/struct/json_to_structs.sql new file mode 100644 index 0000000000..a0ae20b655 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/struct/json_to_structs.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_from_json(j string) USING parquet + +statement +INSERT INTO test_from_json VALUES ('{"a": 1, "b": "hello"}'), ('{}'), (NULL), ('{"a": null}'), ('invalid json') + +query spark_answer_only +SELECT from_json(j, 'a INT, b STRING') FROM test_from_json + +-- literal arguments +query spark_answer_only +SELECT from_json('{"a": 1, "b": "hello"}', 'a INT, b STRING') diff --git a/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql b/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql new file mode 100644 index 0000000000..7f2310f147 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/struct/structs_to_json.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: parquet.enable.dictionary=false,true + +statement +CREATE TABLE test_to_json(a int, b string) USING parquet + +statement +INSERT INTO test_to_json VALUES (1, 'hello'), (NULL, NULL), (0, '') + +query spark_answer_only +SELECT to_json(named_struct('a', a, 'b', b)) FROM test_to_json + +-- literal arguments +query spark_answer_only +SELECT to_json(named_struct('a', 1, 'b', 'hello')) diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala new file mode 100644 index 0000000000..80ccf92557 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet + +import java.io.File + +import org.scalactic.source.Position +import org.scalatest.Tag + +import org.apache.spark.sql.CometTestBase +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper + +class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { + + override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit + pos: Position): Unit = { + super.test(testName, testTags: _*) { + withSQLConf(CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_AUTO) { + testFun + } + } + } + + /** Check if the current Spark version meets a minimum version requirement. */ + private def meetsMinSparkVersion(minVersion: String): Boolean = { + val current = org.apache.spark.SPARK_VERSION.split("[.-]").take(2).map(_.toInt) + val required = minVersion.split("[.-]").take(2).map(_.toInt) + (current(0) > required(0)) || + (current(0) == required(0) && current(1) >= required(1)) + } + + private val testResourceDir = { + val url = getClass.getClassLoader.getResource("sql-tests") + assert(url != null, "Could not find sql-tests resource directory") + new File(url.toURI) + } + + private def discoverTestFiles(dir: File): Seq[File] = { + if (!dir.exists()) return Seq.empty + val files = dir.listFiles().toSeq + val sqlFiles = files.filter(f => f.isFile && f.getName.endsWith(".sql")) + val subDirFiles = files.filter(_.isDirectory).flatMap(discoverTestFiles) + sqlFiles ++ subDirFiles + } + + /** Generate all config combinations from a ConfigMatrix specification. */ + private def configMatrix(matrix: Seq[(String, Seq[String])]): Seq[Seq[(String, String)]] = { + if (matrix.isEmpty) return Seq(Seq.empty) + val (key, values) = matrix.head + val rest = configMatrix(matrix.tail) + for { + value <- values + combo <- rest + } yield (key, value) +: combo + } + + // Disable constant folding so that literal expressions are evaluated by Comet's + // native engine rather than being folded away by Spark's optimizer at plan time. + private val constantFoldingExcluded = Seq( + "spark.sql.optimizer.excludedRules" -> + "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") + + private def runTestFile(file: SqlTestFile): Unit = { + val allConfigs = file.configs ++ constantFoldingExcluded + withSQLConf(allConfigs: _*) { + withTable(file.tables: _*) { + file.records.foreach { + case SqlStatement(sql) => + spark.sql(sql) + case SqlQuery(sql, mode) => + mode match { + case CheckCoverageAndAnswer => + checkSparkAnswerAndOperator(sql) + case SparkAnswerOnly => + checkSparkAnswer(sql) + case WithTolerance(tol) => + checkSparkAnswerWithTolerance(sql, tol) + case ExpectFallback(reason) => + checkSparkAnswerAndFallbackReason(sql, reason) + case Ignore(reason) => + logInfo(s"IGNORED query (${reason}): $sql") + } + } + } + } + } + + // Discover and register all .sql test files + discoverTestFiles(testResourceDir).foreach { file => + val relativePath = testResourceDir.toURI.relativize(file.toURI).getPath + val parsed = SqlFileTestParser.parse(file) + val combinations = configMatrix(parsed.configMatrix) + + // Skip tests that require a newer Spark version + val skip = parsed.minSparkVersion.exists(!meetsMinSparkVersion(_)) + + if (combinations.size <= 1) { + // No matrix or single combination + test(s"sql-file: $relativePath") { + if (skip) { + logInfo(s"SKIPPED (requires Spark ${parsed.minSparkVersion.get}): $relativePath") + } else { + val effectiveConfigs = parsed.configs ++ combinations.headOption.getOrElse(Seq.empty) + runTestFile(parsed.copy(configs = effectiveConfigs)) + } + } + } else { + // Multiple combinations: generate one test per combination + combinations.foreach { matrixConfigs => + val label = matrixConfigs.map { case (k, v) => s"$k=$v" }.mkString(", ") + test(s"sql-file: $relativePath [$label]") { + if (skip) { + logInfo(s"SKIPPED (requires Spark ${parsed.minSparkVersion.get}): $relativePath") + } else { + runTestFile(parsed.copy(configs = parsed.configs ++ matrixConfigs)) + } + } + } + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala b/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala new file mode 100644 index 0000000000..62a349cdea --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet + +import java.io.File + +import scala.io.Source + +/** A record in a SQL test file: either a statement (DDL/DML) or a query (SELECT). */ +sealed trait SqlTestRecord + +/** A SQL statement to execute (CREATE TABLE, INSERT, etc.). */ +case class SqlStatement(sql: String) extends SqlTestRecord + +/** A SQL query whose results are compared between Spark and Comet. */ +case class SqlQuery(sql: String, mode: QueryAssertionMode = CheckCoverageAndAnswer) + extends SqlTestRecord + +sealed trait QueryAssertionMode +case object CheckCoverageAndAnswer extends QueryAssertionMode +case object SparkAnswerOnly extends QueryAssertionMode +case class WithTolerance(tol: Double) extends QueryAssertionMode +case class ExpectFallback(reason: String) extends QueryAssertionMode +case class Ignore(reason: String) extends QueryAssertionMode + +/** + * Parsed representation of a .sql test file. + * + * @param configs + * Spark SQL configs to set for this test file. + * @param configMatrix + * Map of config key to list of values. The test will run once per combination. + * @param records + * Ordered list of statements and queries. + * @param tables + * Table names extracted from CREATE TABLE statements (for cleanup). + * @param minSparkVersion + * Optional minimum Spark version required to run this test (e.g. "3.5"). + */ +case class SqlTestFile( + configs: Seq[(String, String)], + configMatrix: Seq[(String, Seq[String])], + records: Seq[SqlTestRecord], + tables: Seq[String], + minSparkVersion: Option[String] = None) + +object SqlFileTestParser { + + private val ConfigPattern = """--\s*Config:\s*(.+)=(.+)""".r + private val ConfigMatrixPattern = """--\s*ConfigMatrix:\s*(.+)=(.+)""".r + private val MinSparkVersionPattern = """--\s*MinSparkVersion:\s*(.+)""".r + private val CreateTablePattern = """(?i)CREATE\s+TABLE\s+(\w+)""".r.unanchored + + def parse(file: File): SqlTestFile = { + val source = Source.fromFile(file, "UTF-8") + try { + parse(source.getLines().toSeq) + } finally { + source.close() + } + } + + def parse(lines: Seq[String]): SqlTestFile = { + var configs = Seq.empty[(String, String)] + var configMatrix = Seq.empty[(String, Seq[String])] + var minSparkVersion: Option[String] = None + val records = Seq.newBuilder[SqlTestRecord] + val tables = Seq.newBuilder[String] + + var lineIdx = 0 + while (lineIdx < lines.length) { + val line = lines(lineIdx).trim + + line match { + case ConfigPattern(key, value) => + configs :+= (key.trim -> value.trim) + lineIdx += 1 + + case ConfigMatrixPattern(key, values) => + configMatrix :+= (key.trim -> values.split(",").map(_.trim).toSeq) + lineIdx += 1 + + case MinSparkVersionPattern(version) => + minSparkVersion = Some(version.trim) + lineIdx += 1 + + case "statement" => + lineIdx += 1 + val (sql, nextIdx) = collectSql(lines, lineIdx) + // Extract table names for cleanup + CreateTablePattern.findFirstMatchIn(sql).foreach(m => tables += m.group(1)) + records += SqlStatement(sql) + lineIdx = nextIdx + + case s if s.startsWith("query") => + val mode = parseQueryAssertionMode(s) + lineIdx += 1 + val (sql, nextIdx) = collectSql(lines, lineIdx) + records += SqlQuery(sql, mode) + lineIdx = nextIdx + + case _ => + // Skip blank lines and comments + lineIdx += 1 + } + } + + SqlTestFile(configs, configMatrix, records.result(), tables.result(), minSparkVersion) + } + + private val FallbackPattern = """query\s+expect_fallback\((.+)\)""".r + private val IgnorePattern = """query\s+ignore\((.+)\)""".r + + private def parseQueryAssertionMode(directive: String): QueryAssertionMode = { + directive match { + case FallbackPattern(reason) => + ExpectFallback(reason.trim) + case IgnorePattern(reason) => + Ignore(reason.trim) + case _ => + val parts = directive.split("\\s+") + if (parts.length == 1) return CheckCoverageAndAnswer + parts(1) match { + case "spark_answer_only" => SparkAnswerOnly + case s if s.startsWith("tolerance=") => + WithTolerance(s.stripPrefix("tolerance=").toDouble) + case _ => CheckCoverageAndAnswer + } + } + } + + /** Collect SQL lines until a blank line or end of file. */ + private def collectSql(lines: Seq[String], start: Int): (String, Int) = { + val sb = new StringBuilder + var lineIdx = start + while (lineIdx < lines.length && lines(lineIdx).trim.nonEmpty) { + if (sb.nonEmpty) sb.append("\n") + sb.append(lines(lineIdx)) + lineIdx += 1 + } + (sb.toString, lineIdx) + } +}