From c3d5f0061e0d0b0fc933c392e38cd65b869996c2 Mon Sep 17 00:00:00 2001 From: parkma99 Date: Fri, 5 May 2023 15:51:26 +0800 Subject: [PATCH 1/2] Port tests in identifiers.rs to sqllogictest --- .../sqllogictests/test_files/identifiers.slt | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 datafusion/core/tests/sqllogictests/test_files/identifiers.slt diff --git a/datafusion/core/tests/sqllogictests/test_files/identifiers.slt b/datafusion/core/tests/sqllogictests/test_files/identifiers.slt new file mode 100644 index 0000000000000..c4605979d1515 --- /dev/null +++ b/datafusion/core/tests/sqllogictests/test_files/identifiers.slt @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +statement ok +CREATE EXTERNAL TABLE case_insensitive_test ( + a INT, + b INT, + c INT +) +STORED AS CSV +WITH HEADER ROW +LOCATION './tests/data/example.csv' + +# normalized column identifiers +query II +SELECT A, b FROM case_insensitive_test +---- +1 2 + +query II +SELECT t.A, b FROM case_insensitive_test AS t +---- +1 2 + +query II +SELECT t.A as x, b FROM case_insensitive_test AS t +---- +1 2 + +query II +SELECT t.A AS X, b FROM case_insensitive_test AS t +---- +1 2 + + +query II +SELECT t.A AS "X", b FROM case_insensitive_test AS t +---- +1 2 + +query II +SELECT t.A AS x, b FROM case_insensitive_test AS t ORDER BY x +---- +1 2 + +query II +SELECT t.A AS x, b FROM case_insensitive_test AS t ORDER BY X +---- +1 2 + +query II +SELECT t.A AS "X", b FROM case_insensitive_test AS t ORDER BY "X" +---- +1 2 + +query II +SELECT a, b FROM case_insensitive_test where A IS NOT null +---- +1 2 + +query II +SELECT a as x, count(*) as c FROM case_insensitive_test GROUP BY X +---- +1 1 + +query II +SELECT a as "X", count(*) as c FROM case_insensitive_test GROUP BY "X" +---- +1 1 + +statement ok +drop table case_insensitive_test + +# case insensitive in sql errors + +statement ok +CREATE TABLE test("Column1" string) AS VALUES ('content1'); + +statement error DataFusion error: Schema error: No field named column1\. Valid fields are test\."Column1"\. +SELECT COLumn1 from test + +statement error DataFusion error: Schema error: No field named column1\. Valid fields are test\."Column1"\. +SELECT Column1 from test + +statement error DataFusion error: Schema error: No field named column1\. Valid fields are test\."Column1"\. +SELECT column1 from test + +statement error DataFusion error: Schema error: No field named column1\. Valid fields are test\."Column1"\. +SELECT "column1" from test + +statement ok +SELECT "Column1" from test From 8f6900392158a2e0a85e622b894ce16b86d46828 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 5 May 2023 08:04:35 -0400 Subject: [PATCH 2/2] remove identifiers.rs --- datafusion/core/tests/sql/idenfifers.rs | 239 ------------------------ datafusion/core/tests/sql/mod.rs | 1 - 2 files changed, 240 deletions(-) delete mode 100644 datafusion/core/tests/sql/idenfifers.rs diff --git a/datafusion/core/tests/sql/idenfifers.rs b/datafusion/core/tests/sql/idenfifers.rs deleted file mode 100644 index 23fb26eddecc5..0000000000000 --- a/datafusion/core/tests/sql/idenfifers.rs +++ /dev/null @@ -1,239 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::sync::Arc; - -use arrow::{array::StringArray, record_batch::RecordBatch}; -use datafusion::{assert_batches_sorted_eq, prelude::*}; -use datafusion_common::assert_contains; - -use crate::sql::plan_and_collect; - -#[tokio::test] -async fn normalized_column_identifiers() { - // create local execution context - let ctx = SessionContext::new(); - - // register csv file with the execution context - ctx.register_csv( - "case_insensitive_test", - "tests/data/example.csv", - CsvReadOptions::new(), - ) - .await - .unwrap(); - - let sql = "SELECT A, b FROM case_insensitive_test"; - let result = plan_and_collect(&ctx, sql) - .await - .expect("ran plan correctly"); - let expected = vec![ - "+---+---+", - "| a | b |", - "+---+---+", - "| 1 | 2 |", - "+---+---+", - ]; - assert_batches_sorted_eq!(expected, &result); - - let sql = "SELECT t.A, b FROM case_insensitive_test AS t"; - let result = plan_and_collect(&ctx, sql) - .await - .expect("ran plan correctly"); - let expected = vec![ - "+---+---+", - "| a | b |", - "+---+---+", - "| 1 | 2 |", - "+---+---+", - ]; - assert_batches_sorted_eq!(expected, &result); - - // Aliases - - let sql = "SELECT t.A as x, b FROM case_insensitive_test AS t"; - let result = plan_and_collect(&ctx, sql) - .await - .expect("ran plan correctly"); - let expected = vec![ - "+---+---+", - "| x | b |", - "+---+---+", - "| 1 | 2 |", - "+---+---+", - ]; - assert_batches_sorted_eq!(expected, &result); - - let sql = "SELECT t.A AS X, b FROM case_insensitive_test AS t"; - let result = plan_and_collect(&ctx, sql) - .await - .expect("ran plan correctly"); - let expected = vec![ - "+---+---+", - "| x | b |", - "+---+---+", - "| 1 | 2 |", - "+---+---+", - ]; - assert_batches_sorted_eq!(expected, &result); - - let sql = r#"SELECT t.A AS "X", b FROM case_insensitive_test AS t"#; - let result = plan_and_collect(&ctx, sql) - .await - .expect("ran plan correctly"); - let expected = vec![ - "+---+---+", - "| X | b |", - "+---+---+", - "| 1 | 2 |", - "+---+---+", - ]; - assert_batches_sorted_eq!(expected, &result); - - // Order by - - let sql = "SELECT t.A AS x, b FROM case_insensitive_test AS t ORDER BY x"; - let result = plan_and_collect(&ctx, sql) - .await - .expect("ran plan correctly"); - let expected = vec![ - "+---+---+", - "| x | b |", - "+---+---+", - "| 1 | 2 |", - "+---+---+", - ]; - assert_batches_sorted_eq!(expected, &result); - - let sql = "SELECT t.A AS x, b FROM case_insensitive_test AS t ORDER BY X"; - let result = plan_and_collect(&ctx, sql) - .await - .expect("ran plan correctly"); - let expected = vec![ - "+---+---+", - "| x | b |", - "+---+---+", - "| 1 | 2 |", - "+---+---+", - ]; - assert_batches_sorted_eq!(expected, &result); - - let sql = r#"SELECT t.A AS "X", b FROM case_insensitive_test AS t ORDER BY "X""#; - let result = plan_and_collect(&ctx, sql) - .await - .expect("ran plan correctly"); - let expected = vec![ - "+---+---+", - "| X | b |", - "+---+---+", - "| 1 | 2 |", - "+---+---+", - ]; - assert_batches_sorted_eq!(expected, &result); - - // Where - - let sql = "SELECT a, b FROM case_insensitive_test where A IS NOT null"; - let result = plan_and_collect(&ctx, sql) - .await - .expect("ran plan correctly"); - let expected = vec![ - "+---+---+", - "| a | b |", - "+---+---+", - "| 1 | 2 |", - "+---+---+", - ]; - assert_batches_sorted_eq!(expected, &result); - - // Group by - - let sql = "SELECT a as x, count(*) as c FROM case_insensitive_test GROUP BY X"; - let result = plan_and_collect(&ctx, sql) - .await - .expect("ran plan correctly"); - let expected = vec![ - "+---+---+", - "| x | c |", - "+---+---+", - "| 1 | 1 |", - "+---+---+", - ]; - assert_batches_sorted_eq!(expected, &result); - - let sql = r#"SELECT a as "X", count(*) as c FROM case_insensitive_test GROUP BY "X""#; - let result = plan_and_collect(&ctx, sql) - .await - .expect("ran plan correctly"); - let expected = vec![ - "+---+---+", - "| X | c |", - "+---+---+", - "| 1 | 1 |", - "+---+---+", - ]; - assert_batches_sorted_eq!(expected, &result); -} - -#[tokio::test] -async fn case_insensitive_in_sql_errors() { - let record_batch = RecordBatch::try_from_iter(vec![ - // The proper way to refer to this column is "Column1" -- it - // should not be possible to use `column1` or `COLUMN1` or - // other variants - ( - "Column1", - Arc::new(StringArray::from(vec!["content1"])) as _, - ), - ]) - .unwrap(); - - let ctx = SessionContext::new(); - ctx.register_batch("test", record_batch).unwrap(); - - // None of these tests shoud pass - let actual = ctx - .sql("SELECT COLumn1 from test") - .await - .unwrap_err() - .to_string(); - assert_contains!(actual, r#"No field named column1"#); - - let actual = ctx - .sql("SELECT Column1 from test") - .await - .unwrap_err() - .to_string(); - assert_contains!(actual, r#"No field named column1"#); - - let actual = ctx - .sql("SELECT column1 from test") - .await - .unwrap_err() - .to_string(); - assert_contains!(actual, r#"No field named column1"#); - - let actual = ctx - .sql(r#"SELECT "column1" from test"#) - .await - .unwrap_err() - .to_string(); - assert_contains!(actual, r#"No field named column1"#); - - // This should pass (note the quotes) - ctx.sql(r#"SELECT "Column1" from test"#).await.unwrap(); -} diff --git a/datafusion/core/tests/sql/mod.rs b/datafusion/core/tests/sql/mod.rs index 8c7f50c9a6494..6e666752a1b2b 100644 --- a/datafusion/core/tests/sql/mod.rs +++ b/datafusion/core/tests/sql/mod.rs @@ -104,7 +104,6 @@ pub mod wildcard; pub mod window; pub mod explain; -pub mod idenfifers; pub mod information_schema; pub mod parquet_schema; pub mod partitioned_csv;