diff --git a/Cargo.toml b/Cargo.toml index f32478970d2ba..673625434a52f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [workspace] exclude = ["datafusion-cli"] -members = ["datafusion/common", "datafusion/core", "datafusion/expr", "datafusion/execution", "datafusion/optimizer", "datafusion/physical-expr", "datafusion/proto", "datafusion/proto/gen", "datafusion/sql", "datafusion/substrait", "datafusion-examples", "test-utils", "benchmarks", +members = ["datafusion/common", "datafusion/core", "datafusion/expr", "datafusion/execution", "datafusion/optimizer", "datafusion/physical-expr", "datafusion/proto", "datafusion/proto/gen", "datafusion/sql", "datafusion/sqllogictest", "datafusion/substrait", "datafusion-examples", "test-utils", "benchmarks", ] resolver = "2" @@ -56,4 +56,4 @@ lto = false opt-level = 3 overflow-checks = false panic = 'unwind' -rpath = false \ No newline at end of file +rpath = false diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 81a7b91059028..c499c349c096a 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -102,6 +102,7 @@ bigdecimal = "0.4.1" criterion = { version = "0.5", features = ["async_tokio"] } csv = "1.1.6" ctor = "0.2.0" +datafusion-sqllogictest = { path = "../sqllogictest", version = "28.0.0", features = ["postgres"] } doc-comment = "0.3" env_logger = "0.10" half = "2.2.1" diff --git a/datafusion/core/tests/sqllogictests/src/engines/datafusion/util.rs b/datafusion/core/tests/sqllogictests/src/engines/datafusion/util.rs deleted file mode 100644 index 424a297f24ec2..0000000000000 --- a/datafusion/core/tests/sqllogictests/src/engines/datafusion/util.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use arrow::datatypes::DataType; -use datafusion_common::config::ConfigOptions; -use datafusion_common::TableReference; -use datafusion_expr::{AggregateUDF, ScalarUDF, TableSource, WindowUDF}; -use datafusion_sql::planner::ContextProvider; -use std::sync::Arc; - -pub struct LogicTestContextProvider {} - -// Only a mock, don't need to implement -impl ContextProvider for LogicTestContextProvider { - fn get_table_provider( - &self, - _name: TableReference, - ) -> datafusion_common::Result> { - todo!() - } - - fn get_function_meta(&self, _name: &str) -> Option> { - todo!() - } - - fn get_aggregate_meta(&self, _name: &str) -> Option> { - todo!() - } - - fn get_variable_type(&self, _variable_names: &[String]) -> Option { - todo!() - } - - fn options(&self) -> &ConfigOptions { - todo!() - } - - fn get_window_meta(&self, _name: &str) -> Option> { - todo!() - } -} diff --git a/datafusion/core/tests/sqllogictests/src/main.rs b/datafusion/core/tests/sqllogictests/src/main.rs index c74d1cb11a471..7d23971fbfca5 100644 --- a/datafusion/core/tests/sqllogictests/src/main.rs +++ b/datafusion/core/tests/sqllogictests/src/main.rs @@ -20,6 +20,7 @@ use std::path::{Path, PathBuf}; #[cfg(target_family = "windows")] use std::thread; +use datafusion_sqllogictest::{DataFusion, Postgres}; use futures::stream::StreamExt; use log::info; use sqllogictest::strict_column_validator; @@ -28,10 +29,6 @@ use tempfile::TempDir; use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_common::{DataFusionError, Result}; -use crate::engines::datafusion::DataFusion; -use crate::engines::postgres::Postgres; - -mod engines; mod setup; const TEST_DIRECTORY: &str = "tests/sqllogictests/test_files/"; diff --git a/datafusion/core/tests/sqllogictests/test_files/information_schema_columns.slt b/datafusion/core/tests/sqllogictests/test_files/information_schema_columns.slt index fcb653cedd167..7cf845c16d738 100644 --- a/datafusion/core/tests/sqllogictests/test_files/information_schema_columns.slt +++ b/datafusion/core/tests/sqllogictests/test_files/information_schema_columns.slt @@ -50,4 +50,4 @@ statement ok drop table t1 statement ok -drop table t2 \ No newline at end of file +drop table t2 diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.toml new file mode 100644 index 0000000000000..66961abdfaca5 --- /dev/null +++ b/datafusion/sqllogictest/Cargo.toml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +name = "datafusion-sqllogictest" +readme.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[lib] +name = "datafusion_sqllogictest" +path = "src/lib.rs" + +[dependencies] +arrow = {workspace = true} +async-trait = "0.1.41" +bigdecimal = "0.4.1" +datafusion = {path = "../core", version = "28.0.0"} +datafusion-common = {path = "../common", version = "28.0.0"} +half = "2.2.1" +itertools = "0.11" +lazy_static = {version = "^1.4.0"} +object_store = "0.6.1" +rust_decimal = {version = "1.27.0"} +log = "^0.4" +sqllogictest = "0.15.0" +sqlparser.workspace = true +thiserror = "1.0.44" +tokio = {version = "1.0"} +bytes = {version = "1.4.0", optional = true} +futures = {version = "0.3.28", optional = true} +chrono = {version = "0.4.26", optional = true} +tokio-postgres = {version = "0.7.7", optional = true} +postgres-types = {version = "0.2.4", optional = true} +postgres-protocol = {version = "0.6.4", optional = true} + +[features] +postgres = ["bytes", "futures", "chrono", "tokio-postgres", "postgres-types", "postgres-protocol"] diff --git a/datafusion/core/tests/sqllogictests/src/engines/conversion.rs b/datafusion/sqllogictest/src/engines/conversion.rs similarity index 82% rename from datafusion/core/tests/sqllogictests/src/engines/conversion.rs rename to datafusion/sqllogictest/src/engines/conversion.rs index c069c2d4a48df..a44783b098c9b 100644 --- a/datafusion/core/tests/sqllogictests/src/engines/conversion.rs +++ b/datafusion/sqllogictest/src/engines/conversion.rs @@ -19,11 +19,11 @@ use arrow::datatypes::{Decimal128Type, DecimalType}; use bigdecimal::BigDecimal; use half::f16; use rust_decimal::prelude::*; -use rust_decimal::Decimal; +/// Represents a constant for NULL string in your database. pub const NULL_STR: &str = "NULL"; -pub fn bool_to_str(value: bool) -> String { +pub(crate) fn bool_to_str(value: bool) -> String { if value { "true".to_string() } else { @@ -31,7 +31,7 @@ pub fn bool_to_str(value: bool) -> String { } } -pub fn varchar_to_str(value: &str) -> String { +pub(crate) fn varchar_to_str(value: &str) -> String { if value.is_empty() { "(empty)".to_string() } else { @@ -39,7 +39,7 @@ pub fn varchar_to_str(value: &str) -> String { } } -pub fn f16_to_str(value: f16) -> String { +pub(crate) fn f16_to_str(value: f16) -> String { if value.is_nan() { "NaN".to_string() } else if value == f16::INFINITY { @@ -51,7 +51,7 @@ pub fn f16_to_str(value: f16) -> String { } } -pub fn f32_to_str(value: f32) -> String { +pub(crate) fn f32_to_str(value: f32) -> String { if value.is_nan() { "NaN".to_string() } else if value == f32::INFINITY { @@ -63,7 +63,7 @@ pub fn f32_to_str(value: f32) -> String { } } -pub fn f64_to_str(value: f64) -> String { +pub(crate) fn f64_to_str(value: f64) -> String { if value.is_nan() { "NaN".to_string() } else if value == f64::INFINITY { @@ -75,17 +75,17 @@ pub fn f64_to_str(value: f64) -> String { } } -pub fn i128_to_str(value: i128, precision: &u8, scale: &i8) -> String { +pub(crate) fn i128_to_str(value: i128, precision: &u8, scale: &i8) -> String { big_decimal_to_str( BigDecimal::from_str(&Decimal128Type::format_decimal(value, *precision, *scale)) .unwrap(), ) } -pub fn decimal_to_str(value: Decimal) -> String { +pub(crate) fn decimal_to_str(value: Decimal) -> String { big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap()) } -pub fn big_decimal_to_str(value: BigDecimal) -> String { +pub(crate) fn big_decimal_to_str(value: BigDecimal) -> String { value.round(12).normalized().to_string() } diff --git a/datafusion/core/tests/sqllogictests/src/engines/datafusion/error.rs b/datafusion/sqllogictest/src/engines/datafusion_engine/error.rs similarity index 100% rename from datafusion/core/tests/sqllogictests/src/engines/datafusion/error.rs rename to datafusion/sqllogictest/src/engines/datafusion_engine/error.rs diff --git a/datafusion/sqllogictest/src/engines/datafusion_engine/mod.rs b/datafusion/sqllogictest/src/engines/datafusion_engine/mod.rs new file mode 100644 index 0000000000000..663bbdd5a3c7c --- /dev/null +++ b/datafusion/sqllogictest/src/engines/datafusion_engine/mod.rs @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// DataFusion engine implementation for sqllogictest. +mod error; +mod normalize; +mod runner; + +pub use error::*; +pub use normalize::*; +pub use runner::*; diff --git a/datafusion/core/tests/sqllogictests/src/engines/datafusion/normalize.rs b/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs similarity index 97% rename from datafusion/core/tests/sqllogictests/src/engines/datafusion/normalize.rs rename to datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs index 6dd4e17d7dd70..954926ae3310e 100644 --- a/datafusion/core/tests/sqllogictests/src/engines/datafusion/normalize.rs +++ b/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs @@ -27,7 +27,7 @@ use super::super::conversion::*; use super::error::{DFSqlLogicTestError, Result}; /// Converts `batches` to a result as expected by sqllogicteset. -pub fn convert_batches(batches: Vec) -> Result>> { +pub(crate) fn convert_batches(batches: Vec) -> Result>> { if batches.is_empty() { Ok(vec![]) } else { @@ -113,13 +113,13 @@ fn expand_row(mut row: Vec) -> impl Iterator> { /// normalize path references /// -/// ``` +/// ```text /// CsvExec: files={1 group: [[path/to/datafusion/testing/data/csv/aggregate_test_100.csv]]}, ... /// ``` /// /// into: /// -/// ``` +/// ```text /// CsvExec: files={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, ... /// ``` fn normalize_paths(mut row: Vec) -> Vec { @@ -230,7 +230,7 @@ pub fn cell_to_string(col: &ArrayRef, row: usize) -> Result { } /// Converts columns to a result as expected by sqllogicteset. -pub fn convert_schema_to_types(columns: &[DFField]) -> Vec { +pub(crate) fn convert_schema_to_types(columns: &[DFField]) -> Vec { columns .iter() .map(|f| f.data_type()) diff --git a/datafusion/core/tests/sqllogictests/src/engines/datafusion/mod.rs b/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs similarity index 91% rename from datafusion/core/tests/sqllogictests/src/engines/datafusion/mod.rs rename to datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs index dd30ef494d497..afd0a241ca5ef 100644 --- a/datafusion/core/tests/sqllogictests/src/engines/datafusion/mod.rs +++ b/datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs @@ -15,21 +15,17 @@ // specific language governing permissions and limitations // under the License. -use std::path::PathBuf; -use std::time::Duration; +use std::{path::PathBuf, time::Duration}; -use crate::engines::output::{DFColumnType, DFOutput}; - -use self::error::{DFSqlLogicTestError, Result}; +use arrow::record_batch::RecordBatch; use async_trait::async_trait; -use datafusion::arrow::record_batch::RecordBatch; use datafusion::prelude::SessionContext; use log::info; use sqllogictest::DBOutput; -mod error; -mod normalize; -mod util; +use super::{error::Result, normalize, DFSqlLogicTestError}; + +use crate::engines::output::{DFColumnType, DFOutput}; pub struct DataFusion { ctx: SessionContext, @@ -61,7 +57,7 @@ impl sqllogictest::AsyncDB for DataFusion { "DataFusion" } - /// [`Runner`] calls this function to perform sleep. + /// [`DataFusion`] calls this function to perform sleep. /// /// The default implementation is `std::thread::sleep`, which is universal to any async runtime /// but would block the current thread. If you are running in tokio runtime, you should override diff --git a/datafusion/sqllogictest/src/engines/mod.rs b/datafusion/sqllogictest/src/engines/mod.rs new file mode 100644 index 0000000000000..a6a0886332ed7 --- /dev/null +++ b/datafusion/sqllogictest/src/engines/mod.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// Implementation of sqllogictest for datafusion. +mod conversion; +mod datafusion_engine; +mod output; + +pub use datafusion_engine::DataFusion; + +#[cfg(feature = "postgres")] +mod postgres_engine; + +#[cfg(feature = "postgres")] +pub use postgres_engine::Postgres; diff --git a/datafusion/core/tests/sqllogictests/src/engines/output.rs b/datafusion/sqllogictest/src/engines/output.rs similarity index 97% rename from datafusion/core/tests/sqllogictests/src/engines/output.rs rename to datafusion/sqllogictest/src/engines/output.rs index 0682f5df97c19..24299856e00d5 100644 --- a/datafusion/core/tests/sqllogictests/src/engines/output.rs +++ b/datafusion/sqllogictest/src/engines/output.rs @@ -54,4 +54,4 @@ impl ColumnType for DFColumnType { } } -pub type DFOutput = DBOutput; +pub(crate) type DFOutput = DBOutput; diff --git a/datafusion/core/tests/sqllogictests/src/engines/postgres/mod.rs b/datafusion/sqllogictest/src/engines/postgres_engine/mod.rs similarity index 99% rename from datafusion/core/tests/sqllogictests/src/engines/postgres/mod.rs rename to datafusion/sqllogictest/src/engines/postgres_engine/mod.rs index 2c6287b97bfd1..fe2785603e76d 100644 --- a/datafusion/core/tests/sqllogictests/src/engines/postgres/mod.rs +++ b/datafusion/sqllogictest/src/engines/postgres_engine/mod.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +/// Postgres engine implementation for sqllogictest. use std::path::{Path, PathBuf}; use std::str::FromStr; diff --git a/datafusion/core/tests/sqllogictests/src/engines/postgres/types.rs b/datafusion/sqllogictest/src/engines/postgres_engine/types.rs similarity index 100% rename from datafusion/core/tests/sqllogictests/src/engines/postgres/types.rs rename to datafusion/sqllogictest/src/engines/postgres_engine/types.rs diff --git a/datafusion/core/tests/sqllogictests/src/engines/mod.rs b/datafusion/sqllogictest/src/lib.rs similarity index 88% rename from datafusion/core/tests/sqllogictests/src/engines/mod.rs rename to datafusion/sqllogictest/src/lib.rs index a2657bb60017b..b739d75777de0 100644 --- a/datafusion/core/tests/sqllogictests/src/engines/mod.rs +++ b/datafusion/sqllogictest/src/lib.rs @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -mod conversion; -pub mod datafusion; -mod output; -pub mod postgres; +mod engines; + +pub use engines::DataFusion; + +#[cfg(feature = "postgres")] +pub use engines::Postgres;