From 99f33cbccf5c7deca46e50ecc6bdef26f4e47b63 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 13 Aug 2022 08:04:21 -0600 Subject: [PATCH 1/3] Changes needed to support recent sqlparser handling of LIKE --- datafusion/common/Cargo.toml | 2 +- datafusion/core/Cargo.toml | 2 +- datafusion/expr/Cargo.toml | 2 +- datafusion/sql/Cargo.toml | 2 +- datafusion/sql/src/planner.rs | 34 ++++++++++++++++++++++++++++++++-- 5 files changed, 36 insertions(+), 6 deletions(-) diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 35ed2bb2ac3f..ec12365f581f 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -47,4 +47,4 @@ ordered-float = "3.0" parquet = { version = "20.0.0", features = ["arrow"], optional = true } pyo3 = { version = "0.16", optional = true } serde_json = "1.0" -sqlparser = "0.20" +sqlparser = { git="https://github.com/sqlparser-rs/sqlparser-rs", rev="f07063f0cdf06c56df7c3dd65f9062a2ce439c1c" } diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 4ef1af708504..db4b5ce2b778 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -85,7 +85,7 @@ pyo3 = { version = "0.16", optional = true } rand = "0.8" rayon = { version = "1.5", optional = true } smallvec = { version = "1.6", features = ["union"] } -sqlparser = "0.20" +sqlparser = { git="https://github.com/sqlparser-rs/sqlparser-rs", rev="f07063f0cdf06c56df7c3dd65f9062a2ce439c1c" } tempfile = "3" tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "fs", "parking_lot"] } tokio-stream = "0.1" diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml index a3c4dd4acb0f..71d429014dec 100644 --- a/datafusion/expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -38,4 +38,4 @@ path = "src/lib.rs" ahash = { version = "0.7", default-features = false } arrow = { version = "20.0.0", features = ["prettyprint"] } datafusion-common = { path = "../common", version = "10.0.0" } -sqlparser = "0.20" +sqlparser = { git="https://github.com/sqlparser-rs/sqlparser-rs", rev="f07063f0cdf06c56df7c3dd65f9062a2ce439c1c" } diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml index a6499a7f998d..fbc31ae450d6 100644 --- a/datafusion/sql/Cargo.toml +++ b/datafusion/sql/Cargo.toml @@ -42,5 +42,5 @@ arrow = { version = "20.0.0", features = ["prettyprint"] } datafusion-common = { path = "../common", version = "10.0.0" } datafusion-expr = { path = "../expr", version = "10.0.0" } hashbrown = "0.12" -sqlparser = "0.20" +sqlparser = { git="https://github.com/sqlparser-rs/sqlparser-rs", rev="f07063f0cdf06c56df7c3dd65f9062a2ce439c1c" } tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "fs", "parking_lot"] } diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs index daa4753e4bfd..ec63731bb508 100644 --- a/datafusion/sql/src/planner.rs +++ b/datafusion/sql/src/planner.rs @@ -1527,8 +1527,6 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { BinaryOperator::Modulo => Ok(Operator::Modulo), BinaryOperator::And => Ok(Operator::And), BinaryOperator::Or => Ok(Operator::Or), - BinaryOperator::Like => Ok(Operator::Like), - BinaryOperator::NotLike => Ok(Operator::NotLike), BinaryOperator::PGRegexMatch => Ok(Operator::RegexMatch), BinaryOperator::PGRegexIMatch => Ok(Operator::RegexIMatch), BinaryOperator::PGRegexNotMatch => Ok(Operator::RegexNotMatch), @@ -1876,6 +1874,38 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { }) } + SQLExpr::Like { negated, expr, pattern, escape_char } => { + match escape_char { + Some(_) => { + // to support this we will need to introduce `Expr::Like` instead + // of treating it like a binary expression + Err(DataFusionError::NotImplemented("LIKE with ESCAPE is not yet supported".to_string())) + }, + _ => { + Ok(Expr::BinaryExpr { + left: Box::new(self.sql_expr_to_logical_expr(*expr, schema, ctes)?), + op: if negated { Operator::NotLike } else { Operator::Like }, + right: match *pattern { + Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => { + Ok(Box::new(Expr::Literal(ScalarValue::Utf8(Some(s))))) + } + _ => Err(DataFusionError::NotImplemented("Unsupported syntax for LIKE pattern ".to_string())) + }? + }) + } + } + } + + SQLExpr::ILike { .. } => { + // https://github.com/apache/arrow-datafusion/issues/3099 + Err(DataFusionError::NotImplemented("ILIKE is not yet supported".to_string())) + } + + SQLExpr::SimilarTo { .. } => { + // https://github.com/apache/arrow-datafusion/issues/3099 + Err(DataFusionError::NotImplemented("SIMILAR TO is not yet supported".to_string())) + } + SQLExpr::BinaryOp { left, op, From 4539299e5526462b01279dc3ab9a155991b1cb82 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 17 Aug 2022 11:06:58 -0600 Subject: [PATCH 2/3] placeholder for new ShowTables ast --- datafusion/sql/src/planner.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs index 39ff5bc4efcd..979a2bf07c2c 100644 --- a/datafusion/sql/src/planner.rs +++ b/datafusion/sql/src/planner.rs @@ -241,6 +241,13 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { })) } + Statement::ShowTables { + extended, + full, + db_name, + filter, + } => todo!(), + Statement::ShowColumns { extended, full, From d6af947229144aa2cb069015f01c6014c429546e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 17 Aug 2022 11:38:51 -0600 Subject: [PATCH 3/3] add support for sqlparser ast for ShowTables --- datafusion/sql/src/planner.rs | 54 ++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs index 979a2bf07c2c..73ba86180e03 100644 --- a/datafusion/sql/src/planner.rs +++ b/datafusion/sql/src/planner.rs @@ -246,7 +246,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { full, db_name, filter, - } => todo!(), + } => self.show_tables_to_plan(extended, full, db_name, filter), Statement::ShowColumns { extended, @@ -261,6 +261,35 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } } + /// Generate a logical plan from a "SHOW TABLES" query + fn show_tables_to_plan( + &self, + extended: bool, + full: bool, + db_name: Option, + filter: Option, + ) -> Result { + if self.has_table("information_schema", "tables") { + // we only support the basic "SHOW TABLES" + // https://github.com/apache/arrow-datafusion/issues/3188 + if db_name.is_some() || filter.is_some() || full || extended { + Err(DataFusionError::Plan( + "Unsupported parameters to SHOW TABLES".to_string(), + )) + } else { + let query = "SELECT * FROM information_schema.tables;"; + let mut rewrite = DFParser::parse_sql(query)?; + assert_eq!(rewrite.len(), 1); + self.statement_to_plan(rewrite.pop_front().unwrap()) + } + } else { + Err(DataFusionError::Plan( + "SHOW TABLES is not supported unless information_schema is enabled" + .to_string(), + )) + } + } + /// Generate a logical plan from an SQL query pub fn query_to_plan( &self, @@ -2298,26 +2327,11 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } fn show_variable_to_plan(&self, variable: &[Ident]) -> Result { - // Special case SHOW TABLES let variable = ObjectName(variable.to_vec()).to_string(); - if variable.as_str().eq_ignore_ascii_case("tables") { - if self.has_table("information_schema", "tables") { - let query = "SELECT * FROM information_schema.tables;"; - let mut rewrite = DFParser::parse_sql(query)?; - assert_eq!(rewrite.len(), 1); - self.statement_to_plan(rewrite.pop_front().unwrap()) - } else { - Err(DataFusionError::Plan( - "SHOW TABLES is not supported unless information_schema is enabled" - .to_string(), - )) - } - } else { - Err(DataFusionError::NotImplemented(format!( - "SHOW {} not implemented. Supported syntax: SHOW ", - variable - ))) - } + Err(DataFusionError::NotImplemented(format!( + "SHOW {} not implemented. Supported syntax: SHOW ", + variable + ))) } fn show_columns_to_plan(