From dfc8224abcf5cad218041611506eae2e8c951ec3 Mon Sep 17 00:00:00 2001 From: Lichuan Shang Date: Thu, 4 Nov 2021 16:39:53 +0800 Subject: [PATCH 1/3] Fix `with header row` in sql when creating external table --- datafusion/src/sql/parser.rs | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/datafusion/src/sql/parser.rs b/datafusion/src/sql/parser.rs index c4822c81cad12..8890351963201 100644 --- a/datafusion/src/sql/parser.rs +++ b/datafusion/src/sql/parser.rs @@ -293,19 +293,14 @@ impl<'a> DFParser<'a> { } } - fn consume_token(&mut self, expected: &str) -> bool { - if self.parser.peek_token().to_string() == *expected { - self.parser.next_token(); - true - } else { - false - } + fn consume_token(&mut self, expected: &Token) -> bool { + self.parser.consume_token(expected) } fn parse_csv_has_header(&mut self) -> bool { - self.consume_token("WITH") - & self.consume_token("HEADER") - & self.consume_token("ROW") + self.consume_token(&Token::make_keyword("WITH")) + & self.consume_token(&Token::make_keyword("HEADER")) + & self.consume_token(&Token::make_keyword("ROW")) } } @@ -372,6 +367,22 @@ mod tests { }); expect_parse_ok(sql, expected)?; + // positive case: it is ok for case insensitive sql with `WITH HEADER ROW` tokens + let sqls = vec![ + "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV WITH HEADER ROW LOCATION 'foo.csv'", + "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV with header row LOCATION 'foo.csv'" + ]; + for sql in sqls { + let expected = Statement::CreateExternalTable(CreateExternalTable { + name: "t".into(), + columns: vec![make_column_def("c1", DataType::Int(display))], + file_type: FileType::CSV, + has_header: true, + location: "foo.csv".into(), + }); + expect_parse_ok(sql, expected)?; + } + // positive case: it is ok for parquet files not to have columns specified let sql = "CREATE EXTERNAL TABLE t STORED AS PARQUET LOCATION 'foo.parquet'"; let expected = Statement::CreateExternalTable(CreateExternalTable { From b750be59ecea7f591a8a20e432fdd7017948e72b Mon Sep 17 00:00:00 2001 From: Lichuan Shang Date: Fri, 5 Nov 2021 11:36:18 +0800 Subject: [PATCH 2/3] adjust comment --- datafusion/src/sql/parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/src/sql/parser.rs b/datafusion/src/sql/parser.rs index 8890351963201..ba3e6e9c313ad 100644 --- a/datafusion/src/sql/parser.rs +++ b/datafusion/src/sql/parser.rs @@ -367,7 +367,7 @@ mod tests { }); expect_parse_ok(sql, expected)?; - // positive case: it is ok for case insensitive sql with `WITH HEADER ROW` tokens + // positive case: it is ok for case insensitive sql stmt with `WITH HEADER ROW` tokens let sqls = vec![ "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV WITH HEADER ROW LOCATION 'foo.csv'", "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV with header row LOCATION 'foo.csv'" From 8759ac561603e6b54e903a5ef1d40192072f09f2 Mon Sep 17 00:00:00 2001 From: Lichuan Shang Date: Fri, 5 Nov 2021 13:31:10 +0800 Subject: [PATCH 3/3] fix ci failed --- datafusion/src/sql/parser.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/datafusion/src/sql/parser.rs b/datafusion/src/sql/parser.rs index ba3e6e9c313ad..49ae86906c142 100644 --- a/datafusion/src/sql/parser.rs +++ b/datafusion/src/sql/parser.rs @@ -294,7 +294,14 @@ impl<'a> DFParser<'a> { } fn consume_token(&mut self, expected: &Token) -> bool { - self.parser.consume_token(expected) + let token = self.parser.peek_token().to_string().to_uppercase(); + let token = Token::make_keyword(&token); + if token == *expected { + self.parser.next_token(); + true + } else { + false + } } fn parse_csv_has_header(&mut self) -> bool {