Skip to content

[SPARK-14213][SQL] Migrate HiveQl parsing to ANTLR4 parser #12015

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
hvanhovell wants to merge 7 commits into from
Closed

[SPARK-14213][SQL] Migrate HiveQl parsing to ANTLR4 parser #12015

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f7c8eb5
Add DDL commands to SparkSqlParser.
hvanhovell Mar 28, 2016
575acdb
Fix doc.
hvanhovell Mar 28, 2016
b836eae
SerdeClass is option in tableRowFormat.
hvanhovell Mar 28, 2016
1fc3592
Migrate HiveQl parsing to ANTLR4 parser.
hvanhovell Mar 28, 2016
cb398aa
Merge remote-tracking branch 'apache-github/master' into SPARK-14213
hvanhovell Mar 28, 2016
3d487b9
Merge remote-tracking branch 'apache-github/master' into SPARK-14213
hvanhovell Mar 28, 2016
1232590
Handle unsupported Hive commands.
hvanhovell Mar 29, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 32 additions & 2 deletions sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,9 @@ statement
| UNCACHE TABLE identifier #uncacheTable
| CLEAR CACHE #clearCache
| ADD identifier .*? #addResource
| SET ROLE .*? #failNativeCommand
| SET .*? #setConfiguration
| kws=unsupportedHiveNativeCommands .*? #failNativeCommand
| hiveNativeCommands #executeNativeCommand
;

Expand All @@ -145,7 +147,26 @@ hiveNativeCommands
| ROLLBACK WORK?
| SHOW PARTITIONS tableIdentifier partitionSpec?
| DFS .*?
| (CREATE | ALTER | DROP | SHOW | DESC | DESCRIBE | REVOKE | GRANT | LOCK | UNLOCK | MSCK | EXPORT | IMPORT | LOAD) .*?
| (CREATE | ALTER | DROP | SHOW | DESC | DESCRIBE | LOCK | UNLOCK | MSCK | LOAD) .*?
;

unsupportedHiveNativeCommands
: kw1=CREATE kw2=ROLE
| kw1=DROP kw2=ROLE
| kw1=GRANT kw2=ROLE?
| kw1=REVOKE kw2=ROLE?
| kw1=SHOW kw2=GRANT
| kw1=SHOW kw2=ROLE kw3=GRANT?
| kw1=SHOW kw2=PRINCIPALS
| kw1=SHOW kw2=ROLES
| kw1=SHOW kw2=CURRENT kw3=ROLES
| kw1=EXPORT kw2=TABLE
| kw1=IMPORT kw2=TABLE
| kw1=SHOW kw2=COMPACTIONS
| kw1=SHOW kw2=CREATE kw3=TABLE
| kw1=SHOW kw2=TRANSACTIONS
| kw1=SHOW kw2=INDEXES
| kw1=SHOW kw2=LOCKS
;

createTableHeader
Expand Down Expand Up @@ -619,7 +640,8 @@ nonReserved
| AFTER | CASCADE | RESTRICT | BUCKETS | CLUSTERED | SORTED | PURGE | INPUTFORMAT | OUTPUTFORMAT
| INPUTDRIVER | OUTPUTDRIVER | DBPROPERTIES | DFS | TRUNCATE | METADATA | REPLICATION | COMPUTE
| STATISTICS | ANALYZE | PARTITIONED | EXTERNAL | DEFINED | RECORDWRITER
| REVOKE | GRANT | LOCK | UNLOCK | MSCK | EXPORT | IMPORT | LOAD | VALUES | COMMENT
| REVOKE | GRANT | LOCK | UNLOCK | MSCK | EXPORT | IMPORT | LOAD | VALUES | COMMENT | ROLE
| ROLES | COMPACTIONS | PRINCIPALS | TRANSACTIONS | INDEXES | LOCKS | OPTION
;

SELECT: 'SELECT';
Expand Down Expand Up @@ -834,6 +856,14 @@ MSCK: 'MSCK';
EXPORT: 'EXPORT';
IMPORT: 'IMPORT';
LOAD: 'LOAD';
ROLE: 'ROLE';
ROLES: 'ROLES';
COMPACTIONS: 'COMPACTIONS';
PRINCIPALS: 'PRINCIPALS';
TRANSACTIONS: 'TRANSACTIONS';
INDEXES: 'INDEXES';
LOCKS: 'LOCKS';
OPTION: 'OPTION';

STRING
: '\'' ( ~('\''|'\\') | ('\\' .) )* '\''
Expand Down
13 changes: 12 additions & 1 deletion sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,18 @@ private[hive] object HiveSerDe {
HiveSerDe(
inputFormat = Option("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"),
outputFormat = Option("org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"),
serde = Option("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe")))
serde = Option("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe")),

"textfile" ->
HiveSerDe(
inputFormat = Option("org.apache.hadoop.mapred.TextInputFormat"),
outputFormat = Option("org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat")),

"avro" ->
HiveSerDe(
inputFormat = Option("org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat"),
outputFormat = Option("org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat"),
serde = Option("org.apache.hadoop.hive.serde2.avro.AvroSerDe")))

val key = source.toLowerCase match {
case s if s.startsWith("org.apache.spark.sql.parquet") => "parquet"
Expand Down
3 changes: 2 additions & 1 deletion sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry}
import org.apache.spark.sql.catalyst.parser.ParserInterface
import org.apache.spark.sql.execution.{python, SparkPlanner}
import org.apache.spark.sql.execution.datasources._
import org.apache.spark.sql.hive.execution.HiveSqlParser
import org.apache.spark.sql.internal.{SessionState, SQLConf}


Expand Down Expand Up @@ -70,7 +71,7 @@ private[hive] class HiveSessionState(ctx: HiveContext) extends SessionState(ctx)
/**
* Parser for HiveQl query texts.
*/
override lazy val sqlParser: ParserInterface = new HiveQl(conf)
override lazy val sqlParser: ParserInterface = HiveSqlParser

/**
* Planner that takes into account Hive-specific strategies.
Expand Down
Loading