From b38a21ef6146784e4b93ef4ce8c899f1eee14572 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 16 Nov 2015 18:30:26 -0800 Subject: [PATCH 01/13] SPARK-11633 --- .../spark/sql/catalyst/analysis/Analyzer.scala | 3 ++- .../spark/sql/hive/execution/SQLQuerySuite.scala | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 2f4670b55bdba..5a5b71e52dd79 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -425,7 +425,8 @@ class Analyzer( */ j case Some((oldRelation, newRelation)) => - val attributeRewrites = AttributeMap(oldRelation.output.zip(newRelation.output)) + val attributeRewrites = + AttributeMap(oldRelation.output.zip(newRelation.output).filter(x => x._1 != x._2)) val newRight = right transformUp { case r if r == oldRelation => newRelation } transformUp { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 3427152b2da02..5e00546a74c00 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -51,6 +51,8 @@ case class Order( state: String, month: Int) +case class Individual(F1: Integer, F2: Integer) + case class WindowData( month: Int, area: String, @@ -1479,4 +1481,18 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { |FROM (SELECT '{"f1": "value1", "f2": 12}' json, 'hello' as str) test """.stripMargin), Row("value1", "12", 3.14, "hello")) } + + test ("SPARK-11633: HiveContext throws TreeNode Exception : Failed to Copy Node") { + val rdd1 = sparkContext.parallelize(Seq( Individual(1,3), Individual(2,1))) + val df = hiveContext.createDataFrame(rdd1) + df.registerTempTable("foo") + val df2 = sql("select f1, F2 as F2 from foo") + df2.registerTempTable("foo2") + df2.registerTempTable("foo3") + + checkAnswer(sql( + """ + SELECT a.F1 FROM foo2 a INNER JOIN foo3 b ON a.F2=b.F2 + """.stripMargin), Row(2) :: Row(1) :: Nil) + } } From 0546772f151f83d6d3cf4d000cbe341f52545007 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Fri, 20 Nov 2015 10:56:45 -0800 Subject: [PATCH 02/13] converge --- .../spark/sql/catalyst/analysis/Analyzer.scala | 3 +-- .../spark/sql/hive/execution/SQLQuerySuite.scala | 15 --------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 7c9512fbd00aa..47962ebe6ef82 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -417,8 +417,7 @@ class Analyzer( */ j case Some((oldRelation, newRelation)) => - val attributeRewrites = - AttributeMap(oldRelation.output.zip(newRelation.output).filter(x => x._1 != x._2)) + val attributeRewrites = AttributeMap(oldRelation.output.zip(newRelation.output)) val newRight = right transformUp { case r if r == oldRelation => newRelation } transformUp { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 5e00546a74c00..61d9dcd37572b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -51,8 +51,6 @@ case class Order( state: String, month: Int) -case class Individual(F1: Integer, F2: Integer) - case class WindowData( month: Int, area: String, @@ -1481,18 +1479,5 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { |FROM (SELECT '{"f1": "value1", "f2": 12}' json, 'hello' as str) test """.stripMargin), Row("value1", "12", 3.14, "hello")) } - - test ("SPARK-11633: HiveContext throws TreeNode Exception : Failed to Copy Node") { - val rdd1 = sparkContext.parallelize(Seq( Individual(1,3), Individual(2,1))) - val df = hiveContext.createDataFrame(rdd1) - df.registerTempTable("foo") - val df2 = sql("select f1, F2 as F2 from foo") - df2.registerTempTable("foo2") - df2.registerTempTable("foo3") - - checkAnswer(sql( - """ - SELECT a.F1 FROM foo2 a INNER JOIN foo3 b ON a.F2=b.F2 - """.stripMargin), Row(2) :: Row(1) :: Nil) } } From b37a64f13956b6ddd0e38ddfd9fe1caee611f1a8 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Fri, 20 Nov 2015 10:58:37 -0800 Subject: [PATCH 03/13] converge --- .../org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 61d9dcd37572b..3427152b2da02 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -1479,5 +1479,4 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { |FROM (SELECT '{"f1": "value1", "f2": 12}' json, 'hello' as str) test """.stripMargin), Row("value1", "12", 3.14, "hello")) } - } } From b1ffb0d2dfc4676f0a35fe6a3a197179a96dc138 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Sat, 26 Mar 2016 22:58:44 -0700 Subject: [PATCH 04/13] native parsing support for alter view. --- .../apache/spark/sql/execution/SparkQl.scala | 5 +- ...=> AlterTableAlterViewCommandParser.scala} | 67 +++++--- .../spark/sql/execution/command/ddl.scala | 14 +- .../execution/command/DDLCommandSuite.scala | 151 +++++++++++++----- .../org/apache/spark/sql/hive/HiveQl.scala | 28 ++-- 5 files changed, 177 insertions(+), 88 deletions(-) rename sql/core/src/main/scala/org/apache/spark/sql/execution/command/{AlterTableCommandParser.scala => AlterTableAlterViewCommandParser.scala} (86%) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala index b9542c7173505..5fe7747907898 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala @@ -192,8 +192,9 @@ private[sql] class SparkQl(conf: ParserConf = SimpleParserConf()) extends Cataly } CreateFunction(funcName, alias, resources, temp.isDefined)(node.source) - case Token("TOK_ALTERTABLE", alterTableArgs) => - AlterTableCommandParser.parse(node) + case Token(fieldName, alterTableArgs) + if fieldName == "TOK_ALTERTABLE" || fieldName == "TOK_ALTERVIEW" => + AlterTableAlterViewCommandParser.parse(node) case Token("TOK_CREATETABLEUSING", createTableArgs) => val Seq( diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableAlterViewCommandParser.scala similarity index 86% rename from sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala rename to sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableAlterViewCommandParser.scala index 9fbe6db467ffa..632ab357b478e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableAlterViewCommandParser.scala @@ -29,22 +29,26 @@ import org.apache.spark.sql.types.StructType /** - * Helper object to parse alter table commands. + * Helper object to parse alter table or alter view commands. */ -object AlterTableCommandParser { +object AlterTableAlterViewCommandParser { import ParserUtils._ /** - * Parse the given node assuming it is an alter table command. + * Parse the given node assuming it is an alter table/view command. */ def parse(node: ASTNode): LogicalPlan = { + assert(node.token.getText == "TOK_ALTERTABLE" || node.token.getText == "TOK_ALTERVIEW") node.children match { case (tabName @ Token("TOK_TABNAME", _)) :: otherNodes => val tableIdent = extractTableIdent(tabName) val partSpec = getClauseOption("TOK_PARTSPEC", node.children).map(parsePartitionSpec) - matchAlterTableCommands(node, otherNodes, tableIdent, partSpec) + if (partSpec.isDefined && node.token.getText == "TOK_ALTERVIEW") { + parseFailed("Could not parse ALTER VIEW command", node) + } + matchAlterTableAlterViewCommands(node, otherNodes, tableIdent, partSpec) case _ => - parseFailed("Could not parse ALTER TABLE command", node) + parseFailed("Could not parse ALTER TABLE/VIEW command", node) } } @@ -121,8 +125,9 @@ object AlterTableCommandParser { } /** - * Parse an alter table command from a [[ASTNode]] into a [[LogicalPlan]]. - * This follows https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL. + * Parse an alter table/view command from a [[ASTNode]] into a [[LogicalPlan]]. + * This follows https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL + * and https://cwiki.apache.org/confluence/display/Hive/PartitionedViews. * * @param node the original [[ASTNode]] to parse. * @param otherNodes the other [[ASTNode]]s after the first one containing the table name. @@ -130,28 +135,35 @@ object AlterTableCommandParser { * @param partition spec identifying the partition this command is concerned with, if any. */ // TODO: This method is massive. Break it down. - private def matchAlterTableCommands( + private def matchAlterTableAlterViewCommands( node: ASTNode, otherNodes: Seq[ASTNode], tableIdent: TableIdentifier, partition: Option[TablePartitionSpec]): LogicalPlan = { otherNodes match { // ALTER TABLE table_name RENAME TO new_table_name; - case Token("TOK_ALTERTABLE_RENAME", renameArgs) :: _ => + // ALTER VIEW view_name RENAME TO new_view_name; + case Token(fieldName, renameArgs) :: _ + if fieldName == "TOK_ALTERTABLE_RENAME" || fieldName == "TOK_ALTERVIEW_RENAME" => val tableNameClause = getClause("TOK_TABNAME", renameArgs) val newTableIdent = extractTableIdent(tableNameClause) - AlterTableRename(tableIdent, newTableIdent)(node.source) + AlterTableAlterViewRename(tableIdent, newTableIdent)(node.source) // ALTER TABLE table_name SET TBLPROPERTIES ('comment' = new_comment); - case Token("TOK_ALTERTABLE_PROPERTIES", args) :: _ => + // ALTER VIEW view_name SET TBLPROPERTIES ('comment' = new_comment); + case Token(fieldName, args) :: _ + if fieldName == "TOK_ALTERTABLE_PROPERTIES" || fieldName == "TOK_ALTERVIEW_PROPERTIES" => val properties = extractTableProps(args.head) - AlterTableSetProperties(tableIdent, properties)(node.source) + AlterTableAlterViewSetProperties(tableIdent, properties)(node.source) - // ALTER TABLE table_name UNSET TBLPROPERTIES IF EXISTS ('comment', 'key'); - case Token("TOK_ALTERTABLE_DROPPROPERTIES", args) :: _ => + // ALTER TABLE table_name UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key'); + // ALTER VIEW view_name UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key'); + case Token(fieldName, args) :: _ + if fieldName == "TOK_ALTERTABLE_DROPPROPERTIES" || + fieldName == "TOK_ALTERVIEW_DROPPROPERTIES" => val properties = extractTableProps(args.head) val ifExists = getClauseOption("TOK_IFEXISTS", args).isDefined - AlterTableUnsetProperties(tableIdent, properties, ifExists)(node.source) + AlterTableAlterViewUnsetProperties(tableIdent, properties, ifExists)(node.source) // ALTER TABLE table_name [PARTITION spec] SET SERDE serde_name [WITH SERDEPROPERTIES props]; case Token("TOK_ALTERTABLE_SERIALIZER", Token(serdeClassName, Nil) :: serdeArgs) :: _ => @@ -290,9 +302,10 @@ object AlterTableCommandParser { }.toMap AlterTableSkewedLocation(tableIdent, skewedMaps)(node.source) - // ALTER TABLE table_name ADD [IF NOT EXISTS] PARTITION spec [LOCATION 'loc1'] - // spec [LOCATION 'loc2'] ...; - case Token("TOK_ALTERTABLE_ADDPARTS", args) :: _ => + // ALTER TABLE view_name ADD [IF NOT EXISTS] PARTITION spec1[, PARTITION spec2, ...] + // ALTER VIEW view_name ADD [IF NOT EXISTS] PARTITION spec1[, PARTITION spec2, ...] + case Token(fieldName, args) :: _ + if fieldName == "TOK_ALTERTABLE_ADDPARTS" || fieldName == "TOK_ALTERVIEW_ADDPARTS" => val (ifNotExists, parts) = args.head match { case Token("TOK_IFNOTEXISTS", Nil) => (true, args.tail) case _ => (false, args) @@ -310,9 +323,9 @@ object AlterTableCommandParser { parsedParts += ((spec, Some(unquoteString(loc.text)))) } case _ => - parseFailed("Invalid ALTER TABLE command", node) + parseFailed("Invalid ALTER TABLE/VIEW command", node) } - AlterTableAddPartition(tableIdent, parsedParts, ifNotExists)(node.source) + AlterTableAlterViewAddPartition(tableIdent, parsedParts, ifNotExists)(node.source) // ALTER TABLE table_name PARTITION spec1 RENAME TO PARTITION spec2; case Token("TOK_ALTERTABLE_RENAMEPART", spec :: Nil) :: _ => @@ -329,11 +342,17 @@ object AlterTableCommandParser { AlterTableExchangePartition(tableIdent, newTableIdent, parsedSpec)(node.source) // ALTER TABLE table_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...] [PURGE]; - case Token("TOK_ALTERTABLE_DROPPARTS", args) :: _ => + // ALTER VIEW table_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...]; + case Token(fieldName, args) :: _ + if fieldName == "TOK_ALTERTABLE_DROPPARTS" || fieldName == "TOK_ALTERVIEW_DROPPARTS" => val parts = args.collect { case p @ Token("TOK_PARTSPEC", _) => parsePartitionSpec(p) } val ifExists = getClauseOption("TOK_IFEXISTS", args).isDefined - val purge = getClauseOption("PURGE", args).isDefined - AlterTableDropPartition(tableIdent, parts, ifExists, purge)(node.source) + if (fieldName == "TOK_ALTERTABLE_DROPPARTS") { + val purge = getClauseOption("PURGE", args).isDefined + AlterTableDropPartition(tableIdent, parts, ifExists, purge)(node.source) + } else { + AlterViewDropPartition(tableIdent, parts, ifExists)(node.source) + } // ALTER TABLE table_name ARCHIVE PARTITION spec; case Token("TOK_ALTERTABLE_ARCHIVE", spec :: Nil) :: _ => @@ -424,7 +443,7 @@ object AlterTableCommandParser { AlterTableReplaceCol(tableIdent, partition, columns, restrict, cascade)(node.source) case _ => - parseFailed("Unsupported ALTER TABLE command", node) + parseFailed("Unsupported ALTER TABLE/VIEW command", node) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 373b557683f15..e313da496d465 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -77,17 +77,17 @@ case class CreateFunction( isTemp: Boolean)(sql: String) extends NativeDDLCommand(sql) with Logging -case class AlterTableRename( +case class AlterTableAlterViewRename( oldName: TableIdentifier, newName: TableIdentifier)(sql: String) extends NativeDDLCommand(sql) with Logging -case class AlterTableSetProperties( +case class AlterTableAlterViewSetProperties( tableName: TableIdentifier, properties: Map[String, String])(sql: String) extends NativeDDLCommand(sql) with Logging -case class AlterTableUnsetProperties( +case class AlterTableAlterViewUnsetProperties( tableName: TableIdentifier, properties: Map[String, String], ifExists: Boolean)(sql: String) @@ -135,7 +135,7 @@ case class AlterTableSkewedLocation( skewedMap: Map[String, String])(sql: String) extends NativeDDLCommand(sql) with Logging -case class AlterTableAddPartition( +case class AlterTableAlterViewAddPartition( tableName: TableIdentifier, partitionSpecsAndLocs: Seq[(TablePartitionSpec, Option[String])], ifNotExists: Boolean)(sql: String) @@ -160,6 +160,12 @@ case class AlterTableDropPartition( purge: Boolean)(sql: String) extends NativeDDLCommand(sql) with Logging +case class AlterViewDropPartition( + viewName: TableIdentifier, + specs: Seq[TablePartitionSpec], + ifExists: Boolean)(sql: String) + extends NativeDDLCommand(sql) with Logging + case class AlterTableArchivePartition( tableName: TableIdentifier, spec: TablePartitionSpec)(sql: String) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala index a33175aa60b5b..d9d96daceaa5e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala @@ -130,33 +130,60 @@ class DDLCommandSuite extends PlanTest { comparePlans(parsed2, expected2) } - test("alter table: rename table") { - val sql = "ALTER TABLE table_name RENAME TO new_table_name" - val parsed = parser.parsePlan(sql) - val expected = AlterTableRename( + // ALTER TABLE table_name RENAME TO new_table_name; + // ALTER VIEW view_name RENAME TO new_view_name; + test("alter table/view: rename table/view") { + val sql_table = "ALTER TABLE table_name RENAME TO new_table_name" + val sql_view = sql_table.replace("TABLE", "VIEW") + val parsed_table = parser.parsePlan(sql_table) + val parsed_view = parser.parsePlan(sql_view) + val expected_table = AlterTableAlterViewRename( TableIdentifier("table_name", None), - TableIdentifier("new_table_name", None))(sql) - comparePlans(parsed, expected) + TableIdentifier("new_table_name", None))(sql_table) + val expected_view = AlterTableAlterViewRename( + TableIdentifier("table_name", None), + TableIdentifier("new_table_name", None))(sql_view) + comparePlans(parsed_table, expected_table) + comparePlans(parsed_view, expected_view) } - test("alter table: alter table properties") { - val sql1 = "ALTER TABLE table_name SET TBLPROPERTIES ('test' = 'test', " + + // ALTER TABLE table_name SET TBLPROPERTIES ('comment' = new_comment); + // ALTER TABLE table_name UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key'); + // ALTER VIEW view_name SET TBLPROPERTIES ('comment' = new_comment); + // ALTER VIEW view_name UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key'); + test("alter table/view: alter table/view properties") { + val sql1_table = "ALTER TABLE table_name SET TBLPROPERTIES ('test' = 'test', " + "'comment' = 'new_comment')" - val sql2 = "ALTER TABLE table_name UNSET TBLPROPERTIES ('comment', 'test')" - val sql3 = "ALTER TABLE table_name UNSET TBLPROPERTIES IF EXISTS ('comment', 'test')" - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) - val parsed3 = parser.parsePlan(sql3) + val sql2_table = "ALTER TABLE table_name UNSET TBLPROPERTIES ('comment', 'test')" + val sql3_table = "ALTER TABLE table_name UNSET TBLPROPERTIES IF EXISTS ('comment', 'test')" + val sql1_view = sql1_table.replace("TABLE", "VIEW") + val sql2_view = sql2_table.replace("TABLE", "VIEW") + val sql3_view = sql3_table.replace("TABLE", "VIEW") + + val parsed1_table = parser.parsePlan(sql1_table) + val parsed2_table = parser.parsePlan(sql2_table) + val parsed3_table = parser.parsePlan(sql3_table) + val parsed1_view = parser.parsePlan(sql1_view) + val parsed2_view = parser.parsePlan(sql2_view) + val parsed3_view = parser.parsePlan(sql3_view) + val tableIdent = TableIdentifier("table_name", None) - val expected1 = AlterTableSetProperties( - tableIdent, Map("test" -> "test", "comment" -> "new_comment"))(sql1) - val expected2 = AlterTableUnsetProperties( - tableIdent, Map("comment" -> null, "test" -> null), ifExists = false)(sql2) - val expected3 = AlterTableUnsetProperties( - tableIdent, Map("comment" -> null, "test" -> null), ifExists = true)(sql3) - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - comparePlans(parsed3, expected3) + val expected1_table = AlterTableAlterViewSetProperties( + tableIdent, Map("test" -> "test", "comment" -> "new_comment"))(sql1_table) + val expected2_table = AlterTableAlterViewUnsetProperties( + tableIdent, Map("comment" -> null, "test" -> null), ifExists = false)(sql2_table) + val expected3_table = AlterTableAlterViewUnsetProperties( + tableIdent, Map("comment" -> null, "test" -> null), ifExists = true)(sql3_table) + val expected1_view = expected1_table.copy()(sql = sql1_view) + val expected2_view = expected2_table.copy()(sql = sql2_view) + val expected3_view = expected3_table.copy()(sql = sql3_view) + + comparePlans(parsed1_table, expected1_table) + comparePlans(parsed2_table, expected2_table) + comparePlans(parsed3_table, expected3_table) + comparePlans(parsed1_view, expected1_view) + comparePlans(parsed2_view, expected2_view) + comparePlans(parsed3_view, expected3_view) } test("alter table: SerDe properties") { @@ -311,21 +338,42 @@ class DDLCommandSuite extends PlanTest { comparePlans(parsed2, expected2) } - test("alter table: add partition") { - val sql = + // ALTER TABLE view_name ADD [IF NOT EXISTS] PARTITION spec1[, PARTITION spec2, ...] + // ALTER VIEW view_name ADD [IF NOT EXISTS] PARTITION spec1[, PARTITION spec2, ...] + test("alter table/view: add partition") { + val sql1_table = """ |ALTER TABLE table_name ADD IF NOT EXISTS PARTITION |(dt='2008-08-08', country='us') LOCATION 'location1' PARTITION |(dt='2009-09-09', country='uk') """.stripMargin - val parsed = parser.parsePlan(sql) - val expected = AlterTableAddPartition( + val sql1_view = sql1_table.replace("TABLE", "VIEW") + + val sql2_table = "ALTER TABLE table_name ADD PARTITION (dt='2008-08-08') LOCATION 'loc'" + val sql2_view = sql2_table.replace("TABLE", "VIEW") + + val parsed1_table = parser.parsePlan(sql1_table) + val parsed1_view = parser.parsePlan(sql1_view) + val parsed2_table = parser.parsePlan(sql2_table) + val parsed2_view = parser.parsePlan(sql2_view) + + val expected1_table = AlterTableAlterViewAddPartition( TableIdentifier("table_name", None), Seq( (Map("dt" -> "2008-08-08", "country" -> "us"), Some("location1")), (Map("dt" -> "2009-09-09", "country" -> "uk"), None)), - ifNotExists = true)(sql) - comparePlans(parsed, expected) + ifNotExists = true)(sql1_table) + val expected1_view = expected1_table.copy()(sql = sql1_view) + val expected2_table = AlterTableAlterViewAddPartition( + TableIdentifier("table_name", None), + Seq((Map("dt" -> "2008-08-08"), Some("loc"))), + ifNotExists = false)(sql2_table) + val expected2_view = expected2_table.copy()(sql = sql2_view) + + comparePlans(parsed1_table, expected1_table) + comparePlans(parsed2_table, expected2_table) + comparePlans(parsed1_view, expected1_view) + comparePlans(parsed2_view, expected2_view) } test("alter table: rename partition") { @@ -356,36 +404,61 @@ class DDLCommandSuite extends PlanTest { comparePlans(parsed, expected) } - test("alter table: drop partitions") { - val sql1 = + // ALTER TABLE table_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...] [PURGE] + // ALTER VIEW table_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...] + test("alter table/view: drop partitions") { + val sql1_table = """ |ALTER TABLE table_name DROP IF EXISTS PARTITION |(dt='2008-08-08', country='us'), PARTITION (dt='2009-09-09', country='uk') """.stripMargin - val sql2 = + val sql2_table = """ |ALTER TABLE table_name DROP PARTITION |(dt='2008-08-08', country='us'), PARTITION (dt='2009-09-09', country='uk') PURGE """.stripMargin - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) + val sql1_view = sql1_table.replace("TABLE", "VIEW") + // Note: ALTER VIEW DROP PARTITION does not support PURGE + val sql2_view = sql2_table.replace("TABLE", "VIEW").replace("PURGE", "") + + val parsed1_table = parser.parsePlan(sql1_table) + val parsed2_table = parser.parsePlan(sql2_table) + val parsed1_view = parser.parsePlan(sql1_view) + val parsed2_view = parser.parsePlan(sql2_view) + val tableIdent = TableIdentifier("table_name", None) - val expected1 = AlterTableDropPartition( + val expected1_table = AlterTableDropPartition( tableIdent, Seq( Map("dt" -> "2008-08-08", "country" -> "us"), Map("dt" -> "2009-09-09", "country" -> "uk")), ifExists = true, - purge = false)(sql1) - val expected2 = AlterTableDropPartition( + purge = false)(sql1_table) + val expected2_table = AlterTableDropPartition( tableIdent, Seq( Map("dt" -> "2008-08-08", "country" -> "us"), Map("dt" -> "2009-09-09", "country" -> "uk")), ifExists = false, - purge = true)(sql2) - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) + purge = true)(sql2_table) + + val expected1_view = AlterViewDropPartition( + tableIdent, + Seq( + Map("dt" -> "2008-08-08", "country" -> "us"), + Map("dt" -> "2009-09-09", "country" -> "uk")), + ifExists = true)(sql1_view) + val expected2_view = AlterViewDropPartition( + tableIdent, + Seq( + Map("dt" -> "2008-08-08", "country" -> "us"), + Map("dt" -> "2009-09-09", "country" -> "uk")), + ifExists = false)(sql2_table) + + comparePlans(parsed1_table, expected1_table) + comparePlans(parsed2_table, expected2_table) + comparePlans(parsed1_view, expected1_view) + comparePlans(parsed2_view, expected2_view) } test("alter table: archive partition") { diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index 6586b90377eb7..1ba2b4cbb7552 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -90,11 +90,6 @@ private[hive] class HiveQl(conf: ParserConf) extends SparkQl(conf) with Logging "TOK_ALTERINDEX_REBUILD", "TOK_ALTERTABLE_ALTERPARTS", "TOK_ALTERTABLE_PARTITION", - "TOK_ALTERVIEW_ADDPARTS", - "TOK_ALTERVIEW_AS", - "TOK_ALTERVIEW_DROPPARTS", - "TOK_ALTERVIEW_PROPERTIES", - "TOK_ALTERVIEW_RENAME", "TOK_CREATEINDEX", "TOK_CREATEMACRO", @@ -257,21 +252,16 @@ private[hive] class HiveQl(conf: ParserConf) extends SparkQl(conf) with Logging AnalyzeTable(tableName) } - case view @ Token("TOK_ALTERVIEW", children) => - val Some(nameParts) :: maybeQuery :: _ = - getClauses(Seq( - "TOK_TABNAME", - "TOK_QUERY", - "TOK_ALTERVIEW_ADDPARTS", - "TOK_ALTERVIEW_DROPPARTS", - "TOK_ALTERVIEW_PROPERTIES", - "TOK_ALTERVIEW_RENAME"), children) - - // if ALTER VIEW doesn't have query part, let hive to handle it. - maybeQuery.map { query => - createView(view, nameParts, query, Nil, Map(), allowExist = false, replace = true) - }.getOrElse(NativePlaceholder) + // ALTER VIEW [db_name.]view_name AS select_statement + case view @ Token("TOK_ALTERVIEW", children) + if children.collect { case t @ Token("TOK_QUERY", _) => t }.nonEmpty => + val Seq(Some(viewNameParts), Some(query)) = + getClauses(Seq("TOK_TABNAME", "TOK_QUERY"), children) + createView(view, viewNameParts, query, Nil, Map(), allowExist = false, replace = true) + // CREATE VIEW [IF NOT EXISTS] [db_name.]view_name + // [(column_name [COMMENT column_comment], ...) ] [COMMENT view_comment] + // [TBLPROPERTIES (property_name = property_value, ...)] AS select_statement case view @ Token("TOK_CREATEVIEW", children) if children.collect { case t @ Token("TOK_QUERY", _) => t }.nonEmpty => val Seq( From 3b7b41b1c4b33e4ec721131f5578ddbfc0b84035 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Sun, 27 Mar 2016 00:21:24 -0700 Subject: [PATCH 05/13] added comments and fixed bugs. --- .../AlterTableAlterViewCommandParser.scala | 9 ++- .../spark/sql/execution/command/ddl.scala | 25 +++++++- .../execution/command/DDLCommandSuite.scala | 62 ++++++++++++------- 3 files changed, 69 insertions(+), 27 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableAlterViewCommandParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableAlterViewCommandParser.scala index 632ab357b478e..a8be8be6b89da 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableAlterViewCommandParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableAlterViewCommandParser.scala @@ -302,7 +302,8 @@ object AlterTableAlterViewCommandParser { }.toMap AlterTableSkewedLocation(tableIdent, skewedMaps)(node.source) - // ALTER TABLE view_name ADD [IF NOT EXISTS] PARTITION spec1[, PARTITION spec2, ...] + // ALTER TABLE table_name ADD [IF NOT EXISTS] PARTITION spec1 + // [LOCATION 'location1'] PARTITION spec2 [LOCATION 'location2'] ... // ALTER VIEW view_name ADD [IF NOT EXISTS] PARTITION spec1[, PARTITION spec2, ...] case Token(fieldName, args) :: _ if fieldName == "TOK_ALTERTABLE_ADDPARTS" || fieldName == "TOK_ALTERVIEW_ADDPARTS" => @@ -318,7 +319,11 @@ object AlterTableAlterViewCommandParser { parsedParts += ((parsePartitionSpec(t), None)) case Token("TOK_PARTITIONLOCATION", loc :: Nil) => // Update the location of the last partition we just added - if (parsedParts.nonEmpty) { + if (fieldName == "TOK_ALTERVIEW_ADDPARTS") { + // Location clause is not allowed in ALTER VIEW + parseFailed("Invalid ALTER VIEW command", node) + } + else if (parsedParts.nonEmpty) { val (spec, _) = parsedParts.remove(parsedParts.length - 1) parsedParts += ((spec, Some(unquoteString(loc.text)))) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 486be8f77f231..f5806117001d3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -19,8 +19,7 @@ package org.apache.spark.sql.execution.command import org.apache.spark.internal.Logging import org.apache.spark.sql.{Row, SQLContext} -import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} -import org.apache.spark.sql.catalyst.catalog.CatalogFunction +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.ExternalCatalog.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} import org.apache.spark.sql.execution.datasources.BucketSpec @@ -58,7 +57,7 @@ case class CreateDatabase( * * 'ifExists': * - true, if database_name does't exist, no action - * - false (default), if database_name does't exist, a warning message will be issued + * - false (default), if database_name does't exist, an error message will be issued * 'restric': * - true (default), the database cannot be dropped if it is not empty. The inclusive * tables must be dropped at first. @@ -106,16 +105,19 @@ case class DropFunction( isTemp: Boolean)(sql: String) extends NativeDDLCommand(sql) with Logging +/** Rename in ALTER TABLE/VIEW: change the name of a table/view to a different name. */ case class AlterTableAlterViewRename( oldName: TableIdentifier, newName: TableIdentifier)(sql: String) extends NativeDDLCommand(sql) with Logging +/** Set Properties in ALTER TABLE/VIEW: add metadata to a table/view. */ case class AlterTableAlterViewSetProperties( tableName: TableIdentifier, properties: Map[String, String])(sql: String) extends NativeDDLCommand(sql) with Logging +/** Unset Properties in ALTER TABLE/VIEW: remove metadata from a table/view. */ case class AlterTableAlterViewUnsetProperties( tableName: TableIdentifier, properties: Map[String, String], @@ -164,6 +166,12 @@ case class AlterTableSkewedLocation( skewedMap: Map[String, String])(sql: String) extends NativeDDLCommand(sql) with Logging +/** + * Add Partition in ALTER TABLE/VIEW: add the table/view partitions. + * 'partitionSpecsAndLocs': the syntax of ALTER VIEW is identical to ALTER TABLE, + * EXCEPT that it is ILLEGAL to specify a LOCATION clause. + * An error message will be issued if the partition exists, unless 'ifNotExists' is false. + */ case class AlterTableAlterViewAddPartition( tableName: TableIdentifier, partitionSpecsAndLocs: Seq[(TablePartitionSpec, Option[String])], @@ -182,6 +190,13 @@ case class AlterTableExchangePartition( spec: TablePartitionSpec)(sql: String) extends NativeDDLCommand(sql) with Logging +/** + * Drop Partition in ALTER TABLE: to drop a particular partition for a table. + * This removes the data and metadata for this partition. + * The data is actually moved to the .Trash/Current directory if Trash is configured, + * unless 'purge' is true, but the metadata is completely lost. + * An error message will be issued if the partition does not exist, unless 'ifExists' is false. + */ case class AlterTableDropPartition( tableName: TableIdentifier, specs: Seq[TablePartitionSpec], @@ -189,6 +204,10 @@ case class AlterTableDropPartition( purge: Boolean)(sql: String) extends NativeDDLCommand(sql) with Logging +/** + * Drop Partition in ALTER VIEW: drop the related partition metadata for a view. + * An error message will be issued if the partition does not exist, unless 'ifExists' is false. + */ case class AlterViewDropPartition( viewName: TableIdentifier, specs: Seq[TablePartitionSpec], diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala index 8d951088fb4ef..a01c2ca3dba81 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.execution.command import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.expressions.{Ascending, Descending} import org.apache.spark.sql.catalyst.plans.PlanTest import org.apache.spark.sql.execution.SparkQl import org.apache.spark.sql.execution.datasources.BucketSpec @@ -417,41 +416,60 @@ class DDLCommandSuite extends PlanTest { } // ALTER TABLE view_name ADD [IF NOT EXISTS] PARTITION spec1[, PARTITION spec2, ...] - // ALTER VIEW view_name ADD [IF NOT EXISTS] PARTITION spec1[, PARTITION spec2, ...] - test("alter table/view: add partition") { - val sql1_table = + test("alter table: add partition") { + val sql1 = """ |ALTER TABLE table_name ADD IF NOT EXISTS PARTITION |(dt='2008-08-08', country='us') LOCATION 'location1' PARTITION |(dt='2009-09-09', country='uk') """.stripMargin - val sql1_view = sql1_table.replace("TABLE", "VIEW") + val sql2 = "ALTER TABLE table_name ADD PARTITION (dt='2008-08-08') LOCATION 'loc'" - val sql2_table = "ALTER TABLE table_name ADD PARTITION (dt='2008-08-08') LOCATION 'loc'" - val sql2_view = sql2_table.replace("TABLE", "VIEW") - - val parsed1_table = parser.parsePlan(sql1_table) - val parsed1_view = parser.parsePlan(sql1_view) - val parsed2_table = parser.parsePlan(sql2_table) - val parsed2_view = parser.parsePlan(sql2_view) + val parsed1 = parser.parsePlan(sql1) + val parsed2 = parser.parsePlan(sql2) - val expected1_table = AlterTableAlterViewAddPartition( + val expected1 = AlterTableAlterViewAddPartition( TableIdentifier("table_name", None), Seq( (Map("dt" -> "2008-08-08", "country" -> "us"), Some("location1")), (Map("dt" -> "2009-09-09", "country" -> "uk"), None)), - ifNotExists = true)(sql1_table) - val expected1_view = expected1_table.copy()(sql = sql1_view) - val expected2_table = AlterTableAlterViewAddPartition( + ifNotExists = true)(sql1) + val expected2 = AlterTableAlterViewAddPartition( TableIdentifier("table_name", None), Seq((Map("dt" -> "2008-08-08"), Some("loc"))), - ifNotExists = false)(sql2_table) - val expected2_view = expected2_table.copy()(sql = sql2_view) + ifNotExists = false)(sql2) - comparePlans(parsed1_table, expected1_table) - comparePlans(parsed2_table, expected2_table) - comparePlans(parsed1_view, expected1_view) - comparePlans(parsed2_view, expected2_view) + comparePlans(parsed1, expected1) + comparePlans(parsed2, expected2) + } + + // ALTER VIEW view_name ADD [IF NOT EXISTS] PARTITION spec1[, PARTITION spec2, ...] + // Location clause is not allowed. + test("alter view: add partition") { + val sql1 = + """ + |ALTER VIEW view_name ADD IF NOT EXISTS PARTITION + |(dt='2008-08-08', country='us') PARTITION + |(dt='2009-09-09', country='uk') + """.stripMargin + val sql2 = "ALTER VIEW view_name ADD PARTITION (dt='2008-08-08')" + + val parsed1 = parser.parsePlan(sql1) + val parsed2 = parser.parsePlan(sql2) + + val expected1 = AlterTableAlterViewAddPartition( + TableIdentifier("view_name", None), + Seq( + (Map("dt" -> "2008-08-08", "country" -> "us"), None), + (Map("dt" -> "2009-09-09", "country" -> "uk"), None)), + ifNotExists = true)(sql1) + val expected2 = AlterTableAlterViewAddPartition( + TableIdentifier("view_name", None), + Seq((Map("dt" -> "2008-08-08"), None)), + ifNotExists = false)(sql2) + + comparePlans(parsed1, expected1) + comparePlans(parsed2, expected2) } test("alter table: rename partition") { From 7322fb1a5062e841538fea864116c961c1daf3ac Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Sun, 27 Mar 2016 00:32:06 -0700 Subject: [PATCH 06/13] update comments. --- .../execution/command/AlterTableAlterViewCommandParser.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableAlterViewCommandParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableAlterViewCommandParser.scala index a8be8be6b89da..40f8f67116e8c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableAlterViewCommandParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableAlterViewCommandParser.scala @@ -347,7 +347,7 @@ object AlterTableAlterViewCommandParser { AlterTableExchangePartition(tableIdent, newTableIdent, parsedSpec)(node.source) // ALTER TABLE table_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...] [PURGE]; - // ALTER VIEW table_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...]; + // ALTER VIEW view_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...]; case Token(fieldName, args) :: _ if fieldName == "TOK_ALTERTABLE_DROPPARTS" || fieldName == "TOK_ALTERVIEW_DROPPARTS" => val parts = args.collect { case p @ Token("TOK_PARTSPEC", _) => parsePartitionSpec(p) } From 9136b581559cb14b3dbc5f7b0c7fe01755aed487 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Wed, 30 Mar 2016 08:01:01 -0700 Subject: [PATCH 07/13] address comments. --- .../spark/sql/catalyst/parser/ng/SqlBase.g4 | 18 ++-- .../apache/spark/sql/execution/SparkQl.scala | 5 +- .../spark/sql/execution/SparkSqlParser.scala | 88 +++++++++++++++++++ .../command/AlterTableCommandParser.scala | 12 +-- .../execution/command/DDLCommandSuite.scala | 18 ++-- 5 files changed, 114 insertions(+), 27 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4 index 4e77b6db25438..34ec3269d18f5 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4 @@ -102,6 +102,15 @@ statement (PARTITIONED ON identifierList)? (TBLPROPERTIES tablePropertyList)? AS query #createView | ALTER VIEW tableIdentifier AS? query #alterViewQuery + | ALTER VIEW from=tableIdentifier AS? RENAME TO to=tableIdentifier #renameView + | ALTER VIEW from=tableIdentifier AS? + SET TBLPROPERTIES tablePropertyList #setViewProperties + | ALTER VIEW from=tableIdentifier AS? + UNSET TBLPROPERTIES (IF EXISTS)? tablePropertyList #unsetViewProperties + | ALTER VIEW from=tableIdentifier AS? + ADD (IF NOT EXISTS)? partitionSpec+ #addViewPartition + | ALTER VIEW from=tableIdentifier AS? + DROP (IF EXISTS)? partitionSpec (',' partitionSpec)* #dropViewPartitions | CREATE TEMPORARY? FUNCTION qualifiedName AS className=STRING (USING resource (',' resource)*)? #createFunction | DROP TEMPORARY? FUNCTION (IF EXISTS)? qualifiedName #dropFunction @@ -131,15 +140,6 @@ hiveNativeCommands | DELETE FROM tableIdentifier (WHERE booleanExpression)? | TRUNCATE TABLE tableIdentifier partitionSpec? (COLUMNS identifierList)? - | ALTER VIEW from=tableIdentifier AS? RENAME TO to=tableIdentifier - | ALTER VIEW from=tableIdentifier AS? - SET TBLPROPERTIES tablePropertyList - | ALTER VIEW from=tableIdentifier AS? - UNSET TBLPROPERTIES (IF EXISTS)? tablePropertyList - | ALTER VIEW from=tableIdentifier AS? - ADD (IF NOT EXISTS)? partitionSpecLocation+ - | ALTER VIEW from=tableIdentifier AS? - DROP (IF EXISTS)? partitionSpec (',' partitionSpec)* PURGE? | DROP VIEW (IF EXISTS)? qualifiedName | SHOW COLUMNS (FROM | IN) tableIdentifier ((FROM|IN) identifier)? | START TRANSACTION (transactionMode (',' transactionMode)*)? diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala index 9ad2c78d66644..af463ac632c96 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala @@ -250,9 +250,8 @@ private[sql] class SparkQl(conf: ParserConf = SimpleParserConf()) extends Cataly DropFunction(dbName, funcName, ifExists.isDefined, temp.isDefined)(node.source) - case Token(fieldName, alterTableArgs) - if fieldName == "TOK_ALTERTABLE" || fieldName == "TOK_ALTERVIEW" => - AlterTableAlterViewCommandParser.parse(node) + case Token("TOK_ALTERTABLE" | "TOK_ALTERVIEW", alterTableArgs) => + AlterTableCommandParser.parse(node) case Token("TOK_CREATETABLEUSING", createTableArgs) => val Seq( diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 8333074ecaf22..c16397f26cd9a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -344,6 +344,21 @@ class SparkSqlAstBuilder extends AstBuilder { command(ctx)) } + /** + * Create a [[AlterTableRename]] command. + * + * For example: + * {{{ + * ALTER VIEW view1 RENAME TO view2; + * }}} + */ + override def visitRenameView(ctx: RenameViewContext): LogicalPlan = withOrigin(ctx) { + AlterTableRename( + visitTableIdentifier(ctx.from), + visitTableIdentifier(ctx.to))( + command(ctx)) + } + /** * Create an [[AlterTableSetProperties]] command. * @@ -360,6 +375,22 @@ class SparkSqlAstBuilder extends AstBuilder { command(ctx)) } + /** + * Create an [[AlterTableSetProperties]] command. + * + * For example: + * {{{ + * ALTER VIEW view SET TBLPROPERTIES ('comment' = new_comment); + * }}} + */ + override def visitSetViewProperties( + ctx: SetViewPropertiesContext): LogicalPlan = withOrigin(ctx) { + AlterTableSetProperties( + visitTableIdentifier(ctx.tableIdentifier), + visitTablePropertyList(ctx.tablePropertyList))( + command(ctx)) + } + /** * Create an [[AlterTableUnsetProperties]] command. * @@ -377,6 +408,23 @@ class SparkSqlAstBuilder extends AstBuilder { command(ctx)) } + /** + * Create an [[AlterTableUnsetProperties]] command. + * + * For example: + * {{{ + * ALTER VIEW view UNSET TBLPROPERTIES IF EXISTS ('comment', 'key'); + * }}} + */ + override def visitUnsetViewProperties( + ctx: UnsetViewPropertiesContext): LogicalPlan = withOrigin(ctx) { + AlterTableUnsetProperties( + visitTableIdentifier(ctx.tableIdentifier), + visitTablePropertyList(ctx.tablePropertyList), + ctx.EXISTS != null)( + command(ctx)) + } + /** * Create an [[AlterTableSerDeProperties]] command. * @@ -528,6 +576,28 @@ class SparkSqlAstBuilder extends AstBuilder { command(ctx)) } + /** + * Create an [[AlterTableAddPartition]] command. + * + * For example: + * {{{ + * ALTER VIEW view ADD [IF NOT EXISTS] PARTITION spec + * }}} + */ + override def visitAddViewPartition( + ctx: AddViewPartitionContext): LogicalPlan = withOrigin(ctx) { + // Create partition spec to location mapping. + // Different from ALTER TABLE ADD PARTITION, location is not allowed here. + val specsAndLocs = ctx.partitionSpec.asScala.map { + visitNonOptionalPartitionSpec(_) -> None + } + AlterTableAddPartition( + visitTableIdentifier(ctx.tableIdentifier), + specsAndLocs, + ctx.EXISTS != null)( + command(ctx)) + } + /** * Create an [[AlterTableExchangePartition]] command. * @@ -580,6 +650,24 @@ class SparkSqlAstBuilder extends AstBuilder { command(ctx)) } + /** + * Create an [[AlterViewDropPartition]] command + * + * For example: + * {{{ + * ALTER VIEW view DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...]; + * }}} + */ + override def visitDropViewPartitions( + ctx: DropViewPartitionsContext): LogicalPlan = withOrigin(ctx) { + // Different from ALTER TABLE DROP PARTITION, PURGE is not allowed here. + AlterViewDropPartition( + visitTableIdentifier(ctx.tableIdentifier), + ctx.partitionSpec.asScala.map(visitNonOptionalPartitionSpec), + ctx.EXISTS != null)( + command(ctx)) + } + /** * Create an [[AlterTableArchivePartition]] command * diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala index b66dda2a2fd88..3486d9bb8a67e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala @@ -46,7 +46,7 @@ object AlterTableCommandParser { if (partSpec.isDefined && node.token.getText == "TOK_ALTERVIEW") { parseFailed("Could not parse ALTER VIEW command", node) } - matchAlterTableAlterViewCommands(node, otherNodes, tableIdent, partSpec) + matchAlterTableCommands(node, otherNodes, tableIdent, partSpec) case _ => parseFailed("Could not parse ALTER TABLE/VIEW command", node) } @@ -135,7 +135,7 @@ object AlterTableCommandParser { * @param partition spec identifying the partition this command is concerned with, if any. */ // TODO: This method is massive. Break it down. - private def matchAlterTableAlterViewCommands( + private def matchAlterTableCommands( node: ASTNode, otherNodes: Seq[ASTNode], tableIdent: TableIdentifier, @@ -147,14 +147,14 @@ object AlterTableCommandParser { if fieldName == "TOK_ALTERTABLE_RENAME" || fieldName == "TOK_ALTERVIEW_RENAME" => val tableNameClause = getClause("TOK_TABNAME", renameArgs) val newTableIdent = extractTableIdent(tableNameClause) - AlterTableAlterViewRename(tableIdent, newTableIdent)(node.source) + AlterTableRename(tableIdent, newTableIdent)(node.source) // ALTER TABLE table_name SET TBLPROPERTIES ('comment' = new_comment); // ALTER VIEW view_name SET TBLPROPERTIES ('comment' = new_comment); case Token(fieldName, args) :: _ if fieldName == "TOK_ALTERTABLE_PROPERTIES" || fieldName == "TOK_ALTERVIEW_PROPERTIES" => val properties = extractTableProps(args.head) - AlterTableAlterViewSetProperties(tableIdent, properties)(node.source) + AlterTableSetProperties(tableIdent, properties)(node.source) // ALTER TABLE table_name UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key'); // ALTER VIEW view_name UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key'); @@ -163,7 +163,7 @@ object AlterTableCommandParser { fieldName == "TOK_ALTERVIEW_DROPPROPERTIES" => val properties = extractTableProps(args.head) val ifExists = getClauseOption("TOK_IFEXISTS", args).isDefined - AlterTableAlterViewUnsetProperties(tableIdent, properties, ifExists)(node.source) + AlterTableUnsetProperties(tableIdent, properties, ifExists)(node.source) // ALTER TABLE table_name [PARTITION spec] SET SERDE serde_name [WITH SERDEPROPERTIES props]; case Token("TOK_ALTERTABLE_SERIALIZER", Token(serdeClassName, Nil) :: serdeArgs) :: _ => @@ -330,7 +330,7 @@ object AlterTableCommandParser { case _ => parseFailed("Invalid ALTER TABLE/VIEW command", node) } - AlterTableAlterViewAddPartition(tableIdent, parsedParts, ifNotExists)(node.source) + AlterTableAddPartition(tableIdent, parsedParts, ifNotExists)(node.source) // ALTER TABLE table_name PARTITION spec1 RENAME TO PARTITION spec2; case Token("TOK_ALTERTABLE_RENAMEPART", spec :: Nil) :: _ => diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala index 8ebef6af7e2ec..e9621e5520456 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala @@ -202,10 +202,10 @@ class DDLCommandSuite extends PlanTest { val sql_view = sql_table.replace("TABLE", "VIEW") val parsed_table = parser.parsePlan(sql_table) val parsed_view = parser.parsePlan(sql_view) - val expected_table = AlterTableAlterViewRename( + val expected_table = AlterTableRename( TableIdentifier("table_name", None), TableIdentifier("new_table_name", None))(sql_table) - val expected_view = AlterTableAlterViewRename( + val expected_view = AlterTableRename( TableIdentifier("table_name", None), TableIdentifier("new_table_name", None))(sql_view) comparePlans(parsed_table, expected_table) @@ -233,11 +233,11 @@ class DDLCommandSuite extends PlanTest { val parsed3_view = parser.parsePlan(sql3_view) val tableIdent = TableIdentifier("table_name", None) - val expected1_table = AlterTableAlterViewSetProperties( + val expected1_table = AlterTableSetProperties( tableIdent, Map("test" -> "test", "comment" -> "new_comment"))(sql1_table) - val expected2_table = AlterTableAlterViewUnsetProperties( + val expected2_table = AlterTableUnsetProperties( tableIdent, Map("comment" -> null, "test" -> null), ifExists = false)(sql2_table) - val expected3_table = AlterTableAlterViewUnsetProperties( + val expected3_table = AlterTableUnsetProperties( tableIdent, Map("comment" -> null, "test" -> null), ifExists = true)(sql3_table) val expected1_view = expected1_table.copy()(sql = sql1_view) val expected2_view = expected2_table.copy()(sql = sql2_view) @@ -416,13 +416,13 @@ class DDLCommandSuite extends PlanTest { val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) - val expected1 = AlterTableAlterViewAddPartition( + val expected1 = AlterTableAddPartition( TableIdentifier("table_name", None), Seq( (Map("dt" -> "2008-08-08", "country" -> "us"), Some("location1")), (Map("dt" -> "2009-09-09", "country" -> "uk"), None)), ifNotExists = true)(sql1) - val expected2 = AlterTableAlterViewAddPartition( + val expected2 = AlterTableAddPartition( TableIdentifier("table_name", None), Seq((Map("dt" -> "2008-08-08"), Some("loc"))), ifNotExists = false)(sql2) @@ -445,13 +445,13 @@ class DDLCommandSuite extends PlanTest { val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) - val expected1 = AlterTableAlterViewAddPartition( + val expected1 = AlterTableAddPartition( TableIdentifier("view_name", None), Seq( (Map("dt" -> "2008-08-08", "country" -> "us"), None), (Map("dt" -> "2009-09-09", "country" -> "uk"), None)), ifNotExists = true)(sql1) - val expected2 = AlterTableAlterViewAddPartition( + val expected2 = AlterTableAddPartition( TableIdentifier("view_name", None), Seq((Map("dt" -> "2008-08-08"), None)), ifNotExists = false)(sql2) From f379636ed28e5fe78af44dfdb4d3821dc6c964af Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Wed, 30 Mar 2016 08:31:38 -0700 Subject: [PATCH 08/13] add test cases. --- .../spark/sql/execution/command/DDLCommandSuite.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala index e9621e5520456..9fa48579fba2a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala @@ -440,7 +440,12 @@ class DDLCommandSuite extends PlanTest { |(dt='2008-08-08', country='us') PARTITION |(dt='2009-09-09', country='uk') """.stripMargin - val sql2 = "ALTER VIEW view_name ADD PARTITION (dt='2008-08-08')" + // different constant types in partitioning spec + val sql2 = + """ + |ALTER VIEW view_name ADD PARTITION + |(col1=NULL, cOL2='f', col3=5, COL4=true) + """.stripMargin val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) @@ -453,7 +458,7 @@ class DDLCommandSuite extends PlanTest { ifNotExists = true)(sql1) val expected2 = AlterTableAddPartition( TableIdentifier("view_name", None), - Seq((Map("dt" -> "2008-08-08"), None)), + Seq((Map("col1" -> "NULL", "col2" -> "f", "col3" -> "5", "col4" -> "true"), None)), ifNotExists = false)(sql2) comparePlans(parsed1, expected1) From 38ea348cab7ffcf9a107bb56fb905bab7a6f2d00 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Wed, 30 Mar 2016 21:46:31 -0700 Subject: [PATCH 09/13] address comments. --- .../spark/sql/catalyst/parser/ng/SqlBase.g4 | 20 +++--- .../spark/sql/execution/SparkSqlParser.scala | 70 ++----------------- .../command/AlterTableCommandParser.scala | 60 +++++----------- .../spark/sql/execution/command/ddl.scala | 13 +--- .../execution/command/DDLCommandSuite.scala | 16 +++-- .../org/apache/spark/sql/hive/HiveQl.scala | 28 +++++--- 6 files changed, 60 insertions(+), 147 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4 index 34ec3269d18f5..ab9f410a1f644 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4 @@ -55,10 +55,11 @@ statement (AS? query)? #createTable | ANALYZE TABLE tableIdentifier partitionSpec? COMPUTE STATISTICS (identifier | FOR COLUMNS identifierSeq?) #analyze - | ALTER TABLE from=tableIdentifier RENAME TO to=tableIdentifier #renameTable - | ALTER TABLE tableIdentifier + | ALTER (TABLE | VIEW) from=tableIdentifier + RENAME TO to=tableIdentifier #renameTable + | ALTER (TABLE | VIEW) tableIdentifier SET TBLPROPERTIES tablePropertyList #setTableProperties - | ALTER TABLE tableIdentifier + | ALTER (TABLE | VIEW) tableIdentifier UNSET TBLPROPERTIES (IF EXISTS)? tablePropertyList #unsetTableProperties | ALTER TABLE tableIdentifier (partitionSpec)? SET SERDE STRING (WITH SERDEPROPERTIES tablePropertyList)? #setTableSerDe @@ -74,12 +75,16 @@ statement SET SKEWED LOCATION skewedLocationList #setTableSkewLocations | ALTER TABLE tableIdentifier ADD (IF NOT EXISTS)? partitionSpecLocation+ #addTablePartition + | ALTER VIEW tableIdentifier ADD (IF NOT EXISTS)? + partitionSpec+ #addViewPartition | ALTER TABLE tableIdentifier from=partitionSpec RENAME TO to=partitionSpec #renameTablePartition | ALTER TABLE from=tableIdentifier EXCHANGE partitionSpec WITH TABLE to=tableIdentifier #exchangeTablePartition | ALTER TABLE tableIdentifier DROP (IF EXISTS)? partitionSpec (',' partitionSpec)* PURGE? #dropTablePartitions + | ALTER VIEW tableIdentifier + DROP (IF EXISTS)? partitionSpec (',' partitionSpec)* #dropTablePartitions | ALTER TABLE tableIdentifier ARCHIVE partitionSpec #archiveTablePartition | ALTER TABLE tableIdentifier UNARCHIVE partitionSpec #unarchiveTablePartition | ALTER TABLE tableIdentifier partitionSpec? @@ -102,15 +107,6 @@ statement (PARTITIONED ON identifierList)? (TBLPROPERTIES tablePropertyList)? AS query #createView | ALTER VIEW tableIdentifier AS? query #alterViewQuery - | ALTER VIEW from=tableIdentifier AS? RENAME TO to=tableIdentifier #renameView - | ALTER VIEW from=tableIdentifier AS? - SET TBLPROPERTIES tablePropertyList #setViewProperties - | ALTER VIEW from=tableIdentifier AS? - UNSET TBLPROPERTIES (IF EXISTS)? tablePropertyList #unsetViewProperties - | ALTER VIEW from=tableIdentifier AS? - ADD (IF NOT EXISTS)? partitionSpec+ #addViewPartition - | ALTER VIEW from=tableIdentifier AS? - DROP (IF EXISTS)? partitionSpec (',' partitionSpec)* #dropViewPartitions | CREATE TEMPORARY? FUNCTION qualifiedName AS className=STRING (USING resource (',' resource)*)? #createFunction | DROP TEMPORARY? FUNCTION (IF EXISTS)? qualifiedName #dropFunction diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index c16397f26cd9a..d28564fa44623 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -335,24 +335,10 @@ class SparkSqlAstBuilder extends AstBuilder { * For example: * {{{ * ALTER TABLE table1 RENAME TO table2; - * }}} - */ - override def visitRenameTable(ctx: RenameTableContext): LogicalPlan = withOrigin(ctx) { - AlterTableRename( - visitTableIdentifier(ctx.from), - visitTableIdentifier(ctx.to))( - command(ctx)) - } - - /** - * Create a [[AlterTableRename]] command. - * - * For example: - * {{{ * ALTER VIEW view1 RENAME TO view2; * }}} */ - override def visitRenameView(ctx: RenameViewContext): LogicalPlan = withOrigin(ctx) { + override def visitRenameTable(ctx: RenameTableContext): LogicalPlan = withOrigin(ctx) { AlterTableRename( visitTableIdentifier(ctx.from), visitTableIdentifier(ctx.to))( @@ -365,6 +351,7 @@ class SparkSqlAstBuilder extends AstBuilder { * For example: * {{{ * ALTER TABLE table SET TBLPROPERTIES ('comment' = new_comment); + * ALTER VIEW view SET TBLPROPERTIES ('comment' = new_comment); * }}} */ override def visitSetTableProperties( @@ -375,28 +362,13 @@ class SparkSqlAstBuilder extends AstBuilder { command(ctx)) } - /** - * Create an [[AlterTableSetProperties]] command. - * - * For example: - * {{{ - * ALTER VIEW view SET TBLPROPERTIES ('comment' = new_comment); - * }}} - */ - override def visitSetViewProperties( - ctx: SetViewPropertiesContext): LogicalPlan = withOrigin(ctx) { - AlterTableSetProperties( - visitTableIdentifier(ctx.tableIdentifier), - visitTablePropertyList(ctx.tablePropertyList))( - command(ctx)) - } - /** * Create an [[AlterTableUnsetProperties]] command. * * For example: * {{{ * ALTER TABLE table UNSET TBLPROPERTIES IF EXISTS ('comment', 'key'); + * ALTER VIEW view UNSET TBLPROPERTIES IF EXISTS ('comment', 'key'); * }}} */ override def visitUnsetTableProperties( @@ -408,23 +380,6 @@ class SparkSqlAstBuilder extends AstBuilder { command(ctx)) } - /** - * Create an [[AlterTableUnsetProperties]] command. - * - * For example: - * {{{ - * ALTER VIEW view UNSET TBLPROPERTIES IF EXISTS ('comment', 'key'); - * }}} - */ - override def visitUnsetViewProperties( - ctx: UnsetViewPropertiesContext): LogicalPlan = withOrigin(ctx) { - AlterTableUnsetProperties( - visitTableIdentifier(ctx.tableIdentifier), - visitTablePropertyList(ctx.tablePropertyList), - ctx.EXISTS != null)( - command(ctx)) - } - /** * Create an [[AlterTableSerDeProperties]] command. * @@ -638,6 +593,7 @@ class SparkSqlAstBuilder extends AstBuilder { * For example: * {{{ * ALTER TABLE table DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...] [PURGE]; + * ALTER VIEW view DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...]; * }}} */ override def visitDropTablePartitions( @@ -650,24 +606,6 @@ class SparkSqlAstBuilder extends AstBuilder { command(ctx)) } - /** - * Create an [[AlterViewDropPartition]] command - * - * For example: - * {{{ - * ALTER VIEW view DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...]; - * }}} - */ - override def visitDropViewPartitions( - ctx: DropViewPartitionsContext): LogicalPlan = withOrigin(ctx) { - // Different from ALTER TABLE DROP PARTITION, PURGE is not allowed here. - AlterViewDropPartition( - visitTableIdentifier(ctx.tableIdentifier), - ctx.partitionSpec.asScala.map(visitNonOptionalPartitionSpec), - ctx.EXISTS != null)( - command(ctx)) - } - /** * Create an [[AlterTableArchivePartition]] command * diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala index 3486d9bb8a67e..9fbe6db467ffa 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala @@ -29,26 +29,22 @@ import org.apache.spark.sql.types.StructType /** - * Helper object to parse alter table or alter view commands. + * Helper object to parse alter table commands. */ object AlterTableCommandParser { import ParserUtils._ /** - * Parse the given node assuming it is an alter table/view command. + * Parse the given node assuming it is an alter table command. */ def parse(node: ASTNode): LogicalPlan = { - assert(node.token.getText == "TOK_ALTERTABLE" || node.token.getText == "TOK_ALTERVIEW") node.children match { case (tabName @ Token("TOK_TABNAME", _)) :: otherNodes => val tableIdent = extractTableIdent(tabName) val partSpec = getClauseOption("TOK_PARTSPEC", node.children).map(parsePartitionSpec) - if (partSpec.isDefined && node.token.getText == "TOK_ALTERVIEW") { - parseFailed("Could not parse ALTER VIEW command", node) - } matchAlterTableCommands(node, otherNodes, tableIdent, partSpec) case _ => - parseFailed("Could not parse ALTER TABLE/VIEW command", node) + parseFailed("Could not parse ALTER TABLE command", node) } } @@ -125,9 +121,8 @@ object AlterTableCommandParser { } /** - * Parse an alter table/view command from a [[ASTNode]] into a [[LogicalPlan]]. - * This follows https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL - * and https://cwiki.apache.org/confluence/display/Hive/PartitionedViews. + * Parse an alter table command from a [[ASTNode]] into a [[LogicalPlan]]. + * This follows https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL. * * @param node the original [[ASTNode]] to parse. * @param otherNodes the other [[ASTNode]]s after the first one containing the table name. @@ -142,25 +137,18 @@ object AlterTableCommandParser { partition: Option[TablePartitionSpec]): LogicalPlan = { otherNodes match { // ALTER TABLE table_name RENAME TO new_table_name; - // ALTER VIEW view_name RENAME TO new_view_name; - case Token(fieldName, renameArgs) :: _ - if fieldName == "TOK_ALTERTABLE_RENAME" || fieldName == "TOK_ALTERVIEW_RENAME" => + case Token("TOK_ALTERTABLE_RENAME", renameArgs) :: _ => val tableNameClause = getClause("TOK_TABNAME", renameArgs) val newTableIdent = extractTableIdent(tableNameClause) AlterTableRename(tableIdent, newTableIdent)(node.source) // ALTER TABLE table_name SET TBLPROPERTIES ('comment' = new_comment); - // ALTER VIEW view_name SET TBLPROPERTIES ('comment' = new_comment); - case Token(fieldName, args) :: _ - if fieldName == "TOK_ALTERTABLE_PROPERTIES" || fieldName == "TOK_ALTERVIEW_PROPERTIES" => + case Token("TOK_ALTERTABLE_PROPERTIES", args) :: _ => val properties = extractTableProps(args.head) AlterTableSetProperties(tableIdent, properties)(node.source) - // ALTER TABLE table_name UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key'); - // ALTER VIEW view_name UNSET TBLPROPERTIES [IF EXISTS] ('comment', 'key'); - case Token(fieldName, args) :: _ - if fieldName == "TOK_ALTERTABLE_DROPPROPERTIES" || - fieldName == "TOK_ALTERVIEW_DROPPROPERTIES" => + // ALTER TABLE table_name UNSET TBLPROPERTIES IF EXISTS ('comment', 'key'); + case Token("TOK_ALTERTABLE_DROPPROPERTIES", args) :: _ => val properties = extractTableProps(args.head) val ifExists = getClauseOption("TOK_IFEXISTS", args).isDefined AlterTableUnsetProperties(tableIdent, properties, ifExists)(node.source) @@ -302,11 +290,9 @@ object AlterTableCommandParser { }.toMap AlterTableSkewedLocation(tableIdent, skewedMaps)(node.source) - // ALTER TABLE table_name ADD [IF NOT EXISTS] PARTITION spec1 - // [LOCATION 'location1'] PARTITION spec2 [LOCATION 'location2'] ... - // ALTER VIEW view_name ADD [IF NOT EXISTS] PARTITION spec1[, PARTITION spec2, ...] - case Token(fieldName, args) :: _ - if fieldName == "TOK_ALTERTABLE_ADDPARTS" || fieldName == "TOK_ALTERVIEW_ADDPARTS" => + // ALTER TABLE table_name ADD [IF NOT EXISTS] PARTITION spec [LOCATION 'loc1'] + // spec [LOCATION 'loc2'] ...; + case Token("TOK_ALTERTABLE_ADDPARTS", args) :: _ => val (ifNotExists, parts) = args.head match { case Token("TOK_IFNOTEXISTS", Nil) => (true, args.tail) case _ => (false, args) @@ -319,16 +305,12 @@ object AlterTableCommandParser { parsedParts += ((parsePartitionSpec(t), None)) case Token("TOK_PARTITIONLOCATION", loc :: Nil) => // Update the location of the last partition we just added - if (fieldName == "TOK_ALTERVIEW_ADDPARTS") { - // Location clause is not allowed in ALTER VIEW - parseFailed("Invalid ALTER VIEW command", node) - } - else if (parsedParts.nonEmpty) { + if (parsedParts.nonEmpty) { val (spec, _) = parsedParts.remove(parsedParts.length - 1) parsedParts += ((spec, Some(unquoteString(loc.text)))) } case _ => - parseFailed("Invalid ALTER TABLE/VIEW command", node) + parseFailed("Invalid ALTER TABLE command", node) } AlterTableAddPartition(tableIdent, parsedParts, ifNotExists)(node.source) @@ -347,17 +329,11 @@ object AlterTableCommandParser { AlterTableExchangePartition(tableIdent, newTableIdent, parsedSpec)(node.source) // ALTER TABLE table_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...] [PURGE]; - // ALTER VIEW view_name DROP [IF EXISTS] PARTITION spec1[, PARTITION spec2, ...]; - case Token(fieldName, args) :: _ - if fieldName == "TOK_ALTERTABLE_DROPPARTS" || fieldName == "TOK_ALTERVIEW_DROPPARTS" => + case Token("TOK_ALTERTABLE_DROPPARTS", args) :: _ => val parts = args.collect { case p @ Token("TOK_PARTSPEC", _) => parsePartitionSpec(p) } val ifExists = getClauseOption("TOK_IFEXISTS", args).isDefined - if (fieldName == "TOK_ALTERTABLE_DROPPARTS") { - val purge = getClauseOption("PURGE", args).isDefined - AlterTableDropPartition(tableIdent, parts, ifExists, purge)(node.source) - } else { - AlterViewDropPartition(tableIdent, parts, ifExists)(node.source) - } + val purge = getClauseOption("PURGE", args).isDefined + AlterTableDropPartition(tableIdent, parts, ifExists, purge)(node.source) // ALTER TABLE table_name ARCHIVE PARTITION spec; case Token("TOK_ALTERTABLE_ARCHIVE", spec :: Nil) :: _ => @@ -448,7 +424,7 @@ object AlterTableCommandParser { AlterTableReplaceCol(tableIdent, partition, columns, restrict, cascade)(node.source) case _ => - parseFailed("Unsupported ALTER TABLE/VIEW command", node) + parseFailed("Unsupported ALTER TABLE command", node) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 428ff32ebd71d..552f1a6c8d1dd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -281,11 +281,12 @@ case class AlterTableExchangePartition( extends NativeDDLCommand(sql) with Logging /** - * Drop Partition in ALTER TABLE: to drop a particular partition for a table. + * Drop Partition in ALTER TABLE/VIEW: to drop a particular partition for a table/view. * This removes the data and metadata for this partition. * The data is actually moved to the .Trash/Current directory if Trash is configured, * unless 'purge' is true, but the metadata is completely lost. * An error message will be issued if the partition does not exist, unless 'ifExists' is false. + * Note: purge is always false when the target is a view. */ case class AlterTableDropPartition( tableName: TableIdentifier, @@ -294,16 +295,6 @@ case class AlterTableDropPartition( purge: Boolean)(sql: String) extends NativeDDLCommand(sql) with Logging -/** - * Drop Partition in ALTER VIEW: drop the related partition metadata for a view. - * An error message will be issued if the partition does not exist, unless 'ifExists' is false. - */ -case class AlterViewDropPartition( - viewName: TableIdentifier, - specs: Seq[TablePartitionSpec], - ifExists: Boolean)(sql: String) - extends NativeDDLCommand(sql) with Logging - case class AlterTableArchivePartition( tableName: TableIdentifier, spec: TablePartitionSpec)(sql: String) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala index 9fa48579fba2a..cebf9c856d749 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala @@ -403,7 +403,8 @@ class DDLCommandSuite extends PlanTest { comparePlans(parsed2, expected2) } - // ALTER TABLE view_name ADD [IF NOT EXISTS] PARTITION spec1[, PARTITION spec2, ...] + // ALTER TABLE table_name ADD [IF NOT EXISTS] PARTITION partition_spec + // [LOCATION 'location1'] partition_spec [LOCATION 'location2'] ...; test("alter table: add partition") { val sql1 = """ @@ -431,8 +432,7 @@ class DDLCommandSuite extends PlanTest { comparePlans(parsed2, expected2) } - // ALTER VIEW view_name ADD [IF NOT EXISTS] PARTITION spec1[, PARTITION spec2, ...] - // Location clause is not allowed. + // ALTER VIEW view_name ADD [IF NOT EXISTS] PARTITION partition_spec PARTITION partition_spec ...; test("alter view: add partition") { val sql1 = """ @@ -531,18 +531,20 @@ class DDLCommandSuite extends PlanTest { ifExists = false, purge = true)(sql2_table) - val expected1_view = AlterViewDropPartition( + val expected1_view = AlterTableDropPartition( tableIdent, Seq( Map("dt" -> "2008-08-08", "country" -> "us"), Map("dt" -> "2009-09-09", "country" -> "uk")), - ifExists = true)(sql1_view) - val expected2_view = AlterViewDropPartition( + ifExists = true, + purge = false)(sql1_view) + val expected2_view = AlterTableDropPartition( tableIdent, Seq( Map("dt" -> "2008-08-08", "country" -> "us"), Map("dt" -> "2009-09-09", "country" -> "uk")), - ifExists = false)(sql2_table) + ifExists = false, + purge = false)(sql2_table) comparePlans(parsed1_table, expected1_table) comparePlans(parsed2_table, expected2_table) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala index d8eb7cde90466..052c43a3cef51 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala @@ -109,6 +109,11 @@ private[hive] class HiveQl(conf: ParserConf) extends SparkQl(conf) with Logging "TOK_ALTERINDEX_REBUILD", "TOK_ALTERTABLE_ALTERPARTS", "TOK_ALTERTABLE_PARTITION", + "TOK_ALTERVIEW_ADDPARTS", + "TOK_ALTERVIEW_AS", + "TOK_ALTERVIEW_DROPPARTS", + "TOK_ALTERVIEW_PROPERTIES", + "TOK_ALTERVIEW_RENAME", "TOK_CREATEINDEX", "TOK_CREATEMACRO", @@ -250,16 +255,21 @@ private[hive] class HiveQl(conf: ParserConf) extends SparkQl(conf) with Logging AnalyzeTable(tableName) } - // ALTER VIEW [db_name.]view_name AS select_statement - case view @ Token("TOK_ALTERVIEW", children) - if children.collect { case t @ Token("TOK_QUERY", _) => t }.nonEmpty => - val Seq(Some(viewNameParts), Some(query)) = - getClauses(Seq("TOK_TABNAME", "TOK_QUERY"), children) - createView(view, viewNameParts, query, Nil, Map(), allowExist = false, replace = true) + case view @ Token("TOK_ALTERVIEW", children) => + val Some(nameParts) :: maybeQuery :: _ = + getClauses(Seq( + "TOK_TABNAME", + "TOK_QUERY", + "TOK_ALTERVIEW_ADDPARTS", + "TOK_ALTERVIEW_DROPPARTS", + "TOK_ALTERVIEW_PROPERTIES", + "TOK_ALTERVIEW_RENAME"), children) + + // if ALTER VIEW doesn't have query part, let hive to handle it. + maybeQuery.map { query => + createView(view, nameParts, query, Nil, Map(), allowExist = false, replace = true) + }.getOrElse(NativePlaceholder) - // CREATE VIEW [IF NOT EXISTS] [db_name.]view_name - // [(column_name [COMMENT column_comment], ...) ] [COMMENT view_comment] - // [TBLPROPERTIES (property_name = property_value, ...)] AS select_statement case view @ Token("TOK_CREATEVIEW", children) if children.collect { case t @ Token("TOK_QUERY", _) => t }.nonEmpty => val Seq( From dd34529ea4f15b627cdf4dd921bf5a726d543b6d Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Wed, 30 Mar 2016 21:48:28 -0700 Subject: [PATCH 10/13] address comments. --- .../src/main/scala/org/apache/spark/sql/execution/SparkQl.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala index af463ac632c96..6fe04757ba2d1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala @@ -250,7 +250,7 @@ private[sql] class SparkQl(conf: ParserConf = SimpleParserConf()) extends Cataly DropFunction(dbName, funcName, ifExists.isDefined, temp.isDefined)(node.source) - case Token("TOK_ALTERTABLE" | "TOK_ALTERVIEW", alterTableArgs) => + case Token("TOK_ALTERTABLE", alterTableArgs) => AlterTableCommandParser.parse(node) case Token("TOK_CREATETABLEUSING", createTableArgs) => From 48aec92480ec59ed4a965941d56126d9222cb853 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Wed, 30 Mar 2016 22:49:10 -0700 Subject: [PATCH 11/13] address comments. --- .../spark/sql/catalyst/parser/ng/SqlBase.g4 | 2 +- .../spark/sql/execution/SparkSqlParser.scala | 38 ++++++------------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4 index ab9f410a1f644..237db78f42560 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/ng/SqlBase.g4 @@ -76,7 +76,7 @@ statement | ALTER TABLE tableIdentifier ADD (IF NOT EXISTS)? partitionSpecLocation+ #addTablePartition | ALTER VIEW tableIdentifier ADD (IF NOT EXISTS)? - partitionSpec+ #addViewPartition + partitionSpec+ #addTablePartition | ALTER TABLE tableIdentifier from=partitionSpec RENAME TO to=partitionSpec #renameTablePartition | ALTER TABLE from=tableIdentifier diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index d28564fa44623..410cefd4c94ea 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -513,38 +513,22 @@ class SparkSqlAstBuilder extends AstBuilder { * For example: * {{{ * ALTER TABLE table ADD [IF NOT EXISTS] PARTITION spec [LOCATION 'loc1'] + * ALTER VIEW view ADD [IF NOT EXISTS] PARTITION spec * }}} */ override def visitAddTablePartition( ctx: AddTablePartitionContext): LogicalPlan = withOrigin(ctx) { // Create partition spec to location mapping. - val specsAndLocs = ctx.partitionSpecLocation.asScala.map { - splCtx => - val spec = visitNonOptionalPartitionSpec(splCtx.partitionSpec) - val location = Option(splCtx.locationSpec).map(visitLocationSpec) - spec -> location - } - AlterTableAddPartition( - visitTableIdentifier(ctx.tableIdentifier), - specsAndLocs, - ctx.EXISTS != null)( - command(ctx)) - } - - /** - * Create an [[AlterTableAddPartition]] command. - * - * For example: - * {{{ - * ALTER VIEW view ADD [IF NOT EXISTS] PARTITION spec - * }}} - */ - override def visitAddViewPartition( - ctx: AddViewPartitionContext): LogicalPlan = withOrigin(ctx) { - // Create partition spec to location mapping. - // Different from ALTER TABLE ADD PARTITION, location is not allowed here. - val specsAndLocs = ctx.partitionSpec.asScala.map { - visitNonOptionalPartitionSpec(_) -> None + val specsAndLocs = if (ctx.partitionSpecLocation.isEmpty) { + // Alter View: the location clauses are not allowed. + ctx.partitionSpec.asScala.map(visitNonOptionalPartitionSpec(_) -> None) + } else { + ctx.partitionSpecLocation.asScala.map { + splCtx => + val spec = visitNonOptionalPartitionSpec(splCtx.partitionSpec) + val location = Option(splCtx.locationSpec).map(visitLocationSpec) + spec -> location + } } AlterTableAddPartition( visitTableIdentifier(ctx.tableIdentifier), From cdb8cc0e0151f29d8436cfc693a10f92bbfde9cd Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Thu, 31 Mar 2016 07:07:38 -0700 Subject: [PATCH 12/13] update the comments. --- .../scala/org/apache/spark/sql/execution/command/ddl.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 552f1a6c8d1dd..cd7e0eed65e78 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -260,7 +260,7 @@ case class AlterTableSkewedLocation( * Add Partition in ALTER TABLE/VIEW: add the table/view partitions. * 'partitionSpecsAndLocs': the syntax of ALTER VIEW is identical to ALTER TABLE, * EXCEPT that it is ILLEGAL to specify a LOCATION clause. - * An error message will be issued if the partition exists, unless 'ifNotExists' is false. + * An error message will be issued if the partition exists, unless 'ifNotExists' is true. */ case class AlterTableAddPartition( tableName: TableIdentifier, @@ -285,7 +285,7 @@ case class AlterTableExchangePartition( * This removes the data and metadata for this partition. * The data is actually moved to the .Trash/Current directory if Trash is configured, * unless 'purge' is true, but the metadata is completely lost. - * An error message will be issued if the partition does not exist, unless 'ifExists' is false. + * An error message will be issued if the partition does not exist, unless 'ifExists' is true. * Note: purge is always false when the target is a view. */ case class AlterTableDropPartition( From 315de9097d7f8b04e62afa735ebcefa61ad17031 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Thu, 31 Mar 2016 07:52:46 -0700 Subject: [PATCH 13/13] address comments. --- .../org/apache/spark/sql/execution/SparkSqlParser.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 410cefd4c94ea..fbf6873239efd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -519,16 +519,16 @@ class SparkSqlAstBuilder extends AstBuilder { override def visitAddTablePartition( ctx: AddTablePartitionContext): LogicalPlan = withOrigin(ctx) { // Create partition spec to location mapping. - val specsAndLocs = if (ctx.partitionSpecLocation.isEmpty) { - // Alter View: the location clauses are not allowed. - ctx.partitionSpec.asScala.map(visitNonOptionalPartitionSpec(_) -> None) - } else { + val specsAndLocs = if (ctx.partitionSpec.isEmpty) { ctx.partitionSpecLocation.asScala.map { splCtx => val spec = visitNonOptionalPartitionSpec(splCtx.partitionSpec) val location = Option(splCtx.locationSpec).map(visitLocationSpec) spec -> location } + } else { + // Alter View: the location clauses are not allowed. + ctx.partitionSpec.asScala.map(visitNonOptionalPartitionSpec(_) -> None) } AlterTableAddPartition( visitTableIdentifier(ctx.tableIdentifier),