From a20c29d5d358482179af4efeb1db5db72b2929e5 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Sun, 19 Jul 2020 16:46:59 +0800 Subject: [PATCH 1/2] bug fix init --- .../sql/catalyst/analysis/ResolveHints.scala | 21 +++++++++++++++++-- .../sql/hive/execution/SQLQuerySuite.scala | 7 +++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala index 120842b0c4a07..8fddf4551ea6b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala @@ -20,9 +20,8 @@ package org.apache.spark.sql.catalyst.analysis import java.util.Locale import scala.collection.mutable - import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.catalyst.expressions.{Ascending, Expression, IntegerLiteral, SortOrder} +import org.apache.spark.sql.catalyst.expressions.{Ascending, Expression, IntegerLiteral, SortOrder, SubqueryExpression} import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.CurrentOrigin @@ -165,6 +164,24 @@ object ResolveHints { hintErrorHandler.hintRelationsNotFound(h.name, h.parameters, unmatchedIdents) applied } + case With(child, relations) => resolveCTEHint(child, + relations.foldLeft(Seq.empty[(String, LogicalPlan)]) { + case (resolved, (name, relation)) => + resolved :+ name -> apply(resolveCTEHint(relation, resolved)) + }) + } + + def resolveCTEHint(plan: LogicalPlan, cteRelations: Seq[(String, LogicalPlan)]): LogicalPlan = { + plan resolveOperatorsDown { + case u: UnresolvedRelation => + cteRelations.find(x => resolver(x._1, u.tableName)).map(_._2).getOrElse(u) + case other => + // This cannot be done in ResolveSubquery because ResolveSubquery does not know the CTE. + other transformExpressions { + case e: SubqueryExpression => + e.withNewPlan(resolveCTEHint(e.plan, cteRelations)) + } + } } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 920f6385f8e19..32cecaf13ab1a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -2558,6 +2558,13 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi } } } + + test("SPARK-32347: cte hint regression") { + withTempView("t") { + sql("create temporary view t as select 1 as id") + sql("with cte as (select /*+ BROADCAST(id) */ id from t) select id from cte") + } + } } @SlowHiveTest From be0711aa5582ea27d3d1f6b89a712ba7edcea057 Mon Sep 17 00:00:00 2001 From: TJX2014 Date: Mon, 20 Jul 2020 06:54:19 +0800 Subject: [PATCH 2/2] style fix --- .../org/apache/spark/sql/catalyst/analysis/ResolveHints.scala | 1 + .../org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala index 8fddf4551ea6b..6de01f555563c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.analysis import java.util.Locale import scala.collection.mutable + import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.expressions.{Ascending, Expression, IntegerLiteral, SortOrder, SubqueryExpression} import org.apache.spark.sql.catalyst.plans.logical._ diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 32cecaf13ab1a..faa582d40382b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -2559,10 +2559,11 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi } } - test("SPARK-32347: cte hint regression") { + test("SPARK-32347: cte hint should be resolved in Hints batch rule") { withTempView("t") { sql("create temporary view t as select 1 as id") sql("with cte as (select /*+ BROADCAST(id) */ id from t) select id from cte") + sql("with cte as (select /*+ COALESCE(3) */ id from t) select id from cte") } } }