From ca958493957e7cc5f49dc318bcbf213793a991c0 Mon Sep 17 00:00:00 2001 From: jeanlyn Date: Wed, 18 Mar 2015 10:02:35 +0800 Subject: [PATCH 1/3] use spark physical plan when add jar --- .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index 895688ab2ec2e..5b9b3eca4d157 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -33,7 +33,7 @@ import org.apache.hadoop.hive.common.{HiveInterruptCallback, HiveInterruptUtils, import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.ql.Driver import org.apache.hadoop.hive.ql.exec.Utilities -import org.apache.hadoop.hive.ql.processors.{SetProcessor, CommandProcessor, CommandProcessorFactory} +import org.apache.hadoop.hive.ql.processors.{AddResourceProcessor, SetProcessor, CommandProcessor, CommandProcessorFactory} import org.apache.hadoop.hive.ql.session.SessionState import org.apache.hadoop.hive.shims.ShimLoader import org.apache.thrift.transport.TSocket @@ -264,7 +264,9 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging { val proc: CommandProcessor = HiveShim.getCommandProcessor(Array(tokens(0)), hconf) if (proc != null) { - if (proc.isInstanceOf[Driver] || proc.isInstanceOf[SetProcessor]) { + if (proc.isInstanceOf[Driver] + || proc.isInstanceOf[SetProcessor] + || proc.isInstanceOf[AddResourceProcessor]) { val driver = new SparkSQLDriver driver.init() From ca78d7285b9bfe79550d553b40ff7ff3fe1152e9 Mon Sep 17 00:00:00 2001 From: jeanlyn Date: Wed, 18 Mar 2015 18:40:28 +0800 Subject: [PATCH 2/3] add test --- .../src/test/resources/data/files/hello.jar | Bin 0 -> 1995 bytes .../sql/hive/thriftserver/CliSuite.scala | 24 ++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 sql/hive-thriftserver/src/test/resources/data/files/hello.jar diff --git a/sql/hive-thriftserver/src/test/resources/data/files/hello.jar b/sql/hive-thriftserver/src/test/resources/data/files/hello.jar new file mode 100644 index 0000000000000000000000000000000000000000..7b1a3456a061457e34f65077105aca5471fd8571 GIT binary patch literal 1995 zcmWIWW@h1H0D*OpMQ&gQl;8x?zOEsTx}JV+`TuQ}lbAEG>!4=~NPm8q9p3%Hmx@t?%2``|Em1mkc zmcFQ3n$kRF#%swVvuA!TvJ_vcHf_nWhEOXp6}9vwT0hjLEq$gb@mlhwBv4XJH9ck8 zQZdCI{u7>SHFU!oJUeyF z_5jOko#*^Hj>gsdXw03}sdS~?Do{TCzFx>T&$uH!CT(g{K1QV7nm0W`v;T|gL1{Ow zzO#Z0S3NVDcXCd4e%0aiMS89}lUGVSo$0sC{qKK8Cc~Yj>kH~cSD9Q_dl>c6==_^E zqPhQ0oY@q>yT$a8H8cA(u>aSGZ=Zo8hNnt+X(E z)5-}ed6ZR^EDhB}C(hIUXe;*0tM$?6n+>62 z1?Ei23%K{Zw)xao_FZMYzBwB%IF;SKSnHORv9V?9dxL9_c0CncbE9a&->k&TKeMfq zqw5)A@rIfH_>mJ96rP<>biKWf! z7I_L?Sac=&fmumT>aoDATtg+5bWx2nT*rbB?7B1IXpNg*;hSA74 z$Gpm@tlt}P?_BtlkmHkCL>9E%7UooYnN-;xfA64j>bbWED^9gslV>estz9^;qbO{O zLYa)fc@>TB9eNKp|66z1@uL9${ZFrWN?3}#-u6co{GOEkb-T&@j=%CVZoDe}{q?k6 z3hVP71tu)KAB?XYt0-7^NA8mBM-Yuv9K*C>FOWfsL10NEN)kmXo)H#9 z0t~&xgy?5r*wT0c$V4hX5gIXbAt=j2fB?h){|&&x7w%$M28OvCQ!^+FBQ&qY)Qpm$ zahsz6GX@xJ|NlS5YYr@f!%V}T#bE|+X;fy$@Dh@%Af`bxJR+R1r$~gMCs~Ly6p=my Vyjj^m`Z<8m4CujnU|Gb#006U9u "OK") } + + test("SPARK-6392:add jar") { + val dataFilePath = + Thread.currentThread().getContextClassLoader.getResource("data/files/small_kv.txt") + val jarPath = + Thread.currentThread().getContextClassLoader.getResource("data/files/hello.jar").toString.split(":")(1) + + runCliWithin(3.minute)( + s""" + |CREATE TABLE hive_test(key INT, val STRING); + |LOAD DATA LOCAL INPATH '$dataFilePath' OVERWRITE INTO TABLE hive_test; + |""".stripMargin + -> "OK", + s""" + |add jar $jarPath ; + |create temporary function hello as 'hello'; + |""".stripMargin + -> "Time taken: ", + "select hello(count(*)) from hive_test;" + -> "hello 5", + "DROP TABLE hive_test;" + -> "Time taken: " + ) + } } From 2de3945607a91a825aa8df9ed960f437bb6cb9b3 Mon Sep 17 00:00:00 2001 From: jeanlyn Date: Thu, 19 Mar 2015 09:36:22 +0800 Subject: [PATCH 3/3] rename udf --- .../{data/files/hello.jar => jar/TestUdf.jar} | Bin .../spark/sql/hive/thriftserver/CliSuite.scala | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename sql/hive-thriftserver/src/test/resources/{data/files/hello.jar => jar/TestUdf.jar} (100%) diff --git a/sql/hive-thriftserver/src/test/resources/data/files/hello.jar b/sql/hive-thriftserver/src/test/resources/jar/TestUdf.jar similarity index 100% rename from sql/hive-thriftserver/src/test/resources/data/files/hello.jar rename to sql/hive-thriftserver/src/test/resources/jar/TestUdf.jar diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index cac66ec695250..f82c8d97a1585 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -128,7 +128,7 @@ class CliSuite extends FunSuite with BeforeAndAfterAll with Logging { val dataFilePath = Thread.currentThread().getContextClassLoader.getResource("data/files/small_kv.txt") val jarPath = - Thread.currentThread().getContextClassLoader.getResource("data/files/hello.jar").toString.split(":")(1) + Thread.currentThread().getContextClassLoader.getResource("jar/TestUdf.jar").toString.split(":")(1) runCliWithin(3.minute)( s"""