From d9f67887f53046f986faf41b8de6717ee138a788 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 7 Dec 2020 17:47:16 +0800 Subject: [PATCH 01/37] [WIP] FIX HiveThriftHttpServerSuite flakiness --- .../java/org/apache/hive/service/cli/HiveSQLException.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/HiveSQLException.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/HiveSQLException.java index c331d5492b04e..24309c4b7a444 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/HiveSQLException.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/HiveSQLException.java @@ -118,7 +118,8 @@ public TStatus toTStatus() { TStatus tStatus = new TStatus(TStatusCode.ERROR_STATUS); tStatus.setSqlState(getSQLState()); tStatus.setErrorCode(getErrorCode()); - tStatus.setErrorMessage(getMessage()); + tStatus.setErrorMessage(org.apache.hadoop.util.StringUtils + .stringifyException(this)); tStatus.setInfoMessages(toString(this)); return tStatus; } @@ -133,7 +134,8 @@ public static TStatus toTStatus(Exception e) { return ((HiveSQLException)e).toTStatus(); } TStatus tStatus = new TStatus(TStatusCode.ERROR_STATUS); - tStatus.setErrorMessage(e.getMessage()); + tStatus.setErrorMessage(org.apache.hadoop.util.StringUtils + .stringifyException(e)); tStatus.setInfoMessages(toString(e)); return tStatus; } From 13b2b8f8bf28decd483bb0274419d179247bfd9b Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 7 Dec 2020 19:03:45 +0800 Subject: [PATCH 02/37] sep toTStatus --- .../org/apache/hive/service/cli/HiveSQLException.java | 6 ++---- .../hive/service/cli/thrift/ThriftCLIService.java | 10 +++++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/HiveSQLException.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/HiveSQLException.java index 24309c4b7a444..c331d5492b04e 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/HiveSQLException.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/HiveSQLException.java @@ -118,8 +118,7 @@ public TStatus toTStatus() { TStatus tStatus = new TStatus(TStatusCode.ERROR_STATUS); tStatus.setSqlState(getSQLState()); tStatus.setErrorCode(getErrorCode()); - tStatus.setErrorMessage(org.apache.hadoop.util.StringUtils - .stringifyException(this)); + tStatus.setErrorMessage(getMessage()); tStatus.setInfoMessages(toString(this)); return tStatus; } @@ -134,8 +133,7 @@ public static TStatus toTStatus(Exception e) { return ((HiveSQLException)e).toTStatus(); } TStatus tStatus = new TStatus(TStatusCode.ERROR_STATUS); - tStatus.setErrorMessage(org.apache.hadoop.util.StringUtils - .stringifyException(e)); + tStatus.setErrorMessage(e.getMessage()); tStatus.setInfoMessages(toString(e)); return tStatus; } diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java index 150f1d60fc466..be38771305ebe 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java @@ -257,11 +257,19 @@ public TOpenSessionResp OpenSession(TOpenSessionReq req) throws TException { } } catch (Exception e) { LOG.warn("Error opening session: ", e); - resp.setStatus(HiveSQLException.toTStatus(e)); + resp.setStatus(toTStatus(e)); } return resp; } + public static TStatus toTStatus(Exception e) { + TStatus tStatus = new TStatus(TStatusCode.ERROR_STATUS); + tStatus.setErrorMessage(org.apache.hadoop.util.StringUtils + .stringifyException(e)); + tStatus.setInfoMessages(HiveSQLException.toString(e)); + return tStatus; + } + @Override public TSetClientInfoResp SetClientInfo(TSetClientInfoReq req) throws TException { // TODO: We don't do anything for now, just log this for debugging. From 772e544c32b25668919e2a177138372c2e791849 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 7 Dec 2020 19:14:14 +0800 Subject: [PATCH 03/37] simplify libthrift dep --- dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 2 +- dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 2 +- pom.xml | 14 +------------- resource-managers/yarn/pom.xml | 5 ----- sql/hive/pom.xml | 8 -------- 5 files changed, 3 insertions(+), 28 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index 3f1199478bc67..2687775f95e67 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -178,7 +178,7 @@ kubernetes-model-settings/4.12.0//kubernetes-model-settings-4.12.0.jar kubernetes-model-storageclass/4.12.0//kubernetes-model-storageclass-4.12.0.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar -libthrift/0.12.0//libthrift-0.12.0.jar +libthrift/0.9.3//libthrift-0.9.3.jar log4j/1.2.17//log4j-1.2.17.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.7.1//lz4-java-1.7.1.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index d16235339897e..14f6a00d40551 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -190,7 +190,7 @@ kubernetes-model-settings/4.12.0//kubernetes-model-settings-4.12.0.jar kubernetes-model-storageclass/4.12.0//kubernetes-model-storageclass-4.12.0.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar -libthrift/0.12.0//libthrift-0.12.0.jar +libthrift/0.9.3//libthrift-0.9.3.jar log4j/1.2.17//log4j-1.2.17.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.7.1//lz4-java-1.7.1.jar diff --git a/pom.xml b/pom.xml index 8aaa4a504ef0c..4c204858c209b 100644 --- a/pom.xml +++ b/pom.xml @@ -189,7 +189,6 @@ 2.10.5 3.5.2 3.0.0 - 0.12.0 4.8-1 1.1 3.141.59 @@ -2051,7 +2050,7 @@ 3.1.2 - * + org.apache.hive * @@ -2289,17 +2288,6 @@ datanucleus-core ${datanucleus-core.version} - - org.apache.thrift - libthrift - ${libthrift.version} - - - org.slf4j - slf4j-api - - - org.apache.thrift libfb303 diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index 1d3856742f520..73bf3976daec7 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -208,11 +208,6 @@ hive-metastore provided - - org.apache.thrift - libthrift - provided - org.apache.thrift libfb303 diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 27d2756c741ef..752d76b8c85ea 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -162,14 +162,6 @@ org.datanucleus datanucleus-core - - org.apache.thrift - libthrift - - - org.apache.thrift - libfb303 - org.apache.derby derby From 053b45260a5a3eab552b6bb51bef539352292e68 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 7 Dec 2020 19:50:10 +0800 Subject: [PATCH 04/37] fix build --- sql/hive/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 752d76b8c85ea..7679408e1d162 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -117,6 +117,11 @@ hive-llap-client ${hive.llap.scope} + + org.apache.hive + hive-service-rpc + ${hive.deps.scope} + org.apache.avro From f889060a48cdc3552a1f48f1ac8f07462b8b75b9 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 7 Dec 2020 20:28:05 +0800 Subject: [PATCH 05/37] random port --- .../HiveThriftServer2Suites.scala | 37 ++++++++++--------- .../hive/thriftserver/UISeleniumSuite.scala | 4 +- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 5bf7892478082..9c0718ecd8bb3 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -29,7 +29,7 @@ import scala.collection.mutable.ArrayBuffer import scala.concurrent.{ExecutionContext, Future, Promise} import scala.concurrent.duration._ import scala.io.Source -import scala.util.{Random, Try} +import scala.util.Try import com.google.common.io.Files import org.apache.hadoop.hive.conf.HiveConf.ConfVars @@ -1207,7 +1207,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl protected def extraConf: Seq[String] = Nil - protected def serverStartCommand(port: Int) = { + protected def serverStartCommand(): Seq[String] = { val portConf = if (mode == ServerMode.binary) { ConfVars.HIVE_SERVER2_THRIFT_PORT } else { @@ -1240,7 +1240,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl | --hiveconf ${ConfVars.HIVE_SERVER2_TRANSPORT_MODE}=$mode | --hiveconf ${ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION}=$operationLogPath | --hiveconf ${ConfVars.LOCALSCRATCHDIR}=$lScratchDir - | --hiveconf $portConf=$port + | --hiveconf $portConf=0 | --driver-class-path $driverClassPath | --driver-java-options -Dlog4j.debug | --conf spark.ui.enabled=false @@ -1262,7 +1262,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl val SERVER_STARTUP_TIMEOUT = 3.minutes - private def startThriftServer(port: Int, attempt: Int) = { + private def startThriftServer(attempt: Int) = { warehousePath = Utils.createTempDir() warehousePath.delete() metastorePath = Utils.createTempDir() @@ -1274,18 +1274,16 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl logPath = null logTailingProcess = null - val command = serverStartCommand(port) + val command = serverStartCommand() diagnosisBuffer ++= s""" |### Attempt $attempt ### |HiveThriftServer2 command line: $command - |Listening port: $port + |Listening port: 0 |System user: $user """.stripMargin.split("\n") - logInfo(s"Trying to start HiveThriftServer2: port=$port, mode=$mode, attempt=$attempt") - logPath = { val lines = Utils.executeAndGetOutput( command = command, @@ -1312,7 +1310,11 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl // Ensures that the following "tail" command won't fail. logPath.createNewFile() - val successLines = Seq(THRIFT_BINARY_SERVICE_LIVE, THRIFT_HTTP_SERVICE_LIVE) + val successLines = if (mode == ServerMode.http) { + THRIFT_HTTP_SERVICE_LIVE + } else { + THRIFT_BINARY_SERVICE_LIVE + } logTailingProcess = { val command = s"/usr/bin/env tail -n +0 -f ${logPath.getCanonicalPath}".split(" ") @@ -1321,14 +1323,15 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl val captureOutput = (line: String) => diagnosisBuffer.synchronized { diagnosisBuffer += line - successLines.foreach { r => - if (line.contains(r)) { - serverStarted.trySuccess(()) - } + if (line.contains(successLines)) { + listeningPort = line.split(" on port ")(1).split(' ').head.toInt + logInfo(s"Started HiveThriftServer2: port=$listeningPort, mode=$mode, attempt=$attempt") + serverStarted.trySuccess(()) + () } } - val process = builder.start() + val process = builder.start() new ProcessOutputCapturer(process.getInputStream, captureOutput).start() new ProcessOutputCapturer(process.getErrorStream, captureOutput).start() @@ -1379,16 +1382,14 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl override protected def beforeAll(): Unit = { super.beforeAll() - // Chooses a random port between 10000 and 19999 - listeningPort = 10000 + Random.nextInt(10000) diagnosisBuffer.clear() // Retries up to 3 times with different port numbers if the server fails to start - (1 to 3).foldLeft(Try(startThriftServer(listeningPort, 0))) { case (started, attempt) => + (1 to 3).foldLeft(Try(startThriftServer(0))) { case (started, attempt) => started.orElse { listeningPort += 1 stopThriftServer() - Try(startThriftServer(listeningPort, attempt)) + Try(startThriftServer(attempt)) } }.recover { case cause: Throwable => diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala index d0b829c240327..91a829239ac9b 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala @@ -57,7 +57,7 @@ class UISeleniumSuite } } - override protected def serverStartCommand(port: Int) = { + override protected def serverStartCommand(): Seq[String] = { val portConf = if (mode == ServerMode.binary) { ConfVars.HIVE_SERVER2_THRIFT_PORT } else { @@ -71,7 +71,7 @@ class UISeleniumSuite | --hiveconf ${ConfVars.METASTOREWAREHOUSE}=$warehousePath | --hiveconf ${ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST}=localhost | --hiveconf ${ConfVars.HIVE_SERVER2_TRANSPORT_MODE}=$mode - | --hiveconf $portConf=$port + | --hiveconf $portConf=0 | --driver-class-path ${sys.props("java.class.path")} | --conf spark.ui.enabled=true | --conf spark.ui.port=$uiPort From 869d24bce987dcc4980ac07ce2d2c08b9c9a0eba Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 7 Dec 2020 23:16:11 +0800 Subject: [PATCH 06/37] dumplogs --- .../spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 9c0718ecd8bb3..3fe5edbc0a55c 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1397,6 +1397,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl throw cause }.get + dumpLogs() logInfo(s"HiveThriftServer2 started successfully") } From 7be1980761315079fc6cda528a804a083da10f68 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 8 Dec 2020 01:00:34 +0800 Subject: [PATCH 07/37] dumplogs after --- .../HiveThriftServer2Suites.scala | 132 +++++++++--------- .../thriftserver/JdbcConnectionUriSuite.scala | 70 ---------- .../SparkMetadataOperationSuite.scala | 2 +- ...arkThriftServerProtocolVersionsSuite.scala | 2 +- .../hive/thriftserver/UISeleniumSuite.scala | 2 +- 5 files changed, 71 insertions(+), 137 deletions(-) delete mode 100644 sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/JdbcConnectionUriSuite.scala diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 3fe5edbc0a55c..def8a259af788 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -41,6 +41,7 @@ import org.apache.hive.service.rpc.thrift.TCLIService.Client import org.apache.thrift.protocol.TBinaryProtocol import org.apache.thrift.transport.TSocket import org.scalatest.BeforeAndAfterAll +import org.scalatest.concurrent.Eventually._ import org.apache.spark.{SparkException, SparkFunSuite} import org.apache.spark.internal.Logging @@ -60,7 +61,7 @@ object TestData { val smallKvWithNull = getTestDataFilePath("small_kv_with_null.txt") } -class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { +class HiveThriftBinaryServerSuite extends HiveThriftServer2Test { override def mode: ServerMode.Value = ServerMode.binary private def withCLIServiceClient(f: ThriftCLIServiceClient => Unit): Unit = { @@ -935,7 +936,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { } } -class SingleSessionSuite extends HiveThriftJdbcTest { +class SingleSessionSuite extends HiveThriftServer2Test { override def mode: ServerMode.Value = ServerMode.binary override protected def extraConf: Seq[String] = @@ -1046,7 +1047,7 @@ class SingleSessionSuite extends HiveThriftJdbcTest { } } -class HiveThriftCleanUpScratchDirSuite extends HiveThriftJdbcTest{ +class HiveThriftCleanUpScratchDirSuite extends HiveThriftServer2Test{ var tempScratchDir: File = _ override protected def beforeAll(): Unit = { @@ -1079,7 +1080,7 @@ class HiveThriftCleanUpScratchDirSuite extends HiveThriftJdbcTest{ } } -class HiveThriftHttpServerSuite extends HiveThriftJdbcTest { +class HiveThriftHttpServerSuite extends HiveThriftServer2Test { override def mode: ServerMode.Value = ServerMode.http test("JDBC query execution") { @@ -1122,62 +1123,6 @@ object ServerMode extends Enumeration { val binary, http = Value } -abstract class HiveThriftJdbcTest extends HiveThriftServer2Test { - Utils.classForName(classOf[HiveDriver].getCanonicalName) - - private def jdbcUri = if (mode == ServerMode.http) { - s"""jdbc:hive2://localhost:$serverPort/ - |default? - |hive.server2.transport.mode=http; - |hive.server2.thrift.http.path=cliservice; - |${hiveConfList}#${hiveVarList} - """.stripMargin.split("\n").mkString.trim - } else { - s"jdbc:hive2://localhost:$serverPort/?${hiveConfList}#${hiveVarList}" - } - - def withMultipleConnectionJdbcStatement(tableNames: String*)(fs: (Statement => Unit)*): Unit = { - val user = System.getProperty("user.name") - val connections = fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") } - val statements = connections.map(_.createStatement()) - - try { - statements.zip(fs).foreach { case (s, f) => f(s) } - } finally { - tableNames.foreach { name => - // TODO: Need a better way to drop the view. - if (name.toUpperCase(Locale.ROOT).startsWith("VIEW")) { - statements(0).execute(s"DROP VIEW IF EXISTS $name") - } else { - statements(0).execute(s"DROP TABLE IF EXISTS $name") - } - } - statements.foreach(_.close()) - connections.foreach(_.close()) - } - } - - def withDatabase(dbNames: String*)(fs: (Statement => Unit)*): Unit = { - val user = System.getProperty("user.name") - val connections = fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") } - val statements = connections.map(_.createStatement()) - - try { - statements.zip(fs).foreach { case (s, f) => f(s) } - } finally { - dbNames.foreach { name => - statements(0).execute(s"DROP DATABASE IF EXISTS $name") - } - statements.foreach(_.close()) - connections.foreach(_.close()) - } - } - - def withJdbcStatement(tableNames: String*)(f: Statement => Unit): Unit = { - withMultipleConnectionJdbcStatement(tableNames: _*)(f) - } -} - abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAll with Logging { def mode: ServerMode.Value @@ -1220,7 +1165,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl val tempLog4jConf = Utils.createTempDir().getCanonicalPath Files.write( - """log4j.rootCategory=DEBUG, console + """log4j.rootCategory=INFO, console |log4j.appender.console=org.apache.log4j.ConsoleAppender |log4j.appender.console.target=System.err |log4j.appender.console.layout=org.apache.log4j.PatternLayout @@ -1242,7 +1187,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl | --hiveconf ${ConfVars.LOCALSCRATCHDIR}=$lScratchDir | --hiveconf $portConf=0 | --driver-class-path $driverClassPath - | --driver-java-options -Dlog4j.debug + | --driver-java-options -Dlog4j.info | --conf spark.ui.enabled=false | ${extraConf.mkString("\n")} """.stripMargin.split("\\s+").toSeq @@ -1387,9 +1332,13 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl // Retries up to 3 times with different port numbers if the server fails to start (1 to 3).foldLeft(Try(startThriftServer(0))) { case (started, attempt) => started.orElse { - listeningPort += 1 stopThriftServer() - Try(startThriftServer(attempt)) + Try { + startThriftServer(attempt) + eventually(timeout(30.seconds), interval(1.seconds)) { + withJdbcStatement() { _.execute("SELECT 1") } + } + } } }.recover { case cause: Throwable => @@ -1403,10 +1352,65 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl override protected def afterAll(): Unit = { try { + dumpLogs() stopThriftServer() logInfo("HiveThriftServer2 stopped") } finally { super.afterAll() } } + + Utils.classForName(classOf[HiveDriver].getCanonicalName) + + private def jdbcUri = if (mode == ServerMode.http) { + s"""jdbc:hive2://localhost:$serverPort/ + |default? + |hive.server2.transport.mode=http; + |hive.server2.thrift.http.path=cliservice; + |${hiveConfList}#${hiveVarList} + """.stripMargin.split("\n").mkString.trim + } else { + s"jdbc:hive2://localhost:$serverPort/?${hiveConfList}#${hiveVarList}" + } + + def withMultipleConnectionJdbcStatement(tableNames: String*)(fs: (Statement => Unit)*): Unit = { + val user = System.getProperty("user.name") + val connections = fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") } + val statements = connections.map(_.createStatement()) + + try { + statements.zip(fs).foreach { case (s, f) => f(s) } + } finally { + tableNames.foreach { name => + // TODO: Need a better way to drop the view. + if (name.toUpperCase(Locale.ROOT).startsWith("VIEW")) { + statements(0).execute(s"DROP VIEW IF EXISTS $name") + } else { + statements(0).execute(s"DROP TABLE IF EXISTS $name") + } + } + statements.foreach(_.close()) + connections.foreach(_.close()) + } + } + + def withDatabase(dbNames: String*)(fs: (Statement => Unit)*): Unit = { + val user = System.getProperty("user.name") + val connections = fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") } + val statements = connections.map(_.createStatement()) + + try { + statements.zip(fs).foreach { case (s, f) => f(s) } + } finally { + dbNames.foreach { name => + statements(0).execute(s"DROP DATABASE IF EXISTS $name") + } + statements.foreach(_.close()) + connections.foreach(_.close()) + } + } + + def withJdbcStatement(tableNames: String*)(f: Statement => Unit): Unit = { + withMultipleConnectionJdbcStatement(tableNames: _*)(f) + } } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/JdbcConnectionUriSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/JdbcConnectionUriSuite.scala deleted file mode 100644 index fb8a7e273ae44..0000000000000 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/JdbcConnectionUriSuite.scala +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.hive.thriftserver - -import java.sql.DriverManager - -import org.apache.hive.jdbc.HiveDriver - -import org.apache.spark.util.Utils - -class JdbcConnectionUriSuite extends HiveThriftServer2Test { - Utils.classForName(classOf[HiveDriver].getCanonicalName) - - override def mode: ServerMode.Value = ServerMode.binary - - val JDBC_TEST_DATABASE = "jdbc_test_database" - val USER = System.getProperty("user.name") - val PASSWORD = "" - - override protected def beforeAll(): Unit = { - super.beforeAll() - - val jdbcUri = s"jdbc:hive2://localhost:$serverPort/" - val connection = DriverManager.getConnection(jdbcUri, USER, PASSWORD) - val statement = connection.createStatement() - statement.execute(s"CREATE DATABASE $JDBC_TEST_DATABASE") - connection.close() - } - - override protected def afterAll(): Unit = { - try { - val jdbcUri = s"jdbc:hive2://localhost:$serverPort/" - val connection = DriverManager.getConnection(jdbcUri, USER, PASSWORD) - val statement = connection.createStatement() - statement.execute(s"DROP DATABASE $JDBC_TEST_DATABASE") - connection.close() - } finally { - super.afterAll() - } - } - - test("SPARK-17819 Support default database in connection URIs") { - val jdbcUri = s"jdbc:hive2://localhost:$serverPort/$JDBC_TEST_DATABASE" - val connection = DriverManager.getConnection(jdbcUri, USER, PASSWORD) - val statement = connection.createStatement() - try { - val resultSet = statement.executeQuery("select current_database()") - resultSet.next() - assert(resultSet.getString(1) === JDBC_TEST_DATABASE) - } finally { - statement.close() - connection.close() - } - } -} diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala index b413b46adcaa1..e60851e3d65bf 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.analysis.FunctionRegistry import org.apache.spark.sql.types._ import org.apache.spark.util.VersionUtils -class SparkMetadataOperationSuite extends HiveThriftJdbcTest { +class SparkMetadataOperationSuite extends HiveThriftServer2Test { override def mode: ServerMode.Value = ServerMode.binary diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala index 52cf429441d16..7c9f2e97591db 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala @@ -31,7 +31,7 @@ import org.apache.thrift.transport.TSocket import org.apache.spark.sql.catalyst.util.NumberConverter import org.apache.spark.unsafe.types.UTF8String -class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest { +class SparkThriftServerProtocolVersionsSuite extends HiveThriftServer2Test { override def mode: ServerMode.Value = ServerMode.binary diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala index 91a829239ac9b..e6907247a99fb 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala @@ -32,7 +32,7 @@ import org.scalatestplus.selenium.WebBrowser import org.apache.spark.ui.SparkUICssErrorHandler class UISeleniumSuite - extends HiveThriftJdbcTest + extends HiveThriftServer2Test with WebBrowser with Matchers with BeforeAndAfterAll { implicit var webDriver: WebDriver = _ From b73943cdbd400671b61828d3fd1d849e207ec4de Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 8 Dec 2020 23:12:04 +0800 Subject: [PATCH 08/37] exclude jsr311 from sbt build --- project/SparkBuild.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 23fb73d228e01..d25a78ded8b96 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -654,7 +654,10 @@ object DependencyOverrides { */ object ExcludedDependencies { lazy val settings = Seq( - libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") } + libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") }, + excludeDependencies ++= Seq( + ExclusionRule("javax.ws.rs", "jsr311-api") + ) ) } From 09811e1cef3ec6a952f4ecb1da79decac34a260f Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 9 Dec 2020 00:46:58 +0800 Subject: [PATCH 09/37] revert and trigger --- .../hive/service/cli/thrift/ThriftCLIService.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java index be38771305ebe..150f1d60fc466 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java @@ -257,19 +257,11 @@ public TOpenSessionResp OpenSession(TOpenSessionReq req) throws TException { } } catch (Exception e) { LOG.warn("Error opening session: ", e); - resp.setStatus(toTStatus(e)); + resp.setStatus(HiveSQLException.toTStatus(e)); } return resp; } - public static TStatus toTStatus(Exception e) { - TStatus tStatus = new TStatus(TStatusCode.ERROR_STATUS); - tStatus.setErrorMessage(org.apache.hadoop.util.StringUtils - .stringifyException(e)); - tStatus.setInfoMessages(HiveSQLException.toString(e)); - return tStatus; - } - @Override public TSetClientInfoResp SetClientInfo(TSetClientInfoReq req) throws TException { // TODO: We don't do anything for now, just log this for debugging. From 34944a096581335a72973cb6554b8e1f710f8628 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 9 Dec 2020 01:17:50 +0800 Subject: [PATCH 10/37] restore thrift version --- pom.xml | 14 +++++++++++++- resource-managers/yarn/pom.xml | 5 +++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4c204858c209b..8aaa4a504ef0c 100644 --- a/pom.xml +++ b/pom.xml @@ -189,6 +189,7 @@ 2.10.5 3.5.2 3.0.0 + 0.12.0 4.8-1 1.1 3.141.59 @@ -2050,7 +2051,7 @@ 3.1.2 - org.apache.hive + * * @@ -2288,6 +2289,17 @@ datanucleus-core ${datanucleus-core.version} + + org.apache.thrift + libthrift + ${libthrift.version} + + + org.slf4j + slf4j-api + + + org.apache.thrift libfb303 diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index 73bf3976daec7..1d3856742f520 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -208,6 +208,11 @@ hive-metastore provided + + org.apache.thrift + libthrift + provided + org.apache.thrift libfb303 From 7fe3c7b6753faa00f47d2adbc727d45016170c14 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 9 Dec 2020 01:30:03 +0800 Subject: [PATCH 11/37] restore thrift version --- dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 2 -- dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 2 -- 2 files changed, 4 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index 2687775f95e67..adfbe83406485 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -177,8 +177,6 @@ kubernetes-model-scheduling/4.12.0//kubernetes-model-scheduling-4.12.0.jar kubernetes-model-settings/4.12.0//kubernetes-model-settings-4.12.0.jar kubernetes-model-storageclass/4.12.0//kubernetes-model-storageclass-4.12.0.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar -libfb303/0.9.3//libfb303-0.9.3.jar -libthrift/0.9.3//libthrift-0.9.3.jar log4j/1.2.17//log4j-1.2.17.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.7.1//lz4-java-1.7.1.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index 14f6a00d40551..974ad0d2c75fc 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -189,8 +189,6 @@ kubernetes-model-scheduling/4.12.0//kubernetes-model-scheduling-4.12.0.jar kubernetes-model-settings/4.12.0//kubernetes-model-settings-4.12.0.jar kubernetes-model-storageclass/4.12.0//kubernetes-model-storageclass-4.12.0.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar -libfb303/0.9.3//libfb303-0.9.3.jar -libthrift/0.9.3//libthrift-0.9.3.jar log4j/1.2.17//log4j-1.2.17.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.7.1//lz4-java-1.7.1.jar From 5c65a55e7e6258048e2404262cc1b51758e51d81 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 9 Dec 2020 01:56:54 +0800 Subject: [PATCH 12/37] fix --- project/SparkBuild.scala | 15 +++++++++++---- sql/hive/pom.xml | 13 ++++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index d25a78ded8b96..3865137415a96 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -395,6 +395,8 @@ object SparkBuild extends PomBuild { enable(KubernetesIntegrationTests.settings)(kubernetesIntegrationTests) + enable(HiveThriftServer.settings)(hiveThriftServer) + /** * Adds the ability to run the spark shell directly from SBT without building an assembly * jar. @@ -654,10 +656,7 @@ object DependencyOverrides { */ object ExcludedDependencies { lazy val settings = Seq( - libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") }, - excludeDependencies ++= Seq( - ExclusionRule("javax.ws.rs", "jsr311-api") - ) + libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") } ) } @@ -761,6 +760,14 @@ object Hive { ) } +object HiveThriftServer { + lazy val settings = Seq( + excludeDependencies ++= Seq( + ExclusionRule("javax.ws.rs", "jsr311-api") + ) + ) +} + object Assembly { import sbtassembly.AssemblyUtils._ import sbtassembly.AssemblyPlugin.autoImport._ diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 7679408e1d162..27d2756c741ef 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -117,11 +117,6 @@ hive-llap-client ${hive.llap.scope} - - org.apache.hive - hive-service-rpc - ${hive.deps.scope} - org.apache.avro @@ -167,6 +162,14 @@ org.datanucleus datanucleus-core + + org.apache.thrift + libthrift + + + org.apache.thrift + libfb303 + org.apache.derby derby From 7a33e56c738d9412fa013f040128c4aaf1d93f57 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 9 Dec 2020 02:13:42 +0800 Subject: [PATCH 13/37] refine --- .../HiveThriftServer2Suites.scala | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index def8a259af788..8a3477729f7d3 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1165,7 +1165,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl val tempLog4jConf = Utils.createTempDir().getCanonicalPath Files.write( - """log4j.rootCategory=INFO, console + """log4j.rootCategory=DEBUG, console |log4j.appender.console=org.apache.log4j.ConsoleAppender |log4j.appender.console.target=System.err |log4j.appender.console.layout=org.apache.log4j.PatternLayout @@ -1187,7 +1187,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl | --hiveconf ${ConfVars.LOCALSCRATCHDIR}=$lScratchDir | --hiveconf $portConf=0 | --driver-class-path $driverClassPath - | --driver-java-options -Dlog4j.info + | --driver-java-options -Dlog4j.debug | --conf spark.ui.enabled=false | ${extraConf.mkString("\n")} """.stripMargin.split("\\s+").toSeq @@ -1346,13 +1346,11 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl throw cause }.get - dumpLogs() logInfo(s"HiveThriftServer2 started successfully") } override protected def afterAll(): Unit = { try { - dumpLogs() stopThriftServer() logInfo("HiveThriftServer2 stopped") } finally { @@ -1373,7 +1371,17 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl s"jdbc:hive2://localhost:$serverPort/?${hiveConfList}#${hiveVarList}" } - def withMultipleConnectionJdbcStatement(tableNames: String*)(fs: (Statement => Unit)*): Unit = { + private def tryCaptureSysLog(f: => Unit): Unit = { + try f catch { + case e: Exception => + // Dump the HiveThriftServer2 log if error occurs, e.g. getConnection failure. + dumpLogs() + throw e + } + } + + def withMultipleConnectionJdbcStatement( + tableNames: String*)(fs: (Statement => Unit)*): Unit = tryCaptureSysLog { val user = System.getProperty("user.name") val connections = fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") } val statements = connections.map(_.createStatement()) @@ -1394,7 +1402,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl } } - def withDatabase(dbNames: String*)(fs: (Statement => Unit)*): Unit = { + def withDatabase(dbNames: String*)(fs: (Statement => Unit)*): Unit = tryCaptureSysLog { val user = System.getProperty("user.name") val connections = fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") } val statements = connections.map(_.createStatement()) From 47dca2889e221d188590554b3a7364d5f2b5ff64 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 9 Dec 2020 02:17:38 +0800 Subject: [PATCH 14/37] deps regen --- dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 2 ++ dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 2 ++ 2 files changed, 4 insertions(+) diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index adfbe83406485..3f1199478bc67 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -177,6 +177,8 @@ kubernetes-model-scheduling/4.12.0//kubernetes-model-scheduling-4.12.0.jar kubernetes-model-settings/4.12.0//kubernetes-model-settings-4.12.0.jar kubernetes-model-storageclass/4.12.0//kubernetes-model-storageclass-4.12.0.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar +libfb303/0.9.3//libfb303-0.9.3.jar +libthrift/0.12.0//libthrift-0.12.0.jar log4j/1.2.17//log4j-1.2.17.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.7.1//lz4-java-1.7.1.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index 974ad0d2c75fc..d16235339897e 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -189,6 +189,8 @@ kubernetes-model-scheduling/4.12.0//kubernetes-model-scheduling-4.12.0.jar kubernetes-model-settings/4.12.0//kubernetes-model-settings-4.12.0.jar kubernetes-model-storageclass/4.12.0//kubernetes-model-storageclass-4.12.0.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar +libfb303/0.9.3//libfb303-0.9.3.jar +libthrift/0.12.0//libthrift-0.12.0.jar log4j/1.2.17//log4j-1.2.17.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.7.1//lz4-java-1.7.1.jar From 2f89a6828ef34eaabacf42e5de630d53d35cc1a5 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 9 Dec 2020 10:15:33 +0800 Subject: [PATCH 15/37] info --- .../spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 8a3477729f7d3..f1ef127bf05ea 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1165,7 +1165,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl val tempLog4jConf = Utils.createTempDir().getCanonicalPath Files.write( - """log4j.rootCategory=DEBUG, console + """log4j.rootCategory=INFO, console |log4j.appender.console=org.apache.log4j.ConsoleAppender |log4j.appender.console.target=System.err |log4j.appender.console.layout=org.apache.log4j.PatternLayout From 614e650c2bdc993db4841b7e4c70fa4e0ca9ef54 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 9 Dec 2020 14:35:54 +0800 Subject: [PATCH 16/37] add exclusion --- project/SparkBuild.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 3865137415a96..3b276a8139ca1 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -763,7 +763,8 @@ object Hive { object HiveThriftServer { lazy val settings = Seq( excludeDependencies ++= Seq( - ExclusionRule("javax.ws.rs", "jsr311-api") + ExclusionRule("javax.ws.rs", "jsr311-api"), + ExclusionRule(organization = "com.sun.jersey") ) ) } From 0447184363cfd845cac675b74b227fa7c54cca57 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 9 Dec 2020 15:29:48 +0800 Subject: [PATCH 17/37] eclude jetty-web app --- project/SparkBuild.scala | 3 ++- .../spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 3b276a8139ca1..7d2a9b6006a9e 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -764,7 +764,8 @@ object HiveThriftServer { lazy val settings = Seq( excludeDependencies ++= Seq( ExclusionRule("javax.ws.rs", "jsr311-api"), - ExclusionRule(organization = "com.sun.jersey") + ExclusionRule(organization = "com.sun.jersey"), + ExclusionRule("org.eclipse.jetty", "jetty-webapp") ) ) } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index f1ef127bf05ea..27de643a464ac 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1351,6 +1351,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl override protected def afterAll(): Unit = { try { + dumpLogs() stopThriftServer() logInfo("HiveThriftServer2 stopped") } finally { From abebd302b5373a0ee99cd6ad4a04ef0ecafe9903 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 9 Dec 2020 16:21:35 +0800 Subject: [PATCH 18/37] log protential jar conflicts --- .../thriftserver/HiveThriftServer2Suites.scala | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 27de643a464ac..cc095a2aa40f8 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1207,6 +1207,8 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl val SERVER_STARTUP_TIMEOUT = 3.minutes + private var sparkCmd: String = _ + private def startThriftServer(attempt: Int) = { warehousePath = Utils.createTempDir() warehousePath.delete() @@ -1268,6 +1270,9 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl val captureOutput = (line: String) => diagnosisBuffer.synchronized { diagnosisBuffer += line + if (line.startsWith("Spark Command: ")) { + sparkCmd = line + } if (line.contains(successLines)) { listeningPort = line.split(" on port ")(1).split(' ').head.toInt logInfo(s"Started HiveThriftServer2: port=$listeningPort, mode=$mode, attempt=$attempt") @@ -1352,6 +1357,18 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl override protected def afterAll(): Unit = { try { dumpLogs() + if (sparkCmd != null) { + import scala.sys.process._ + val jars = sparkCmd.split(":").filter(_.endsWith("jar")) + jars.foreach { jar => + val lineStream = s"jar vtf $jar".lineStream + val classes = lineStream.filter(_.contains("NewCookie.class")) + if (classes.nonEmpty) { + info(s"!!!!!!!!! Suspect???????? ---> $jar --> ${classes.mkString(", ")}") + } + } + + } stopThriftServer() logInfo("HiveThriftServer2 stopped") } finally { From 927ff869083779e6aebc0444172d6178d1acff84 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Wed, 9 Dec 2020 19:49:50 +0800 Subject: [PATCH 19/37] improve log --- .../service/cli/thrift/ThriftHttpServlet.java | 2 + .../HiveThriftServer2Suites.scala | 38 +++++++++++-------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java index 0cfa84d0971b3..3c644b7a1754a 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java @@ -315,6 +315,8 @@ private static String getHttpOnlyCookieHeader(Cookie cookie) { NewCookie newCookie = new NewCookie(cookie.getName(), cookie.getValue(), cookie.getPath(), cookie.getDomain(), cookie.getVersion(), cookie.getComment(), cookie.getMaxAge(), cookie.getSecure()); + LOG.error("!!!!!!!!! Suspect???????? ---> " + + newCookie.getClass().getProtectionDomain().getCodeSource().getLocation()); return newCookie + "; HttpOnly"; } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index cc095a2aa40f8..b2d9f0a00c908 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1083,6 +1083,27 @@ class HiveThriftCleanUpScratchDirSuite extends HiveThriftServer2Test{ class HiveThriftHttpServerSuite extends HiveThriftServer2Test { override def mode: ServerMode.Value = ServerMode.http + override def afterAll(): Unit = { + dumpLogs() + if (sparkCmd != null) { + import scala.sys.process._ + val cmds = sparkCmd.split(":") + val jars = cmds.filter(_.endsWith("jar")) + val compileClasses = cmds.filter(_.endsWith("classes")) + .map(new File(_)) + .filter(_.isDirectory).map(_.list()) + jars.foreach { jar => + val lineStream = s"jar vtf $jar".lineStream + val classes = lineStream.filter(_.contains("NewCookie.class")) + if (classes.nonEmpty) { + info(s"!!!!!!!!! Suspect???????? ---> $jar --> ${classes.mkString(", ")}") + } + } + + } + super.afterAll() + } + test("JDBC query execution") { withJdbcStatement("test") { statement => val queries = Seq( @@ -1207,7 +1228,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl val SERVER_STARTUP_TIMEOUT = 3.minutes - private var sparkCmd: String = _ + var sparkCmd: String = _ private def startThriftServer(attempt: Int) = { warehousePath = Utils.createTempDir() @@ -1317,7 +1338,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl logTailingProcess = null } - private def dumpLogs(): Unit = { + def dumpLogs(): Unit = { logError( s""" |===================================== @@ -1356,19 +1377,6 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl override protected def afterAll(): Unit = { try { - dumpLogs() - if (sparkCmd != null) { - import scala.sys.process._ - val jars = sparkCmd.split(":").filter(_.endsWith("jar")) - jars.foreach { jar => - val lineStream = s"jar vtf $jar".lineStream - val classes = lineStream.filter(_.contains("NewCookie.class")) - if (classes.nonEmpty) { - info(s"!!!!!!!!! Suspect???????? ---> $jar --> ${classes.mkString(", ")}") - } - } - - } stopThriftServer() logInfo("HiveThriftServer2 stopped") } finally { From 893d8edb1658db6553a99d909da984eecbeea62f Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 10 Dec 2020 02:47:10 +0800 Subject: [PATCH 20/37] eclude for assembly --- project/SparkBuild.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 7d2a9b6006a9e..b4144c7356d9e 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -802,7 +802,12 @@ object Assembly { => MergeStrategy.filterDistinctLines case "reference.conf" => MergeStrategy.concat case _ => MergeStrategy.first - } + }, + excludeDependencies ++= Seq( + ExclusionRule("javax.ws.rs", "jsr311-api"), + ExclusionRule(organization = "com.sun.jersey"), + ExclusionRule("org.eclipse.jetty", "jetty-webapp") + ) ) } From b6ebd4c74383dc25336a1c73889d266d92ccfaf3 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 10 Dec 2020 15:23:52 +0800 Subject: [PATCH 21/37] only yarn --- project/SparkBuild.scala | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index b4144c7356d9e..5b36906e6d639 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -395,7 +395,7 @@ object SparkBuild extends PomBuild { enable(KubernetesIntegrationTests.settings)(kubernetesIntegrationTests) - enable(HiveThriftServer.settings)(hiveThriftServer) + enable(YARN.settings)(yarn) /** * Adds the ability to run the spark shell directly from SBT without building an assembly @@ -656,7 +656,13 @@ object DependencyOverrides { */ object ExcludedDependencies { lazy val settings = Seq( - libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") } + libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") }, + excludeDependencies ++= Seq( + ExclusionRule("javax.ws.rs", "jsr311-api") +// ExclusionRule("javax.ws.rs", "jsr311-api"), +// ExclusionRule(organization = "com.sun.jersey"), +// ExclusionRule("org.eclipse.jetty", "jetty-webapp") + ) ) } @@ -760,13 +766,9 @@ object Hive { ) } -object HiveThriftServer { +object YARN { lazy val settings = Seq( - excludeDependencies ++= Seq( - ExclusionRule("javax.ws.rs", "jsr311-api"), - ExclusionRule(organization = "com.sun.jersey"), - ExclusionRule("org.eclipse.jetty", "jetty-webapp") - ) + libraryDependencies += "javax.ws.rs" % "jsr311-api" % "1.1.1" % "test" ) } @@ -802,12 +804,7 @@ object Assembly { => MergeStrategy.filterDistinctLines case "reference.conf" => MergeStrategy.concat case _ => MergeStrategy.first - }, - excludeDependencies ++= Seq( - ExclusionRule("javax.ws.rs", "jsr311-api"), - ExclusionRule(organization = "com.sun.jersey"), - ExclusionRule("org.eclipse.jetty", "jetty-webapp") - ) + } ) } From 2a45b03fe438164f12d960ace53f282e62ac23e8 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 10 Dec 2020 19:39:23 +0800 Subject: [PATCH 22/37] fix yarn --- project/SparkBuild.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 5b36906e6d639..39c0b795eaf12 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -768,7 +768,10 @@ object Hive { object YARN { lazy val settings = Seq( - libraryDependencies += "javax.ws.rs" % "jsr311-api" % "1.1.1" % "test" + libraryDependencies += "javax.ws.rs" % "jsr311-api" % "1.1.1" % "test", + excludeDependencies --= Seq( + ExclusionRule("javax.ws.rs", "jsr311-api") + ) ) } From 436d3fb478cc21391a30a78113afb03f7f34010d Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 10 Dec 2020 20:50:16 +0800 Subject: [PATCH 23/37] refine --- project/SparkBuild.scala | 11 ++---- .../service/cli/thrift/ThriftHttpServlet.java | 2 -- .../HiveThriftServer2Suites.scala | 34 +++---------------- 3 files changed, 6 insertions(+), 41 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 39c0b795eaf12..61dc29fa60500 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -657,12 +657,7 @@ object DependencyOverrides { object ExcludedDependencies { lazy val settings = Seq( libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") }, - excludeDependencies ++= Seq( - ExclusionRule("javax.ws.rs", "jsr311-api") -// ExclusionRule("javax.ws.rs", "jsr311-api"), -// ExclusionRule(organization = "com.sun.jersey"), -// ExclusionRule("org.eclipse.jetty", "jetty-webapp") - ) + excludeDependencies += ExclusionRule("javax.ws.rs", "jsr311-api") ) } @@ -769,9 +764,7 @@ object Hive { object YARN { lazy val settings = Seq( libraryDependencies += "javax.ws.rs" % "jsr311-api" % "1.1.1" % "test", - excludeDependencies --= Seq( - ExclusionRule("javax.ws.rs", "jsr311-api") - ) + excludeDependencies -= ExclusionRule("javax.ws.rs", "jsr311-api") ) } diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java index 3c644b7a1754a..0cfa84d0971b3 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java @@ -315,8 +315,6 @@ private static String getHttpOnlyCookieHeader(Cookie cookie) { NewCookie newCookie = new NewCookie(cookie.getName(), cookie.getValue(), cookie.getPath(), cookie.getDomain(), cookie.getVersion(), cookie.getComment(), cookie.getMaxAge(), cookie.getSecure()); - LOG.error("!!!!!!!!! Suspect???????? ---> " + - newCookie.getClass().getProtectionDomain().getCodeSource().getLocation()); return newCookie + "; HttpOnly"; } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index b2d9f0a00c908..84f2b94ed27a5 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1047,7 +1047,7 @@ class SingleSessionSuite extends HiveThriftServer2Test { } } -class HiveThriftCleanUpScratchDirSuite extends HiveThriftServer2Test{ +class HiveThriftCleanUpScratchDirSuite extends HiveThriftServer2Test { var tempScratchDir: File = _ override protected def beforeAll(): Unit = { @@ -1083,27 +1083,6 @@ class HiveThriftCleanUpScratchDirSuite extends HiveThriftServer2Test{ class HiveThriftHttpServerSuite extends HiveThriftServer2Test { override def mode: ServerMode.Value = ServerMode.http - override def afterAll(): Unit = { - dumpLogs() - if (sparkCmd != null) { - import scala.sys.process._ - val cmds = sparkCmd.split(":") - val jars = cmds.filter(_.endsWith("jar")) - val compileClasses = cmds.filter(_.endsWith("classes")) - .map(new File(_)) - .filter(_.isDirectory).map(_.list()) - jars.foreach { jar => - val lineStream = s"jar vtf $jar".lineStream - val classes = lineStream.filter(_.contains("NewCookie.class")) - if (classes.nonEmpty) { - info(s"!!!!!!!!! Suspect???????? ---> $jar --> ${classes.mkString(", ")}") - } - } - - } - super.afterAll() - } - test("JDBC query execution") { withJdbcStatement("test") { statement => val queries = Seq( @@ -1228,8 +1207,6 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl val SERVER_STARTUP_TIMEOUT = 3.minutes - var sparkCmd: String = _ - private def startThriftServer(attempt: Int) = { warehousePath = Utils.createTempDir() warehousePath.delete() @@ -1278,7 +1255,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl // Ensures that the following "tail" command won't fail. logPath.createNewFile() - val successLines = if (mode == ServerMode.http) { + val successLine = if (mode == ServerMode.http) { THRIFT_HTTP_SERVICE_LIVE } else { THRIFT_BINARY_SERVICE_LIVE @@ -1291,10 +1268,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl val captureOutput = (line: String) => diagnosisBuffer.synchronized { diagnosisBuffer += line - if (line.startsWith("Spark Command: ")) { - sparkCmd = line - } - if (line.contains(successLines)) { + if (line.contains(successLine)) { listeningPort = line.split(" on port ")(1).split(' ').head.toInt logInfo(s"Started HiveThriftServer2: port=$listeningPort, mode=$mode, attempt=$attempt") serverStarted.trySuccess(()) @@ -1338,7 +1312,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl logTailingProcess = null } - def dumpLogs(): Unit = { + private def dumpLogs(): Unit = { logError( s""" |===================================== From c893d56621127f8551f026f0cacb028fb6adcb65 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Fri, 11 Dec 2020 13:39:06 +0800 Subject: [PATCH 24/37] add one missing test --- .../HiveThriftServer2Suites.scala | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 84f2b94ed27a5..d87fe227819ca 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1360,15 +1360,15 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl Utils.classForName(classOf[HiveDriver].getCanonicalName) - private def jdbcUri = if (mode == ServerMode.http) { + private def jdbcUri(database: String = "default"): String = if (mode == ServerMode.http) { s"""jdbc:hive2://localhost:$serverPort/ - |default? + |$database? |hive.server2.transport.mode=http; |hive.server2.thrift.http.path=cliservice; |${hiveConfList}#${hiveVarList} """.stripMargin.split("\n").mkString.trim } else { - s"jdbc:hive2://localhost:$serverPort/?${hiveConfList}#${hiveVarList}" + s"jdbc:hive2://localhost:$serverPort/$database?${hiveConfList}#${hiveVarList}" } private def tryCaptureSysLog(f: => Unit): Unit = { @@ -1383,7 +1383,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl def withMultipleConnectionJdbcStatement( tableNames: String*)(fs: (Statement => Unit)*): Unit = tryCaptureSysLog { val user = System.getProperty("user.name") - val connections = fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") } + val connections = fs.map { _ => DriverManager.getConnection(jdbcUri(), user, "") } val statements = connections.map(_.createStatement()) try { @@ -1404,7 +1404,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl def withDatabase(dbNames: String*)(fs: (Statement => Unit)*): Unit = tryCaptureSysLog { val user = System.getProperty("user.name") - val connections = fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") } + val connections = fs.map { _ => DriverManager.getConnection(jdbcUri(), user, "") } val statements = connections.map(_.createStatement()) try { @@ -1421,4 +1421,16 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl def withJdbcStatement(tableNames: String*)(f: Statement => Unit): Unit = { withMultipleConnectionJdbcStatement(tableNames: _*)(f) } + + test("SPARK-17819: Support default database in connection URIs") { + withDatabase("spark17819") { statement => + statement.execute(s"CREATE DATABASE IF NOT EXISTS spark17819") + val jdbcStr = jdbcUri("spark17819") + val connection = DriverManager.getConnection(jdbcStr, user, "") + val statementN = connection.createStatement() + val resultSet = statementN.executeQuery("select current_database()") + resultSet.next() + assert(resultSet.getString(1) === "spark17819") + } + } } From e4b52644faddca9e9eed0cf860a80dbbc50d1bf0 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Fri, 11 Dec 2020 13:47:18 +0800 Subject: [PATCH 25/37] add one missing test 2 --- .../hive/thriftserver/HiveThriftServer2Suites.scala | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index d87fe227819ca..914ce40afb570 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1428,9 +1428,14 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl val jdbcStr = jdbcUri("spark17819") val connection = DriverManager.getConnection(jdbcStr, user, "") val statementN = connection.createStatement() - val resultSet = statementN.executeQuery("select current_database()") - resultSet.next() - assert(resultSet.getString(1) === "spark17819") + try { + val resultSet = statementN.executeQuery("select current_database()") + resultSet.next() + assert(resultSet.getString(1) === "spark17819") + } finally { + statementN.close() + connection.close() + } } } } From ea16b7ad469a947c36baf55151e81acc14b96be7 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Fri, 11 Dec 2020 15:33:17 +0800 Subject: [PATCH 26/37] fix tests --- .../thriftserver/HiveThriftServer2Suites.scala | 14 ++++++++++---- .../thriftserver/SparkMetadataOperationSuite.scala | 2 +- .../SparkThriftServerProtocolVersionsSuite.scala | 2 +- .../sql/hive/thriftserver/UISeleniumSuite.scala | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 914ce40afb570..bd0db743b8d4c 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -936,7 +936,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftServer2Test { } } -class SingleSessionSuite extends HiveThriftServer2Test { +class SingleSessionSuite extends HiveThriftServer2TestBase { override def mode: ServerMode.Value = ServerMode.binary override protected def extraConf: Seq[String] = @@ -1047,7 +1047,7 @@ class SingleSessionSuite extends HiveThriftServer2Test { } } -class HiveThriftCleanUpScratchDirSuite extends HiveThriftServer2Test { +class HiveThriftCleanUpScratchDirSuite extends HiveThriftServer2TestBase { var tempScratchDir: File = _ override protected def beforeAll(): Unit = { @@ -1123,7 +1123,7 @@ object ServerMode extends Enumeration { val binary, http = Value } -abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAll with Logging { +abstract class HiveThriftServer2TestBase extends SparkFunSuite with BeforeAndAfterAll with Logging { def mode: ServerMode.Value private val CLASS_NAME = HiveThriftServer2.getClass.getCanonicalName.stripSuffix("$") @@ -1360,7 +1360,7 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl Utils.classForName(classOf[HiveDriver].getCanonicalName) - private def jdbcUri(database: String = "default"): String = if (mode == ServerMode.http) { + protected def jdbcUri(database: String = "default"): String = if (mode == ServerMode.http) { s"""jdbc:hive2://localhost:$serverPort/ |$database? |hive.server2.transport.mode=http; @@ -1421,7 +1421,13 @@ abstract class HiveThriftServer2Test extends SparkFunSuite with BeforeAndAfterAl def withJdbcStatement(tableNames: String*)(f: Statement => Unit): Unit = { withMultipleConnectionJdbcStatement(tableNames: _*)(f) } +} +/** + * Common tests for both binary and http mode thrift server + * TODO: SPARK-31914: Move common tests from subclasses to this trait + */ +abstract class HiveThriftServer2Test extends HiveThriftServer2TestBase { test("SPARK-17819: Support default database in connection URIs") { withDatabase("spark17819") { statement => statement.execute(s"CREATE DATABASE IF NOT EXISTS spark17819") diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala index e60851e3d65bf..bb7448293f559 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.analysis.FunctionRegistry import org.apache.spark.sql.types._ import org.apache.spark.util.VersionUtils -class SparkMetadataOperationSuite extends HiveThriftServer2Test { +class SparkMetadataOperationSuite extends HiveThriftServer2TestBase { override def mode: ServerMode.Value = ServerMode.binary diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala index 7c9f2e97591db..fd4d7231e8989 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala @@ -31,7 +31,7 @@ import org.apache.thrift.transport.TSocket import org.apache.spark.sql.catalyst.util.NumberConverter import org.apache.spark.unsafe.types.UTF8String -class SparkThriftServerProtocolVersionsSuite extends HiveThriftServer2Test { +class SparkThriftServerProtocolVersionsSuite extends HiveThriftServer2TestBase { override def mode: ServerMode.Value = ServerMode.binary diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala index e6907247a99fb..2d0edb8eb8d48 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala @@ -32,7 +32,7 @@ import org.scalatestplus.selenium.WebBrowser import org.apache.spark.ui.SparkUICssErrorHandler class UISeleniumSuite - extends HiveThriftServer2Test + extends HiveThriftServer2TestBase with WebBrowser with Matchers with BeforeAndAfterAll { implicit var webDriver: WebDriver = _ From c227bc8decc4c16bcf5a675a651a2d5ef1b074cd Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sat, 12 Dec 2020 12:01:22 +0800 Subject: [PATCH 27/37] debug hadoop 27 --- .../apache/hive/service/cli/thrift/ThriftHttpServlet.java | 1 + .../sql/hive/thriftserver/HiveThriftServer2Suites.scala | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java index 0cfa84d0971b3..a80cf035cc530 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java @@ -315,6 +315,7 @@ private static String getHttpOnlyCookieHeader(Cookie cookie) { NewCookie newCookie = new NewCookie(cookie.getName(), cookie.getValue(), cookie.getPath(), cookie.getDomain(), cookie.getVersion(), cookie.getComment(), cookie.getMaxAge(), cookie.getSecure()); + LOG.error("====>" + newCookie.getClass().getProtectionDomain().getCodeSource().getLocation()); return newCookie + "; HttpOnly"; } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index bd0db743b8d4c..1cde38feb0b35 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1117,6 +1117,11 @@ class HiveThriftHttpServerSuite extends HiveThriftServer2Test { assert(resultSet.getString(1) === "4.56") } } + + override def afterAll(): Unit = { + dumpLogs + super.afterAll() + } } object ServerMode extends Enumeration { @@ -1312,7 +1317,7 @@ abstract class HiveThriftServer2TestBase extends SparkFunSuite with BeforeAndAft logTailingProcess = null } - private def dumpLogs(): Unit = { + def dumpLogs(): Unit = { logError( s""" |===================================== From a756cfb395d7bc0bd874f5af6c93e0e944af1c1b Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sat, 12 Dec 2020 15:04:33 +0800 Subject: [PATCH 28/37] Revert "debug hadoop 27" This reverts commit c227bc8decc4c16bcf5a675a651a2d5ef1b074cd. --- .../apache/hive/service/cli/thrift/ThriftHttpServlet.java | 1 - .../sql/hive/thriftserver/HiveThriftServer2Suites.scala | 7 +------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java index a80cf035cc530..0cfa84d0971b3 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java @@ -315,7 +315,6 @@ private static String getHttpOnlyCookieHeader(Cookie cookie) { NewCookie newCookie = new NewCookie(cookie.getName(), cookie.getValue(), cookie.getPath(), cookie.getDomain(), cookie.getVersion(), cookie.getComment(), cookie.getMaxAge(), cookie.getSecure()); - LOG.error("====>" + newCookie.getClass().getProtectionDomain().getCodeSource().getLocation()); return newCookie + "; HttpOnly"; } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 1cde38feb0b35..bd0db743b8d4c 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1117,11 +1117,6 @@ class HiveThriftHttpServerSuite extends HiveThriftServer2Test { assert(resultSet.getString(1) === "4.56") } } - - override def afterAll(): Unit = { - dumpLogs - super.afterAll() - } } object ServerMode extends Enumeration { @@ -1317,7 +1312,7 @@ abstract class HiveThriftServer2TestBase extends SparkFunSuite with BeforeAndAft logTailingProcess = null } - def dumpLogs(): Unit = { + private def dumpLogs(): Unit = { logError( s""" |===================================== From c64c94ed78d6a70e58a6fb0a4045ab5f7e02f697 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sat, 12 Dec 2020 19:58:41 +0800 Subject: [PATCH 29/37] Revert "Revert "debug hadoop 27"" This reverts commit a756cfb395d7bc0bd874f5af6c93e0e944af1c1b. --- .../apache/hive/service/cli/thrift/ThriftHttpServlet.java | 1 + .../sql/hive/thriftserver/HiveThriftServer2Suites.scala | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java index 0cfa84d0971b3..a80cf035cc530 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java @@ -315,6 +315,7 @@ private static String getHttpOnlyCookieHeader(Cookie cookie) { NewCookie newCookie = new NewCookie(cookie.getName(), cookie.getValue(), cookie.getPath(), cookie.getDomain(), cookie.getVersion(), cookie.getComment(), cookie.getMaxAge(), cookie.getSecure()); + LOG.error("====>" + newCookie.getClass().getProtectionDomain().getCodeSource().getLocation()); return newCookie + "; HttpOnly"; } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index bd0db743b8d4c..1cde38feb0b35 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1117,6 +1117,11 @@ class HiveThriftHttpServerSuite extends HiveThriftServer2Test { assert(resultSet.getString(1) === "4.56") } } + + override def afterAll(): Unit = { + dumpLogs + super.afterAll() + } } object ServerMode extends Enumeration { @@ -1312,7 +1317,7 @@ abstract class HiveThriftServer2TestBase extends SparkFunSuite with BeforeAndAft logTailingProcess = null } - private def dumpLogs(): Unit = { + def dumpLogs(): Unit = { logError( s""" |===================================== From 0ad4e9db401228d72b5f16372f7c4f99ea8ede41 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sat, 12 Dec 2020 20:16:00 +0800 Subject: [PATCH 30/37] Revert "Revert "Revert "debug hadoop 27""" This reverts commit c64c94ed78d6a70e58a6fb0a4045ab5f7e02f697. --- .../apache/hive/service/cli/thrift/ThriftHttpServlet.java | 1 - .../sql/hive/thriftserver/HiveThriftServer2Suites.scala | 7 +------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java index a80cf035cc530..0cfa84d0971b3 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java @@ -315,7 +315,6 @@ private static String getHttpOnlyCookieHeader(Cookie cookie) { NewCookie newCookie = new NewCookie(cookie.getName(), cookie.getValue(), cookie.getPath(), cookie.getDomain(), cookie.getVersion(), cookie.getComment(), cookie.getMaxAge(), cookie.getSecure()); - LOG.error("====>" + newCookie.getClass().getProtectionDomain().getCodeSource().getLocation()); return newCookie + "; HttpOnly"; } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 1cde38feb0b35..bd0db743b8d4c 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -1117,11 +1117,6 @@ class HiveThriftHttpServerSuite extends HiveThriftServer2Test { assert(resultSet.getString(1) === "4.56") } } - - override def afterAll(): Unit = { - dumpLogs - super.afterAll() - } } object ServerMode extends Enumeration { @@ -1317,7 +1312,7 @@ abstract class HiveThriftServer2TestBase extends SparkFunSuite with BeforeAndAft logTailingProcess = null } - def dumpLogs(): Unit = { + private def dumpLogs(): Unit = { logError( s""" |===================================== From 4efd4b8ae86a7d466fad3c26177d2c13e10ccc39 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sat, 12 Dec 2020 20:45:28 +0800 Subject: [PATCH 31/37] exclude jersey core --- project/SparkBuild.scala | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 61dc29fa60500..24795c1e107d9 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -657,7 +657,10 @@ object DependencyOverrides { object ExcludedDependencies { lazy val settings = Seq( libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") }, - excludeDependencies += ExclusionRule("javax.ws.rs", "jsr311-api") + excludeDependencies ++= Seq( + ExclusionRule("com.sun.jersey", "jersey-core"), + ExclusionRule("javax.ws.rs", "jsr311-api") + ) ) } @@ -763,8 +766,13 @@ object Hive { object YARN { lazy val settings = Seq( - libraryDependencies += "javax.ws.rs" % "jsr311-api" % "1.1.1" % "test", - excludeDependencies -= ExclusionRule("javax.ws.rs", "jsr311-api") + libraryDependencies ++= Seq( + "javax.ws.rs" % "jsr311-api" % "1.1.1" % "test", + "com.sun.jersey" % "jersey-core" % "1.9" % "test"), + excludeDependencies --= Seq( + ExclusionRule("com.sun.jersey", "jersey-core"), + ExclusionRule("javax.ws.rs", "jsr311-api") + ) ) } From c63acffb813ca209d70fc64b80910d202e1397f4 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sat, 12 Dec 2020 21:15:52 +0800 Subject: [PATCH 32/37] exclude jersey core --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 24795c1e107d9..dd4fcfc07ecaf 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -768,7 +768,7 @@ object YARN { lazy val settings = Seq( libraryDependencies ++= Seq( "javax.ws.rs" % "jsr311-api" % "1.1.1" % "test", - "com.sun.jersey" % "jersey-core" % "1.9" % "test"), + "com.sun.jersey" % "jersey-core" % "1.19" % "test"), excludeDependencies --= Seq( ExclusionRule("com.sun.jersey", "jersey-core"), ExclusionRule("javax.ws.rs", "jsr311-api") From fee0e31a7f8924468c8269a317eac26a1757c255 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sun, 13 Dec 2020 02:29:15 +0800 Subject: [PATCH 33/37] servlet-api --- core/pom.xml | 4 ++-- pom.xml | 2 +- project/SparkBuild.scala | 9 ++++----- resource-managers/yarn/pom.xml | 7 ------- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 84ca852d1f30a..314cd71cda088 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -161,8 +161,8 @@ compile - javax.servlet - javax.servlet-api + jakarta.servlet + jakarta.servlet-api ${javaxservlet.version} diff --git a/pom.xml b/pom.xml index 8aaa4a504ef0c..a7fd38776d3ff 100644 --- a/pom.xml +++ b/pom.xml @@ -138,7 +138,7 @@ 1.10.1 1.6.6 9.4.28.v20200408 - 3.1.0 + 4.0.3 0.9.5 2.4.0 2.0.8 diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index dd4fcfc07ecaf..08b71722a4a2b 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -658,7 +658,8 @@ object ExcludedDependencies { lazy val settings = Seq( libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") }, excludeDependencies ++= Seq( - ExclusionRule("com.sun.jersey", "jersey-core"), + ExclusionRule(organization = "com.sun.jersey"), + ExclusionRule("javax.servlet", "javax.servlet-api"), ExclusionRule("javax.ws.rs", "jsr311-api") ) ) @@ -766,11 +767,9 @@ object Hive { object YARN { lazy val settings = Seq( - libraryDependencies ++= Seq( - "javax.ws.rs" % "jsr311-api" % "1.1.1" % "test", - "com.sun.jersey" % "jersey-core" % "1.19" % "test"), excludeDependencies --= Seq( - ExclusionRule("com.sun.jersey", "jersey-core"), + ExclusionRule(organization = "com.sun.jersey"), + ExclusionRule("javax.servlet", "javax.servlet-api"), ExclusionRule("javax.ws.rs", "jsr311-api") ) ) diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index 1d3856742f520..c0ce1c8e151ed 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -88,13 +88,6 @@ hadoop-client - - jakarta.servlet - jakarta.servlet-api - 4.0.3 - test - - com.google.guava From 47791c91ed60cd8443e51e1f7c6f7bd032f9049a Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sun, 13 Dec 2020 03:03:30 +0800 Subject: [PATCH 34/37] deps regen --- dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 2 +- dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index 3f1199478bc67..df6bcac5cd24e 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -119,6 +119,7 @@ jackson-xc/1.9.13//jackson-xc-1.9.13.jar jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar +jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar @@ -126,7 +127,6 @@ janino/3.0.16//janino-3.0.16.jar javassist/3.25.0-GA//javassist-3.25.0-GA.jar javax.inject/1//javax.inject-1.jar javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar -javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar javolution/5.5.1//javolution-5.5.1.jar jaxb-api/2.2.2//jaxb-api-2.2.2.jar jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index d16235339897e..47d1c9c3fa363 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -118,6 +118,7 @@ jackson-module-scala_2.12/2.10.5//jackson-module-scala_2.12-2.10.5.jar jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar +jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar From afbfdba730fc50ff92ab5bafc5402c7ea919073d Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sun, 13 Dec 2020 12:32:45 +0800 Subject: [PATCH 35/37] trigger ga --- project/SparkBuild.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 08b71722a4a2b..c6a371c7b05c7 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -660,8 +660,7 @@ object ExcludedDependencies { excludeDependencies ++= Seq( ExclusionRule(organization = "com.sun.jersey"), ExclusionRule("javax.servlet", "javax.servlet-api"), - ExclusionRule("javax.ws.rs", "jsr311-api") - ) + ExclusionRule("javax.ws.rs", "jsr311-api")) ) } @@ -770,8 +769,7 @@ object YARN { excludeDependencies --= Seq( ExclusionRule(organization = "com.sun.jersey"), ExclusionRule("javax.servlet", "javax.servlet-api"), - ExclusionRule("javax.ws.rs", "jsr311-api") - ) + ExclusionRule("javax.ws.rs", "jsr311-api")) ) } From 986686df21b878792dced064ee31edfdd83c0c65 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 14 Dec 2020 10:27:19 +0800 Subject: [PATCH 36/37] license --- LICENSE-binary | 2 +- core/pom.xml | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index d363661b1cc7e..2a5434e14a3f5 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -521,7 +521,6 @@ Common Development and Distribution License (CDDL) 1.1 ------------------------------------------------------ javax.el:javax.el-api https://javaee.github.io/uel-ri/ -javax.servlet:javax.servlet-api https://javaee.github.io/servlet-spec/ javax.servlet.jsp:jsp-api javax.transaction:jta http://www.oracle.com/technetwork/java/index.html javax.xml.bind:jaxb-api https://github.com/javaee/jaxb-v2 @@ -553,6 +552,7 @@ Eclipse Public License (EPL) 2.0 -------------------------------- jakarta.annotation:jakarta-annotation-api https://projects.eclipse.org/projects/ee4j.ca +jakarta.servlet:jakarta.servlet-api https://projects.eclipse.org/projects/ee4j.servlet jakarta.ws.rs:jakarta.ws.rs-api https://github.com/eclipse-ee4j/jaxrs-api org.glassfish.hk2.external:jakarta.inject diff --git a/core/pom.xml b/core/pom.xml index 314cd71cda088..1f24c5273ad0b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -163,7 +163,7 @@ jakarta.servlet jakarta.servlet-api - ${javaxservlet.version} + ${jakartaservlet.version} org.apache.commons diff --git a/pom.xml b/pom.xml index a7fd38776d3ff..706c48d471753 100644 --- a/pom.xml +++ b/pom.xml @@ -138,7 +138,7 @@ 1.10.1 1.6.6 9.4.28.v20200408 - 4.0.3 + 4.0.3 0.9.5 2.4.0 2.0.8 From f0ac81782132276a389470a8cf627f4cc12a919a Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 14 Dec 2020 11:38:19 +0800 Subject: [PATCH 37/37] comments --- project/SparkBuild.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index c6a371c7b05c7..a28c2b55b3789 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -657,6 +657,16 @@ object DependencyOverrides { object ExcludedDependencies { lazy val settings = Seq( libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") }, + // SPARK-33705: Due to sbt compiler issues, it brings exclusions defined in maven pom back to + // the classpath directly and assemble test scope artifacts to assembly/target/scala-xx/jars, + // which is also will be added to the classpath of some unit tests that will build a subprocess + // to run `spark-submit`, e.g. HiveThriftServer2Test. + // + // These artifacts are for the jersey-1 API but Spark use jersey-2 ones, so it cause test + // flakiness w/ jar conflicts issues. + // + // Also jersey-1 is only used by yarn module(see resource-managers/yarn/pom.xml) for testing + // purpose only. Here we exclude them from the whole project scope and add them w/ yarn only. excludeDependencies ++= Seq( ExclusionRule(organization = "com.sun.jersey"), ExclusionRule("javax.servlet", "javax.servlet-api"),