From c980cbed80e605ad580504be81e27399a2c1e790 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sun, 20 Aug 2023 00:43:18 +0800 Subject: [PATCH 01/13] init work --- .../org/apache/spark/status/api/v1/api.scala | 31 ++++++++++---- .../ui/exec/ExecutorThreadDumpPage.scala | 3 ++ .../scala/org/apache/spark/util/Utils.scala | 41 +++++++++++-------- 3 files changed, 52 insertions(+), 23 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index 8d648b9df38fa..c87c1386185ab 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -527,13 +527,30 @@ case class StackTrace(elems: Seq[String]) { } case class ThreadStackTrace( - val threadId: Long, - val threadName: String, - val threadState: Thread.State, - val stackTrace: StackTrace, - val blockedByThreadId: Option[Long], - val blockedByLock: String, - val holdingLocks: Seq[String]) + threadId: Long, + threadName: String, + threadState: Thread.State, + stackTrace: StackTrace, + blockedByThreadId: Option[Long], + blockedByLock: String, + holdingLocks: Seq[String], + lockName: Option[String], + lockOwnerName: Option[String], + suspended: Boolean, + inNative: Boolean) { + + override def toString: String = { + val sb = new StringBuilder(s"\"$threadName\" Id=$threadName $threadState") + lockName.foreach(lock => sb.append(s" on $lock")) + lockOwnerName.foreach(owner => sb.append(s" owned by \"$owner\"")) + blockedByThreadId.foreach(id => s" Id=${id}") + if (suspended) sb.append(" (suspended)") + if (inNative) sb.append(" (in native)") + sb.append('\n') + + sb.toString + } +} class ProcessSummary private[spark]( val id: String, diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala index c3246dc90976c..da1842f475178 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala @@ -73,6 +73,9 @@ private[ui] class ExecutorThreadDumpPage(

Collapse All

+

+ Download +

diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index c211d6bba5dfa..63952935ff648 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -2169,14 +2169,20 @@ private[spark] object Utils } private def threadInfoToThreadStackTrace(threadInfo: ThreadInfo): ThreadStackTrace = { - val monitors = threadInfo.getLockedMonitors.map(m => m.getLockedStackFrame -> m).toMap - val stackTrace = StackTrace(threadInfo.getStackTrace.map { frame => - monitors.get(frame) match { - case Some(monitor) => - monitor.getLockedStackFrame.toString + s" => holding ${monitor.lockString}" - case None => - frame.toString - } + val threadState = threadInfo.getThreadState + val monitors = threadInfo.getLockedMonitors.map(m => m.getLockedStackDepth -> m).toMap + val stackTrace = StackTrace(threadInfo.getStackTrace.zipWithIndex.map { case (frame, idx) => + val locked = if (idx == 0 && threadInfo.getLockInfo != null) { + threadState match { + case Thread.State.BLOCKED => + s"\t- blocked on ${threadInfo.getLockInfo}" + case Thread.State.WAITING | Thread.State.TIMED_WAITING => + s"\t- waiting on ${threadInfo.getLockInfo}" + case _ => "" + } + } else "" + val locking = monitors.get(idx).map(mi => s"\t- locked $mi") + frame.toString + locked + locking }) // use a set to dedup re-entrant locks that are held at multiple places @@ -2184,14 +2190,17 @@ private[spark] object Utils (threadInfo.getLockedSynchronizers ++ threadInfo.getLockedMonitors).map(_.lockString).toSet ThreadStackTrace( - threadId = threadInfo.getThreadId, - threadName = threadInfo.getThreadName, - threadState = threadInfo.getThreadState, - stackTrace = stackTrace, - blockedByThreadId = - if (threadInfo.getLockOwnerId < 0) None else Some(threadInfo.getLockOwnerId), - blockedByLock = Option(threadInfo.getLockInfo).map(_.lockString).getOrElse(""), - holdingLocks = heldLocks.toSeq) + threadInfo.getThreadId, + threadInfo.getThreadName, + threadState, + stackTrace, + if (threadInfo.getLockOwnerId < 0) None else Some(threadInfo.getLockOwnerId), + Option(threadInfo.getLockInfo).map(_.lockString).getOrElse(""), + heldLocks.toSeq, + Option(threadInfo.getLockName), + Option(threadInfo.getLockOwnerName), + threadInfo.isSuspended, + threadInfo.isInNative) } /** From a30a3248e61ef07a164bb60a0bcfd20037ea091a Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sun, 20 Aug 2023 21:46:41 +0800 Subject: [PATCH 02/13] [WIP][SPARK-44863][UI] Add a button to download thread dump as a txt file in Spark UI --- .../org/apache/spark/status/api/v1/api.scala | 29 ++++++++++++++++--- .../ui/exec/ExecutorThreadDumpPage.scala | 6 ++-- .../scala/org/apache/spark/util/Utils.scala | 18 +++++------- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index c87c1386185ab..6b81f4ed78c9c 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -533,21 +533,42 @@ case class ThreadStackTrace( stackTrace: StackTrace, blockedByThreadId: Option[Long], blockedByLock: String, - holdingLocks: Seq[String], + synchronizers: Seq[String], + monitors: Seq[String], lockName: Option[String], lockOwnerName: Option[String], suspended: Boolean, inNative: Boolean) { + /** + * Returns a string representation of this thread stack trace + * w.r.t java.lang.management.ThreadInfo(JDK 8)'s toString. + * + * TODO(Kent Yao): Considering 'daemon', 'priority' from higher JDKs + * + * TODO(Kent Yao): Also considering adding information os_prio, cpu, elapsed, tid, nid, etc., + * from the jstack tool + */ override def toString: String = { - val sb = new StringBuilder(s"\"$threadName\" Id=$threadName $threadState") + val sb = new StringBuilder() + val basic = "\"" + threadName + "\" Id=" + threadId + " " + threadState + sb.append(basic) lockName.foreach(lock => sb.append(s" on $lock")) - lockOwnerName.foreach(owner => sb.append(s" owned by \"$owner\"")) - blockedByThreadId.foreach(id => s" Id=${id}") + lockOwnerName.foreach { + owner => sb.append(s"""owned by "$owner"""") + } + blockedByThreadId.foreach(id => s" Id=$id") if (suspended) sb.append(" (suspended)") if (inNative) sb.append(" (in native)") sb.append('\n') + sb.append(stackTrace.elems.map(e => s"\tat $e").mkString) + + if (synchronizers.nonEmpty) { + sb.append(s"\n\tNumber of locked synchronizers = ${synchronizers.length}\n") + synchronizers.foreach(sync => sb.append(s"\t- $sync\n")) + } + sb.append('\n') sb.toString } } diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala index da1842f475178..ad19e93ba7927 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala @@ -48,7 +48,9 @@ private[ui] class ExecutorThreadDumpPage(
case None => Text("") } - val heldLocks = thread.holdingLocks.mkString(", ") + val synchronizers = thread.synchronizers.map(l => s"Lock($l)") + val monitors = thread.monitors.map(m => s"Monitor($m)") + val heldLocks = (synchronizers ++ monitors).mkString(", ") Collapse All

-

+

Download

diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 63952935ff648..3fb87cc757199 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -2170,25 +2170,22 @@ private[spark] object Utils private def threadInfoToThreadStackTrace(threadInfo: ThreadInfo): ThreadStackTrace = { val threadState = threadInfo.getThreadState - val monitors = threadInfo.getLockedMonitors.map(m => m.getLockedStackDepth -> m).toMap + val monitors = threadInfo.getLockedMonitors.map(m => m.getLockedStackDepth -> m.toString).toMap val stackTrace = StackTrace(threadInfo.getStackTrace.zipWithIndex.map { case (frame, idx) => val locked = if (idx == 0 && threadInfo.getLockInfo != null) { threadState match { case Thread.State.BLOCKED => - s"\t- blocked on ${threadInfo.getLockInfo}" + s"\t- blocked on ${threadInfo.getLockInfo}\n" case Thread.State.WAITING | Thread.State.TIMED_WAITING => - s"\t- waiting on ${threadInfo.getLockInfo}" + s"\t- waiting on ${threadInfo.getLockInfo}\n" case _ => "" } } else "" - val locking = monitors.get(idx).map(mi => s"\t- locked $mi") - frame.toString + locked + locking + val locking = monitors.get(idx).map(mi => s"\t- locked $mi\n").getOrElse("") + s"${frame.toString}\n$locked$locking" }) - // use a set to dedup re-entrant locks that are held at multiple places - val heldLocks = - (threadInfo.getLockedSynchronizers ++ threadInfo.getLockedMonitors).map(_.lockString).toSet - + val synchronizers = threadInfo.getLockedSynchronizers.map(_.toString) ThreadStackTrace( threadInfo.getThreadId, threadInfo.getThreadName, @@ -2196,7 +2193,8 @@ private[spark] object Utils stackTrace, if (threadInfo.getLockOwnerId < 0) None else Some(threadInfo.getLockOwnerId), Option(threadInfo.getLockInfo).map(_.lockString).getOrElse(""), - heldLocks.toSeq, + synchronizers, + monitors.values.toSeq, Option(threadInfo.getLockName), Option(threadInfo.getLockOwnerName), threadInfo.isSuspended, From 06a201cf90788662e8d6e970ff987bd280fcb756 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 21 Aug 2023 10:26:52 +0800 Subject: [PATCH 03/13] mima --- core/src/main/scala/org/apache/spark/status/api/v1/api.scala | 2 ++ core/src/main/scala/org/apache/spark/util/Utils.scala | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index 6b81f4ed78c9c..fc016eca21077 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -533,6 +533,8 @@ case class ThreadStackTrace( stackTrace: StackTrace, blockedByThreadId: Option[Long], blockedByLock: String, + @deprecated("using synchronizers and monitors instead", "4.0.0") + holdingLocks: Seq[String], synchronizers: Seq[String], monitors: Seq[String], lockName: Option[String], diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 3fb87cc757199..7e4d9b78af2d6 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -2186,6 +2186,7 @@ private[spark] object Utils }) val synchronizers = threadInfo.getLockedSynchronizers.map(_.toString) + val monitorStrs = monitors.values.toSeq ThreadStackTrace( threadInfo.getThreadId, threadInfo.getThreadName, @@ -2193,8 +2194,9 @@ private[spark] object Utils stackTrace, if (threadInfo.getLockOwnerId < 0) None else Some(threadInfo.getLockOwnerId), Option(threadInfo.getLockInfo).map(_.lockString).getOrElse(""), + synchronizers ++ monitorStrs, synchronizers, - monitors.values.toSeq, + monitorStrs, Option(threadInfo.getLockName), Option(threadInfo.getLockOwnerName), threadInfo.isSuspended, From 89f0af57aa745f2c953b920059f2f3b519c96b74 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 21 Aug 2023 11:18:57 +0800 Subject: [PATCH 04/13] css --- .../org/apache/spark/ui/static/webui.css | 6 +++++ .../ui/exec/ExecutorThreadDumpPage.scala | 26 ++++++++----------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.css b/core/src/main/resources/org/apache/spark/ui/static/webui.css index f952f86503e30..ca7c1f8ba65e2 100755 --- a/core/src/main/resources/org/apache/spark/ui/static/webui.css +++ b/core/src/main/resources/org/apache/spark/ui/static/webui.css @@ -286,6 +286,12 @@ span.expand-dag-viz, .collapse-table { a.expandbutton { cursor: pointer; + margin-right: 10px; +} + +a.downloadbutton { + cursor: pointer; + margin-right: 10px; } .threaddump-td-mouseover { diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala index ad19e93ba7927..0be9df921d1b8 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala @@ -69,21 +69,17 @@ private[ui] class ExecutorThreadDumpPage(

Updated at {UIUtils.formatDate(time)}

{ // scalastyle:off -

- Expand All -

-

- Collapse All -

-

- Download -

-
-
-
-
- - +
+ Expand All + Collapse All + Download +
+
+
+
+ + +
From 59813b0edc66877b9644e30839cfdc0c62498705 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 21 Aug 2023 11:26:52 +0800 Subject: [PATCH 05/13] mima --- .../org/apache/spark/status/api/v1/api.scala | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index fc016eca21077..f4b64f4919aeb 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -527,20 +527,20 @@ case class StackTrace(elems: Seq[String]) { } case class ThreadStackTrace( - threadId: Long, - threadName: String, - threadState: Thread.State, - stackTrace: StackTrace, - blockedByThreadId: Option[Long], - blockedByLock: String, + val threadId: Long, + val threadName: String, + val threadState: Thread.State, + val stackTrace: StackTrace, + val blockedByThreadId: Option[Long], + val blockedByLock: String, @deprecated("using synchronizers and monitors instead", "4.0.0") - holdingLocks: Seq[String], - synchronizers: Seq[String], - monitors: Seq[String], - lockName: Option[String], - lockOwnerName: Option[String], - suspended: Boolean, - inNative: Boolean) { + val holdingLocks: Seq[String], + val synchronizers: Seq[String], + val monitors: Seq[String], + val lockName: Option[String], + val lockOwnerName: Option[String], + val suspended: Boolean, + val inNative: Boolean) { /** * Returns a string representation of this thread stack trace From 3411da875c3351b85149517be07941a1c23d2474 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 21 Aug 2023 12:24:20 +0800 Subject: [PATCH 06/13] Revert "css" This reverts commit 89f0af57aa745f2c953b920059f2f3b519c96b74. --- .../org/apache/spark/ui/static/webui.css | 6 ----- .../ui/exec/ExecutorThreadDumpPage.scala | 26 +++++++++++-------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.css b/core/src/main/resources/org/apache/spark/ui/static/webui.css index ca7c1f8ba65e2..f952f86503e30 100755 --- a/core/src/main/resources/org/apache/spark/ui/static/webui.css +++ b/core/src/main/resources/org/apache/spark/ui/static/webui.css @@ -286,12 +286,6 @@ span.expand-dag-viz, .collapse-table { a.expandbutton { cursor: pointer; - margin-right: 10px; -} - -a.downloadbutton { - cursor: pointer; - margin-right: 10px; } .threaddump-td-mouseover { diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala index 0be9df921d1b8..ad19e93ba7927 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala @@ -69,17 +69,21 @@ private[ui] class ExecutorThreadDumpPage(

Updated at {UIUtils.formatDate(time)}

{ // scalastyle:off -
- Expand All - Collapse All - Download -
-
-
-
- - -
+

+ Expand All +

+

+ Collapse All +

+

+ Download +

+
+
+
+
+ +
From 43ccfa6894982078fab414dcc0c478320ddf433a Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 21 Aug 2023 12:24:38 +0800 Subject: [PATCH 07/13] Revert "Revert "css"" This reverts commit 3411da875c3351b85149517be07941a1c23d2474. --- .../org/apache/spark/ui/static/webui.css | 6 +++++ .../ui/exec/ExecutorThreadDumpPage.scala | 26 ++++++++----------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.css b/core/src/main/resources/org/apache/spark/ui/static/webui.css index f952f86503e30..ca7c1f8ba65e2 100755 --- a/core/src/main/resources/org/apache/spark/ui/static/webui.css +++ b/core/src/main/resources/org/apache/spark/ui/static/webui.css @@ -286,6 +286,12 @@ span.expand-dag-viz, .collapse-table { a.expandbutton { cursor: pointer; + margin-right: 10px; +} + +a.downloadbutton { + cursor: pointer; + margin-right: 10px; } .threaddump-td-mouseover { diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala index ad19e93ba7927..0be9df921d1b8 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala @@ -69,21 +69,17 @@ private[ui] class ExecutorThreadDumpPage(

Updated at {UIUtils.formatDate(time)}

{ // scalastyle:off -

- Expand All -

-

- Collapse All -

-

- Download -

-
-
-
-
- - +
+ Expand All + Collapse All + Download +
+
+
+
+ + +
From 4935a6d061b26d538669958dc2c2830ee9cff6ee Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 21 Aug 2023 12:24:49 +0800 Subject: [PATCH 08/13] Revert "mima" This reverts commit 59813b0edc66877b9644e30839cfdc0c62498705. --- .../org/apache/spark/status/api/v1/api.scala | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index f4b64f4919aeb..fc016eca21077 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -527,20 +527,20 @@ case class StackTrace(elems: Seq[String]) { } case class ThreadStackTrace( - val threadId: Long, - val threadName: String, - val threadState: Thread.State, - val stackTrace: StackTrace, - val blockedByThreadId: Option[Long], - val blockedByLock: String, + threadId: Long, + threadName: String, + threadState: Thread.State, + stackTrace: StackTrace, + blockedByThreadId: Option[Long], + blockedByLock: String, @deprecated("using synchronizers and monitors instead", "4.0.0") - val holdingLocks: Seq[String], - val synchronizers: Seq[String], - val monitors: Seq[String], - val lockName: Option[String], - val lockOwnerName: Option[String], - val suspended: Boolean, - val inNative: Boolean) { + holdingLocks: Seq[String], + synchronizers: Seq[String], + monitors: Seq[String], + lockName: Option[String], + lockOwnerName: Option[String], + suspended: Boolean, + inNative: Boolean) { /** * Returns a string representation of this thread stack trace From c485f0a2ae6e9e6c8dade168958bcc7f5c7f3918 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 21 Aug 2023 12:25:58 +0800 Subject: [PATCH 09/13] mima skip --- project/MimaExcludes.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index f818d9aa2c8a0..6bca1d9a13821 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -81,6 +81,8 @@ object MimaExcludes { ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.GroupState"), // [SPARK-44705][PYTHON] Make PythonRunner single-threaded ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.api.python.BasePythonRunner#ReaderIterator.this") + //[SPARK-44863][UI] Add a button to download thread dump as a txt in Spark UI + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ThreadStackTrace.*") ) // Default exclude rules From fe5d0673b5706580259cf795f9e110a17350e421 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 21 Aug 2023 13:39:03 +0800 Subject: [PATCH 10/13] fix --- project/MimaExcludes.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 6bca1d9a13821..18d9d160a0ff8 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -80,7 +80,7 @@ object MimaExcludes { ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SaveMode"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.GroupState"), // [SPARK-44705][PYTHON] Make PythonRunner single-threaded - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.api.python.BasePythonRunner#ReaderIterator.this") + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.api.python.BasePythonRunner#ReaderIterator.this"), //[SPARK-44863][UI] Add a button to download thread dump as a txt in Spark UI ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ThreadStackTrace.*") ) From 3eaa7aff9ed61a1012856a8344b28b0a7f2a3dd4 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 21 Aug 2023 15:40:21 +0800 Subject: [PATCH 11/13] fix --- project/MimaExcludes.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 18d9d160a0ff8..6bdf8d460e01e 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -36,6 +36,9 @@ object MimaExcludes { // Exclude rules for 4.0.x lazy val v40excludes = v35excludes ++ Seq( + // [SPARK-44863][UI] Add a button to download thread dump as a txt in Spark UI + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ThreadStackTrace.*"), + ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.status.api.v1.ThreadStackTrace$") ) // Exclude rules for 3.5.x from 3.4.0 @@ -80,9 +83,7 @@ object MimaExcludes { ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SaveMode"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.GroupState"), // [SPARK-44705][PYTHON] Make PythonRunner single-threaded - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.api.python.BasePythonRunner#ReaderIterator.this"), - //[SPARK-44863][UI] Add a button to download thread dump as a txt in Spark UI - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ThreadStackTrace.*") + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.api.python.BasePythonRunner#ReaderIterator.this") ) // Default exclude rules From 1bb469272de4628a92ace6690ca14b384a269358 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 21 Aug 2023 19:12:18 +0800 Subject: [PATCH 12/13] address comments --- .../main/scala/org/apache/spark/status/api/v1/api.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index fc016eca21077..3e4e2f17a77e1 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -546,15 +546,13 @@ case class ThreadStackTrace( * Returns a string representation of this thread stack trace * w.r.t java.lang.management.ThreadInfo(JDK 8)'s toString. * - * TODO(Kent Yao): Considering 'daemon', 'priority' from higher JDKs + * TODO(SPARK-44895): Considering 'daemon', 'priority' from higher JDKs * - * TODO(Kent Yao): Also considering adding information os_prio, cpu, elapsed, tid, nid, etc., + * TODO(SPARK-44896): Also considering adding information os_prio, cpu, elapsed, tid, nid, etc., * from the jstack tool */ override def toString: String = { - val sb = new StringBuilder() - val basic = "\"" + threadName + "\" Id=" + threadId + " " + threadState - sb.append(basic) + val sb = new StringBuilder(s""""$threadName" Id=$threadId $threadState""") lockName.foreach(lock => sb.append(s" on $lock")) lockOwnerName.foreach { owner => sb.append(s"""owned by "$owner"""") From fc2f4726cfb29bd9460e7f176f0e8e88a22b1b3c Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 24 Aug 2023 11:53:49 +0800 Subject: [PATCH 13/13] addr comments - v35excludes --- project/MimaExcludes.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 6bdf8d460e01e..6200f24eed1cc 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -36,9 +36,6 @@ object MimaExcludes { // Exclude rules for 4.0.x lazy val v40excludes = v35excludes ++ Seq( - // [SPARK-44863][UI] Add a button to download thread dump as a txt in Spark UI - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ThreadStackTrace.*"), - ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.status.api.v1.ThreadStackTrace$") ) // Exclude rules for 3.5.x from 3.4.0 @@ -83,7 +80,10 @@ object MimaExcludes { ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SaveMode"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.GroupState"), // [SPARK-44705][PYTHON] Make PythonRunner single-threaded - ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.api.python.BasePythonRunner#ReaderIterator.this") + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.api.python.BasePythonRunner#ReaderIterator.this"), + // [SPARK-44863][UI] Add a button to download thread dump as a txt in Spark UI + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ThreadStackTrace.*"), + ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.status.api.v1.ThreadStackTrace$") ) // Default exclude rules