From 84d617cab34c77dd1665f2ca3a788d2b2b706cf7 Mon Sep 17 00:00:00 2001 From: Kay Ousterhout Date: Sat, 28 Feb 2015 17:51:28 -0800 Subject: [PATCH 1/4] [SPARK-6088] Correct how tasks that get remote results are shown in the UI. --- .../org/apache/spark/scheduler/TaskInfo.scala | 2 +- .../org/apache/spark/ui/jobs/StagePage.scala | 20 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskInfo.scala index 6fa1f2c880f7..28948f3f5396 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskInfo.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskInfo.scala @@ -82,7 +82,7 @@ class TaskInfo( def status: String = { if (running) { "RUNNING" - } else if (gettingResult) { + } else if (gettingResult && running) { "GET RESULT" } else if (failed) { "FAILED" diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index 110f8780a9a1..d512647b4ef2 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -268,11 +268,7 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { +: getFormattedTimeQuantiles(serializationTimes) val gettingResultTimes = validTasks.map { case TaskUIData(info, _, _) => - if (info.gettingResultTime > 0) { - (info.finishTime - info.gettingResultTime).toDouble - } else { - 0.0 - } + getGettingResultTime(info) } val gettingResultQuantiles = @@ -462,7 +458,7 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { val gcTime = metrics.map(_.jvmGCTime).getOrElse(0L) val taskDeserializationTime = metrics.map(_.executorDeserializeTime).getOrElse(0L) val serializationTime = metrics.map(_.resultSerializationTime).getOrElse(0L) - val gettingResultTime = info.gettingResultTime + val gettingResultTime = getGettingResultTime(info) val maybeAccumulators = info.accumulables val accumulatorsReadable = maybeAccumulators.map{acc => s"${acc.name}: ${acc.update.get}"} @@ -625,6 +621,14 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { {errorSummary}{details} } + private def getGettingResultTime(info: TaskInfo): Long = { + if (info.gettingResultTime > 0 && info.finishTime > 0) { + (info.finishTime - info.gettingResultTime).toDouble + } else { + 0.0 + } + } + private def getSchedulerDelay(info: TaskInfo, metrics: TaskMetrics): Long = { val totalExecutionTime = if (info.gettingResult) { @@ -636,6 +640,8 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { } val executorOverhead = (metrics.executorDeserializeTime + metrics.resultSerializationTime) - math.max(0, totalExecutionTime - metrics.executorRunTime - executorOverhead) + math.max( + 0, + totalExecutionTime - metrics.executorRunTime - executorOverhead - getGettingResultTime(info)) } } From 748ea6bfd33b084587dd6f26c6a184daf2125355 Mon Sep 17 00:00:00 2001 From: Kay Ousterhout Date: Sat, 28 Feb 2015 18:36:41 -0800 Subject: [PATCH 2/4] Fixed build failure --- core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index d512647b4ef2..e4570c19a8c4 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -268,7 +268,7 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { +: getFormattedTimeQuantiles(serializationTimes) val gettingResultTimes = validTasks.map { case TaskUIData(info, _, _) => - getGettingResultTime(info) + getGettingResultTime(info).toDouble } val gettingResultQuantiles = From f346b49bd02a31b4e3f64adc53a00a03dcf8a24b Mon Sep 17 00:00:00 2001 From: Kay Ousterhout Date: Sat, 28 Feb 2015 22:33:53 -0800 Subject: [PATCH 3/4] Typos --- core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index e4570c19a8c4..1b0ef972c924 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -623,9 +623,9 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { private def getGettingResultTime(info: TaskInfo): Long = { if (info.gettingResultTime > 0 && info.finishTime > 0) { - (info.finishTime - info.gettingResultTime).toDouble + info.finishTime - info.gettingResultTime } else { - 0.0 + 0L } } From 3ab012c892ababa3fd629a56944d5b6651d23c76 Mon Sep 17 00:00:00 2001 From: Kay Ousterhout Date: Sun, 1 Mar 2015 20:43:07 -0800 Subject: [PATCH 4/4] Update getting result time incrementally, correctly set GET_RESULT status --- .../main/scala/org/apache/spark/scheduler/TaskInfo.scala | 8 +++++--- .../main/scala/org/apache/spark/ui/jobs/StagePage.scala | 9 +++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskInfo.scala index 28948f3f5396..132a9ced7770 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskInfo.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskInfo.scala @@ -81,9 +81,11 @@ class TaskInfo( def status: String = { if (running) { - "RUNNING" - } else if (gettingResult && running) { - "GET RESULT" + if (gettingResult) { + "GET RESULT" + } else { + "RUNNING" + } } else if (failed) { "FAILED" } else if (successful) { diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index 1b0ef972c924..a1215562eb42 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -622,8 +622,13 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { } private def getGettingResultTime(info: TaskInfo): Long = { - if (info.gettingResultTime > 0 && info.finishTime > 0) { - info.finishTime - info.gettingResultTime + if (info.gettingResultTime > 0) { + if (info.finishTime > 0) { + info.finishTime - info.gettingResultTime + } else { + // The task is still fetching the result. + System.currentTimeMillis - info.gettingResultTime + } } else { 0L }