From 5c401c4a74eb21ede2e27ec8376527705a7f05a8 Mon Sep 17 00:00:00 2001 From: shivusondur Date: Thu, 27 Sep 2018 14:49:40 +0530 Subject: [PATCH 1/2] [SPARK-25550] Environment Page of Spark Job History UI showing wrong value for spark.ui.retainedJobs --- .../scala/org/apache/spark/status/AppStatusListener.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala index 36aaf67b57298..e65e2b0af19c7 100644 --- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala +++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala @@ -141,10 +141,14 @@ private[spark] class AppStatusListener( jvmInfo.get("Java Version").orNull, jvmInfo.get("Java Home").orNull, jvmInfo.get("Scala Version").orNull) - + val sparkProperties = if (!live) { + (details.getOrElse("Spark Properties", Nil).toMap ++ conf.getAll.toMap).toSeq + } else { + details.getOrElse("Spark Properties", Nil) + } val envInfo = new v1.ApplicationEnvironmentInfo( runtime, - details.getOrElse("Spark Properties", Nil), + sparkProperties, details.getOrElse("System Properties", Nil), details.getOrElse("Classpath Entries", Nil)) From 77e9de005636154beccdedb78b90d516800cf36f Mon Sep 17 00:00:00 2001 From: shivusondur Date: Sun, 7 Oct 2018 23:41:21 +0530 Subject: [PATCH 2/2] [SPARK-25550][WEBUI]Environment Page of Spark Job History UI showing wrong value for spark.ui.retainedJobs --- .../resources/org/apache/spark/ui/static/webui.js | 1 + .../apache/spark/status/AppStatusListener.scala | 11 ++++------- .../org/apache/spark/status/api/v1/api.scala | 3 ++- .../org/apache/spark/ui/env/EnvironmentPage.scala | 15 +++++++++++++++ docs/configuration.md | 2 +- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js b/core/src/main/resources/org/apache/spark/ui/static/webui.js index f01c567ba58ad..38234b4ef4cb6 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/webui.js +++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js @@ -63,6 +63,7 @@ $(function() { collapseTablePageLoad('collapse-aggregated-finishedDrivers','aggregated-finishedDrivers'); collapseTablePageLoad('collapse-aggregated-runtimeInformation','aggregated-runtimeInformation'); collapseTablePageLoad('collapse-aggregated-sparkProperties','aggregated-sparkProperties'); + collapseTablePageLoad('collapse-aggregated-historyProperties','aggregated-historyProperties'); collapseTablePageLoad('collapse-aggregated-systemProperties','aggregated-systemProperties'); collapseTablePageLoad('collapse-aggregated-classpathEntries','aggregated-classpathEntries'); collapseTablePageLoad('collapse-aggregated-activeJobs','aggregated-activeJobs'); diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala index e65e2b0af19c7..772e92c989d39 100644 --- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala +++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala @@ -141,16 +141,13 @@ private[spark] class AppStatusListener( jvmInfo.get("Java Version").orNull, jvmInfo.get("Java Home").orNull, jvmInfo.get("Scala Version").orNull) - val sparkProperties = if (!live) { - (details.getOrElse("Spark Properties", Nil).toMap ++ conf.getAll.toMap).toSeq - } else { - details.getOrElse("Spark Properties", Nil) - } + val envInfo = new v1.ApplicationEnvironmentInfo( runtime, - sparkProperties, + details.getOrElse("Spark Properties", Nil), details.getOrElse("System Properties", Nil), - details.getOrElse("Classpath Entries", Nil)) + details.getOrElse("Classpath Entries", Nil), + if (!live) conf.getAll else Nil) coresPerTask = envInfo.sparkProperties.toMap.get("spark.task.cpus").map(_.toInt) .getOrElse(coresPerTask) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index 77466b62ff6ed..490f2206457d8 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -347,7 +347,8 @@ class ApplicationEnvironmentInfo private[spark] ( val runtime: RuntimeInfo, val sparkProperties: Seq[(String, String)], val systemProperties: Seq[(String, String)], - val classpathEntries: Seq[(String, String)]) + val classpathEntries: Seq[(String, String)], + val historyProperties: Seq[(String, String)]) class RuntimeInfo private[spark]( val javaVersion: String, diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala index 3d465a34e44aa..26a60085f7fd5 100644 --- a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala @@ -42,6 +42,8 @@ private[ui] class EnvironmentPage( propertyHeader, jvmRow, jvmInformation, fixedWidth = true) val sparkPropertiesTable = UIUtils.listingTable(propertyHeader, propertyRow, Utils.redact(conf, appEnv.sparkProperties.toSeq), fixedWidth = true) + val historyPropertiesTable = UIUtils.listingTable(propertyHeader, propertyRow, + Utils.redact(conf, appEnv.historyProperties.toSeq), fixedWidth = true) val systemPropertiesTable = UIUtils.listingTable( propertyHeader, propertyRow, appEnv.systemProperties, fixedWidth = true) val classpathEntriesTable = UIUtils.listingTable( @@ -70,6 +72,19 @@ private[ui] class EnvironmentPage(
{sparkPropertiesTable}
+ {if (appEnv.historyProperties.nonEmpty) { + +

+ + History Properties +

+ +
+ {historyPropertiesTable} +
+ }} diff --git a/docs/configuration.md b/docs/configuration.md index 782ccff667076..42d201258fb94 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -938,7 +938,7 @@ Apart from these, the following properties are also available, and may be useful org.apache.spark.io.LZ4CompressionCodec, org.apache.spark.io.LZFCompressionCodec, org.apache.spark.io.SnappyCompressionCodec, - and org.apache.spark.io.ZstdCompressionCodec. + and org.apache.spark.io.ZStdCompressionCodec.