From 43024bbc91cfc1b05a03b0030e72bc8ccf1f8e50 Mon Sep 17 00:00:00 2001
From: Ziqi Liu <ziqi.liu@databricks.com>
Date: Mon, 26 Aug 2024 17:39:51 +0800
Subject: [PATCH 1/3] expose task peak onheap/offheap execution memory to API

---
 .../spark/status/protobuf/store_types.proto   | 10 ++++
 .../org/apache/spark/ui/static/stagepage.js   | 51 ++++++++++++++++++-
 .../spark/ui/static/stagespage-template.html  |  2 +
 .../apache/spark/status/AppStatusStore.scala  | 10 ++++
 .../org/apache/spark/status/LiveEntity.scala  | 16 +++++-
 .../spark/status/api/v1/StagesResource.scala  |  2 +
 .../org/apache/spark/status/api/v1/api.scala  |  6 +++
 .../protobuf/CachedQuantileSerializer.scala   |  4 ++
 .../protobuf/StageDataWrapperSerializer.scala | 14 +++++
 .../protobuf/TaskDataWrapperSerializer.scala  |  4 ++
 .../org/apache/spark/status/storeTypes.scala  | 10 ++++
 .../org/apache/spark/ui/jobs/JobPage.scala    |  2 +
 .../org/apache/spark/ui/jobs/StagePage.scala  | 18 +++++++
 .../spark/executor/TaskMetricsSuite.scala     |  6 +++
 .../spark/status/AppStatusStoreSuite.scala    |  4 +-
 .../spark/status/AppStatusUtilsSuite.scala    |  4 ++
 .../KVStoreProtobufSerializerSuite.scala      | 20 ++++++++
 .../org/apache/spark/ui/StagePageSuite.scala  |  2 +
 .../apache/spark/util/JsonProtocolSuite.scala |  2 +
 19 files changed, 183 insertions(+), 4 deletions(-)

diff --git a/core/src/main/protobuf/org/apache/spark/status/protobuf/store_types.proto b/core/src/main/protobuf/org/apache/spark/status/protobuf/store_types.proto
index 386c660b16def..f962d71f5d43d 100644
--- a/core/src/main/protobuf/org/apache/spark/status/protobuf/store_types.proto
+++ b/core/src/main/protobuf/org/apache/spark/status/protobuf/store_types.proto
@@ -128,6 +128,8 @@ message TaskDataWrapper {
   int64 shuffle_merged_local_bytes_read = 49;
   int64 shuffle_remote_reqs_duration = 50;
   int64 shuffle_merged_remote_req_duration = 51;
+  int64 peak_on_heap_execution_memory = 52;
+  int64 peak_off_heap_execution_memory = 53;
 }
 
 message ExecutorMetrics {
@@ -323,6 +325,8 @@ message CachedQuantile {
   double shuffle_merged_local_bytes_read = 40;
   double shuffle_remote_reqs_duration = 41;
   double shuffle_merged_remote_reqs_duration = 42;
+  double peak_on_heap_execution_memory = 43;
+  double peak_off_heap_execution_memory = 44;
 }
 
 message SpeculationStageSummary {
@@ -611,6 +615,8 @@ message StageData {
   int64 shuffle_merged_remote_reqs_duration = 62;
   bool is_shuffle_push_enabled = 63;
   int32 shuffle_mergers_count = 64;
+  int64 peak_on_heap_execution_memory = 65;
+  int64 peak_off_heap_execution_memory = 66;
 }
 
 message TaskMetrics {
@@ -628,6 +634,8 @@ message TaskMetrics {
   OutputMetrics output_metrics = 12;
   ShuffleReadMetrics shuffle_read_metrics = 13;
   ShuffleWriteMetrics shuffle_write_metrics = 14;
+  int64 peak_on_heap_execution_memory = 15;
+  int64 peak_off_heap_execution_memory = 16;
 }
 
 message InputMetrics {
@@ -689,6 +697,8 @@ message TaskMetricDistributions {
   OutputMetricDistributions output_metrics = 16;
   ShuffleReadMetricDistributions shuffle_read_metrics = 17;
   ShuffleWriteMetricDistributions shuffle_write_metrics = 18;
+  repeated double peak_on_heap_execution_memory = 19;
+  repeated double peak_off_heap_execution_memory = 20;
 }
 
 message InputMetricDistributions {
diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
index 55305dee3899c..3489cacca6e01 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
@@ -129,6 +129,12 @@ function getColumnNameForTaskMetricSummary(columnKey) {
     case "peakExecutionMemory":
       return "Peak Execution Memory";
 
+    case "peakOnHeapExecutionMemory":
+      return "Peak OnHeap Execution Memory";
+
+    case "peakOffHeapExecutionMemory":
+      return "Peak OffHeap Execution Memory";
+
     case "resultSerializationTime":
       return "Result Serialization Time";
 
@@ -201,7 +207,8 @@ function displayRowsForSummaryMetricsTable(row, type, columnIndex) {
  
     default:
       return (row.columnKey == 'peakExecutionMemory' || row.columnKey == 'memoryBytesSpilled'
-        || row.columnKey == 'diskBytesSpilled') ? formatBytes(
+        || row.columnKey == 'diskBytesSpilled' || row.columnKey == 'peakOnHeapExecutionMemory'
+        || row.columnKey == 'peakOffHeapExecutionMemory') ? formatBytes(
           row.data[columnIndex], type) : (formatDuration(row.data[columnIndex]));
 
   }
@@ -326,7 +333,7 @@ function getStageAttemptId() {
 var taskSummaryMetricsTableArray = [];
 var taskSummaryMetricsTableCurrentStateArray = [];
 var taskSummaryMetricsDataTable;
-var optionalColumns = [11, 12, 13, 14, 15, 16, 17, 21];
+var optionalColumns = [11, 12, 13, 14, 15, 16, 17, 21, 26, 27];
 var taskTableSelector;
 
 var executorOptionalColumns = [15, 16, 17, 18];
@@ -350,6 +357,8 @@ $(document).ready(function () {
     "<div id='result_serialization_time' class='result-serialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-15' data-column='15' data-metrics-type='task'> Result Serialization Time</div>" +
     "<div id='getting_result_time' class='getting-result-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-16' data-column='16' data-metrics-type='task'> Getting Result Time</div>" +
     "<div id='peak_execution_memory' class='peak-execution-memory-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-17' data-column='17' data-metrics-type='task'> Peak Execution Memory</div>" +
+    "<div id='peak_onheap_execution_memory' class='peak-onheap-execution-memory-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-26' data-column='26' data-metrics-type='task'> Peak OnHeap Execution Memory</div>" +
+    "<div id='peak_offheap_execution_memory' class='peak-offheap-execution-memory-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-27' data-column='27' data-metrics-type='task'> Peak OffHeap Execution Memory</div>" +
     "<div id='executor_jvm_on_off_heap_memory' class='executor-jvm-metrics-checkbox-div'><input type='checkbox' class='toggle-vis' id='executor-box-15'  data-column='15' data-metrics-type='executor'> Peak JVM Memory OnHeap / OffHeap</div>" +
     "<div id='executor_on_off_heap_execution_memory' class='executor-jvm-metrics-checkbox-div'><input type='checkbox' class='toggle-vis' id='executor-box-16' data-column='16' data-metrics-type='executor'> Peak Execution Memory OnHeap / OffHeap</div>" +
     "<div id='executor_on_off_heap_storage_memory' class='executor-jvm-metrics-checkbox-div'><input type='checkbox' class='toggle-vis' id='executor-box-17' data-column='17' data-metrics-type='executor'> Peak Storage Memory OnHeap / OffHeap</div>" +
@@ -385,6 +394,12 @@ $(document).ready(function () {
       "should be approximately the sum of the peak sizes across all such data structures created " +
       "in this task. For SQL jobs, this only tracks all unsafe operators, broadcast joins, and " +
       "external sort.");
+  $('#peak_onheap_execution_memory').attr("data-toggle", "tooltip")
+    .attr("data-placement", "top")
+    .attr("title", "Peak OnHeap Execution memory of this stage");
+  $('#peak_offheap_execution_memory').attr("data-toggle", "tooltip")
+    .attr("data-placement", "top")
+    .attr("title", "Peak OffHeap Execution memory of this stage");
   $('[data-toggle="tooltip"]').tooltip();
   var tasksSummary = $("#parent-container");
   getStandAloneAppId(function (appId) {
@@ -747,6 +762,18 @@ $(document).ready(function () {
                   taskSummaryMetricsTableArray.push(row);
                   break;
 
+                case "peakOnHeapExecutionMemory":
+                  row = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 26);
+                  taskSummaryMetricsTableArray.push(row);
+                  break;
+
+                case "peakOffHeapExecutionMemory":
+                  row = createRowMetadataForColumn(
+                    columnKey, taskMetricsResponse[columnKey], 27);
+                  taskSummaryMetricsTableArray.push(row);
+                  break;
+
                 case "inputMetrics":
                   row = createRowMetadataForColumn(
                     columnKey, taskMetricsResponse[columnKey], 1);
@@ -1069,6 +1096,26 @@ $(document).ready(function () {
               },
               name: "Spill (Disk)"
             },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.peakOnHeapExecutionMemory) {
+                  return type === 'display' ? formatBytes(row.taskMetrics.peakOnHeapExecutionMemory, type) : row.taskMetrics.peakOnHeapExecutionMemory;
+                } else {
+                  return "";
+                }
+              },
+              name: "Peak OnHeap Execution Memory"
+            },
+            {
+              data : function (row, type) {
+                if (row.taskMetrics && row.taskMetrics.peakOffHeapExecutionMemory) {
+                  return type === 'display' ? formatBytes(row.taskMetrics.peakOffHeapExecutionMemory, type) : row.taskMetrics.peakOffHeapExecutionMemory;
+                } else {
+                  return "";
+                }
+              },
+              name: "Peak OffHeap Execution Memory"
+            },
             {
               data : function (row, _ignored_type) {
                 var msg = row.errorMessage;
diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html b/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html
index 86ec73e91a76c..95868603f5c61 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html
@@ -129,6 +129,8 @@ <h4 id="tasksTitle" class="title-table"></h4>
                     <th>Shuffle Read Size / Records</th>
                     <th>Spill (Memory)</th>
                     <th>Spill (Disk)</th>
+                    <th>Peak OnHeap Execution Memory</th>
+                    <th>Peak OffHeap Execution Memory</th>
                     <th>Errors</th>
                 </tr>
                 </thead>
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index 87f876467c30e..7b3afb5455629 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -308,6 +308,8 @@ private[spark] class AppStatusStore(
         gettingResultTime = toValues(_.gettingResultTime),
         schedulerDelay = toValues(_.schedulerDelay),
         peakExecutionMemory = toValues(_.peakExecutionMemory),
+        peakOnHeapExecutionMemory = toValues(_.peakOnHeapExecutionMemory),
+        peakOffHeapExecutionMemory = toValues(_.peakOffHeapExecutionMemory),
         memoryBytesSpilled = toValues(_.memoryBytesSpilled),
         diskBytesSpilled = toValues(_.diskBytesSpilled),
         inputMetrics = new v1.InputMetricDistributions(
@@ -399,6 +401,10 @@ private[spark] class AppStatusStore(
       },
       schedulerDelay = scanTasks(TaskIndexNames.SCHEDULER_DELAY) { t => t.schedulerDelay },
       peakExecutionMemory = scanTasks(TaskIndexNames.PEAK_MEM) { t => t.peakExecutionMemory },
+      peakOnHeapExecutionMemory =
+        scanTasks(TaskIndexNames.PEAK_ON_HEAP_MEM) { t => t.peakOnHeapExecutionMemory },
+      peakOffHeapExecutionMemory =
+        scanTasks(TaskIndexNames.PEAK_OFF_HEAP_MEM) { t => t.peakOffHeapExecutionMemory },
       memoryBytesSpilled = scanTasks(TaskIndexNames.MEM_SPILL) { t => t.memoryBytesSpilled },
       diskBytesSpilled = scanTasks(TaskIndexNames.DISK_SPILL) { t => t.diskBytesSpilled },
       inputMetrics = new v1.InputMetricDistributions(
@@ -474,6 +480,8 @@ private[spark] class AppStatusStore(
           gettingResultTime = computedQuantiles.gettingResultTime(idx),
           schedulerDelay = computedQuantiles.schedulerDelay(idx),
           peakExecutionMemory = computedQuantiles.peakExecutionMemory(idx),
+          peakOnHeapExecutionMemory = computedQuantiles.peakOnHeapExecutionMemory(idx),
+          peakOffHeapExecutionMemory = computedQuantiles.peakOffHeapExecutionMemory(idx),
           memoryBytesSpilled = computedQuantiles.memoryBytesSpilled(idx),
           diskBytesSpilled = computedQuantiles.diskBytesSpilled(idx),
 
@@ -692,6 +700,8 @@ private[spark] class AppStatusStore(
         memoryBytesSpilled = stage.memoryBytesSpilled,
         diskBytesSpilled = stage.diskBytesSpilled,
         peakExecutionMemory = stage.peakExecutionMemory,
+        peakOnHeapExecutionMemory = stage.peakOnHeapExecutionMemory,
+        peakOffHeapExecutionMemory = stage.peakOffHeapExecutionMemory,
         inputBytes = stage.inputBytes,
         inputRecords = stage.inputRecords,
         outputBytes = stage.outputBytes,
diff --git a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
index efc670440bc64..c92a36311e6df 100644
--- a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
+++ b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
@@ -150,6 +150,8 @@ private class LiveTask(
         metrics.memoryBytesSpilled,
         metrics.diskBytesSpilled,
         metrics.peakExecutionMemory,
+        metrics.peakOnHeapExecutionMemory,
+        metrics.peakOffHeapExecutionMemory,
         metrics.inputMetrics.bytesRead,
         metrics.inputMetrics.recordsRead,
         metrics.outputMetrics.bytesWritten,
@@ -236,6 +238,8 @@ private class LiveTask(
       taskMetrics.memoryBytesSpilled,
       taskMetrics.diskBytesSpilled,
       taskMetrics.peakExecutionMemory,
+      taskMetrics.peakOnHeapExecutionMemory,
+      taskMetrics.peakOffHeapExecutionMemory,
       taskMetrics.inputMetrics.bytesRead,
       taskMetrics.inputMetrics.recordsRead,
       taskMetrics.outputMetrics.bytesWritten,
@@ -511,6 +515,8 @@ private class LiveStage(var info: StageInfo) extends LiveEntity {
       memoryBytesSpilled = metrics.memoryBytesSpilled,
       diskBytesSpilled = metrics.diskBytesSpilled,
       peakExecutionMemory = metrics.peakExecutionMemory,
+      peakOnHeapExecutionMemory = metrics.peakOnHeapExecutionMemory,
+      peakOffHeapExecutionMemory = metrics.peakOffHeapExecutionMemory,
       inputBytes = metrics.inputMetrics.bytesRead,
       inputRecords = metrics.inputMetrics.recordsRead,
       outputBytes = metrics.outputMetrics.bytesWritten,
@@ -762,6 +768,8 @@ private[spark] object LiveEntityHelpers {
       memoryBytesSpilled: Long,
       diskBytesSpilled: Long,
       peakExecutionMemory: Long,
+      peakOnHeapExecutionMemory: Long,
+      peakOffHeapExecutionMemory: Long,
       inputBytesRead: Long,
       inputRecordsRead: Long,
       outputBytesWritten: Long,
@@ -797,6 +805,8 @@ private[spark] object LiveEntityHelpers {
       memoryBytesSpilled,
       diskBytesSpilled,
       peakExecutionMemory,
+      peakOnHeapExecutionMemory,
+      peakOffHeapExecutionMemory,
       new v1.InputMetrics(
         inputBytesRead,
         inputRecordsRead),
@@ -834,7 +844,7 @@ private[spark] object LiveEntityHelpers {
     createMetrics(default, default, default, default, default, default, default, default,
       default, default, default, default, default, default, default, default, default,
       default, default, default, default, default, default, default, default, default,
-      default, default, default, default, default, default, default, default)
+      default, default, default, default, default, default, default, default, default, default)
   }
 
   /** Add m2 values to m1. */
@@ -869,6 +879,8 @@ private[spark] object LiveEntityHelpers {
       updateMetricValue(m.memoryBytesSpilled),
       updateMetricValue(m.diskBytesSpilled),
       updateMetricValue(m.peakExecutionMemory),
+      updateMetricValue(m.peakOnHeapExecutionMemory),
+      updateMetricValue(m.peakOffHeapExecutionMemory),
       updateMetricValue(m.inputMetrics.bytesRead),
       updateMetricValue(m.inputMetrics.recordsRead),
       updateMetricValue(m.shuffleReadMetrics.shufflePushReadMetrics.corruptMergedBlockChunks),
@@ -907,6 +919,8 @@ private[spark] object LiveEntityHelpers {
       m1.memoryBytesSpilled + m2.memoryBytesSpilled * mult,
       m1.diskBytesSpilled + m2.diskBytesSpilled * mult,
       m1.peakExecutionMemory + m2.peakExecutionMemory * mult,
+      m1.peakOnHeapExecutionMemory + m2.peakOnHeapExecutionMemory * mult,
+      m1.peakOffHeapExecutionMemory + m2.peakOffHeapExecutionMemory * mult,
       m1.inputMetrics.bytesRead + m2.inputMetrics.bytesRead * mult,
       m1.inputMetrics.recordsRead + m2.inputMetrics.recordsRead * mult,
       m1.outputMetrics.bytesWritten + m2.outputMetrics.bytesWritten * mult,
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
index ee18d5044ea7a..fdb47c94f4ccc 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
@@ -227,6 +227,8 @@ private[v1] class StagesResource extends BaseAppResource {
         || containsValue(Utils.bytesToString(task.taskMetrics.get.memoryBytesSpilled))
         || containsValue(Utils.bytesToString(task.taskMetrics.get.diskBytesSpilled))
         || containsValue(Utils.bytesToString(task.taskMetrics.get.peakExecutionMemory))
+        || containsValue(Utils.bytesToString(task.taskMetrics.get.peakOnHeapExecutionMemory))
+        || containsValue(Utils.bytesToString(task.taskMetrics.get.peakOffHeapExecutionMemory))
         || containsValue(Utils.bytesToString(task.taskMetrics.get.inputMetrics.bytesRead))
         || containsValue(task.taskMetrics.get.inputMetrics.recordsRead)
         || containsValue(Utils.bytesToString(
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 6ae1dce57f31c..62b586c852113 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -272,6 +272,8 @@ class StageData private[spark](
     val memoryBytesSpilled: Long,
     val diskBytesSpilled: Long,
     val peakExecutionMemory: Long,
+    val peakOnHeapExecutionMemory: Long,
+    val peakOffHeapExecutionMemory: Long,
     val inputBytes: Long,
     val inputRecords: Long,
     val outputBytes: Long,
@@ -350,6 +352,8 @@ class TaskMetrics private[spark](
     val memoryBytesSpilled: Long,
     val diskBytesSpilled: Long,
     val peakExecutionMemory: Long,
+    val peakOnHeapExecutionMemory: Long,
+    val peakOffHeapExecutionMemory: Long,
     val inputMetrics: InputMetrics,
     val outputMetrics: OutputMetrics,
     val shuffleReadMetrics: ShuffleReadMetrics,
@@ -404,6 +408,8 @@ class TaskMetricDistributions private[spark](
     val gettingResultTime: IndexedSeq[Double],
     val schedulerDelay: IndexedSeq[Double],
     val peakExecutionMemory: IndexedSeq[Double],
+    val peakOnHeapExecutionMemory: IndexedSeq[Double],
+    val peakOffHeapExecutionMemory: IndexedSeq[Double],
     val memoryBytesSpilled: IndexedSeq[Double],
     val diskBytesSpilled: IndexedSeq[Double],
 
diff --git a/core/src/main/scala/org/apache/spark/status/protobuf/CachedQuantileSerializer.scala b/core/src/main/scala/org/apache/spark/status/protobuf/CachedQuantileSerializer.scala
index 89acf8e242570..e10b78d5aa4f5 100644
--- a/core/src/main/scala/org/apache/spark/status/protobuf/CachedQuantileSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/status/protobuf/CachedQuantileSerializer.scala
@@ -38,6 +38,8 @@ private[protobuf] class CachedQuantileSerializer extends ProtobufSerDe[CachedQua
       .setGettingResultTime(data.gettingResultTime)
       .setSchedulerDelay(data.schedulerDelay)
       .setPeakExecutionMemory(data.peakExecutionMemory)
+      .setPeakOnHeapExecutionMemory(data.peakOnHeapExecutionMemory)
+      .setPeakOffHeapExecutionMemory(data.peakOffHeapExecutionMemory)
       .setMemoryBytesSpilled(data.memoryBytesSpilled)
       .setDiskBytesSpilled(data.diskBytesSpilled)
       .setBytesRead(data.bytesRead)
@@ -87,6 +89,8 @@ private[protobuf] class CachedQuantileSerializer extends ProtobufSerDe[CachedQua
       gettingResultTime = binary.getGettingResultTime,
       schedulerDelay = binary.getSchedulerDelay,
       peakExecutionMemory = binary.getPeakExecutionMemory,
+      peakOnHeapExecutionMemory = binary.getPeakOnHeapExecutionMemory,
+      peakOffHeapExecutionMemory = binary.getPeakOffHeapExecutionMemory,
       memoryBytesSpilled = binary.getMemoryBytesSpilled,
       diskBytesSpilled = binary.getDiskBytesSpilled,
       bytesRead = binary.getBytesRead,
diff --git a/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala b/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala
index d83cff5f23a27..a3248e1711bab 100644
--- a/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala
@@ -62,6 +62,8 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
       .setMemoryBytesSpilled(stageData.memoryBytesSpilled)
       .setDiskBytesSpilled(stageData.diskBytesSpilled)
       .setPeakExecutionMemory(stageData.peakExecutionMemory)
+      .setPeakOnHeapExecutionMemory(stageData.peakOnHeapExecutionMemory)
+      .setPeakOffHeapExecutionMemory(stageData.peakOffHeapExecutionMemory)
       .setInputBytes(stageData.inputBytes)
       .setInputRecords(stageData.inputRecords)
       .setOutputBytes(stageData.outputBytes)
@@ -192,6 +194,8 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
       .setMemoryBytesSpilled(tm.memoryBytesSpilled)
       .setDiskBytesSpilled(tm.diskBytesSpilled)
       .setPeakExecutionMemory(tm.peakExecutionMemory)
+      .setPeakOnHeapExecutionMemory(tm.peakOnHeapExecutionMemory)
+      .setPeakOffHeapExecutionMemory(tm.peakOffHeapExecutionMemory)
       .setInputMetrics(serializeInputMetrics(tm.inputMetrics))
       .setOutputMetrics(serializeOutputMetrics(tm.outputMetrics))
       .setShuffleReadMetrics(serializeShuffleReadMetrics(tm.shuffleReadMetrics))
@@ -278,6 +282,8 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
     tmd.gettingResultTime.foreach(grt => builder.addGettingResultTime(grt))
     tmd.schedulerDelay.foreach(sd => builder.addSchedulerDelay(sd))
     tmd.peakExecutionMemory.foreach(pem => builder.addPeakExecutionMemory(pem))
+    tmd.peakOnHeapExecutionMemory.foreach(peonm => builder.addPeakOnHeapExecutionMemory(peonm))
+    tmd.peakOffHeapExecutionMemory.foreach(peoffm => builder.addPeakOffHeapExecutionMemory(peoffm))
     tmd.memoryBytesSpilled.foreach(mbs => builder.addMemoryBytesSpilled(mbs))
     tmd.diskBytesSpilled.foreach(dbs => builder.addDiskBytesSpilled(dbs))
     builder
@@ -441,6 +447,8 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
       memoryBytesSpilled = binary.getMemoryBytesSpilled,
       diskBytesSpilled = binary.getDiskBytesSpilled,
       peakExecutionMemory = binary.getPeakExecutionMemory,
+      peakOnHeapExecutionMemory = binary.getPeakOnHeapExecutionMemory,
+      peakOffHeapExecutionMemory = binary.getPeakOffHeapExecutionMemory,
       inputBytes = binary.getInputBytes,
       inputRecords = binary.getInputRecords,
       outputBytes = binary.getOutputBytes,
@@ -515,6 +523,10 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
       gettingResultTime = binary.getGettingResultTimeList.asScala.map(_.toDouble).toIndexedSeq,
       schedulerDelay = binary.getSchedulerDelayList.asScala.map(_.toDouble).toIndexedSeq,
       peakExecutionMemory = binary.getPeakExecutionMemoryList.asScala.map(_.toDouble).toIndexedSeq,
+      peakOnHeapExecutionMemory = binary.getPeakOnHeapExecutionMemoryList.asScala
+        .map(_.toDouble).toIndexedSeq,
+      peakOffHeapExecutionMemory = binary.getPeakOffHeapExecutionMemoryList.asScala
+        .map(_.toDouble).toIndexedSeq,
       memoryBytesSpilled = binary.getMemoryBytesSpilledList.asScala.map(_.toDouble).toIndexedSeq,
       diskBytesSpilled = binary.getDiskBytesSpilledList.asScala.map(_.toDouble).toIndexedSeq,
       inputMetrics = deserializeInputMetricDistributions(binary.getInputMetrics),
@@ -664,6 +676,8 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
       binary.getMemoryBytesSpilled,
       binary.getDiskBytesSpilled,
       binary.getPeakExecutionMemory,
+      binary.getPeakOnHeapExecutionMemory,
+      binary.getPeakOffHeapExecutionMemory,
       deserializeInputMetrics(binary.getInputMetrics),
       deserializeOutputMetrics(binary.getOutputMetrics),
       deserializeShuffleReadMetrics(binary.getShuffleReadMetrics),
diff --git a/core/src/main/scala/org/apache/spark/status/protobuf/TaskDataWrapperSerializer.scala b/core/src/main/scala/org/apache/spark/status/protobuf/TaskDataWrapperSerializer.scala
index 8c729fa947810..4396a1f556323 100644
--- a/core/src/main/scala/org/apache/spark/status/protobuf/TaskDataWrapperSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/status/protobuf/TaskDataWrapperSerializer.scala
@@ -44,6 +44,8 @@ private[protobuf] class TaskDataWrapperSerializer extends ProtobufSerDe[TaskData
       .setMemoryBytesSpilled(input.memoryBytesSpilled)
       .setDiskBytesSpilled(input.diskBytesSpilled)
       .setPeakExecutionMemory(input.peakExecutionMemory)
+      .setPeakOnHeapExecutionMemory(input.peakOnHeapExecutionMemory)
+      .setPeakOffHeapExecutionMemory(input.peakOffHeapExecutionMemory)
       .setInputBytesRead(input.inputBytesRead)
       .setInputRecordsRead(input.inputRecordsRead)
       .setOutputBytesWritten(input.outputBytesWritten)
@@ -111,6 +113,8 @@ private[protobuf] class TaskDataWrapperSerializer extends ProtobufSerDe[TaskData
       memoryBytesSpilled = binary.getMemoryBytesSpilled,
       diskBytesSpilled = binary.getDiskBytesSpilled,
       peakExecutionMemory = binary.getPeakExecutionMemory,
+      peakOnHeapExecutionMemory = binary.getPeakOnHeapExecutionMemory,
+      peakOffHeapExecutionMemory = binary.getPeakOffHeapExecutionMemory,
       inputBytesRead = binary.getInputBytesRead,
       inputRecordsRead = binary.getInputRecordsRead,
       outputBytesWritten = binary.getOutputBytesWritten,
diff --git a/core/src/main/scala/org/apache/spark/status/storeTypes.scala b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
index b53455207a0d3..ca64ebcec0148 100644
--- a/core/src/main/scala/org/apache/spark/status/storeTypes.scala
+++ b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
@@ -124,6 +124,8 @@ private[spark] object TaskIndexNames {
   final val OUTPUT_RECORDS = "or"
   final val OUTPUT_SIZE = "os"
   final val PEAK_MEM = "pem"
+  final val PEAK_ON_HEAP_MEM = "peonm"
+  final val PEAK_OFF_HEAP_MEM = "peoffm"
   final val RESULT_SIZE = "rs"
   final val SCHEDULER_DELAY = "dly"
   final val SER_TIME = "rst"
@@ -220,6 +222,10 @@ private[spark] class TaskDataWrapper(
     val diskBytesSpilled: Long,
     @KVIndexParam(value = TaskIndexNames.PEAK_MEM, parent = TaskIndexNames.STAGE)
     val peakExecutionMemory: Long,
+    @KVIndexParam(value = TaskIndexNames.PEAK_ON_HEAP_MEM, parent = TaskIndexNames.STAGE)
+    val peakOnHeapExecutionMemory: Long,
+    @KVIndexParam(value = TaskIndexNames.PEAK_OFF_HEAP_MEM, parent = TaskIndexNames.STAGE)
+    val peakOffHeapExecutionMemory: Long,
     @KVIndexParam(value = TaskIndexNames.INPUT_SIZE, parent = TaskIndexNames.STAGE)
     val inputBytesRead: Long,
     @KVIndexParam(value = TaskIndexNames.INPUT_RECORDS, parent = TaskIndexNames.STAGE)
@@ -307,6 +313,8 @@ private[spark] class TaskDataWrapper(
         getMetricValue(memoryBytesSpilled),
         getMetricValue(diskBytesSpilled),
         getMetricValue(peakExecutionMemory),
+        getMetricValue(peakOnHeapExecutionMemory),
+        getMetricValue(peakOffHeapExecutionMemory),
         new InputMetrics(
           getMetricValue(inputBytesRead),
           getMetricValue(inputRecordsRead)),
@@ -557,6 +565,8 @@ private[spark] class CachedQuantile(
     val gettingResultTime: Double,
     val schedulerDelay: Double,
     val peakExecutionMemory: Double,
+    val peakOnHeapExecutionMemory: Double,
+    val peakOffHeapExecutionMemory: Double,
     val memoryBytesSpilled: Double,
     val diskBytesSpilled: Double,
 
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
index 793e65f44ba90..f37310148ed1d 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
@@ -266,6 +266,8 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
           memoryBytesSpilled = 0L,
           diskBytesSpilled = 0L,
           peakExecutionMemory = 0L,
+          peakOnHeapExecutionMemory = 0L,
+          peakOffHeapExecutionMemory = 0L,
           inputBytes = 0L,
           inputRecords = 0L,
           outputBytes = 0L,
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 257f7b37758a7..a371a4171837c 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -189,6 +189,16 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
               {Utils.bytesToString(stageData.diskBytesSpilled)}
             </li>
           }}
+          {if (hasPeakExecutionMemory(stageData)) {
+            <li>
+              <strong>Peak OnHeap Execution Memory: </strong>
+              {Utils.bytesToString(stageData.peakOnHeapExecutionMemory)}
+            </li>
+            <li>
+              <strong>Peak OffHeap Execution Memory: </strong>
+              {Utils.bytesToString(stageData.peakOffHeapExecutionMemory)}
+            </li>
+        }}
           {if (!stageJobIds.isEmpty) {
             <li>
               <strong>Associated Job Ids: </strong>
@@ -465,6 +475,8 @@ private[spark] object ApiHelper {
   val HEADER_SER_TIME = "Result Serialization Time"
   val HEADER_GETTING_RESULT_TIME = "Getting Result Time"
   val HEADER_PEAK_MEM = "Peak Execution Memory"
+  val HEADER_ON_HEAP_PEAK_MEM = "Peak OnHeap Execution Memory"
+  val HEADER_OFF_HEAP_PEAK_MEM = "Peak OffHeap Execution Memory"
   val HEADER_ACCUMULATORS = "Accumulators"
   val HEADER_INPUT_SIZE = "Input Size / Records"
   val HEADER_OUTPUT_SIZE = "Output Size / Records"
@@ -495,6 +507,8 @@ private[spark] object ApiHelper {
     HEADER_SER_TIME -> TaskIndexNames.SER_TIME,
     HEADER_GETTING_RESULT_TIME -> TaskIndexNames.GETTING_RESULT_TIME,
     HEADER_PEAK_MEM -> TaskIndexNames.PEAK_MEM,
+    HEADER_ON_HEAP_PEAK_MEM -> TaskIndexNames.PEAK_ON_HEAP_MEM,
+    HEADER_OFF_HEAP_PEAK_MEM -> TaskIndexNames.PEAK_OFF_HEAP_MEM,
     HEADER_ACCUMULATORS -> TaskIndexNames.ACCUMULATORS,
     HEADER_INPUT_SIZE -> TaskIndexNames.INPUT_SIZE,
     HEADER_OUTPUT_SIZE -> TaskIndexNames.OUTPUT_SIZE,
@@ -527,6 +541,10 @@ private[spark] object ApiHelper {
     stageData.diskBytesSpilled > 0 || stageData.memoryBytesSpilled > 0
   }
 
+  def hasPeakExecutionMemory(stageData: StageData): Boolean = {
+    stageData.peakOnHeapExecutionMemory > 0 || stageData.peakOffHeapExecutionMemory > 0
+  }
+
   def totalBytesRead(metrics: ShuffleReadMetrics): Long = {
     metrics.localBytesRead + metrics.remoteBytesRead
   }
diff --git a/core/src/test/scala/org/apache/spark/executor/TaskMetricsSuite.scala b/core/src/test/scala/org/apache/spark/executor/TaskMetricsSuite.scala
index 7bcc2fb5231db..49ff2b8732e5c 100644
--- a/core/src/test/scala/org/apache/spark/executor/TaskMetricsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/TaskMetricsSuite.scala
@@ -37,6 +37,8 @@ class TaskMetricsSuite extends SparkFunSuite {
     assert(tm.memoryBytesSpilled == 0L)
     assert(tm.diskBytesSpilled == 0L)
     assert(tm.peakExecutionMemory == 0L)
+    assert(tm.peakOnHeapExecutionMemory == 0L)
+    assert(tm.peakOffHeapExecutionMemory == 0L)
     assert(tm.updatedBlockStatuses.isEmpty)
     // set or increment values
     tm.setExecutorDeserializeTime(100L)
@@ -55,6 +57,8 @@ class TaskMetricsSuite extends SparkFunSuite {
     tm.incDiskBytesSpilled(7L)
     tm.incPeakExecutionMemory(800L)
     tm.incPeakExecutionMemory(8L)
+    tm.setPeakOnHeapExecutionMemory(809L)
+    tm.setPeakOffHeapExecutionMemory(810L)
     val block1 = (TestBlockId("a"), BlockStatus(MEMORY_ONLY, 1L, 2L))
     val block2 = (TestBlockId("b"), BlockStatus(MEMORY_ONLY, 3L, 4L))
     tm.incUpdatedBlockStatuses(block1)
@@ -68,6 +72,8 @@ class TaskMetricsSuite extends SparkFunSuite {
     assert(tm.memoryBytesSpilled == 606L)
     assert(tm.diskBytesSpilled == 707L)
     assert(tm.peakExecutionMemory == 808L)
+    assert(tm.peakOnHeapExecutionMemory == 809L)
+    assert(tm.peakOffHeapExecutionMemory == 810L)
     assert(tm.updatedBlockStatuses == Seq(block1, block2))
   }
 
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
index f2b795764b7e8..e9446a190883d 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
@@ -181,6 +181,8 @@ class AppStatusStoreSuite extends SparkFunSuite {
         assertQuantiles(_.memoryBytesSpilled.toDouble, summary.memoryBytesSpilled)
         assertQuantiles(_.diskBytesSpilled.toDouble, summary.diskBytesSpilled)
         assertQuantiles(_.peakExecutionMemory.toDouble, summary.peakExecutionMemory)
+        assertQuantiles(_.peakOnHeapExecutionMemory.toDouble, summary.peakOnHeapExecutionMemory)
+        assertQuantiles(_.peakOffHeapExecutionMemory.toDouble, summary.peakOffHeapExecutionMemory)
         assertQuantiles(_.inputMetrics.bytesRead.toDouble, summary.inputMetrics.bytesRead)
         assertQuantiles(_.inputMetrics.recordsRead.toDouble, summary.inputMetrics.recordsRead)
         assertQuantiles(_.outputMetrics.bytesWritten.toDouble, summary.outputMetrics.bytesWritten)
@@ -282,7 +284,7 @@ class AppStatusStoreSuite extends SparkFunSuite {
       i, i, i, i, i, i, i, i, i, i,
       i, i, i, i, i, i, i, i, i, i,
       i, i, i, i, i, i, i, i, i, i,
-      i, i, i, i, stageId, attemptId)
+      i, i, i, i, i, i, stageId, attemptId)
   }
 
   private def writeTaskDataToStore(i: Int, store: KVStore, status: String): Unit = {
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala
index 9c1a280d2b68e..779c5772f85e5 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala
@@ -51,6 +51,8 @@ class AppStatusUtilsSuite extends SparkFunSuite {
         memoryBytesSpilled = 0L,
         diskBytesSpilled = 0L,
         peakExecutionMemory = 0L,
+        peakOnHeapExecutionMemory = 0L,
+        peakOffHeapExecutionMemory = 0L,
         inputMetrics = null,
         outputMetrics = null,
         shuffleReadMetrics = null,
@@ -86,6 +88,8 @@ class AppStatusUtilsSuite extends SparkFunSuite {
         memoryBytesSpilled = 0L,
         diskBytesSpilled = 0L,
         peakExecutionMemory = 100L,
+        peakOnHeapExecutionMemory = 110L,
+        peakOffHeapExecutionMemory = 120L,
         inputMetrics = null,
         outputMetrics = null,
         shuffleReadMetrics = null,
diff --git a/core/src/test/scala/org/apache/spark/status/protobuf/KVStoreProtobufSerializerSuite.scala b/core/src/test/scala/org/apache/spark/status/protobuf/KVStoreProtobufSerializerSuite.scala
index ac568fee1ad45..52d13da6beada 100644
--- a/core/src/test/scala/org/apache/spark/status/protobuf/KVStoreProtobufSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/protobuf/KVStoreProtobufSerializerSuite.scala
@@ -158,6 +158,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
         memoryBytesSpilled = 14L,
         diskBytesSpilled = 15L,
         peakExecutionMemory = 16L,
+        peakOnHeapExecutionMemory = 43L,
+        peakOffHeapExecutionMemory = 44L,
         inputBytesRead = 17L,
         inputRecordsRead = 18L,
         outputBytesWritten = 19L,
@@ -212,6 +214,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
       assert(result.memoryBytesSpilled == input.memoryBytesSpilled)
       assert(result.diskBytesSpilled == input.diskBytesSpilled)
       assert(result.peakExecutionMemory == input.peakExecutionMemory)
+      assert(result.peakOnHeapExecutionMemory == input.peakOnHeapExecutionMemory)
+      assert(result.peakOffHeapExecutionMemory == input.peakOffHeapExecutionMemory)
       assert(result.inputBytesRead == input.inputBytesRead)
       assert(result.inputRecordsRead == input.inputRecordsRead)
       assert(result.outputBytesWritten == input.outputBytesWritten)
@@ -645,6 +649,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
         gettingResultTime = 12.1,
         schedulerDelay = 13.1,
         peakExecutionMemory = 14.1,
+        peakOnHeapExecutionMemory = 14.2,
+        peakOffHeapExecutionMemory = 14.3,
         memoryBytesSpilled = 15.1,
         diskBytesSpilled = 16.1,
         bytesRead = 17.1,
@@ -689,6 +695,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
       assert(result.gettingResultTime == input.gettingResultTime)
       assert(result.schedulerDelay == input.schedulerDelay)
       assert(result.peakExecutionMemory == input.peakExecutionMemory)
+      assert(result.peakOnHeapExecutionMemory == input.peakOnHeapExecutionMemory)
+      assert(result.peakOffHeapExecutionMemory == input.peakOffHeapExecutionMemory)
       assert(result.memoryBytesSpilled == input.memoryBytesSpilled)
       assert(result.diskBytesSpilled == input.diskBytesSpilled)
       assert(result.bytesRead == input.bytesRead)
@@ -1064,6 +1072,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
       memoryBytesSpilled = 8L,
       diskBytesSpilled = 9L,
       peakExecutionMemory = 10L,
+      peakOnHeapExecutionMemory = 11L,
+      peakOffHeapExecutionMemory = 12L,
       inputMetrics = inputMetrics,
       outputMetrics = outputMetrics,
       shuffleReadMetrics = shuffleReadMetrics,
@@ -1209,6 +1219,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
       gettingResultTime = IndexedSeq(19.001D, 20.001D),
       schedulerDelay = IndexedSeq(21.001D, 22.001D),
       peakExecutionMemory = IndexedSeq(23.001D, 24.001D),
+      peakOnHeapExecutionMemory = IndexedSeq(29.001D, 30.001D),
+      peakOffHeapExecutionMemory = IndexedSeq(31.001D, 32.001D),
       memoryBytesSpilled = IndexedSeq(25.001D, 26.001D),
       diskBytesSpilled = IndexedSeq(27.001D, 28.001D),
       inputMetrics = inputMetricDistributions,
@@ -1263,6 +1275,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
       memoryBytesSpilled = 16L,
       diskBytesSpilled = 17L,
       peakExecutionMemory = 18L,
+      peakOnHeapExecutionMemory = 44L,
+      peakOffHeapExecutionMemory = 45L,
       inputBytes = 19L,
       inputRecords = 20L,
       outputBytes = 21L,
@@ -1345,6 +1359,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
     assert(result.info.memoryBytesSpilled == input.info.memoryBytesSpilled)
     assert(result.info.diskBytesSpilled == input.info.diskBytesSpilled)
     assert(result.info.peakExecutionMemory == input.info.peakExecutionMemory)
+    assert(result.info.peakOnHeapExecutionMemory == input.info.peakOnHeapExecutionMemory)
+    assert(result.info.peakOffHeapExecutionMemory == input.info.peakOffHeapExecutionMemory)
     assert(result.info.inputBytes == input.info.inputBytes)
     assert(result.info.inputRecords == input.info.inputRecords)
     assert(result.info.outputBytes == input.info.outputBytes)
@@ -1461,6 +1477,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
     assert(result.memoryBytesSpilled == expected.memoryBytesSpilled)
     assert(result.diskBytesSpilled == expected.diskBytesSpilled)
     assert(result.peakExecutionMemory == expected.peakExecutionMemory)
+    assert(result.peakOnHeapExecutionMemory == expected.peakOnHeapExecutionMemory)
+    assert(result.peakOffHeapExecutionMemory == expected.peakOffHeapExecutionMemory)
     checkAnswer(result.inputMetrics, expected.inputMetrics)
     checkAnswer(result.outputMetrics, expected.outputMetrics)
     checkAnswer(result.shuffleReadMetrics, expected.shuffleReadMetrics)
@@ -1613,6 +1631,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
     assert(result.gettingResultTime == expected.gettingResultTime)
     assert(result.schedulerDelay == expected.schedulerDelay)
     assert(result.peakExecutionMemory == expected.peakExecutionMemory)
+    assert(result.peakOnHeapExecutionMemory == expected.peakOnHeapExecutionMemory)
+    assert(result.peakOffHeapExecutionMemory == expected.peakOffHeapExecutionMemory)
     assert(result.memoryBytesSpilled == expected.memoryBytesSpilled)
     assert(result.diskBytesSpilled == expected.diskBytesSpilled)
 
diff --git a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
index c76c97d071418..d288837b3b2d8 100644
--- a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
@@ -65,6 +65,8 @@ class StagePageSuite extends SparkFunSuite with LocalSparkContext {
         memoryBytesSpilled = 1L,
         diskBytesSpilled = 1L,
         peakExecutionMemory = 1L,
+        peakOnHeapExecutionMemory = 1L,
+        peakOffHeapExecutionMemory = 1L,
         inputBytes = 1L,
         inputRecords = 1L,
         outputBytes = 1L,
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index cdee6ccda706e..31942809e8264 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -371,6 +371,8 @@ class JsonProtocolSuite extends SparkFunSuite {
     assert(newMetrics.executorDeserializeCpuTime == 0)
     assert(newMetrics.executorCpuTime == 0)
     assert(newMetrics.peakExecutionMemory == 0)
+    assert(newMetrics.peakOnHeapExecutionMemory == 0)
+    assert(newMetrics.peakOffHeapExecutionMemory == 0)
   }
 
   test("StorageLevel backward compatibility") {

From 5c3121a8b0ad20be2d42443d04ba8b15c9830143 Mon Sep 17 00:00:00 2001
From: Ziqi Liu <ziqi.liu@databricks.com>
Date: Mon, 16 Sep 2024 19:25:38 -0700
Subject: [PATCH 2/3] generate ground truth

---
 .../complete_stage_list_json_expectation.json |   6 ++
 ...xcludeOnFailure_for_stage_expectation.json |  26 +++++
 ...eOnFailure_node_for_stage_expectation.json |  30 ++++++
 .../failed_stage_list_json_expectation.json   |   2 +
 ..._details_with_failed_task_expectation.json |   2 +
 .../one_stage_attempt_json_expectation.json   |  18 ++++
 .../one_stage_json_expectation.json           |  18 ++++
 ...e_stage_json_with_details_expectation.json |  18 ++++
 ...age_json_with_partitionId_expectation.json |  22 ++++
 .../stage_list_json_expectation.json          |   8 ++
 ...ist_with_accumulable_json_expectation.json |   2 +
 ...ge_list_with_peak_metrics_expectation.json |   6 ++
 .../stage_task_list_expectation.json          |  40 +++++++
 ...multi_attempt_app_json_1__expectation.json |  16 +++
 ...multi_attempt_app_json_2__expectation.json |  16 +++
 ...k_list_w__offset___length_expectation.json | 100 ++++++++++++++++++
 ...stage_task_list_w__sortBy_expectation.json |  40 +++++++
 ...tBy_short_names___runtime_expectation.json |  40 +++++++
 ...rtBy_short_names__runtime_expectation.json |  40 +++++++
 ..._status___offset___length_expectation.json |   4 +
 ...rtBy_short_names__runtime_expectation.json |  40 +++++++
 ...stage_task_list_w__status_expectation.json |  20 ++++
 ...ask_list_with_partitionId_expectation.json |  16 +++
 ...mmary_w__custom_quantiles_expectation.json |   2 +
 ...sk_summary_w_shuffle_read_expectation.json |   2 +
 ...k_summary_w_shuffle_write_expectation.json |   2 +
 ...age_with_accumulable_json_expectation.json |  18 ++++
 .../stage_with_peak_metrics_expectation.json  |  34 ++++++
 ..._with_speculation_summary_expectation.json |  12 +++
 .../stage_with_summaries_expectation.json     |  36 +++++++
 30 files changed, 636 insertions(+)

diff --git a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
index ac0f2ce26051d..faf0abb8fd104 100644
--- a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 160,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -103,6 +105,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 28000128,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -185,6 +189,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
index d614bb000e4ce..fb02769768f8c 100644
--- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -79,6 +81,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -145,6 +149,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -211,6 +217,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -277,6 +285,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -343,6 +353,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -409,6 +421,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -476,6 +490,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -542,6 +558,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -608,6 +626,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -674,6 +694,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -740,6 +762,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -806,6 +830,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
index 475dee00a2654..95de6a9af9231 100644
--- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -78,6 +80,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -145,6 +149,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -211,6 +217,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -278,6 +286,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -345,6 +355,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -411,6 +423,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -477,6 +491,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -543,6 +559,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -609,6 +627,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -675,6 +695,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -741,6 +763,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -807,6 +831,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -873,6 +899,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -940,6 +968,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
index dc1bcd6a39625..30668430861c6 100644
--- a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
@@ -22,6 +22,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
index e24ac4f82b8a8..e24ce60de812b 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 28000128,
   "inputRecords" : 0,
   "outputBytes" : 0,
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
index 659e3c41d9289..b191b468ab54e 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 28000128,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -76,6 +78,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -139,6 +143,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -202,6 +208,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -265,6 +273,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -328,6 +338,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -391,6 +403,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -454,6 +468,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -517,6 +533,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
index f84cf26fcf1d6..788d1eccb3f89 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+"peakOnHeapExecutionMemory" : 0,
+"peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 28000128,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -76,6 +78,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -139,6 +143,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -202,6 +208,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -265,6 +273,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -328,6 +338,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -391,6 +403,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -454,6 +468,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -517,6 +533,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
index 564f3eadd1cc2..1fc695303b2c4 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 28000128,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -78,6 +80,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -141,6 +145,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -204,6 +210,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -267,6 +275,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -330,6 +340,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -393,6 +405,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -456,6 +470,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -519,6 +535,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json
index 2bf7f34803775..d4b92a2d33114 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -76,6 +78,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -139,6 +143,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -202,6 +208,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -265,6 +273,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -328,6 +338,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -391,6 +403,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -454,6 +468,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -517,6 +533,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -580,6 +598,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -643,6 +663,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
index 8df41bfcc8d78..c52215d3c3609 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 160,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -104,6 +106,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -186,6 +190,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 28000128,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -268,6 +274,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
index 730df3fbd5341..2c3ae38a65783 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
index 16d92244ee0d0..4ae08d9cf667c 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 384640,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -103,6 +105,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 16256,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -186,6 +190,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
index 2347b687d3dd1..3e94a2d6208bb 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
@@ -22,6 +22,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 49294,
       "recordsRead" : 10000
@@ -84,6 +86,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -146,6 +150,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -209,6 +215,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -271,6 +279,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -333,6 +343,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -395,6 +407,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -457,6 +471,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -519,6 +535,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -581,6 +599,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60489,
       "recordsRead" : 10000
@@ -643,6 +663,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -705,6 +727,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -767,6 +791,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -829,6 +855,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -891,6 +919,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -953,6 +983,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1015,6 +1047,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1077,6 +1111,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1139,6 +1175,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1201,6 +1239,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
index eaec1c32ba306..56aee9389f5c7 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
@@ -27,6 +27,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -94,6 +96,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -161,6 +165,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -228,6 +234,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -295,6 +303,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -362,6 +372,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -429,6 +441,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -496,6 +510,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
index 78566cbce9889..00dd2f0396820 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
@@ -27,6 +27,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -94,6 +96,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -161,6 +165,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -228,6 +234,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -295,6 +303,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -362,6 +372,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -429,6 +441,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -496,6 +510,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
index f9cf705d06c11..64d5c93ecf193 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
@@ -22,6 +22,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -84,6 +86,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -146,6 +150,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -208,6 +214,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -270,6 +278,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -332,6 +342,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -394,6 +406,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -456,6 +470,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -518,6 +534,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -580,6 +598,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -642,6 +662,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -704,6 +726,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -766,6 +790,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -828,6 +854,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -890,6 +918,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -952,6 +982,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1014,6 +1046,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1076,6 +1110,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1138,6 +1174,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1200,6 +1238,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1262,6 +1302,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1324,6 +1366,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1386,6 +1430,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1448,6 +1494,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1510,6 +1558,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1572,6 +1622,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1634,6 +1686,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1696,6 +1750,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1758,6 +1814,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1820,6 +1878,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1882,6 +1942,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1944,6 +2006,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2006,6 +2070,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2068,6 +2134,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2130,6 +2198,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2192,6 +2262,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2254,6 +2326,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2316,6 +2390,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2378,6 +2454,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2440,6 +2518,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2502,6 +2582,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2564,6 +2646,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2626,6 +2710,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2688,6 +2774,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2750,6 +2838,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2812,6 +2902,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2874,6 +2966,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2936,6 +3030,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2998,6 +3094,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -3060,6 +3158,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
index 2a52ea95807ed..d4ab4b351c39a 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
@@ -22,6 +22,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -84,6 +86,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -146,6 +150,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -208,6 +214,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -270,6 +278,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -332,6 +342,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -394,6 +406,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 49294,
       "recordsRead" : 10000
@@ -456,6 +470,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -518,6 +534,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -580,6 +598,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -642,6 +662,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -704,6 +726,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -766,6 +790,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -828,6 +854,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -890,6 +918,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60489,
       "recordsRead" : 10000
@@ -952,6 +982,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1014,6 +1046,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1076,6 +1110,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -1138,6 +1174,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1200,6 +1238,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
index 718d1e7a9512f..b0177cd2a8e8b 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
@@ -22,6 +22,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -84,6 +86,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -146,6 +150,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -208,6 +214,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -270,6 +278,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -332,6 +342,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -394,6 +406,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 49294,
       "recordsRead" : 10000
@@ -456,6 +470,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -518,6 +534,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -580,6 +598,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -642,6 +662,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -704,6 +726,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -766,6 +790,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -828,6 +854,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -890,6 +918,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60489,
       "recordsRead" : 10000
@@ -952,6 +982,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1014,6 +1046,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1076,6 +1110,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -1138,6 +1174,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1200,6 +1238,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
index 38a4a209c0452..40e6fe9d1b017 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
@@ -22,6 +22,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -84,6 +86,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -146,6 +150,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -208,6 +214,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -270,6 +278,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -332,6 +342,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -394,6 +406,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -456,6 +470,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -518,6 +534,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -580,6 +598,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -642,6 +662,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -704,6 +726,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -766,6 +790,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -828,6 +854,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -890,6 +918,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -952,6 +982,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70565,
       "recordsRead" : 10000
@@ -1014,6 +1046,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1076,6 +1110,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1138,6 +1174,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1200,6 +1238,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json
index f9cc66295f28a..e89b6d65079a6 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json
@@ -22,6 +22,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -84,6 +86,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json
index 368bbd862d7af..8f226fa316ba0 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json
@@ -22,6 +22,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -84,6 +86,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -146,6 +150,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -208,6 +214,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -270,6 +278,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -332,6 +342,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -394,6 +406,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -456,6 +470,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -518,6 +534,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -580,6 +598,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -642,6 +662,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -704,6 +726,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -766,6 +790,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -828,6 +854,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -890,6 +918,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -952,6 +982,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70565,
       "recordsRead" : 10000
@@ -1014,6 +1046,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1076,6 +1110,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1138,6 +1174,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1200,6 +1238,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json
index b07f86e30fabd..c80864dd72942 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json
@@ -23,6 +23,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -89,6 +91,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -155,6 +159,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -221,6 +227,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -287,6 +295,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -353,6 +363,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -419,6 +431,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -485,6 +499,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -551,6 +567,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -617,6 +635,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_with_partitionId_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_with_partitionId_expectation.json
index c4f1ec3624108..51fcdb41cf562 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_with_partitionId_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_with_partitionId_expectation.json
@@ -22,6 +22,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 12
@@ -84,6 +86,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 13
@@ -146,6 +150,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 12
@@ -208,6 +214,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 13
@@ -270,6 +278,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 12
@@ -332,6 +342,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 13
@@ -394,6 +406,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 12
@@ -456,6 +470,8 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "peakExecutionMemory" : 0,
+    "peakOnHeapExecutionMemory" : 0,
+    "peakOffHeapExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 13
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
index e3dba103bbd50..e8b2868ab0328 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
@@ -11,6 +11,8 @@
   "gettingResultTime" : [ 0.0, 0.0, 0.0 ],
   "schedulerDelay" : [ 2.0, 6.0, 53.0 ],
   "peakExecutionMemory" : [ 0.0, 0.0, 0.0 ],
+  "peakOnHeapExecutionMemory" : [ 0.0, 0.0, 0.0 ],
+  "peakOffHeapExecutionMemory" : [ 0.0, 0.0, 0.0 ],
   "memoryBytesSpilled" : [ 0.0, 0.0, 0.0 ],
   "diskBytesSpilled" : [ 0.0, 0.0, 0.0 ],
   "inputMetrics" : {
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
index ba7c794482640..37d99d2ff33b1 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
@@ -11,6 +11,8 @@
   "gettingResultTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
   "schedulerDelay" : [ 4.0, 4.0, 6.0, 7.0, 9.0 ],
   "peakExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+  "peakOnHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+  "peakOffHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
   "memoryBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
   "diskBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
   "inputMetrics" : {
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
index b463314150bad..0e184fb797ede 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
@@ -11,6 +11,8 @@
   "gettingResultTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
   "schedulerDelay" : [ 2.0, 4.0, 6.0, 13.0, 40.0 ],
   "peakExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+  "peakOnHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+  "peakOffHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
   "memoryBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
   "diskBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
   "inputMetrics" : {
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
index e38741b7bf6e7..ed583833cc2eb 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -85,6 +87,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -153,6 +157,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -221,6 +227,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -289,6 +297,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -357,6 +367,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -425,6 +437,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -493,6 +507,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -561,6 +577,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
index 630b0512e8f98..a8a925892b5ab 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 384640,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -76,6 +78,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -142,6 +146,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -208,6 +214,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -274,6 +282,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -340,6 +350,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -406,6 +418,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -472,6 +486,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -538,6 +554,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -604,6 +622,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -670,6 +690,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -736,6 +758,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -802,6 +826,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -868,6 +894,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -934,6 +962,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -1000,6 +1030,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -1066,6 +1098,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
index 23770480ad62a..10253b44c0c88 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 0,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -78,6 +80,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -144,6 +148,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -210,6 +216,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -276,6 +284,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -343,6 +353,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 0,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
index c8458a409589e..a059ec16908e7 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
@@ -21,6 +21,8 @@
   "memoryBytesSpilled" : 0,
   "diskBytesSpilled" : 0,
   "peakExecutionMemory" : 384640,
+  "peakOnHeapExecutionMemory" : 0,
+  "peakOffHeapExecutionMemory" : 0,
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -76,6 +78,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -142,6 +146,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -208,6 +214,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -274,6 +282,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -340,6 +350,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -406,6 +418,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -472,6 +486,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -538,6 +554,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -604,6 +622,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -670,6 +690,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -736,6 +758,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -802,6 +826,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -868,6 +894,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -934,6 +962,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -1000,6 +1030,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -1066,6 +1098,8 @@
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
         "peakExecutionMemory" : 24040,
+        "peakOnHeapExecutionMemory" : 0,
+        "peakOffHeapExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -1237,6 +1271,8 @@
     "gettingResultTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
     "schedulerDelay" : [ 62.0, 64.0, 70.0, 71.0, 79.0 ],
     "peakExecutionMemory" : [ 24040.0, 24040.0, 24040.0, 24040.0, 24040.0 ],
+    "peakOnHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "peakOffHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
     "memoryBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
     "diskBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
     "inputMetrics" : {

From 1fd59fe9082003bfcff3e38fc42fd156dabbece5 Mon Sep 17 00:00:00 2001
From: Ziqi Liu <ziqi.liu@databricks.com>
Date: Tue, 17 Sep 2024 11:34:30 -0700
Subject: [PATCH 3/3] fix JsonProtocolSuite

---
 .../test/scala/org/apache/spark/util/JsonProtocolSuite.scala  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index 31942809e8264..f497826eacd48 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -358,6 +358,8 @@ class JsonProtocolSuite extends SparkFunSuite {
   test("TaskMetrics backward compatibility") {
     // "Executor Deserialize CPU Time" and "Executor CPU Time" were introduced in Spark 2.1.0
     // "Peak Execution Memory" was introduced in Spark 3.0.0
+    // "Peak On Heap Execution Memory" and "Peak Off Heap Execution Memory" were introduced in
+    // Spark 3.5.0
     val metrics = makeTaskMetrics(1L, 2L, 3L, 4L, 5, 6, 0, hasHadoopInput = false, hasOutput = true)
     metrics.setExecutorDeserializeCpuTime(100L)
     metrics.setExecutorCpuTime(100L)
@@ -367,6 +369,8 @@ class JsonProtocolSuite extends SparkFunSuite {
       .removeField("Executor Deserialize CPU Time")
       .removeField("Executor CPU Time")
       .removeField("Peak Execution Memory")
+      .removeField("Peak On Heap Execution Memory")
+      .removeField("Peak Off Heap Execution Memory")
     val newMetrics = JsonProtocol.taskMetricsFromJson(oldJson)
     assert(newMetrics.executorDeserializeCpuTime == 0)
     assert(newMetrics.executorCpuTime == 0)