diff --git a/core/src/main/protobuf/org/apache/spark/status/protobuf/store_types.proto b/core/src/main/protobuf/org/apache/spark/status/protobuf/store_types.proto
index 386c660b16def..f962d71f5d43d 100644
--- a/core/src/main/protobuf/org/apache/spark/status/protobuf/store_types.proto
+++ b/core/src/main/protobuf/org/apache/spark/status/protobuf/store_types.proto
@@ -128,6 +128,8 @@ message TaskDataWrapper {
int64 shuffle_merged_local_bytes_read = 49;
int64 shuffle_remote_reqs_duration = 50;
int64 shuffle_merged_remote_req_duration = 51;
+ int64 peak_on_heap_execution_memory = 52;
+ int64 peak_off_heap_execution_memory = 53;
}
message ExecutorMetrics {
@@ -323,6 +325,8 @@ message CachedQuantile {
double shuffle_merged_local_bytes_read = 40;
double shuffle_remote_reqs_duration = 41;
double shuffle_merged_remote_reqs_duration = 42;
+ double peak_on_heap_execution_memory = 43;
+ double peak_off_heap_execution_memory = 44;
}
message SpeculationStageSummary {
@@ -611,6 +615,8 @@ message StageData {
int64 shuffle_merged_remote_reqs_duration = 62;
bool is_shuffle_push_enabled = 63;
int32 shuffle_mergers_count = 64;
+ int64 peak_on_heap_execution_memory = 65;
+ int64 peak_off_heap_execution_memory = 66;
}
message TaskMetrics {
@@ -628,6 +634,8 @@ message TaskMetrics {
OutputMetrics output_metrics = 12;
ShuffleReadMetrics shuffle_read_metrics = 13;
ShuffleWriteMetrics shuffle_write_metrics = 14;
+ int64 peak_on_heap_execution_memory = 15;
+ int64 peak_off_heap_execution_memory = 16;
}
message InputMetrics {
@@ -689,6 +697,8 @@ message TaskMetricDistributions {
OutputMetricDistributions output_metrics = 16;
ShuffleReadMetricDistributions shuffle_read_metrics = 17;
ShuffleWriteMetricDistributions shuffle_write_metrics = 18;
+ repeated double peak_on_heap_execution_memory = 19;
+ repeated double peak_off_heap_execution_memory = 20;
}
message InputMetricDistributions {
diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
index 55305dee3899c..3489cacca6e01 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
@@ -129,6 +129,12 @@ function getColumnNameForTaskMetricSummary(columnKey) {
case "peakExecutionMemory":
return "Peak Execution Memory";
+ case "peakOnHeapExecutionMemory":
+ return "Peak OnHeap Execution Memory";
+
+ case "peakOffHeapExecutionMemory":
+ return "Peak OffHeap Execution Memory";
+
case "resultSerializationTime":
return "Result Serialization Time";
@@ -201,7 +207,8 @@ function displayRowsForSummaryMetricsTable(row, type, columnIndex) {
default:
return (row.columnKey == 'peakExecutionMemory' || row.columnKey == 'memoryBytesSpilled'
- || row.columnKey == 'diskBytesSpilled') ? formatBytes(
+ || row.columnKey == 'diskBytesSpilled' || row.columnKey == 'peakOnHeapExecutionMemory'
+ || row.columnKey == 'peakOffHeapExecutionMemory') ? formatBytes(
row.data[columnIndex], type) : (formatDuration(row.data[columnIndex]));
}
@@ -326,7 +333,7 @@ function getStageAttemptId() {
var taskSummaryMetricsTableArray = [];
var taskSummaryMetricsTableCurrentStateArray = [];
var taskSummaryMetricsDataTable;
-var optionalColumns = [11, 12, 13, 14, 15, 16, 17, 21];
+var optionalColumns = [11, 12, 13, 14, 15, 16, 17, 21, 26, 27];
var taskTableSelector;
var executorOptionalColumns = [15, 16, 17, 18];
@@ -350,6 +357,8 @@ $(document).ready(function () {
"
Result Serialization Time
" +
" Getting Result Time
" +
" Peak Execution Memory
" +
+ " Peak OnHeap Execution Memory
" +
+ " Peak OffHeap Execution Memory
" +
" Peak JVM Memory OnHeap / OffHeap
" +
" Peak Execution Memory OnHeap / OffHeap
" +
" Peak Storage Memory OnHeap / OffHeap
" +
@@ -385,6 +394,12 @@ $(document).ready(function () {
"should be approximately the sum of the peak sizes across all such data structures created " +
"in this task. For SQL jobs, this only tracks all unsafe operators, broadcast joins, and " +
"external sort.");
+ $('#peak_onheap_execution_memory').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Peak OnHeap Execution memory of this stage");
+ $('#peak_offheap_execution_memory').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Peak OffHeap Execution memory of this stage");
$('[data-toggle="tooltip"]').tooltip();
var tasksSummary = $("#parent-container");
getStandAloneAppId(function (appId) {
@@ -747,6 +762,18 @@ $(document).ready(function () {
taskSummaryMetricsTableArray.push(row);
break;
+ case "peakOnHeapExecutionMemory":
+ row = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 26);
+ taskSummaryMetricsTableArray.push(row);
+ break;
+
+ case "peakOffHeapExecutionMemory":
+ row = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 27);
+ taskSummaryMetricsTableArray.push(row);
+ break;
+
case "inputMetrics":
row = createRowMetadataForColumn(
columnKey, taskMetricsResponse[columnKey], 1);
@@ -1069,6 +1096,26 @@ $(document).ready(function () {
},
name: "Spill (Disk)"
},
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.peakOnHeapExecutionMemory) {
+ return type === 'display' ? formatBytes(row.taskMetrics.peakOnHeapExecutionMemory, type) : row.taskMetrics.peakOnHeapExecutionMemory;
+ } else {
+ return "";
+ }
+ },
+ name: "Peak OnHeap Execution Memory"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.peakOffHeapExecutionMemory) {
+ return type === 'display' ? formatBytes(row.taskMetrics.peakOffHeapExecutionMemory, type) : row.taskMetrics.peakOffHeapExecutionMemory;
+ } else {
+ return "";
+ }
+ },
+ name: "Peak OffHeap Execution Memory"
+ },
{
data : function (row, _ignored_type) {
var msg = row.errorMessage;
diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html b/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html
index 86ec73e91a76c..95868603f5c61 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html
@@ -129,6 +129,8 @@
Shuffle Read Size / Records |
Spill (Memory) |
Spill (Disk) |
+ Peak OnHeap Execution Memory |
+ Peak OffHeap Execution Memory |
Errors |
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index 87f876467c30e..7b3afb5455629 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -308,6 +308,8 @@ private[spark] class AppStatusStore(
gettingResultTime = toValues(_.gettingResultTime),
schedulerDelay = toValues(_.schedulerDelay),
peakExecutionMemory = toValues(_.peakExecutionMemory),
+ peakOnHeapExecutionMemory = toValues(_.peakOnHeapExecutionMemory),
+ peakOffHeapExecutionMemory = toValues(_.peakOffHeapExecutionMemory),
memoryBytesSpilled = toValues(_.memoryBytesSpilled),
diskBytesSpilled = toValues(_.diskBytesSpilled),
inputMetrics = new v1.InputMetricDistributions(
@@ -399,6 +401,10 @@ private[spark] class AppStatusStore(
},
schedulerDelay = scanTasks(TaskIndexNames.SCHEDULER_DELAY) { t => t.schedulerDelay },
peakExecutionMemory = scanTasks(TaskIndexNames.PEAK_MEM) { t => t.peakExecutionMemory },
+ peakOnHeapExecutionMemory =
+ scanTasks(TaskIndexNames.PEAK_ON_HEAP_MEM) { t => t.peakOnHeapExecutionMemory },
+ peakOffHeapExecutionMemory =
+ scanTasks(TaskIndexNames.PEAK_OFF_HEAP_MEM) { t => t.peakOffHeapExecutionMemory },
memoryBytesSpilled = scanTasks(TaskIndexNames.MEM_SPILL) { t => t.memoryBytesSpilled },
diskBytesSpilled = scanTasks(TaskIndexNames.DISK_SPILL) { t => t.diskBytesSpilled },
inputMetrics = new v1.InputMetricDistributions(
@@ -474,6 +480,8 @@ private[spark] class AppStatusStore(
gettingResultTime = computedQuantiles.gettingResultTime(idx),
schedulerDelay = computedQuantiles.schedulerDelay(idx),
peakExecutionMemory = computedQuantiles.peakExecutionMemory(idx),
+ peakOnHeapExecutionMemory = computedQuantiles.peakOnHeapExecutionMemory(idx),
+ peakOffHeapExecutionMemory = computedQuantiles.peakOffHeapExecutionMemory(idx),
memoryBytesSpilled = computedQuantiles.memoryBytesSpilled(idx),
diskBytesSpilled = computedQuantiles.diskBytesSpilled(idx),
@@ -692,6 +700,8 @@ private[spark] class AppStatusStore(
memoryBytesSpilled = stage.memoryBytesSpilled,
diskBytesSpilled = stage.diskBytesSpilled,
peakExecutionMemory = stage.peakExecutionMemory,
+ peakOnHeapExecutionMemory = stage.peakOnHeapExecutionMemory,
+ peakOffHeapExecutionMemory = stage.peakOffHeapExecutionMemory,
inputBytes = stage.inputBytes,
inputRecords = stage.inputRecords,
outputBytes = stage.outputBytes,
diff --git a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
index efc670440bc64..c92a36311e6df 100644
--- a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
+++ b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
@@ -150,6 +150,8 @@ private class LiveTask(
metrics.memoryBytesSpilled,
metrics.diskBytesSpilled,
metrics.peakExecutionMemory,
+ metrics.peakOnHeapExecutionMemory,
+ metrics.peakOffHeapExecutionMemory,
metrics.inputMetrics.bytesRead,
metrics.inputMetrics.recordsRead,
metrics.outputMetrics.bytesWritten,
@@ -236,6 +238,8 @@ private class LiveTask(
taskMetrics.memoryBytesSpilled,
taskMetrics.diskBytesSpilled,
taskMetrics.peakExecutionMemory,
+ taskMetrics.peakOnHeapExecutionMemory,
+ taskMetrics.peakOffHeapExecutionMemory,
taskMetrics.inputMetrics.bytesRead,
taskMetrics.inputMetrics.recordsRead,
taskMetrics.outputMetrics.bytesWritten,
@@ -511,6 +515,8 @@ private class LiveStage(var info: StageInfo) extends LiveEntity {
memoryBytesSpilled = metrics.memoryBytesSpilled,
diskBytesSpilled = metrics.diskBytesSpilled,
peakExecutionMemory = metrics.peakExecutionMemory,
+ peakOnHeapExecutionMemory = metrics.peakOnHeapExecutionMemory,
+ peakOffHeapExecutionMemory = metrics.peakOffHeapExecutionMemory,
inputBytes = metrics.inputMetrics.bytesRead,
inputRecords = metrics.inputMetrics.recordsRead,
outputBytes = metrics.outputMetrics.bytesWritten,
@@ -762,6 +768,8 @@ private[spark] object LiveEntityHelpers {
memoryBytesSpilled: Long,
diskBytesSpilled: Long,
peakExecutionMemory: Long,
+ peakOnHeapExecutionMemory: Long,
+ peakOffHeapExecutionMemory: Long,
inputBytesRead: Long,
inputRecordsRead: Long,
outputBytesWritten: Long,
@@ -797,6 +805,8 @@ private[spark] object LiveEntityHelpers {
memoryBytesSpilled,
diskBytesSpilled,
peakExecutionMemory,
+ peakOnHeapExecutionMemory,
+ peakOffHeapExecutionMemory,
new v1.InputMetrics(
inputBytesRead,
inputRecordsRead),
@@ -834,7 +844,7 @@ private[spark] object LiveEntityHelpers {
createMetrics(default, default, default, default, default, default, default, default,
default, default, default, default, default, default, default, default, default,
default, default, default, default, default, default, default, default, default,
- default, default, default, default, default, default, default, default)
+ default, default, default, default, default, default, default, default, default, default)
}
/** Add m2 values to m1. */
@@ -869,6 +879,8 @@ private[spark] object LiveEntityHelpers {
updateMetricValue(m.memoryBytesSpilled),
updateMetricValue(m.diskBytesSpilled),
updateMetricValue(m.peakExecutionMemory),
+ updateMetricValue(m.peakOnHeapExecutionMemory),
+ updateMetricValue(m.peakOffHeapExecutionMemory),
updateMetricValue(m.inputMetrics.bytesRead),
updateMetricValue(m.inputMetrics.recordsRead),
updateMetricValue(m.shuffleReadMetrics.shufflePushReadMetrics.corruptMergedBlockChunks),
@@ -907,6 +919,8 @@ private[spark] object LiveEntityHelpers {
m1.memoryBytesSpilled + m2.memoryBytesSpilled * mult,
m1.diskBytesSpilled + m2.diskBytesSpilled * mult,
m1.peakExecutionMemory + m2.peakExecutionMemory * mult,
+ m1.peakOnHeapExecutionMemory + m2.peakOnHeapExecutionMemory * mult,
+ m1.peakOffHeapExecutionMemory + m2.peakOffHeapExecutionMemory * mult,
m1.inputMetrics.bytesRead + m2.inputMetrics.bytesRead * mult,
m1.inputMetrics.recordsRead + m2.inputMetrics.recordsRead * mult,
m1.outputMetrics.bytesWritten + m2.outputMetrics.bytesWritten * mult,
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
index ee18d5044ea7a..fdb47c94f4ccc 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
@@ -227,6 +227,8 @@ private[v1] class StagesResource extends BaseAppResource {
|| containsValue(Utils.bytesToString(task.taskMetrics.get.memoryBytesSpilled))
|| containsValue(Utils.bytesToString(task.taskMetrics.get.diskBytesSpilled))
|| containsValue(Utils.bytesToString(task.taskMetrics.get.peakExecutionMemory))
+ || containsValue(Utils.bytesToString(task.taskMetrics.get.peakOnHeapExecutionMemory))
+ || containsValue(Utils.bytesToString(task.taskMetrics.get.peakOffHeapExecutionMemory))
|| containsValue(Utils.bytesToString(task.taskMetrics.get.inputMetrics.bytesRead))
|| containsValue(task.taskMetrics.get.inputMetrics.recordsRead)
|| containsValue(Utils.bytesToString(
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 6ae1dce57f31c..62b586c852113 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -272,6 +272,8 @@ class StageData private[spark](
val memoryBytesSpilled: Long,
val diskBytesSpilled: Long,
val peakExecutionMemory: Long,
+ val peakOnHeapExecutionMemory: Long,
+ val peakOffHeapExecutionMemory: Long,
val inputBytes: Long,
val inputRecords: Long,
val outputBytes: Long,
@@ -350,6 +352,8 @@ class TaskMetrics private[spark](
val memoryBytesSpilled: Long,
val diskBytesSpilled: Long,
val peakExecutionMemory: Long,
+ val peakOnHeapExecutionMemory: Long,
+ val peakOffHeapExecutionMemory: Long,
val inputMetrics: InputMetrics,
val outputMetrics: OutputMetrics,
val shuffleReadMetrics: ShuffleReadMetrics,
@@ -404,6 +408,8 @@ class TaskMetricDistributions private[spark](
val gettingResultTime: IndexedSeq[Double],
val schedulerDelay: IndexedSeq[Double],
val peakExecutionMemory: IndexedSeq[Double],
+ val peakOnHeapExecutionMemory: IndexedSeq[Double],
+ val peakOffHeapExecutionMemory: IndexedSeq[Double],
val memoryBytesSpilled: IndexedSeq[Double],
val diskBytesSpilled: IndexedSeq[Double],
diff --git a/core/src/main/scala/org/apache/spark/status/protobuf/CachedQuantileSerializer.scala b/core/src/main/scala/org/apache/spark/status/protobuf/CachedQuantileSerializer.scala
index 89acf8e242570..e10b78d5aa4f5 100644
--- a/core/src/main/scala/org/apache/spark/status/protobuf/CachedQuantileSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/status/protobuf/CachedQuantileSerializer.scala
@@ -38,6 +38,8 @@ private[protobuf] class CachedQuantileSerializer extends ProtobufSerDe[CachedQua
.setGettingResultTime(data.gettingResultTime)
.setSchedulerDelay(data.schedulerDelay)
.setPeakExecutionMemory(data.peakExecutionMemory)
+ .setPeakOnHeapExecutionMemory(data.peakOnHeapExecutionMemory)
+ .setPeakOffHeapExecutionMemory(data.peakOffHeapExecutionMemory)
.setMemoryBytesSpilled(data.memoryBytesSpilled)
.setDiskBytesSpilled(data.diskBytesSpilled)
.setBytesRead(data.bytesRead)
@@ -87,6 +89,8 @@ private[protobuf] class CachedQuantileSerializer extends ProtobufSerDe[CachedQua
gettingResultTime = binary.getGettingResultTime,
schedulerDelay = binary.getSchedulerDelay,
peakExecutionMemory = binary.getPeakExecutionMemory,
+ peakOnHeapExecutionMemory = binary.getPeakOnHeapExecutionMemory,
+ peakOffHeapExecutionMemory = binary.getPeakOffHeapExecutionMemory,
memoryBytesSpilled = binary.getMemoryBytesSpilled,
diskBytesSpilled = binary.getDiskBytesSpilled,
bytesRead = binary.getBytesRead,
diff --git a/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala b/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala
index d83cff5f23a27..a3248e1711bab 100644
--- a/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala
@@ -62,6 +62,8 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
.setMemoryBytesSpilled(stageData.memoryBytesSpilled)
.setDiskBytesSpilled(stageData.diskBytesSpilled)
.setPeakExecutionMemory(stageData.peakExecutionMemory)
+ .setPeakOnHeapExecutionMemory(stageData.peakOnHeapExecutionMemory)
+ .setPeakOffHeapExecutionMemory(stageData.peakOffHeapExecutionMemory)
.setInputBytes(stageData.inputBytes)
.setInputRecords(stageData.inputRecords)
.setOutputBytes(stageData.outputBytes)
@@ -192,6 +194,8 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
.setMemoryBytesSpilled(tm.memoryBytesSpilled)
.setDiskBytesSpilled(tm.diskBytesSpilled)
.setPeakExecutionMemory(tm.peakExecutionMemory)
+ .setPeakOnHeapExecutionMemory(tm.peakOnHeapExecutionMemory)
+ .setPeakOffHeapExecutionMemory(tm.peakOffHeapExecutionMemory)
.setInputMetrics(serializeInputMetrics(tm.inputMetrics))
.setOutputMetrics(serializeOutputMetrics(tm.outputMetrics))
.setShuffleReadMetrics(serializeShuffleReadMetrics(tm.shuffleReadMetrics))
@@ -278,6 +282,8 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
tmd.gettingResultTime.foreach(grt => builder.addGettingResultTime(grt))
tmd.schedulerDelay.foreach(sd => builder.addSchedulerDelay(sd))
tmd.peakExecutionMemory.foreach(pem => builder.addPeakExecutionMemory(pem))
+ tmd.peakOnHeapExecutionMemory.foreach(peonm => builder.addPeakOnHeapExecutionMemory(peonm))
+ tmd.peakOffHeapExecutionMemory.foreach(peoffm => builder.addPeakOffHeapExecutionMemory(peoffm))
tmd.memoryBytesSpilled.foreach(mbs => builder.addMemoryBytesSpilled(mbs))
tmd.diskBytesSpilled.foreach(dbs => builder.addDiskBytesSpilled(dbs))
builder
@@ -441,6 +447,8 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
memoryBytesSpilled = binary.getMemoryBytesSpilled,
diskBytesSpilled = binary.getDiskBytesSpilled,
peakExecutionMemory = binary.getPeakExecutionMemory,
+ peakOnHeapExecutionMemory = binary.getPeakOnHeapExecutionMemory,
+ peakOffHeapExecutionMemory = binary.getPeakOffHeapExecutionMemory,
inputBytes = binary.getInputBytes,
inputRecords = binary.getInputRecords,
outputBytes = binary.getOutputBytes,
@@ -515,6 +523,10 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
gettingResultTime = binary.getGettingResultTimeList.asScala.map(_.toDouble).toIndexedSeq,
schedulerDelay = binary.getSchedulerDelayList.asScala.map(_.toDouble).toIndexedSeq,
peakExecutionMemory = binary.getPeakExecutionMemoryList.asScala.map(_.toDouble).toIndexedSeq,
+ peakOnHeapExecutionMemory = binary.getPeakOnHeapExecutionMemoryList.asScala
+ .map(_.toDouble).toIndexedSeq,
+ peakOffHeapExecutionMemory = binary.getPeakOffHeapExecutionMemoryList.asScala
+ .map(_.toDouble).toIndexedSeq,
memoryBytesSpilled = binary.getMemoryBytesSpilledList.asScala.map(_.toDouble).toIndexedSeq,
diskBytesSpilled = binary.getDiskBytesSpilledList.asScala.map(_.toDouble).toIndexedSeq,
inputMetrics = deserializeInputMetricDistributions(binary.getInputMetrics),
@@ -664,6 +676,8 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
binary.getMemoryBytesSpilled,
binary.getDiskBytesSpilled,
binary.getPeakExecutionMemory,
+ binary.getPeakOnHeapExecutionMemory,
+ binary.getPeakOffHeapExecutionMemory,
deserializeInputMetrics(binary.getInputMetrics),
deserializeOutputMetrics(binary.getOutputMetrics),
deserializeShuffleReadMetrics(binary.getShuffleReadMetrics),
diff --git a/core/src/main/scala/org/apache/spark/status/protobuf/TaskDataWrapperSerializer.scala b/core/src/main/scala/org/apache/spark/status/protobuf/TaskDataWrapperSerializer.scala
index 8c729fa947810..4396a1f556323 100644
--- a/core/src/main/scala/org/apache/spark/status/protobuf/TaskDataWrapperSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/status/protobuf/TaskDataWrapperSerializer.scala
@@ -44,6 +44,8 @@ private[protobuf] class TaskDataWrapperSerializer extends ProtobufSerDe[TaskData
.setMemoryBytesSpilled(input.memoryBytesSpilled)
.setDiskBytesSpilled(input.diskBytesSpilled)
.setPeakExecutionMemory(input.peakExecutionMemory)
+ .setPeakOnHeapExecutionMemory(input.peakOnHeapExecutionMemory)
+ .setPeakOffHeapExecutionMemory(input.peakOffHeapExecutionMemory)
.setInputBytesRead(input.inputBytesRead)
.setInputRecordsRead(input.inputRecordsRead)
.setOutputBytesWritten(input.outputBytesWritten)
@@ -111,6 +113,8 @@ private[protobuf] class TaskDataWrapperSerializer extends ProtobufSerDe[TaskData
memoryBytesSpilled = binary.getMemoryBytesSpilled,
diskBytesSpilled = binary.getDiskBytesSpilled,
peakExecutionMemory = binary.getPeakExecutionMemory,
+ peakOnHeapExecutionMemory = binary.getPeakOnHeapExecutionMemory,
+ peakOffHeapExecutionMemory = binary.getPeakOffHeapExecutionMemory,
inputBytesRead = binary.getInputBytesRead,
inputRecordsRead = binary.getInputRecordsRead,
outputBytesWritten = binary.getOutputBytesWritten,
diff --git a/core/src/main/scala/org/apache/spark/status/storeTypes.scala b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
index b53455207a0d3..ca64ebcec0148 100644
--- a/core/src/main/scala/org/apache/spark/status/storeTypes.scala
+++ b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
@@ -124,6 +124,8 @@ private[spark] object TaskIndexNames {
final val OUTPUT_RECORDS = "or"
final val OUTPUT_SIZE = "os"
final val PEAK_MEM = "pem"
+ final val PEAK_ON_HEAP_MEM = "peonm"
+ final val PEAK_OFF_HEAP_MEM = "peoffm"
final val RESULT_SIZE = "rs"
final val SCHEDULER_DELAY = "dly"
final val SER_TIME = "rst"
@@ -220,6 +222,10 @@ private[spark] class TaskDataWrapper(
val diskBytesSpilled: Long,
@KVIndexParam(value = TaskIndexNames.PEAK_MEM, parent = TaskIndexNames.STAGE)
val peakExecutionMemory: Long,
+ @KVIndexParam(value = TaskIndexNames.PEAK_ON_HEAP_MEM, parent = TaskIndexNames.STAGE)
+ val peakOnHeapExecutionMemory: Long,
+ @KVIndexParam(value = TaskIndexNames.PEAK_OFF_HEAP_MEM, parent = TaskIndexNames.STAGE)
+ val peakOffHeapExecutionMemory: Long,
@KVIndexParam(value = TaskIndexNames.INPUT_SIZE, parent = TaskIndexNames.STAGE)
val inputBytesRead: Long,
@KVIndexParam(value = TaskIndexNames.INPUT_RECORDS, parent = TaskIndexNames.STAGE)
@@ -307,6 +313,8 @@ private[spark] class TaskDataWrapper(
getMetricValue(memoryBytesSpilled),
getMetricValue(diskBytesSpilled),
getMetricValue(peakExecutionMemory),
+ getMetricValue(peakOnHeapExecutionMemory),
+ getMetricValue(peakOffHeapExecutionMemory),
new InputMetrics(
getMetricValue(inputBytesRead),
getMetricValue(inputRecordsRead)),
@@ -557,6 +565,8 @@ private[spark] class CachedQuantile(
val gettingResultTime: Double,
val schedulerDelay: Double,
val peakExecutionMemory: Double,
+ val peakOnHeapExecutionMemory: Double,
+ val peakOffHeapExecutionMemory: Double,
val memoryBytesSpilled: Double,
val diskBytesSpilled: Double,
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
index 793e65f44ba90..f37310148ed1d 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
@@ -266,6 +266,8 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
memoryBytesSpilled = 0L,
diskBytesSpilled = 0L,
peakExecutionMemory = 0L,
+ peakOnHeapExecutionMemory = 0L,
+ peakOffHeapExecutionMemory = 0L,
inputBytes = 0L,
inputRecords = 0L,
outputBytes = 0L,
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 257f7b37758a7..a371a4171837c 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -189,6 +189,16 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
{Utils.bytesToString(stageData.diskBytesSpilled)}
}}
+ {if (hasPeakExecutionMemory(stageData)) {
+
+ Peak OnHeap Execution Memory:
+ {Utils.bytesToString(stageData.peakOnHeapExecutionMemory)}
+
+
+ Peak OffHeap Execution Memory:
+ {Utils.bytesToString(stageData.peakOffHeapExecutionMemory)}
+
+ }}
{if (!stageJobIds.isEmpty) {
Associated Job Ids:
@@ -465,6 +475,8 @@ private[spark] object ApiHelper {
val HEADER_SER_TIME = "Result Serialization Time"
val HEADER_GETTING_RESULT_TIME = "Getting Result Time"
val HEADER_PEAK_MEM = "Peak Execution Memory"
+ val HEADER_ON_HEAP_PEAK_MEM = "Peak OnHeap Execution Memory"
+ val HEADER_OFF_HEAP_PEAK_MEM = "Peak OffHeap Execution Memory"
val HEADER_ACCUMULATORS = "Accumulators"
val HEADER_INPUT_SIZE = "Input Size / Records"
val HEADER_OUTPUT_SIZE = "Output Size / Records"
@@ -495,6 +507,8 @@ private[spark] object ApiHelper {
HEADER_SER_TIME -> TaskIndexNames.SER_TIME,
HEADER_GETTING_RESULT_TIME -> TaskIndexNames.GETTING_RESULT_TIME,
HEADER_PEAK_MEM -> TaskIndexNames.PEAK_MEM,
+ HEADER_ON_HEAP_PEAK_MEM -> TaskIndexNames.PEAK_ON_HEAP_MEM,
+ HEADER_OFF_HEAP_PEAK_MEM -> TaskIndexNames.PEAK_OFF_HEAP_MEM,
HEADER_ACCUMULATORS -> TaskIndexNames.ACCUMULATORS,
HEADER_INPUT_SIZE -> TaskIndexNames.INPUT_SIZE,
HEADER_OUTPUT_SIZE -> TaskIndexNames.OUTPUT_SIZE,
@@ -527,6 +541,10 @@ private[spark] object ApiHelper {
stageData.diskBytesSpilled > 0 || stageData.memoryBytesSpilled > 0
}
+ def hasPeakExecutionMemory(stageData: StageData): Boolean = {
+ stageData.peakOnHeapExecutionMemory > 0 || stageData.peakOffHeapExecutionMemory > 0
+ }
+
def totalBytesRead(metrics: ShuffleReadMetrics): Long = {
metrics.localBytesRead + metrics.remoteBytesRead
}
diff --git a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
index ac0f2ce26051d..faf0abb8fd104 100644
--- a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 160,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -103,6 +105,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 28000128,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -185,6 +189,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
index d614bb000e4ce..fb02769768f8c 100644
--- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -79,6 +81,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -145,6 +149,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -211,6 +217,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -277,6 +285,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -343,6 +353,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -409,6 +421,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -476,6 +490,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -542,6 +558,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -608,6 +626,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -674,6 +694,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -740,6 +762,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -806,6 +830,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
index 475dee00a2654..95de6a9af9231 100644
--- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -78,6 +80,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -145,6 +149,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -211,6 +217,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -278,6 +286,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -345,6 +355,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -411,6 +423,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -477,6 +491,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -543,6 +559,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -609,6 +627,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -675,6 +695,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -741,6 +763,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -807,6 +831,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -873,6 +899,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -940,6 +968,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
index dc1bcd6a39625..30668430861c6 100644
--- a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
@@ -22,6 +22,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
index e24ac4f82b8a8..e24ce60de812b 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 28000128,
"inputRecords" : 0,
"outputBytes" : 0,
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
index 659e3c41d9289..b191b468ab54e 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 28000128,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -76,6 +78,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -139,6 +143,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -202,6 +208,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -265,6 +273,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -328,6 +338,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -391,6 +403,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -454,6 +468,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -517,6 +533,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
index f84cf26fcf1d6..788d1eccb3f89 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+"peakOnHeapExecutionMemory" : 0,
+"peakOffHeapExecutionMemory" : 0,
"inputBytes" : 28000128,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -76,6 +78,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -139,6 +143,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -202,6 +208,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -265,6 +273,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -328,6 +338,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -391,6 +403,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -454,6 +468,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -517,6 +533,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
index 564f3eadd1cc2..1fc695303b2c4 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 28000128,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -78,6 +80,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -141,6 +145,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -204,6 +210,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -267,6 +275,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -330,6 +340,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -393,6 +405,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -456,6 +470,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
@@ -519,6 +535,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 3500016,
"recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json
index 2bf7f34803775..d4b92a2d33114 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -76,6 +78,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -139,6 +143,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -202,6 +208,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -265,6 +273,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -328,6 +338,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -391,6 +403,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -454,6 +468,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -517,6 +533,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -580,6 +598,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -643,6 +663,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
index 8df41bfcc8d78..c52215d3c3609 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 160,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -104,6 +106,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -186,6 +190,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 28000128,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -268,6 +274,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
index 730df3fbd5341..2c3ae38a65783 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
index 16d92244ee0d0..4ae08d9cf667c 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 384640,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -103,6 +105,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 16256,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -186,6 +190,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
index 2347b687d3dd1..3e94a2d6208bb 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
@@ -22,6 +22,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 49294,
"recordsRead" : 10000
@@ -84,6 +86,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -146,6 +150,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -209,6 +215,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -271,6 +279,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -333,6 +343,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -395,6 +407,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -457,6 +471,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -519,6 +535,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -581,6 +599,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60489,
"recordsRead" : 10000
@@ -643,6 +663,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -705,6 +727,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -767,6 +791,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -829,6 +855,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -891,6 +919,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -953,6 +983,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1015,6 +1047,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1077,6 +1111,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1139,6 +1175,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1201,6 +1239,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
index eaec1c32ba306..56aee9389f5c7 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
@@ -27,6 +27,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -94,6 +96,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -161,6 +165,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -228,6 +234,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -295,6 +303,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -362,6 +372,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -429,6 +441,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -496,6 +510,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
index 78566cbce9889..00dd2f0396820 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
@@ -27,6 +27,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -94,6 +96,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -161,6 +165,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -228,6 +234,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -295,6 +303,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -362,6 +372,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -429,6 +441,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -496,6 +510,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
index f9cf705d06c11..64d5c93ecf193 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
@@ -22,6 +22,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -84,6 +86,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -146,6 +150,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -208,6 +214,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -270,6 +278,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -332,6 +342,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -394,6 +406,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -456,6 +470,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -518,6 +534,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -580,6 +598,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -642,6 +662,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -704,6 +726,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -766,6 +790,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -828,6 +854,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -890,6 +918,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -952,6 +982,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1014,6 +1046,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1076,6 +1110,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1138,6 +1174,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1200,6 +1238,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1262,6 +1302,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1324,6 +1366,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1386,6 +1430,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1448,6 +1494,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1510,6 +1558,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1572,6 +1622,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1634,6 +1686,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1696,6 +1750,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1758,6 +1814,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1820,6 +1878,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1882,6 +1942,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1944,6 +2006,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2006,6 +2070,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2068,6 +2134,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2130,6 +2198,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2192,6 +2262,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2254,6 +2326,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2316,6 +2390,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2378,6 +2454,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2440,6 +2518,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2502,6 +2582,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2564,6 +2646,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2626,6 +2710,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2688,6 +2774,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2750,6 +2838,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2812,6 +2902,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2874,6 +2966,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2936,6 +3030,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -2998,6 +3094,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -3060,6 +3158,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
index 2a52ea95807ed..d4ab4b351c39a 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
@@ -22,6 +22,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -84,6 +86,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -146,6 +150,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -208,6 +214,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -270,6 +278,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -332,6 +342,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -394,6 +406,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 49294,
"recordsRead" : 10000
@@ -456,6 +470,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -518,6 +534,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -580,6 +598,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -642,6 +662,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -704,6 +726,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -766,6 +790,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -828,6 +854,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -890,6 +918,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60489,
"recordsRead" : 10000
@@ -952,6 +982,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1014,6 +1046,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1076,6 +1110,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -1138,6 +1174,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1200,6 +1238,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
index 718d1e7a9512f..b0177cd2a8e8b 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
@@ -22,6 +22,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -84,6 +86,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -146,6 +150,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -208,6 +214,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -270,6 +278,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -332,6 +342,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -394,6 +406,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 49294,
"recordsRead" : 10000
@@ -456,6 +470,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -518,6 +534,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -580,6 +598,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -642,6 +662,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -704,6 +726,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -766,6 +790,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -828,6 +854,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -890,6 +918,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60489,
"recordsRead" : 10000
@@ -952,6 +982,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1014,6 +1046,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1076,6 +1110,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -1138,6 +1174,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1200,6 +1238,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
index 38a4a209c0452..40e6fe9d1b017 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
@@ -22,6 +22,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -84,6 +86,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -146,6 +150,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -208,6 +214,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -270,6 +278,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -332,6 +342,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -394,6 +406,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -456,6 +470,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -518,6 +534,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -580,6 +598,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -642,6 +662,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -704,6 +726,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -766,6 +790,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -828,6 +854,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -890,6 +918,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -952,6 +982,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70565,
"recordsRead" : 10000
@@ -1014,6 +1046,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1076,6 +1110,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1138,6 +1174,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1200,6 +1238,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json
index f9cc66295f28a..e89b6d65079a6 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___offset___length_expectation.json
@@ -22,6 +22,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
@@ -84,6 +86,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 60488,
"recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json
index 368bbd862d7af..8f226fa316ba0 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status___sortBy_short_names__runtime_expectation.json
@@ -22,6 +22,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -84,6 +86,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -146,6 +150,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -208,6 +214,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -270,6 +278,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -332,6 +342,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -394,6 +406,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -456,6 +470,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -518,6 +534,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -580,6 +598,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -642,6 +662,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -704,6 +726,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -766,6 +790,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -828,6 +854,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -890,6 +918,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -952,6 +982,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70565,
"recordsRead" : 10000
@@ -1014,6 +1046,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1076,6 +1110,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1138,6 +1174,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
@@ -1200,6 +1238,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 70564,
"recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json
index b07f86e30fabd..c80864dd72942 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__status_expectation.json
@@ -23,6 +23,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -89,6 +91,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -155,6 +159,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -221,6 +227,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -287,6 +295,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -353,6 +363,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -419,6 +431,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -485,6 +499,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -551,6 +567,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -617,6 +635,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_with_partitionId_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_with_partitionId_expectation.json
index c4f1ec3624108..51fcdb41cf562 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_with_partitionId_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_with_partitionId_expectation.json
@@ -22,6 +22,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 12
@@ -84,6 +86,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 13
@@ -146,6 +150,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 12
@@ -208,6 +214,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 13
@@ -270,6 +278,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 12
@@ -332,6 +342,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 13
@@ -394,6 +406,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 12
@@ -456,6 +470,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 13
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
index e3dba103bbd50..e8b2868ab0328 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
@@ -11,6 +11,8 @@
"gettingResultTime" : [ 0.0, 0.0, 0.0 ],
"schedulerDelay" : [ 2.0, 6.0, 53.0 ],
"peakExecutionMemory" : [ 0.0, 0.0, 0.0 ],
+ "peakOnHeapExecutionMemory" : [ 0.0, 0.0, 0.0 ],
+ "peakOffHeapExecutionMemory" : [ 0.0, 0.0, 0.0 ],
"memoryBytesSpilled" : [ 0.0, 0.0, 0.0 ],
"diskBytesSpilled" : [ 0.0, 0.0, 0.0 ],
"inputMetrics" : {
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
index ba7c794482640..37d99d2ff33b1 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
@@ -11,6 +11,8 @@
"gettingResultTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"schedulerDelay" : [ 4.0, 4.0, 6.0, 7.0, 9.0 ],
"peakExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "peakOnHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "peakOffHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"memoryBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"diskBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"inputMetrics" : {
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
index b463314150bad..0e184fb797ede 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
@@ -11,6 +11,8 @@
"gettingResultTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"schedulerDelay" : [ 2.0, 4.0, 6.0, 13.0, 40.0 ],
"peakExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "peakOnHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "peakOffHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"memoryBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"diskBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"inputMetrics" : {
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
index e38741b7bf6e7..ed583833cc2eb 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -85,6 +87,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -153,6 +157,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -221,6 +227,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -289,6 +297,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -357,6 +367,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -425,6 +437,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -493,6 +507,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -561,6 +577,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
index 630b0512e8f98..a8a925892b5ab 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 384640,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -76,6 +78,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -142,6 +146,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -208,6 +214,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -274,6 +282,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -340,6 +350,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -406,6 +418,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -472,6 +486,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -538,6 +554,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -604,6 +622,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -670,6 +690,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -736,6 +758,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -802,6 +826,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -868,6 +894,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -934,6 +962,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -1000,6 +1030,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -1066,6 +1098,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
index 23770480ad62a..10253b44c0c88 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -78,6 +80,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -144,6 +148,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -210,6 +216,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -276,6 +284,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -343,6 +353,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 0,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
index c8458a409589e..a059ec16908e7 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json
@@ -21,6 +21,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 384640,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputBytes" : 0,
"inputRecords" : 0,
"outputBytes" : 0,
@@ -76,6 +78,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -142,6 +146,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -208,6 +214,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -274,6 +282,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -340,6 +350,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -406,6 +418,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -472,6 +486,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -538,6 +554,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -604,6 +622,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -670,6 +690,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -736,6 +758,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -802,6 +826,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -868,6 +894,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -934,6 +962,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -1000,6 +1030,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -1066,6 +1098,8 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"peakExecutionMemory" : 24040,
+ "peakOnHeapExecutionMemory" : 0,
+ "peakOffHeapExecutionMemory" : 0,
"inputMetrics" : {
"bytesRead" : 0,
"recordsRead" : 0
@@ -1237,6 +1271,8 @@
"gettingResultTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"schedulerDelay" : [ 62.0, 64.0, 70.0, 71.0, 79.0 ],
"peakExecutionMemory" : [ 24040.0, 24040.0, 24040.0, 24040.0, 24040.0 ],
+ "peakOnHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "peakOffHeapExecutionMemory" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"memoryBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"diskBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"inputMetrics" : {
diff --git a/core/src/test/scala/org/apache/spark/executor/TaskMetricsSuite.scala b/core/src/test/scala/org/apache/spark/executor/TaskMetricsSuite.scala
index 7bcc2fb5231db..49ff2b8732e5c 100644
--- a/core/src/test/scala/org/apache/spark/executor/TaskMetricsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/TaskMetricsSuite.scala
@@ -37,6 +37,8 @@ class TaskMetricsSuite extends SparkFunSuite {
assert(tm.memoryBytesSpilled == 0L)
assert(tm.diskBytesSpilled == 0L)
assert(tm.peakExecutionMemory == 0L)
+ assert(tm.peakOnHeapExecutionMemory == 0L)
+ assert(tm.peakOffHeapExecutionMemory == 0L)
assert(tm.updatedBlockStatuses.isEmpty)
// set or increment values
tm.setExecutorDeserializeTime(100L)
@@ -55,6 +57,8 @@ class TaskMetricsSuite extends SparkFunSuite {
tm.incDiskBytesSpilled(7L)
tm.incPeakExecutionMemory(800L)
tm.incPeakExecutionMemory(8L)
+ tm.setPeakOnHeapExecutionMemory(809L)
+ tm.setPeakOffHeapExecutionMemory(810L)
val block1 = (TestBlockId("a"), BlockStatus(MEMORY_ONLY, 1L, 2L))
val block2 = (TestBlockId("b"), BlockStatus(MEMORY_ONLY, 3L, 4L))
tm.incUpdatedBlockStatuses(block1)
@@ -68,6 +72,8 @@ class TaskMetricsSuite extends SparkFunSuite {
assert(tm.memoryBytesSpilled == 606L)
assert(tm.diskBytesSpilled == 707L)
assert(tm.peakExecutionMemory == 808L)
+ assert(tm.peakOnHeapExecutionMemory == 809L)
+ assert(tm.peakOffHeapExecutionMemory == 810L)
assert(tm.updatedBlockStatuses == Seq(block1, block2))
}
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
index f2b795764b7e8..e9446a190883d 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
@@ -181,6 +181,8 @@ class AppStatusStoreSuite extends SparkFunSuite {
assertQuantiles(_.memoryBytesSpilled.toDouble, summary.memoryBytesSpilled)
assertQuantiles(_.diskBytesSpilled.toDouble, summary.diskBytesSpilled)
assertQuantiles(_.peakExecutionMemory.toDouble, summary.peakExecutionMemory)
+ assertQuantiles(_.peakOnHeapExecutionMemory.toDouble, summary.peakOnHeapExecutionMemory)
+ assertQuantiles(_.peakOffHeapExecutionMemory.toDouble, summary.peakOffHeapExecutionMemory)
assertQuantiles(_.inputMetrics.bytesRead.toDouble, summary.inputMetrics.bytesRead)
assertQuantiles(_.inputMetrics.recordsRead.toDouble, summary.inputMetrics.recordsRead)
assertQuantiles(_.outputMetrics.bytesWritten.toDouble, summary.outputMetrics.bytesWritten)
@@ -282,7 +284,7 @@ class AppStatusStoreSuite extends SparkFunSuite {
i, i, i, i, i, i, i, i, i, i,
i, i, i, i, i, i, i, i, i, i,
i, i, i, i, i, i, i, i, i, i,
- i, i, i, i, stageId, attemptId)
+ i, i, i, i, i, i, stageId, attemptId)
}
private def writeTaskDataToStore(i: Int, store: KVStore, status: String): Unit = {
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala
index 9c1a280d2b68e..779c5772f85e5 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala
@@ -51,6 +51,8 @@ class AppStatusUtilsSuite extends SparkFunSuite {
memoryBytesSpilled = 0L,
diskBytesSpilled = 0L,
peakExecutionMemory = 0L,
+ peakOnHeapExecutionMemory = 0L,
+ peakOffHeapExecutionMemory = 0L,
inputMetrics = null,
outputMetrics = null,
shuffleReadMetrics = null,
@@ -86,6 +88,8 @@ class AppStatusUtilsSuite extends SparkFunSuite {
memoryBytesSpilled = 0L,
diskBytesSpilled = 0L,
peakExecutionMemory = 100L,
+ peakOnHeapExecutionMemory = 110L,
+ peakOffHeapExecutionMemory = 120L,
inputMetrics = null,
outputMetrics = null,
shuffleReadMetrics = null,
diff --git a/core/src/test/scala/org/apache/spark/status/protobuf/KVStoreProtobufSerializerSuite.scala b/core/src/test/scala/org/apache/spark/status/protobuf/KVStoreProtobufSerializerSuite.scala
index ac568fee1ad45..52d13da6beada 100644
--- a/core/src/test/scala/org/apache/spark/status/protobuf/KVStoreProtobufSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/protobuf/KVStoreProtobufSerializerSuite.scala
@@ -158,6 +158,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
memoryBytesSpilled = 14L,
diskBytesSpilled = 15L,
peakExecutionMemory = 16L,
+ peakOnHeapExecutionMemory = 43L,
+ peakOffHeapExecutionMemory = 44L,
inputBytesRead = 17L,
inputRecordsRead = 18L,
outputBytesWritten = 19L,
@@ -212,6 +214,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
assert(result.memoryBytesSpilled == input.memoryBytesSpilled)
assert(result.diskBytesSpilled == input.diskBytesSpilled)
assert(result.peakExecutionMemory == input.peakExecutionMemory)
+ assert(result.peakOnHeapExecutionMemory == input.peakOnHeapExecutionMemory)
+ assert(result.peakOffHeapExecutionMemory == input.peakOffHeapExecutionMemory)
assert(result.inputBytesRead == input.inputBytesRead)
assert(result.inputRecordsRead == input.inputRecordsRead)
assert(result.outputBytesWritten == input.outputBytesWritten)
@@ -645,6 +649,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
gettingResultTime = 12.1,
schedulerDelay = 13.1,
peakExecutionMemory = 14.1,
+ peakOnHeapExecutionMemory = 14.2,
+ peakOffHeapExecutionMemory = 14.3,
memoryBytesSpilled = 15.1,
diskBytesSpilled = 16.1,
bytesRead = 17.1,
@@ -689,6 +695,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
assert(result.gettingResultTime == input.gettingResultTime)
assert(result.schedulerDelay == input.schedulerDelay)
assert(result.peakExecutionMemory == input.peakExecutionMemory)
+ assert(result.peakOnHeapExecutionMemory == input.peakOnHeapExecutionMemory)
+ assert(result.peakOffHeapExecutionMemory == input.peakOffHeapExecutionMemory)
assert(result.memoryBytesSpilled == input.memoryBytesSpilled)
assert(result.diskBytesSpilled == input.diskBytesSpilled)
assert(result.bytesRead == input.bytesRead)
@@ -1064,6 +1072,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
memoryBytesSpilled = 8L,
diskBytesSpilled = 9L,
peakExecutionMemory = 10L,
+ peakOnHeapExecutionMemory = 11L,
+ peakOffHeapExecutionMemory = 12L,
inputMetrics = inputMetrics,
outputMetrics = outputMetrics,
shuffleReadMetrics = shuffleReadMetrics,
@@ -1209,6 +1219,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
gettingResultTime = IndexedSeq(19.001D, 20.001D),
schedulerDelay = IndexedSeq(21.001D, 22.001D),
peakExecutionMemory = IndexedSeq(23.001D, 24.001D),
+ peakOnHeapExecutionMemory = IndexedSeq(29.001D, 30.001D),
+ peakOffHeapExecutionMemory = IndexedSeq(31.001D, 32.001D),
memoryBytesSpilled = IndexedSeq(25.001D, 26.001D),
diskBytesSpilled = IndexedSeq(27.001D, 28.001D),
inputMetrics = inputMetricDistributions,
@@ -1263,6 +1275,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
memoryBytesSpilled = 16L,
diskBytesSpilled = 17L,
peakExecutionMemory = 18L,
+ peakOnHeapExecutionMemory = 44L,
+ peakOffHeapExecutionMemory = 45L,
inputBytes = 19L,
inputRecords = 20L,
outputBytes = 21L,
@@ -1345,6 +1359,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
assert(result.info.memoryBytesSpilled == input.info.memoryBytesSpilled)
assert(result.info.diskBytesSpilled == input.info.diskBytesSpilled)
assert(result.info.peakExecutionMemory == input.info.peakExecutionMemory)
+ assert(result.info.peakOnHeapExecutionMemory == input.info.peakOnHeapExecutionMemory)
+ assert(result.info.peakOffHeapExecutionMemory == input.info.peakOffHeapExecutionMemory)
assert(result.info.inputBytes == input.info.inputBytes)
assert(result.info.inputRecords == input.info.inputRecords)
assert(result.info.outputBytes == input.info.outputBytes)
@@ -1461,6 +1477,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
assert(result.memoryBytesSpilled == expected.memoryBytesSpilled)
assert(result.diskBytesSpilled == expected.diskBytesSpilled)
assert(result.peakExecutionMemory == expected.peakExecutionMemory)
+ assert(result.peakOnHeapExecutionMemory == expected.peakOnHeapExecutionMemory)
+ assert(result.peakOffHeapExecutionMemory == expected.peakOffHeapExecutionMemory)
checkAnswer(result.inputMetrics, expected.inputMetrics)
checkAnswer(result.outputMetrics, expected.outputMetrics)
checkAnswer(result.shuffleReadMetrics, expected.shuffleReadMetrics)
@@ -1613,6 +1631,8 @@ class KVStoreProtobufSerializerSuite extends SparkFunSuite {
assert(result.gettingResultTime == expected.gettingResultTime)
assert(result.schedulerDelay == expected.schedulerDelay)
assert(result.peakExecutionMemory == expected.peakExecutionMemory)
+ assert(result.peakOnHeapExecutionMemory == expected.peakOnHeapExecutionMemory)
+ assert(result.peakOffHeapExecutionMemory == expected.peakOffHeapExecutionMemory)
assert(result.memoryBytesSpilled == expected.memoryBytesSpilled)
assert(result.diskBytesSpilled == expected.diskBytesSpilled)
diff --git a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
index c76c97d071418..d288837b3b2d8 100644
--- a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
@@ -65,6 +65,8 @@ class StagePageSuite extends SparkFunSuite with LocalSparkContext {
memoryBytesSpilled = 1L,
diskBytesSpilled = 1L,
peakExecutionMemory = 1L,
+ peakOnHeapExecutionMemory = 1L,
+ peakOffHeapExecutionMemory = 1L,
inputBytes = 1L,
inputRecords = 1L,
outputBytes = 1L,
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index cdee6ccda706e..f497826eacd48 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -358,6 +358,8 @@ class JsonProtocolSuite extends SparkFunSuite {
test("TaskMetrics backward compatibility") {
// "Executor Deserialize CPU Time" and "Executor CPU Time" were introduced in Spark 2.1.0
// "Peak Execution Memory" was introduced in Spark 3.0.0
+ // "Peak On Heap Execution Memory" and "Peak Off Heap Execution Memory" were introduced in
+ // Spark 3.5.0
val metrics = makeTaskMetrics(1L, 2L, 3L, 4L, 5, 6, 0, hasHadoopInput = false, hasOutput = true)
metrics.setExecutorDeserializeCpuTime(100L)
metrics.setExecutorCpuTime(100L)
@@ -367,10 +369,14 @@ class JsonProtocolSuite extends SparkFunSuite {
.removeField("Executor Deserialize CPU Time")
.removeField("Executor CPU Time")
.removeField("Peak Execution Memory")
+ .removeField("Peak On Heap Execution Memory")
+ .removeField("Peak Off Heap Execution Memory")
val newMetrics = JsonProtocol.taskMetricsFromJson(oldJson)
assert(newMetrics.executorDeserializeCpuTime == 0)
assert(newMetrics.executorCpuTime == 0)
assert(newMetrics.peakExecutionMemory == 0)
+ assert(newMetrics.peakOnHeapExecutionMemory == 0)
+ assert(newMetrics.peakOffHeapExecutionMemory == 0)
}
test("StorageLevel backward compatibility") {