From 596934f58fc60cbd5a03665dffa36ee48b2c62f4 Mon Sep 17 00:00:00 2001 From: n1harika Date: Thu, 8 Jan 2026 13:37:10 +0530 Subject: [PATCH 1/4] Disable pre-allocation for dynamic models through onnxruntime_perf_test --- onnxruntime/test/perftest/ort_test_session.cc | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 3468e2e55c7b6..bce400d25d32b 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -1034,11 +1034,35 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); // Switch to custom allocator_ = Ort::UnownedAllocator(custom_allocator_); - // free dimensions are treated as 1 if not overridden - transform_fcn = [](int64_t input) { return (input == -1) ? -input : input; }; - new_value = [](OrtAllocator* allocator, const std::vector& output_shape, Ort::ConstTensorTypeAndShapeInfo& tensor_info) { - return Ort::Value::CreateTensor(allocator, output_shape.data(), output_shape.size(), tensor_info.GetElementType()); - }; + //Do not pre-allocate if dynamic dimensions are present + bool has_dynamic_output = false; + + for (size_t i = 0; i < session_.GetOutputCount(); ++i) { + auto type_info = session_.GetOutputTypeInfo(i); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + auto shape = tensor_info.GetShape(); + for (auto dim : shape) { + if (dim == -1) { + has_dynamic_output = true; + break; + } + } + if (has_dynamic_output) break; + } + + if (has_dynamic_output) { + transform_fcn = [](int64_t input) { return input; }; + new_value = [](OrtAllocator*, const std::vector&, Ort::ConstTensorTypeAndShapeInfo&) { + return Ort::Value(nullptr); + }; + } else { + transform_fcn = [](int64_t input) { return input; }; + new_value = [](OrtAllocator* allocator, const std::vector& output_shape, + Ort::ConstTensorTypeAndShapeInfo& tensor_info) { + return Ort::Value::CreateTensor(allocator, output_shape.data(), output_shape.size(), + tensor_info.GetElementType()); + }; + } } for (size_t i = 0; i < output_names_raw_ptr.size(); i++) { From 6c201f74086795e123275e87750b069714015a00 Mon Sep 17 00:00:00 2001 From: n1harika Date: Thu, 8 Jan 2026 20:11:21 +0530 Subject: [PATCH 2/4] Address review comments --- onnxruntime/test/perftest/ort_test_session.cc | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index bce400d25d32b..4d2a20c11ccf8 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -1018,8 +1018,8 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); auto transform_fcn = std::function(); auto new_value = std::function&, Ort::ConstTensorTypeAndShapeInfo&)>(); + transform_fcn = [](int64_t input) { return input; }; if (device_memory_name_.empty()) { - transform_fcn = [](int64_t input) { return input; }; new_value = [](OrtAllocator*, const std::vector&, Ort::ConstTensorTypeAndShapeInfo&) { return Ort::Value(nullptr); }; @@ -1041,22 +1041,17 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); auto type_info = session_.GetOutputTypeInfo(i); auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); auto shape = tensor_info.GetShape(); - for (auto dim : shape) { - if (dim == -1) { - has_dynamic_output = true; - break; - } + if (std::any_of(shape.begin(), shape.end(), [](int64_t d) { return d == -1; })) { + has_dynamic_output = true; + break; } - if (has_dynamic_output) break; } if (has_dynamic_output) { - transform_fcn = [](int64_t input) { return input; }; new_value = [](OrtAllocator*, const std::vector&, Ort::ConstTensorTypeAndShapeInfo&) { return Ort::Value(nullptr); }; } else { - transform_fcn = [](int64_t input) { return input; }; new_value = [](OrtAllocator* allocator, const std::vector& output_shape, Ort::ConstTensorTypeAndShapeInfo& tensor_info) { return Ort::Value::CreateTensor(allocator, output_shape.data(), output_shape.size(), From 35e276dde2cd708a2779d1917501a38d432cb363 Mon Sep 17 00:00:00 2001 From: n1harika Date: Fri, 9 Jan 2026 11:11:15 +0530 Subject: [PATCH 3/4] Address review comments 2 --- onnxruntime/test/perftest/ort_test_session.cc | 57 +++++-------------- 1 file changed, 15 insertions(+), 42 deletions(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 4d2a20c11ccf8..8e70249df9bac 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -1016,56 +1016,29 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); input_names_[i] = input_names_str_[i].c_str(); } - auto transform_fcn = std::function(); - auto new_value = std::function&, Ort::ConstTensorTypeAndShapeInfo&)>(); - transform_fcn = [](int64_t input) { return input; }; - if (device_memory_name_.empty()) { - new_value = [](OrtAllocator*, const std::vector&, Ort::ConstTensorTypeAndShapeInfo&) { - return Ort::Value(nullptr); - }; - } else { + if (!device_memory_name_.empty()) { + Ort::MemoryInfo memory_info(nullptr); // Default initialize, will be overwritten if (device_memory_name_ == CUDA) { - memory_info = Ort::MemoryInfo(device_memory_name_.data(), OrtArenaAllocator, 0, OrtMemTypeDefault); + memory_info = Ort::MemoryInfo(device_memory_name_.data(), OrtArenaAllocator, 0, OrtMemTypeDefault); } else { - memory_info = Ort::MemoryInfo(device_memory_name_.data(), OrtArenaAllocator, 0, OrtMemTypeCPUOutput); + memory_info = Ort::MemoryInfo(device_memory_name_.data(), OrtArenaAllocator, 0, OrtMemTypeCPUOutput); } custom_allocator_ = Ort::Allocator(session_, memory_info); - // Switch to custom + // Switch to custom allocator allocator_ = Ort::UnownedAllocator(custom_allocator_); - - //Do not pre-allocate if dynamic dimensions are present - bool has_dynamic_output = false; - - for (size_t i = 0; i < session_.GetOutputCount(); ++i) { - auto type_info = session_.GetOutputTypeInfo(i); - auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - auto shape = tensor_info.GetShape(); - if (std::any_of(shape.begin(), shape.end(), [](int64_t d) { return d == -1; })) { - has_dynamic_output = true; - break; - } - } - - if (has_dynamic_output) { - new_value = [](OrtAllocator*, const std::vector&, Ort::ConstTensorTypeAndShapeInfo&) { - return Ort::Value(nullptr); - }; - } else { - new_value = [](OrtAllocator* allocator, const std::vector& output_shape, - Ort::ConstTensorTypeAndShapeInfo& tensor_info) { - return Ort::Value::CreateTensor(allocator, output_shape.data(), output_shape.size(), - tensor_info.GetElementType()); - }; - } } - for (size_t i = 0; i < output_names_raw_ptr.size(); i++) { - Ort::TypeInfo type_info = session_.GetOutputTypeInfo(i); - auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); - std::vector output_shape = tensor_info.GetShape(); - std::transform(output_shape.begin(), output_shape.end(), output_shape.begin(), transform_fcn); - outputs_.emplace_back(new_value(allocator_, output_shape, tensor_info)); + Ort::TypeInfo type_info = session_.GetOutputTypeInfo(i); + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + std::vector output_shape = tensor_info.GetShape(); + auto is_dynamic = std::find(output_shape.begin(), output_shape.end(), -1) != output_shape.end(); + if (is_dynamic || device_memory_name_.empty()) { + outputs_.emplace_back(Ort::Value(nullptr)); + } else { + auto & new_value = Ort::Value::CreateTensor(allocator_, output_shape.data(), output_shape.size(), tensor_info.GetElementType()); + outputs_.emplace_back(std::move(new_value)); + } } } From b9e7509909c647e273d47509e75197c328c54cc0 Mon Sep 17 00:00:00 2001 From: n1harika Date: Fri, 9 Jan 2026 13:39:14 +0530 Subject: [PATCH 4/4] Remove reference type --- onnxruntime/test/perftest/ort_test_session.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 8e70249df9bac..dd0ed75f4782f 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -1036,7 +1036,7 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); if (is_dynamic || device_memory_name_.empty()) { outputs_.emplace_back(Ort::Value(nullptr)); } else { - auto & new_value = Ort::Value::CreateTensor(allocator_, output_shape.data(), output_shape.size(), tensor_info.GetElementType()); + auto new_value = Ort::Value::CreateTensor(allocator_, output_shape.data(), output_shape.size(), tensor_info.GetElementType()); outputs_.emplace_back(std::move(new_value)); } }