From 0304e387088c303a1aefdf7593551517974f170a Mon Sep 17 00:00:00 2001 From: Rajesh Gandham Date: Tue, 10 Jun 2025 06:16:51 -0700 Subject: [PATCH 1/7] Create a script that launches cuopt_cli in parallel --- .../linear_programming/run_mps_files.sh | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100755 benchmarks/linear_programming/run_mps_files.sh diff --git a/benchmarks/linear_programming/run_mps_files.sh b/benchmarks/linear_programming/run_mps_files.sh new file mode 100755 index 0000000000..860807cd76 --- /dev/null +++ b/benchmarks/linear_programming/run_mps_files.sh @@ -0,0 +1,203 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script runs MPS (Mathematical Programming System) files in parallel across multiple GPUs +# It supports running linear programming (LP) and mixed-integer programming (MIP) problems +# +# Usage: +# ./run_mps_files.sh --path /path/to/mps/files --ngpus 4 --time-limit 3600 --output-dir results +# +# Arguments: +# --path : Directory containing .mps files (required) +# --ngpus : Number of GPUs to use (default: 1) +# --time-limit : Time limit in seconds for each problem (default: 360) +# --output-dir : Directory to store output log files (default: current directory) +# +# Examples: +# # Run all MPS files in /data/lp using 2 GPUs with 1 hour time limit +# ./run_mps_files.sh --path /data/lp --ngpus 2 --time-limit 3600 +# +# # Run with specific GPU devices using CUDA_VISIBLE_DEVICES +# CUDA_VISIBLE_DEVICES=0,2,3 ./run_mps_files.sh --path /data/mip +# +# # Run with custom output directory +# ./run_mps_files.sh --path /data/lp --output-dir ~/results/lp_benchmark +# +# Notes: +# - Files are distributed dynamically across available GPUs for load balancing +# - Each problem's results are logged to a separate file in the output directory +# - The script uses file locking to safely distribute work across parallel processes +# - If CUDA_VISIBLE_DEVICES is set, it can override the --ngpus argument + +# Help function +print_help() { + cat << EOF +This script runs MPS (Mathematical Programming System) files in parallel across multiple GPUs. +It supports running linear programming (LP) and mixed-integer programming (MIP) problems. + +Usage: + $(basename "$0") --path /path/to/mps/files [options] + +Required Arguments: + --path PATH Directory containing .mps files + +Optional Arguments: + --ngpus N Number of GPUs to use (default: 1) + --time-limit N Time limit in seconds for each problem (default: 360) + --output-dir PATH Directory to store output log files (default: current directory) + --relaxation Run relaxation instead of solving the MIP + -h, --help Show this help message and exit + +Examples: + # Run all MPS files in /data/lp using 2 GPUs with 1 hour time limit + $(basename "$0") --path /data/lp --ngpus 2 --time-limit 3600 + + # Run with specific GPU devices using CUDA_VISIBLE_DEVICES + CUDA_VISIBLE_DEVICES=0,2,3 $(basename "$0") --path /data/mip + + # Run with custom output directory + $(basename "$0") --path /data/lp --output-dir ~/results/lp_benchmark + +Notes: + - Files are distributed dynamically across available GPUs for load balancing + - Each problem's results are logged to a separate file in the output directory + - The script uses file locking to safely distribute work across parallel processes + - If CUDA_VISIBLE_DEVICES is set, it can override the --ngpus argument +EOF + exit 0 +} + +# Check for help flag +if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]]; then + print_help +fi + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --path) + MPS_DIR="$2" + shift 2 + ;; + --ngpus) + GPU_COUNT="$2" + shift 2 + ;; + --time-limit) + TIME_LIMIT="$2" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --relaxation) + RELAXATION=true + shift + ;; + *) + echo "Unknown argument: $1" + echo "Usage: $0 --path MPS_DIR --ngpus GPU_COUNT --time-limit TIME_LIMIT --output-dir OUTPUT_DIR --relaxation" + exit 1 + ;; + esac +done +# Set defaults and validate required arguments +if [[ -z "$MPS_DIR" ]]; then + echo "Missing required argument: --path MPS_DIR" + echo "Usage: $0 --path MPS_DIR [--ngpus GPU_COUNT] [--time-limit TIME_LIMIT] [--output-dir OUTPUT_DIR] [--relaxation]" + exit 1 +fi + +# Set defaults if not provided +GPU_COUNT=${GPU_COUNT:-1} +TIME_LIMIT=${TIME_LIMIT:-360} +OUTPUT_DIR=${OUTPUT_DIR:-.} +RELAXATION=${RELAXATION:-false} + +# Determine GPU list +if [[ -n "$CUDA_VISIBLE_DEVICES" ]]; then + IFS=',' read -ra GPU_LIST <<< "$CUDA_VISIBLE_DEVICES" +else + GPU_LIST=() + for ((i=0; i "$INDEX_FILE" + +# Adjust GPU_COUNT if there are fewer files than GPUs +if ((GPU_COUNT > file_count)); then + echo "Reducing number of GPUs from $GPU_COUNT to $file_count since there are fewer files than GPUs" + GPU_COUNT=$file_count + # Trim GPU_LIST to match file_count + GPU_LIST=("${GPU_LIST[@]:0:$file_count}") +fi + +# Function for each worker (GPU) +worker() { + local gpu_id=$1 + # echo "GPU $gpu_id Using index file $INDEX_FILE" + while :; do + # Atomically get and increment the index + my_index=$(flock "$INDEX_FILE" bash -c ' + idx=$(<"$0") + if (( idx >= '"$file_count"' )); then + echo -1 + else + echo $((idx+1)) > "$0" + echo $idx + fi + ' "$INDEX_FILE") + + if (( my_index == -1 )); then + return + fi + + mps_file="${mps_files[my_index]}" + echo "GPU $gpu_id processing $my_index" + if [ "$RELAXATION" = true ]; then + CUDA_VISIBLE_DEVICES=$gpu_id cuopt_cli "$mps_file" --time-limit $TIME_LIMIT --log-file "$OUTPUT_DIR/$(basename "${mps_file%.mps}").log" --log-to-console=false --relaxation + else + CUDA_VISIBLE_DEVICES=$gpu_id cuopt_cli "$mps_file" --time-limit $TIME_LIMIT --log-file "$OUTPUT_DIR/$(basename "${mps_file%.mps}").log" --log-to-console=false + fi + done +} + +# Start one worker per GPU in the list +for gpu_id in "${GPU_LIST[@]}"; do + worker "$gpu_id" & +done + +wait + +# Remove the index file +rm -f "$INDEX_FILE" \ No newline at end of file From a828313f2d3a336ef01524dc3b5bd742c4943a5a Mon Sep 17 00:00:00 2001 From: Rajesh Gandham Date: Tue, 10 Jun 2025 10:33:18 -0700 Subject: [PATCH 2/7] Add more options to the bash script --- .../linear_programming/run_mps_files.sh | 68 +++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/benchmarks/linear_programming/run_mps_files.sh b/benchmarks/linear_programming/run_mps_files.sh index 860807cd76..47c14c18df 100755 --- a/benchmarks/linear_programming/run_mps_files.sh +++ b/benchmarks/linear_programming/run_mps_files.sh @@ -110,6 +110,30 @@ while [[ $# -gt 0 ]]; do RELAXATION=true shift ;; + --heuristics-only) + HEURISTICS_ONLY=true + shift + ;; + --write-log-file) + WRITE_LOG_FILE="$2" + shift 2 + ;; + --num-cpu-threads) + NUM_CPU_THREADS="$2" + shift 2 + ;; + --batch-num) + BATCH_NUM="$2" + shift 2 + ;; + --n-batches) + N_BATCHES="$2" + shift 2 + ;; + --log-to-console) + LOG_TO_CONSOLE="$2" + shift 2 + ;; *) echo "Unknown argument: $1" echo "Usage: $0 --path MPS_DIR --ngpus GPU_COUNT --time-limit TIME_LIMIT --output-dir OUTPUT_DIR --relaxation" @@ -129,6 +153,12 @@ GPU_COUNT=${GPU_COUNT:-1} TIME_LIMIT=${TIME_LIMIT:-360} OUTPUT_DIR=${OUTPUT_DIR:-.} RELAXATION=${RELAXATION:-false} +HEURISTICS_ONLY=${HEURISTICS_ONLY:-false} +WRITE_LOG_FILE=${WRITE_LOG_FILE:-false} +NUM_CPU_THREADS=${NUM_CPU_THREADS:-1} +BATCH_NUM=${BATCH_NUM:-0} +N_BATCHES=${N_BATCHES:-1} +LOG_TO_CONSOLE=${LOG_TO_CONSOLE:-true} # Determine GPU list if [[ -n "$CUDA_VISIBLE_DEVICES" ]]; then @@ -143,6 +173,21 @@ GPU_COUNT=${#GPU_LIST[@]} # Gather all mps files into an array mapfile -t mps_files < <(ls "$MPS_DIR"/*.mps) + +# Calculate batch size and start/end indices +batch_size=$(( (${#mps_files[@]} + N_BATCHES - 1) / N_BATCHES )) +start_idx=$((BATCH_NUM * batch_size)) +end_idx=$((start_idx + batch_size)) + +# Ensure end_idx doesn't exceed array length +if ((end_idx > ${#mps_files[@]})); then + end_idx=${#mps_files[@]} +fi + +# Extract subset of files for this batch +mps_files=("${mps_files[@]:$start_idx:$((end_idx-start_idx))}") + + file_count=${#mps_files[@]} # Initialize the index file for locking mechanism @@ -184,11 +229,26 @@ worker() { mps_file="${mps_files[my_index]}" echo "GPU $gpu_id processing $my_index" - if [ "$RELAXATION" = true ]; then - CUDA_VISIBLE_DEVICES=$gpu_id cuopt_cli "$mps_file" --time-limit $TIME_LIMIT --log-file "$OUTPUT_DIR/$(basename "${mps_file%.mps}").log" --log-to-console=false --relaxation - else - CUDA_VISIBLE_DEVICES=$gpu_id cuopt_cli "$mps_file" --time-limit $TIME_LIMIT --log-file "$OUTPUT_DIR/$(basename "${mps_file%.mps}").log" --log-to-console=false + + # Build arguments string + args="" + if [ -n "$NUM_CPU_THREADS" ]; then + args="$args --num-cpu-threads $NUM_CPU_THREADS" + fi + if [ -n "$HEURISTICS_ONLY" ]; then + args="$args --heuristics-only $HEURISTICS_ONLY" + fi + if [ -n "$WRITE_LOG_FILE" ]; then + args="$args --log-file $OUTPUT_DIR/$(basename "${mps_file%.mps}").log" fi + if [ -n "$RELAXATION" ]; then + args="$args --relaxation" + fi + if [ -n "$LOG_TO_CONSOLE" ]; then + args="$args --log-to-console $LOG_TO_CONSOLE" + fi + + CUDA_VISIBLE_DEVICES=$gpu_id cuopt_cli "$mps_file" --time-limit $TIME_LIMIT $args done } From 158377d6b290288fa9c900782510a93cbdbaef66 Mon Sep 17 00:00:00 2001 From: Rajesh Gandham Date: Tue, 10 Jun 2025 10:35:07 -0700 Subject: [PATCH 3/7] Add the newly added options to help --- benchmarks/linear_programming/run_mps_files.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/benchmarks/linear_programming/run_mps_files.sh b/benchmarks/linear_programming/run_mps_files.sh index 47c14c18df..b314634a49 100755 --- a/benchmarks/linear_programming/run_mps_files.sh +++ b/benchmarks/linear_programming/run_mps_files.sh @@ -27,6 +27,13 @@ # --ngpus : Number of GPUs to use (default: 1) # --time-limit : Time limit in seconds for each problem (default: 360) # --output-dir : Directory to store output log files (default: current directory) +# --relaxation : Run relaxation instead of solving the MIP +# --heuristics-only : Run heuristics only +# --write-log-file : Write log file +# --num-cpu-threads : Number of CPU threads to use +# --batch-num : Batch number +# --n-batches : Number of batches +# --log-to-console : Log to console # # Examples: # # Run all MPS files in /data/lp using 2 GPUs with 1 hour time limit @@ -61,6 +68,12 @@ Optional Arguments: --time-limit N Time limit in seconds for each problem (default: 360) --output-dir PATH Directory to store output log files (default: current directory) --relaxation Run relaxation instead of solving the MIP + --heuristics-only Run heuristics only + --write-log-file Write log file + --num-cpu-threads Number of CPU threads to use + --batch-num Batch number + --n-batches Number of batches + --log-to-console Log to console -h, --help Show this help message and exit Examples: From 2276edfc09ce4909db6866b72974d6153dc6cd04 Mon Sep 17 00:00:00 2001 From: Rajesh Gandham Date: Tue, 10 Jun 2025 10:36:46 -0700 Subject: [PATCH 4/7] Cleanup --- benchmarks/linear_programming/run_mps_files.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/linear_programming/run_mps_files.sh b/benchmarks/linear_programming/run_mps_files.sh index b314634a49..1d0c660060 100755 --- a/benchmarks/linear_programming/run_mps_files.sh +++ b/benchmarks/linear_programming/run_mps_files.sh @@ -149,7 +149,7 @@ while [[ $# -gt 0 ]]; do ;; *) echo "Unknown argument: $1" - echo "Usage: $0 --path MPS_DIR --ngpus GPU_COUNT --time-limit TIME_LIMIT --output-dir OUTPUT_DIR --relaxation" + print_help exit 1 ;; esac @@ -157,7 +157,7 @@ done # Set defaults and validate required arguments if [[ -z "$MPS_DIR" ]]; then echo "Missing required argument: --path MPS_DIR" - echo "Usage: $0 --path MPS_DIR [--ngpus GPU_COUNT] [--time-limit TIME_LIMIT] [--output-dir OUTPUT_DIR] [--relaxation]" + print_help exit 1 fi From 95c1ae5a180a2846f3cdd008d342eabeaebdc783 Mon Sep 17 00:00:00 2001 From: Rajesh Gandham Date: Tue, 1 Jul 2025 12:50:26 -0700 Subject: [PATCH 5/7] Fix a typo --- benchmarks/linear_programming/run_mps_files.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/benchmarks/linear_programming/run_mps_files.sh b/benchmarks/linear_programming/run_mps_files.sh index 1d0c660060..27d586baa2 100755 --- a/benchmarks/linear_programming/run_mps_files.sh +++ b/benchmarks/linear_programming/run_mps_files.sh @@ -28,7 +28,7 @@ # --time-limit : Time limit in seconds for each problem (default: 360) # --output-dir : Directory to store output log files (default: current directory) # --relaxation : Run relaxation instead of solving the MIP -# --heuristics-only : Run heuristics only +# --mip-heuristics-only : Run mip heuristics only # --write-log-file : Write log file # --num-cpu-threads : Number of CPU threads to use # --batch-num : Batch number @@ -68,7 +68,7 @@ Optional Arguments: --time-limit N Time limit in seconds for each problem (default: 360) --output-dir PATH Directory to store output log files (default: current directory) --relaxation Run relaxation instead of solving the MIP - --heuristics-only Run heuristics only + --mip-heuristics-only Run mip heuristics only --write-log-file Write log file --num-cpu-threads Number of CPU threads to use --batch-num Batch number @@ -123,8 +123,8 @@ while [[ $# -gt 0 ]]; do RELAXATION=true shift ;; - --heuristics-only) - HEURISTICS_ONLY=true + --mip-heuristics-only) + MIP_HEURISTICS_ONLY=true shift ;; --write-log-file) @@ -166,7 +166,7 @@ GPU_COUNT=${GPU_COUNT:-1} TIME_LIMIT=${TIME_LIMIT:-360} OUTPUT_DIR=${OUTPUT_DIR:-.} RELAXATION=${RELAXATION:-false} -HEURISTICS_ONLY=${HEURISTICS_ONLY:-false} +MIP_HEURISTICS_ONLY=${MIP_HEURISTICS_ONLY:-false} WRITE_LOG_FILE=${WRITE_LOG_FILE:-false} NUM_CPU_THREADS=${NUM_CPU_THREADS:-1} BATCH_NUM=${BATCH_NUM:-0} @@ -248,8 +248,8 @@ worker() { if [ -n "$NUM_CPU_THREADS" ]; then args="$args --num-cpu-threads $NUM_CPU_THREADS" fi - if [ -n "$HEURISTICS_ONLY" ]; then - args="$args --heuristics-only $HEURISTICS_ONLY" + if [ -n "$MIP_HEURISTICS_ONLY" ]; then + args="$args --mip-heuristics-only $MIP_HEURISTICS_ONLY" fi if [ -n "$WRITE_LOG_FILE" ]; then args="$args --log-file $OUTPUT_DIR/$(basename "${mps_file%.mps}").log" From 7b20f7a718ee1a51db8125458d65450aa82d8271 Mon Sep 17 00:00:00 2001 From: Rajesh Gandham Date: Wed, 16 Jul 2025 22:56:51 -0400 Subject: [PATCH 6/7] Update benchmarks/linear_programming/run_mps_files.sh Co-authored-by: Ramakrishnap <42624703+rgsl888prabhu@users.noreply.github.com> --- benchmarks/linear_programming/run_mps_files.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/linear_programming/run_mps_files.sh b/benchmarks/linear_programming/run_mps_files.sh index 27d586baa2..5fc2b1d276 100755 --- a/benchmarks/linear_programming/run_mps_files.sh +++ b/benchmarks/linear_programming/run_mps_files.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); From 3aae928e9ff6ba93e7b6cad08f2ffa4a418a50e2 Mon Sep 17 00:00:00 2001 From: Rajesh Gandham Date: Fri, 18 Jul 2025 06:27:28 -0700 Subject: [PATCH 7/7] Fix bug in handling relaxation --- benchmarks/linear_programming/run_mps_files.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/benchmarks/linear_programming/run_mps_files.sh b/benchmarks/linear_programming/run_mps_files.sh index 5fc2b1d276..7e423f8072 100755 --- a/benchmarks/linear_programming/run_mps_files.sh +++ b/benchmarks/linear_programming/run_mps_files.sh @@ -31,7 +31,7 @@ # --mip-heuristics-only : Run mip heuristics only # --write-log-file : Write log file # --num-cpu-threads : Number of CPU threads to use -# --batch-num : Batch number +# --batch-num : Batch number. This allows to split the work across multiple batches uniformly when resources are limited. # --n-batches : Number of batches # --log-to-console : Log to console # @@ -45,6 +45,9 @@ # # Run with custom output directory # ./run_mps_files.sh --path /data/lp --output-dir ~/results/lp_benchmark # +# # Run with batch number +# ./run_mps_files.sh --path /data/lp --ngpus 2 --time-limit 3600 --batch-num 0 --n-batches 2 +# # Notes: # - Files are distributed dynamically across available GPUs for load balancing # - Each problem's results are logged to a separate file in the output directory @@ -120,6 +123,7 @@ while [[ $# -gt 0 ]]; do shift 2 ;; --relaxation) + echo "Running relaxation" RELAXATION=true shift ;; @@ -248,16 +252,16 @@ worker() { if [ -n "$NUM_CPU_THREADS" ]; then args="$args --num-cpu-threads $NUM_CPU_THREADS" fi - if [ -n "$MIP_HEURISTICS_ONLY" ]; then - args="$args --mip-heuristics-only $MIP_HEURISTICS_ONLY" + if [ "$MIP_HEURISTICS_ONLY" = true ]; then + args="$args --mip-heuristics-only true" fi - if [ -n "$WRITE_LOG_FILE" ]; then + if [ "$WRITE_LOG_FILE" = true ]; then args="$args --log-file $OUTPUT_DIR/$(basename "${mps_file%.mps}").log" fi - if [ -n "$RELAXATION" ]; then + if [ "$RELAXATION" = true ]; then args="$args --relaxation" fi - if [ -n "$LOG_TO_CONSOLE" ]; then + if [ "$LOG_TO_CONSOLE" = true ]; then args="$args --log-to-console $LOG_TO_CONSOLE" fi