From bfe8229f2a54d2748ff65f718efc9d885028a28a Mon Sep 17 00:00:00 2001 From: Miles Lubin Date: Sat, 10 Feb 2024 14:57:20 -0500 Subject: [PATCH 1/5] MPS: avoid creating list of variable names --- src/FileFormats/MPS/MPS.jl | 102 ++++++++++++++++++++++-------------- test/FileFormats/MPS/MPS.jl | 7 +-- 2 files changed, 66 insertions(+), 43 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index 311773f212..dc7879ec00 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -9,6 +9,7 @@ module MPS import ..FileFormats import MathOptInterface as MOI +import DataStructures: OrderedDict const _NUM_TO_STRING = [string(i) for i in -10:10] @@ -209,7 +210,17 @@ Write `model` to `io` in the MPS file format. function Base.write(io::IO, model::Model) options = get_options(model) if options.generic_names - FileFormats.create_generic_names(model) + # just constraints + i = 1 + for (F, S) in MOI.get(model, MOI.ListOfConstraintTypesPresent()) + if F == MOI.VariableIndex + continue # VariableIndex constraints do not need a name. + end + for c in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) + MOI.set(model, MOI.ConstraintName(), c, "R$i") + i += 1 + end + end else FileFormats.create_unique_names( model; @@ -218,11 +229,8 @@ function Base.write(io::IO, model::Model) ) end variables = MOI.get(model, MOI.ListOfVariableIndices()) - ordered_names = Vector{String}(undef, length(variables)) - var_to_column = Dict{MOI.VariableIndex,Int}() + var_to_column = OrderedDict{MOI.VariableIndex,Int}() for (i, x) in enumerate(variables) - n = MOI.get(model, MOI.VariableName(), x) - ordered_names[i] = n var_to_column[x] = i end write_model_name(io, model) @@ -238,19 +246,19 @@ function Base.write(io::IO, model::Model) end write_rows(io, model) obj_const, indicators = - write_columns(io, model, flip_obj, ordered_names, var_to_column) + write_columns(io, model, flip_obj, var_to_column) write_rhs(io, model, obj_const) write_ranges(io, model) - write_bounds(io, model, ordered_names, var_to_column) - write_quadobj(io, model, ordered_names, var_to_column) + write_bounds(io, model, var_to_column) + write_quadobj(io, model, var_to_column) if options.quadratic_format != kQuadraticFormatCPLEX # Gurobi needs qcons _after_ quadobj and _before_ SOS. - write_quadcons(io, model, ordered_names, var_to_column) + write_quadcons(io, model, var_to_column) end - write_sos(io, model, ordered_names, var_to_column) + write_sos(io, model, var_to_column) if options.quadratic_format == kQuadraticFormatCPLEX # CPLEX needs qcons _after_ SOS. - write_quadcons(io, model, ordered_names, var_to_column) + write_quadcons(io, model, var_to_column) end write_indicators(io, indicators) println(io, "ENDATA") @@ -388,7 +396,7 @@ function list_of_integer_variables(model::Model, var_to_column) end function _extract_terms( - var_to_column::Dict{MOI.VariableIndex,Int}, + var_to_column::OrderedDict{MOI.VariableIndex,Int}, coefficients::Vector{Vector{Tuple{String,Float64}}}, row_name::String, func::MOI.ScalarAffineFunction, @@ -403,7 +411,7 @@ function _extract_terms( end function _extract_terms( - var_to_column::Dict{MOI.VariableIndex,Int}, + var_to_column::OrderedDict{MOI.VariableIndex,Int}, coefficients::Vector{Vector{Tuple{String,Float64}}}, row_name::String, func::MOI.ScalarQuadraticFunction, @@ -421,7 +429,7 @@ function _collect_coefficients( model, ::Type{F}, ::Type{S}, - var_to_column::Dict{MOI.VariableIndex,Int}, + var_to_column::OrderedDict{MOI.VariableIndex,Int}, coefficients::Vector{Vector{Tuple{String,Float64}}}, ) where {F,S} for index in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) @@ -468,16 +476,24 @@ function _extract_terms_objective(model, var_to_column, coefficients, flip_obj) return obj_func.constant end +function _var_name(model::Model, variable::MOI.VariableIndex, column::Int, generic_name::Bool)::String + if generic_name + return "C$column" + else + return MOI.get(model, MOI.VariableName(), variable) + end +end + function write_columns( io::IO, model::Model, flip_obj, - ordered_names, var_to_column, ) + options = get_options(model) indicators = Tuple{String,String,MOI.ActivationCondition}[] coefficients = Vector{Tuple{String,Float64}}[ - Tuple{String,Float64}[] for _ in ordered_names + Tuple{String,Float64}[] for _ in 1:length(var_to_column) ] # Build constraint coefficients # The functions and sets are given explicitly so that this function is @@ -511,7 +527,8 @@ function write_columns( integer_variables = list_of_integer_variables(model, var_to_column) println(io, "COLUMNS") int_open = false - for (column, variable) in enumerate(ordered_names) + for (variable, column) in var_to_column + var_name = _var_name(model, variable, column, options.generic_names) is_int = column in integer_variables if is_int && !int_open println(io, Card(f2 = "MARKER", f3 = "'MARKER'", f5 = "'INTORG'")) @@ -523,13 +540,13 @@ function write_columns( if length(coefficients[column]) == 0 # Every variable must appear in the COLUMNS section. Add a 0 # objective coefficient instead. - println(io, Card(f2 = variable, f3 = "OBJ", f4 = "0")) + println(io, Card(f2 = var_name, f3 = "OBJ", f4 = "0")) end for (constraint, coefficient) in coefficients[column] println( io, Card( - f2 = variable, + f2 = var_name, f3 = constraint, f4 = _to_string(coefficient), ), @@ -750,9 +767,10 @@ function _collect_bounds(bounds, model, ::Type{S}, var_to_column) where {S} return end -function write_bounds(io::IO, model::Model, ordered_names, var_to_column) +function write_bounds(io::IO, model::Model, var_to_column) + options = get_options(model) println(io, "BOUNDS") - bounds = [(-Inf, Inf, VTYPE_CONTINUOUS) for _ in ordered_names] + bounds = [(-Inf, Inf, VTYPE_CONTINUOUS) for _ in 1:length(var_to_column)] @_unroll for S in ( MOI.LessThan{Float64}, MOI.GreaterThan{Float64}, @@ -762,7 +780,8 @@ function write_bounds(io::IO, model::Model, ordered_names, var_to_column) ) _collect_bounds(bounds, model, S, var_to_column) end - for (column, var_name) in enumerate(ordered_names) + for (variable, column) in var_to_column + var_name = _var_name(model, variable, column, options.generic_names) lower, upper, vtype = bounds[column] if vtype == VTYPE_BINARY println(io, Card(f1 = "BV", f2 = "bounds", f3 = var_name)) @@ -782,7 +801,7 @@ end # QUADRATIC OBJECTIVE # ============================================================================== -function write_quadobj(io::IO, model::Model, ordered_names, var_to_column) +function write_quadobj(io::IO, model::Model, var_to_column) f = _get_objective(model) if isempty(f.quadratic_terms) return @@ -798,8 +817,8 @@ function write_quadobj(io::IO, model::Model, ordered_names, var_to_column) end _write_q_matrix( io, + model, f, - ordered_names, var_to_column; duplicate_off_diagonal = options.quadratic_format == kQuadraticFormatCPLEX, @@ -809,18 +828,20 @@ end function _write_q_matrix( io::IO, + model::Model, f, - ordered_names, var_to_column; duplicate_off_diagonal::Bool, ) + options = get_options(model) # Convert the quadratic terms into matrix form. We don't need to scale # because MOI uses the same Q/2 format as Gurobi, but we do need to ensure # we collate off-diagonal terms in the lower-triangular. - terms = Dict{Tuple{Int,Int},Float64}() + terms = Dict{Tuple{MOI.VariableIndex,MOI.VariableIndex},Float64}() for term in f.quadratic_terms - x, y = var_to_column[term.variable_1], var_to_column[term.variable_2] - if x > y + x = term.variable_1 + y = term.variable_2 + if var_to_column[x] > var_to_column[y] x, y = y, x end if haskey(terms, (x, y)) @@ -830,12 +851,14 @@ function _write_q_matrix( end end # Use sort for reproducibility, and so the Q matrix is given in order. - for (x, y) in sort!(collect(keys(terms))) + for (x, y) in sort!(collect(keys(terms)), by = ((x,y),) -> (var_to_column[x], var_to_column[y])) + x_name = _var_name(model, x, var_to_column[x], options.generic_names) + y_name = _var_name(model, y, var_to_column[y], options.generic_names) println( io, Card( - f2 = ordered_names[x], - f3 = ordered_names[y], + f2 = x_name, + f3 = y_name, f4 = _to_string(terms[(x, y)]), ), ) @@ -843,8 +866,8 @@ function _write_q_matrix( println( io, Card( - f2 = ordered_names[y], - f3 = ordered_names[x], + f2 = y_name, + f3 = x_name, f4 = _to_string(terms[(x, y)]), ), ) @@ -857,7 +880,7 @@ end # QUADRATIC CONSTRAINTS # ============================================================================== -function write_quadcons(io::IO, model::Model, ordered_names, var_to_column) +function write_quadcons(io::IO, model::Model, var_to_column) options = get_options(model) F = MOI.ScalarQuadraticFunction{Float64} for S in ( @@ -876,8 +899,8 @@ function write_quadcons(io::IO, model::Model, ordered_names, var_to_column) f = MOI.get(model, MOI.ConstraintFunction(), ci) _write_q_matrix( io, + model, f, - ordered_names, var_to_column; duplicate_off_diagonal = options.quadratic_format != kQuadraticFormatMosek, @@ -895,18 +918,18 @@ function write_sos_constraint( io::IO, model::Model, index, - ordered_names, var_to_column, ) + options = get_options(model) func = MOI.get(model, MOI.ConstraintFunction(), index) set = MOI.get(model, MOI.ConstraintSet(), index) for (variable, weight) in zip(func.variables, set.weights) - column = var_to_column[variable] - println(io, Card(f2 = ordered_names[column], f3 = _to_string(weight))) + var_name = _var_name(model, variable, var_to_column[variable], options.generic_names) + println(io, Card(f2 = var_name, f3 = _to_string(weight))) end end -function write_sos(io::IO, model::Model, ordered_names, var_to_column) +function write_sos(io::IO, model::Model, var_to_column) sos1_indices = MOI.get( model, MOI.ListOfConstraintIndices{MOI.VectorOfVariables,MOI.SOS1{Float64}}(), @@ -925,7 +948,6 @@ function write_sos(io::IO, model::Model, ordered_names, var_to_column) io, model, index, - ordered_names, var_to_column, ) idx += 1 diff --git a/test/FileFormats/MPS/MPS.jl b/test/FileFormats/MPS/MPS.jl index 87e395f880..1ac7f91b2a 100644 --- a/test/FileFormats/MPS/MPS.jl +++ b/test/FileFormats/MPS/MPS.jl @@ -10,6 +10,7 @@ using Test import MathOptInterface as MOI import MathOptInterface.Utilities as MOIU +import DataStructures: OrderedDict const MPS = MOI.FileFormats.MPS const MPS_TEST_FILE = "test.mps" @@ -94,7 +95,7 @@ function test_sos() MOI.VectorOfVariables(x), MOI.SOS2([1.25, 2.25, 3.25]), ) - @test sprint(MPS.write_sos, model, ["x1", "x2", "x3"], names) == + @test sprint(MPS.write_sos, model, names) == "SOS\n" * " S1 SOS1\n" * " x1 1.5\n" * @@ -113,7 +114,7 @@ function test_maximization() MOI.set(model, MOI.VariableName(), x, "x") MOI.set(model, MOI.ObjectiveSense(), MOI.MAX_SENSE) MOI.set(model, MOI.ObjectiveFunction{MOI.VariableIndex}(), x) - @test sprint(MPS.write_columns, model, true, ["x"], Dict(x => 1)) == + @test sprint(MPS.write_columns, model, true, OrderedDict(x => 1)) == "COLUMNS\n x OBJ -1\n" end @@ -124,7 +125,7 @@ function test_maximization_objsense_false() MOI.set(model, MOI.ObjectiveSense(), MOI.MAX_SENSE) MOI.set(model, MOI.ObjectiveFunction{MOI.VariableIndex}(), x) sprint(MPS.write, model) - @test sprint(MPS.write_columns, model, false, ["x"], Dict(x => 1)) == + @test sprint(MPS.write_columns, model, false, OrderedDict(x => 1)) == "COLUMNS\n x OBJ 1\n" end From 1fd1a50f232df63532f935e0493ef81f105ad131 Mon Sep 17 00:00:00 2001 From: Miles Lubin Date: Sat, 10 Feb 2024 15:19:59 -0500 Subject: [PATCH 2/5] format --- src/FileFormats/MPS/MPS.jl | 55 +++++++++++++++----------------------- 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index dc7879ec00..3072ff3b58 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -245,8 +245,7 @@ function Base.write(io::IO, model::Model) flip_obj = MOI.get(model, MOI.ObjectiveSense()) == MOI.MAX_SENSE end write_rows(io, model) - obj_const, indicators = - write_columns(io, model, flip_obj, var_to_column) + obj_const, indicators = write_columns(io, model, flip_obj, var_to_column) write_rhs(io, model, obj_const) write_ranges(io, model) write_bounds(io, model, var_to_column) @@ -476,7 +475,12 @@ function _extract_terms_objective(model, var_to_column, coefficients, flip_obj) return obj_func.constant end -function _var_name(model::Model, variable::MOI.VariableIndex, column::Int, generic_name::Bool)::String +function _var_name( + model::Model, + variable::MOI.VariableIndex, + column::Int, + generic_name::Bool, +)::String if generic_name return "C$column" else @@ -484,12 +488,7 @@ function _var_name(model::Model, variable::MOI.VariableIndex, column::Int, gener end end -function write_columns( - io::IO, - model::Model, - flip_obj, - var_to_column, -) +function write_columns(io::IO, model::Model, flip_obj, var_to_column) options = get_options(model) indicators = Tuple{String,String,MOI.ActivationCondition}[] coefficients = Vector{Tuple{String,Float64}}[ @@ -851,25 +850,20 @@ function _write_q_matrix( end end # Use sort for reproducibility, and so the Q matrix is given in order. - for (x, y) in sort!(collect(keys(terms)), by = ((x,y),) -> (var_to_column[x], var_to_column[y])) + for (x, y) in sort!( + collect(keys(terms)), + by = ((x, y),) -> (var_to_column[x], var_to_column[y]), + ) x_name = _var_name(model, x, var_to_column[x], options.generic_names) y_name = _var_name(model, y, var_to_column[y], options.generic_names) println( io, - Card( - f2 = x_name, - f3 = y_name, - f4 = _to_string(terms[(x, y)]), - ), + Card(f2 = x_name, f3 = y_name, f4 = _to_string(terms[(x, y)])), ) if x != y && duplicate_off_diagonal println( io, - Card( - f2 = y_name, - f3 = x_name, - f4 = _to_string(terms[(x, y)]), - ), + Card(f2 = y_name, f3 = x_name, f4 = _to_string(terms[(x, y)])), ) end end @@ -914,17 +908,17 @@ end # SOS # ============================================================================== -function write_sos_constraint( - io::IO, - model::Model, - index, - var_to_column, -) +function write_sos_constraint(io::IO, model::Model, index, var_to_column) options = get_options(model) func = MOI.get(model, MOI.ConstraintFunction(), index) set = MOI.get(model, MOI.ConstraintSet(), index) for (variable, weight) in zip(func.variables, set.weights) - var_name = _var_name(model, variable, var_to_column[variable], options.generic_names) + var_name = _var_name( + model, + variable, + var_to_column[variable], + options.generic_names, + ) println(io, Card(f2 = var_name, f3 = _to_string(weight))) end end @@ -944,12 +938,7 @@ function write_sos(io::IO, model::Model, var_to_column) for (sos_type, indices) in enumerate([sos1_indices, sos2_indices]) for index in indices println(io, Card(f1 = "S$(sos_type)", f2 = "SOS$(idx)")) - write_sos_constraint( - io, - model, - index, - var_to_column, - ) + write_sos_constraint(io, model, index, var_to_column) idx += 1 end end From 9f04a58053a93c9950effc6b655beb788d44abd1 Mon Sep 17 00:00:00 2001 From: Miles Lubin Date: Sat, 10 Feb 2024 17:39:39 -0500 Subject: [PATCH 3/5] fix for indicators --- src/FileFormats/MPS/MPS.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index 3072ff3b58..6a220e65b3 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -448,6 +448,7 @@ function _collect_indicator( coefficients, indicators, ) where {S} + options = get_options(model) F = MOI.VectorAffineFunction{Float64} for index in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) row_name = MOI.get(model, MOI.ConstraintName(), index) @@ -456,9 +457,10 @@ function _collect_indicator( z = convert(MOI.VariableIndex, funcs[1]) _extract_terms(var_to_column, coefficients, row_name, funcs[2]) condition = _activation_condition(S) + var_name = _var_name(model, z, var_to_column[z], options.generic_names) push!( indicators, - (row_name, MOI.get(model, MOI.VariableName(), z), condition), + (row_name, var_name, condition), ) end return From 105fdca2bdff756cd2ed34b0512563f1dd579c82 Mon Sep 17 00:00:00 2001 From: Miles Lubin Date: Sat, 10 Feb 2024 20:32:48 -0500 Subject: [PATCH 4/5] format --- src/FileFormats/MPS/MPS.jl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index 6a220e65b3..2364cff48a 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -458,10 +458,7 @@ function _collect_indicator( _extract_terms(var_to_column, coefficients, row_name, funcs[2]) condition = _activation_condition(S) var_name = _var_name(model, z, var_to_column[z], options.generic_names) - push!( - indicators, - (row_name, var_name, condition), - ) + push!(indicators, (row_name, var_name, condition)) end return end From 696d4780228645094a7b19036e47e0ce48c91593 Mon Sep 17 00:00:00 2001 From: odow Date: Tue, 13 Feb 2024 17:41:35 +1300 Subject: [PATCH 5/5] Refactor create_generic_names --- src/FileFormats/MPS/MPS.jl | 13 ++----------- src/FileFormats/utils.jl | 32 ++++++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/FileFormats/MPS/MPS.jl b/src/FileFormats/MPS/MPS.jl index 2364cff48a..2b21f79fea 100644 --- a/src/FileFormats/MPS/MPS.jl +++ b/src/FileFormats/MPS/MPS.jl @@ -210,17 +210,8 @@ Write `model` to `io` in the MPS file format. function Base.write(io::IO, model::Model) options = get_options(model) if options.generic_names - # just constraints - i = 1 - for (F, S) in MOI.get(model, MOI.ListOfConstraintTypesPresent()) - if F == MOI.VariableIndex - continue # VariableIndex constraints do not need a name. - end - for c in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) - MOI.set(model, MOI.ConstraintName(), c, "R$i") - i += 1 - end - end + # Generic variable names handled in this writer. + FileFormats.create_generic_constraint_names(model) else FileFormats.create_unique_names( model; diff --git a/src/FileFormats/utils.jl b/src/FileFormats/utils.jl index d7f9cdf1da..f33904189b 100644 --- a/src/FileFormats/utils.jl +++ b/src/FileFormats/utils.jl @@ -39,19 +39,39 @@ Rename all variables and constraints in `model` to have generic names. This is helpful for users with proprietary models to avoid leaking information. """ function create_generic_names(model::MOI.ModelLike) + create_generic_variable_names(model) + create_generic_constraint_names(model) + return +end + +function create_generic_variable_names(model::MOI.ModelLike) for (i, x) in enumerate(MOI.get(model, MOI.ListOfVariableIndices())) MOI.set(model, MOI.VariableName(), x, "C$i") end + return +end + +function create_generic_constraint_names(model::MOI.ModelLike) i = 1 for (F, S) in MOI.get(model, MOI.ListOfConstraintTypesPresent()) - if F == MOI.VariableIndex - continue # VariableIndex constraints do not need a name. - end - for c in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) - MOI.set(model, MOI.ConstraintName(), c, "R$i") - i += 1 + if F != MOI.VariableIndex + i = create_generic_constraint_names(model, F, S, i) end end + return +end + +function create_generic_constraint_names( + model::MOI.ModelLike, + ::Type{F}, + ::Type{S}, + i::Int, +) where {F,S} + for c in MOI.get(model, MOI.ListOfConstraintIndices{F,S}()) + MOI.set(model, MOI.ConstraintName(), c, "R$i") + i += 1 + end + return i end function _replace(s::String, replacements::Vector{Function})