From 866ab25a1537f746b49b5288b30f4cd740b46ed3 Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Thu, 13 Jul 2023 18:34:53 -0300 Subject: [PATCH 1/2] add Arrow type --- Project.toml | 3 ++- src/L2O.jl | 7 ++++++- src/arrowrecorder.jl | 17 +++++++++++++++ src/csvrecorder.jl | 35 +++++++++++++++++++++++++++++++ src/datasetgen.jl | 49 ++++++-------------------------------------- test/runtests.jl | 28 +++++++++++++------------ 6 files changed, 81 insertions(+), 58 deletions(-) create mode 100644 src/arrowrecorder.jl create mode 100644 src/csvrecorder.jl diff --git a/Project.toml b/Project.toml index dc49acf..1329a90 100644 --- a/Project.toml +++ b/Project.toml @@ -4,6 +4,7 @@ authors = ["andrewrosemberg and contributors"] version = "1.0.0-DEV" [deps] +Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" ParametricOptInterface = "0ce4ce61-57bf-432b-a095-efac525d185e" @@ -11,9 +12,9 @@ ParametricOptInterface = "0ce4ce61-57bf-432b-a095-efac525d185e" julia = "1" [extras] -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test", "DelimitedFiles", "HiGHS"] diff --git a/src/L2O.jl b/src/L2O.jl index 13ae9e7..498fe41 100644 --- a/src/L2O.jl +++ b/src/L2O.jl @@ -1,10 +1,15 @@ module L2O +using Arrow using JuMP import ParametricOptInterface as POI +import Base: string + +export ArrowFile, CSVFile, ProblemIterator, Recorder, solve_batch -export solve_batch, CSVRecorder, ProblemIterator include("datasetgen.jl") +include("csvrecorder.jl") +include("arrowrecorder.jl") end diff --git a/src/arrowrecorder.jl b/src/arrowrecorder.jl new file mode 100644 index 0000000..f02db47 --- /dev/null +++ b/src/arrowrecorder.jl @@ -0,0 +1,17 @@ +abstract type ArrowFile <: RecorderFile end + +Base.string(::Type{ArrowFile}) = "arrow" + +""" + record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::Int64) + +Record optimization problem solution to an Arrow file. +""" +function record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::Int64) + if !isfile(recorder.filename) + ### NOT WORKING ### + Arrow.write(recorder.filename, (id = Int64[], recorder.primal_variables..., "dual_" .* recorder.dual_variables..., )) + end + + Arrow.write(Arrow.Table(; id = [id], [MOI.get(model, MOI.VariablePrimal(), model[p]) for p in recorder.primal_variables]..., [MOI.get(model, MOI.ConstraintDual(), model[p]) for p in recorder.dual_variables]...) |> Arrow.Table, Arrow.append!(recorder.filename)) +end \ No newline at end of file diff --git a/src/csvrecorder.jl b/src/csvrecorder.jl new file mode 100644 index 0000000..75a3c0a --- /dev/null +++ b/src/csvrecorder.jl @@ -0,0 +1,35 @@ +abstract type CSVFile <: RecorderFile end + +Base.string(::Type{CSVFile}) = "csv" + +""" + record(recorder::Recorder{CSVFile}, model::JuMP.Model, id::Int64) + +Record optimization problem solution to a CSV file. +""" +function record(recorder::Recorder{CSVFile}, model::JuMP.Model, id::Int64) + if !isfile(recorder.filename) + open(recorder.filename, "w") do f + write(f, "id") + for p in recorder.primal_variables + write(f, ",$p") + end + for p in recorder.dual_variables + write(f, ",dual_$p") + end + write(f, "\n") + end + end + open(recorder.filename, "a") do f + write(f, "$id") + for p in recorder.primal_variables + val = MOI.get(model, MOI.VariablePrimal(), model[p]) + write(f, ",$val") + end + for p in recorder.dual_variables + val = MOI.get(model, MOI.ConstraintDual(), model[p]) + write(f, ",$val") + end + write(f, "\n") + end +end \ No newline at end of file diff --git a/src/datasetgen.jl b/src/datasetgen.jl index c566e95..988061e 100644 --- a/src/datasetgen.jl +++ b/src/datasetgen.jl @@ -1,23 +1,18 @@ -""" - Recorder - -Abstract type for recorders of optimization problem solutions. -""" -abstract type Recorder end +abstract type RecorderFile end """ - CSVRecorder(filename; primal_variables=[], dual_variables=[], filterfn=(model)-> termination_status(model) == MOI.OPTIMAL) + Recorder(filename; primal_variables=[], dual_variables=[], filterfn=(model)-> termination_status(model) == MOI.OPTIMAL) -Recorder type of optimization problem solutions to a CSV file. +Recorder of optimization problem solutions. """ -mutable struct CSVRecorder <: Recorder +mutable struct Recorder{T<:RecorderFile} filename::String primal_variables::AbstractArray{Symbol} dual_variables::AbstractArray{Symbol} filterfn::Function - function CSVRecorder(filename::String; primal_variables=[], dual_variables=[], filterfn=(model)-> termination_status(model) == MOI.OPTIMAL) - return new(filename, primal_variables, dual_variables, filterfn) + function Recorder{T}(filename::String; primal_variables=[], dual_variables=[], filterfn=(model)-> termination_status(model) == MOI.OPTIMAL) where T<:RecorderFile + return new{T}(filename, primal_variables, dual_variables, filterfn) end end @@ -37,38 +32,6 @@ struct ProblemIterator{T<:Real, Z<:Integer} end end -""" - record(recorder::CSVRecorder, model::JuMP.Model, id::Int64) - -Record optimization problem solution to a CSV file. -""" -function record(recorder::CSVRecorder, model::JuMP.Model, id::Int64) - if !isfile(recorder.filename) - open(recorder.filename, "w") do f - write(f, "id") - for p in recorder.primal_variables - write(f, ",$p") - end - for p in recorder.dual_variables - write(f, ",dual_$p") - end - write(f, "\n") - end - end - open(recorder.filename, "a") do f - write(f, "$id") - for p in recorder.primal_variables - val = MOI.get(model, MOI.VariablePrimal(), model[p]) - write(f, ",$val") - end - for p in recorder.dual_variables - val = MOI.get(model, MOI.ConstraintDual(), model[p]) - write(f, ",$val") - end - write(f, "\n") - end -end - """ update_model!(model::JuMP.Model, p::VariableRef, val::Real) diff --git a/test/runtests.jl b/test/runtests.jl index b6071c1..9fe7133 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,18 +5,20 @@ using JuMP, HiGHS import ParametricOptInterface as POI @testset "L2O.jl" begin - model = Model(() -> POI.Optimizer(HiGHS.Optimizer())) - @variable(model, x) - p = @variable(model, _p in POI.Parameter(1.0)) - @constraint(model, cons, x + _p >= 3) - @objective(model, Min, 2x) + @testset "Dataset generation: $filetype" for filetype in [CSVFile, ArrowFile] + model = Model(() -> POI.Optimizer(HiGHS.Optimizer())) + @variable(model, x) + p = @variable(model, _p in POI.Parameter(1.0)) + @constraint(model, cons, x + _p >= 3) + @objective(model, Min, 2x) - num_p = 10 - problem_iterator = ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:num_p))) - recorder = CSVRecorder("test.csv", primal_variables=[:x], dual_variables=[:cons]) - solve_batch(model, problem_iterator, recorder) - @test isfile("test.csv") - @test length(readdlm("test.csv", ',')[:, 1]) == num_p+1 - @test length(readdlm("test.csv", ',')[1, :]) == 3 - rm("test.csv") + num_p = 10 + problem_iterator = ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:num_p))) + recorder = Recorder{filetype}("test.$(string(filetype))", primal_variables=[:x], dual_variables=[:cons]) + solve_batch(model, problem_iterator, recorder) + @test isfile("test.$(string(filetype))") + @test length(readdlm("test.$(string(filetype))", ',')[:, 1]) == num_p+1 + @test length(readdlm("test.$(string(filetype))", ',')[1, :]) == 3 + rm("test.$(string(filetype))") + end end From de901f06d7680805b8277c2b25c2cdc0d1d9733b Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Fri, 14 Jul 2023 13:48:42 -0300 Subject: [PATCH 2/2] fix arrow --- src/arrowrecorder.jl | 23 ++++++++++++++++++----- test/runtests.jl | 29 +++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/arrowrecorder.jl b/src/arrowrecorder.jl index f02db47..71a1a44 100644 --- a/src/arrowrecorder.jl +++ b/src/arrowrecorder.jl @@ -3,15 +3,28 @@ abstract type ArrowFile <: RecorderFile end Base.string(::Type{ArrowFile}) = "arrow" """ - record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::Int64) + record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::T) Record optimization problem solution to an Arrow file. """ -function record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::Int64) +function record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::T) where {T<:Integer} if !isfile(recorder.filename) - ### NOT WORKING ### - Arrow.write(recorder.filename, (id = Int64[], recorder.primal_variables..., "dual_" .* recorder.dual_variables..., )) + Arrow.append( + recorder.filename, (; + id = T[], + zip(recorder.primal_variables, fill(Float64[], length(recorder.primal_variables)))..., + zip(Symbol.("dual_" .* string.(recorder.dual_variables)), fill(Float64[], length(recorder.dual_variables)))..., + ) + ) end - Arrow.write(Arrow.Table(; id = [id], [MOI.get(model, MOI.VariablePrimal(), model[p]) for p in recorder.primal_variables]..., [MOI.get(model, MOI.ConstraintDual(), model[p]) for p in recorder.dual_variables]...) |> Arrow.Table, Arrow.append!(recorder.filename)) + Arrow.append( + recorder.filename, (; + id = [id], + zip(recorder.primal_variables, [[MOI.get(model, MOI.VariablePrimal(), model[p])] for p in recorder.primal_variables])..., + zip(Symbol.("dual_" .* string.(recorder.dual_variables)), [[MOI.get(model, MOI.ConstraintDual(), model[p])] for p in recorder.dual_variables])..., + ) + ) + + end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 9fe7133..0c75066 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,10 +1,11 @@ using L2O +using Arrow using Test using DelimitedFiles using JuMP, HiGHS import ParametricOptInterface as POI -@testset "L2O.jl" begin +function testdataset_gen(path) @testset "Dataset generation: $filetype" for filetype in [CSVFile, ArrowFile] model = Model(() -> POI.Optimizer(HiGHS.Optimizer())) @variable(model, x) @@ -14,11 +15,27 @@ import ParametricOptInterface as POI num_p = 10 problem_iterator = ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:num_p))) - recorder = Recorder{filetype}("test.$(string(filetype))", primal_variables=[:x], dual_variables=[:cons]) + file = joinpath(path, "test.$(string(filetype))") + recorder = Recorder{filetype}(file, primal_variables=[:x], dual_variables=[:cons]) solve_batch(model, problem_iterator, recorder) - @test isfile("test.$(string(filetype))") - @test length(readdlm("test.$(string(filetype))", ',')[:, 1]) == num_p+1 - @test length(readdlm("test.$(string(filetype))", ',')[1, :]) == 3 - rm("test.$(string(filetype))") + if filetype == CSVFile + file1 = joinpath(path, "test.csv") + @test isfile(file1) + @test length(readdlm(file1, ',')[:, 1]) == num_p+1 + @test length(readdlm(file1, ',')[1, :]) == 3 + rm(file1) + else + file2 = joinpath(path, "test.arrow") + @test isfile(file2) + df = Arrow.Table(file2) + @test length(df) == 3 + @test length(df[1]) == num_p + end + end +end + +@testset "L2O.jl" begin + mktempdir() do path + testdataset_gen(path) end end