From 343aabf6ffe89005926f93a0169fc02840151ec1 Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Fri, 14 Jul 2023 15:14:10 -0300 Subject: [PATCH 1/9] add powermodels example --- .gitignore | 3 ++ examples/powermodels/pg_lib.jl | 57 ++++++++++++++++++++++++++++++++++ src/datasetgen.jl | 8 ++--- 3 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 examples/powermodels/pg_lib.jl diff --git a/.gitignore b/.gitignore index 4e890ab..897d8cf 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ /docs/build/ Manifest.toml /.vscode/* +*.csv +*.arrow +*.m diff --git a/examples/powermodels/pg_lib.jl b/examples/powermodels/pg_lib.jl new file mode 100644 index 0000000..809ecef --- /dev/null +++ b/examples/powermodels/pg_lib.jl @@ -0,0 +1,57 @@ +using Downloads +using PowerModels +using JuMP, HiGHS +import ParametricOptInterface as POI + +"""create ref for anonimous variables on model""" +function createvarrefs!(sp::JuMP.Model, pm::AbstractPowerModel) + for listvarref in values(PowerModels.var(pm)) + for variableref in values(listvarref) + if typeof(variableref) == JuMP.VariableRef + sp[Symbol(name(variableref))] = variableref + end + end + end +end + +function generate_dataset_pglib(data_dir::String, case_name::String; download_files::Bool=true, filetype::Type{RecorderFile}) + case_file_path = joinpath(data_dir, case_name) + if download_files && !isfile(case_file_path) + Downloads.download( + "https://raw.githubusercontent.com/power-grid-lib/pglib-opf/01681386d084d8bd03b429abcd1ee6966f68b9a3/" * + case_name, + case_file_path, + ) + end + + # Read data + network_data = PowerModels.parse_file(case_file_path) + + # The problem to iterate over + model = Model(() -> POI.Optimizer(HiGHS.Optimizer())) + + # Link POI + network_data["load"]["1"]["pd"] = p = @variable(model, _p in POI.Parameter(1.0)) + pm = instantiate_model( + network_data, + DCPPowerModel, + PowerModels.build_opf; + setting=Dict("output" => Dict("duals" => true)), + jump_model=model, + ) + + num_p = 10 + problem_iterator = ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:num_p))) + + createvarrefs!(model, pm) + + file = joinpath(data_dir, "test.$(string(filetype))") + recorder = Recorder{filetype}(file, primal_variables=[Symbol("0_pg[$i]") for i in 1:5]) + return solve_batch(model, problem_iterator, recorder) +end + +data_dir = joinpath(dirname(@__FILE__), "data") + +case_name = "pglib_opf_case5_pjm.m" + +success_solves = generate_dataset_pglib(data_dir, case_name; download_files=true, filetype=CSVFile) \ No newline at end of file diff --git a/src/datasetgen.jl b/src/datasetgen.jl index 988061e..fac2272 100644 --- a/src/datasetgen.jl +++ b/src/datasetgen.jl @@ -71,8 +71,9 @@ function solve_and_record(model::JuMP.Model, problem_iterator::ProblemIterator, optimize!(model) if recorder.filterfn(model) record(recorder, model, problem_iterator.ids[idx]) + return 1 end - return nothing + return 0 end """ @@ -81,8 +82,5 @@ end Solve a batch of optimization problems and record the solutions. """ function solve_batch(model::JuMP.Model, problem_iterator::ProblemIterator, recorder::Recorder) - for idx in 1:length(problem_iterator.ids) - solve_and_record(model, problem_iterator, recorder, idx) - end - return nothing + return sum(solve_and_record(model, problem_iterator, recorder, idx) for idx in 1:length(problem_iterator.ids)) / length(problem_iterator.ids) end From 8ab9b58f02842840157a6c540b3b9d2f25b998f4 Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Fri, 14 Jul 2023 15:15:52 -0300 Subject: [PATCH 2/9] add csv example --- .gitignore | 1 - examples/powermodels/data/test.csv | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 examples/powermodels/data/test.csv diff --git a/.gitignore b/.gitignore index 897d8cf..515fd15 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,5 @@ /docs/build/ Manifest.toml /.vscode/* -*.csv *.arrow *.m diff --git a/examples/powermodels/data/test.csv b/examples/powermodels/data/test.csv new file mode 100644 index 0000000..4eb488a --- /dev/null +++ b/examples/powermodels/data/test.csv @@ -0,0 +1,8 @@ +id,0_pg[1],0_pg[2],0_pg[3],0_pg[4],0_pg[5] +1,0.4,1.7,1.5965025107920006,0.0,4.303497489207999 +2,0.4,1.7,2.415725486741256,0.0,4.484274513258743 +3,0.4,1.7,3.234948462690511,0.0,4.665051537309488 +4,0.4,1.7,4.054171438639767,0.0,4.845828561360232 +5,0.4,1.7,4.873394414589022,0.0,5.026605585410977 +6,0.4,1.7,5.2,0.32903966105684207,5.370960338943155 +7,0.4,1.7,5.2,0.8762327880428372,5.823767211957159 From 54593d473210ef9eb50fbcffe89beca0873ad7c2 Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Sat, 15 Jul 2023 14:05:43 -0300 Subject: [PATCH 3/9] create testset pglib --- .gitignore | 1 + Project.toml | 9 ++++-- examples/powermodels/data/test.csv | 8 ----- examples/powermodels/pg_lib.jl | 48 ++++++++++++++++++++++++------ src/arrowrecorder.jl | 12 -------- test/runtests.jl | 30 +++++++++++++++---- 6 files changed, 72 insertions(+), 36 deletions(-) delete mode 100644 examples/powermodels/data/test.csv diff --git a/.gitignore b/.gitignore index 515fd15..6cc3760 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ Manifest.toml /.vscode/* *.arrow *.m +*.csv diff --git a/Project.toml b/Project.toml index 1329a90..699437b 100644 --- a/Project.toml +++ b/Project.toml @@ -9,12 +9,17 @@ JuMP = "4076af6c-e467-56ae-b986-b466b2749572" ParametricOptInterface = "0ce4ce61-57bf-432b-a095-efac525d185e" [compat] -julia = "1" +JuMP = "1" +Arrow = "2" +ParametricOptInterface = "0.5" +julia = "1.6" [extras] DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b" +PowerModels = "c36e90e8-916a-50a6-bd94-075b64ef4655" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test", "DelimitedFiles", "HiGHS"] +test = ["Test", "DelimitedFiles", "Downloads", "HiGHS", "PowerModels"] diff --git a/examples/powermodels/data/test.csv b/examples/powermodels/data/test.csv deleted file mode 100644 index 4eb488a..0000000 --- a/examples/powermodels/data/test.csv +++ /dev/null @@ -1,8 +0,0 @@ -id,0_pg[1],0_pg[2],0_pg[3],0_pg[4],0_pg[5] -1,0.4,1.7,1.5965025107920006,0.0,4.303497489207999 -2,0.4,1.7,2.415725486741256,0.0,4.484274513258743 -3,0.4,1.7,3.234948462690511,0.0,4.665051537309488 -4,0.4,1.7,4.054171438639767,0.0,4.845828561360232 -5,0.4,1.7,4.873394414589022,0.0,5.026605585410977 -6,0.4,1.7,5.2,0.32903966105684207,5.370960338943155 -7,0.4,1.7,5.2,0.8762327880428372,5.823767211957159 diff --git a/examples/powermodels/pg_lib.jl b/examples/powermodels/pg_lib.jl index 809ecef..d6eac91 100644 --- a/examples/powermodels/pg_lib.jl +++ b/examples/powermodels/pg_lib.jl @@ -3,7 +3,11 @@ using PowerModels using JuMP, HiGHS import ParametricOptInterface as POI -"""create ref for anonimous variables on model""" +""" + createvarrefs!(sp::JuMP.Model, pm::AbstractPowerModel) + +create ref for anonimous variables on model +""" function createvarrefs!(sp::JuMP.Model, pm::AbstractPowerModel) for listvarref in values(PowerModels.var(pm)) for variableref in values(listvarref) @@ -14,7 +18,16 @@ function createvarrefs!(sp::JuMP.Model, pm::AbstractPowerModel) end end -function generate_dataset_pglib(data_dir::String, case_name::String; download_files::Bool=true, filetype::Type{RecorderFile}) +""" + generate_dataset_pglib(data_dir::String, case_name::String; download_files::Bool=true, filetype::Type{RecorderFile}, + num_p=10 +) + +Generate dataset for pglib case_name with num_p problems and save it in data_dir +""" +function generate_dataset_pglib(data_dir::String, case_name::String; download_files::Bool=true, filetype::Type{T}, + num_p=10 +) where {T<:L2O.RecorderFile} case_file_path = joinpath(data_dir, case_name) if download_files && !isfile(case_file_path) Downloads.download( @@ -30,8 +43,11 @@ function generate_dataset_pglib(data_dir::String, case_name::String; download_fi # The problem to iterate over model = Model(() -> POI.Optimizer(HiGHS.Optimizer())) - # Link POI + # Save original load value and Link POI + original_load = network_data["load"]["1"]["pd"] network_data["load"]["1"]["pd"] = p = @variable(model, _p in POI.Parameter(1.0)) + + # Instantiate the model pm = instantiate_model( network_data, DCPPowerModel, @@ -40,18 +56,32 @@ function generate_dataset_pglib(data_dir::String, case_name::String; download_fi jump_model=model, ) - num_p = 10 - problem_iterator = ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:num_p))) + # The problem iterator + problem_iterator = ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:num_p) .* original_load)) + # Create ref for anonimous variables on model createvarrefs!(model, pm) + # Solve the problem and return the number of successfull solves file = joinpath(data_dir, "test.$(string(filetype))") - recorder = Recorder{filetype}(file, primal_variables=[Symbol("0_pg[$i]") for i in 1:5]) - return solve_batch(model, problem_iterator, recorder) + number_generators = length(network_data["gen"]) + recorder = Recorder{filetype}(file, primal_variables=[Symbol("0_pg[$i]") for i in 1:number_generators]) + return solve_batch(model, problem_iterator, recorder), number_generators end +# Define test case from pglib data_dir = joinpath(dirname(@__FILE__), "data") - case_name = "pglib_opf_case5_pjm.m" -success_solves = generate_dataset_pglib(data_dir, case_name; download_files=true, filetype=CSVFile) \ No newline at end of file +# Define number of problems +num_p=10 + +# Generate dataset +success_solves, number_generators = generate_dataset_pglib(data_dir, case_name; download_files=true, filetype=CSVFile, num_p=num_p) + +# Check if the number of successfull solves is equal to the number of problems saved +file = joinpath(data_dir, "test.csv") +@test isfile(file) +@test length(readdlm(file, ',')[:, 1]) == num_p * success_solves + 1 +@test length(readdlm(file, ',')[1, :]) == number_generators + 1 +rm(file) \ No newline at end of file diff --git a/src/arrowrecorder.jl b/src/arrowrecorder.jl index 71a1a44..2b1dd0a 100644 --- a/src/arrowrecorder.jl +++ b/src/arrowrecorder.jl @@ -8,16 +8,6 @@ Base.string(::Type{ArrowFile}) = "arrow" Record optimization problem solution to an Arrow file. """ function record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::T) where {T<:Integer} - if !isfile(recorder.filename) - Arrow.append( - recorder.filename, (; - id = T[], - zip(recorder.primal_variables, fill(Float64[], length(recorder.primal_variables)))..., - zip(Symbol.("dual_" .* string.(recorder.dual_variables)), fill(Float64[], length(recorder.dual_variables)))..., - ) - ) - end - Arrow.append( recorder.filename, (; id = [id], @@ -25,6 +15,4 @@ function record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::T) where { zip(Symbol.("dual_" .* string.(recorder.dual_variables)), [[MOI.get(model, MOI.ConstraintDual(), model[p])] for p in recorder.dual_variables])..., ) ) - - end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 0c75066..b30cd30 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,19 +5,32 @@ using DelimitedFiles using JuMP, HiGHS import ParametricOptInterface as POI -function testdataset_gen(path) - @testset "Dataset generation: $filetype" for filetype in [CSVFile, ArrowFile] +""" + testdataset_gen(path::String) + +Test dataset generation for different filetypes +""" +function testdataset_gen(path::String) + @testset "Type: $filetype" for filetype in [CSVFile, ArrowFile] + # The problem to iterate over model = Model(() -> POI.Optimizer(HiGHS.Optimizer())) @variable(model, x) p = @variable(model, _p in POI.Parameter(1.0)) @constraint(model, cons, x + _p >= 3) @objective(model, Min, 2x) + # The problem iterator num_p = 10 problem_iterator = ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:num_p))) - file = joinpath(path, "test.$(string(filetype))") + + # The recorder + file = joinpath(path, "test.$(string(filetype))") # file path recorder = Recorder{filetype}(file, primal_variables=[:x], dual_variables=[:cons]) + + # Solve all problems and record solutions solve_batch(model, problem_iterator, recorder) + + # Check if file exists and has the correct number of rows and columns if filetype == CSVFile file1 = joinpath(path, "test.csv") @test isfile(file1) @@ -35,7 +48,14 @@ function testdataset_gen(path) end @testset "L2O.jl" begin - mktempdir() do path - testdataset_gen(path) + @testset "Dataset Generation" begin + # Different filetypes + mktempdir() do path + testdataset_gen(path) + end + # pglib + @testset "pg_lib case" begin + include(joinpath(dirname(dirname(@__FILE__)), "examples", "powermodels", "pg_lib.jl")) + end end end From 008ad11f812c73d480a6bef74a7a9fbe7af4fd77 Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Sat, 15 Jul 2023 14:09:12 -0300 Subject: [PATCH 4/9] format code using JuliaFormatter --- docs/make.jl | 9 ++------ examples/powermodels/pg_lib.jl | 24 ++++++++++++++------- src/L2O.jl | 1 - src/arrowrecorder.jl | 27 +++++++++++++++++------- src/csvrecorder.jl | 2 +- src/datasetgen.jl | 38 ++++++++++++++++++++++++---------- test/runtests.jl | 10 ++++++--- 7 files changed, 74 insertions(+), 37 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index d8e279a..6cbb92d 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -14,12 +14,7 @@ makedocs(; edit_link="main", assets=String[], ), - pages=[ - "Home" => "index.md", - ], + pages=["Home" => "index.md"], ) -deploydocs(; - repo="github.com/andrewrosemberg/L2O.jl", - devbranch="main", -) +deploydocs(; repo="github.com/andrewrosemberg/L2O.jl", devbranch="main") diff --git a/examples/powermodels/pg_lib.jl b/examples/powermodels/pg_lib.jl index d6eac91..71d52cb 100644 --- a/examples/powermodels/pg_lib.jl +++ b/examples/powermodels/pg_lib.jl @@ -25,8 +25,12 @@ end Generate dataset for pglib case_name with num_p problems and save it in data_dir """ -function generate_dataset_pglib(data_dir::String, case_name::String; download_files::Bool=true, filetype::Type{T}, - num_p=10 +function generate_dataset_pglib( + data_dir::String, + case_name::String; + download_files::Bool=true, + filetype::Type{T}, + num_p=10, ) where {T<:L2O.RecorderFile} case_file_path = joinpath(data_dir, case_name) if download_files && !isfile(case_file_path) @@ -57,7 +61,9 @@ function generate_dataset_pglib(data_dir::String, case_name::String; download_fi ) # The problem iterator - problem_iterator = ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:num_p) .* original_load)) + problem_iterator = ProblemIterator( + collect(1:num_p), Dict(p => collect(1.0:num_p) .* original_load) + ) # Create ref for anonimous variables on model createvarrefs!(model, pm) @@ -65,7 +71,9 @@ function generate_dataset_pglib(data_dir::String, case_name::String; download_fi # Solve the problem and return the number of successfull solves file = joinpath(data_dir, "test.$(string(filetype))") number_generators = length(network_data["gen"]) - recorder = Recorder{filetype}(file, primal_variables=[Symbol("0_pg[$i]") for i in 1:number_generators]) + recorder = Recorder{filetype}( + file; primal_variables=[Symbol("0_pg[$i]") for i in 1:number_generators] + ) return solve_batch(model, problem_iterator, recorder), number_generators end @@ -74,14 +82,16 @@ data_dir = joinpath(dirname(@__FILE__), "data") case_name = "pglib_opf_case5_pjm.m" # Define number of problems -num_p=10 +num_p = 10 # Generate dataset -success_solves, number_generators = generate_dataset_pglib(data_dir, case_name; download_files=true, filetype=CSVFile, num_p=num_p) +success_solves, number_generators = generate_dataset_pglib( + data_dir, case_name; download_files=true, filetype=CSVFile, num_p=num_p +) # Check if the number of successfull solves is equal to the number of problems saved file = joinpath(data_dir, "test.csv") @test isfile(file) @test length(readdlm(file, ',')[:, 1]) == num_p * success_solves + 1 @test length(readdlm(file, ',')[1, :]) == number_generators + 1 -rm(file) \ No newline at end of file +rm(file) diff --git a/src/L2O.jl b/src/L2O.jl index 498fe41..5e8cf52 100644 --- a/src/L2O.jl +++ b/src/L2O.jl @@ -7,7 +7,6 @@ import Base: string export ArrowFile, CSVFile, ProblemIterator, Recorder, solve_batch - include("datasetgen.jl") include("csvrecorder.jl") include("arrowrecorder.jl") diff --git a/src/arrowrecorder.jl b/src/arrowrecorder.jl index 2b1dd0a..e690c16 100644 --- a/src/arrowrecorder.jl +++ b/src/arrowrecorder.jl @@ -8,11 +8,24 @@ Base.string(::Type{ArrowFile}) = "arrow" Record optimization problem solution to an Arrow file. """ function record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::T) where {T<:Integer} - Arrow.append( - recorder.filename, (; - id = [id], - zip(recorder.primal_variables, [[MOI.get(model, MOI.VariablePrimal(), model[p])] for p in recorder.primal_variables])..., - zip(Symbol.("dual_" .* string.(recorder.dual_variables)), [[MOI.get(model, MOI.ConstraintDual(), model[p])] for p in recorder.dual_variables])..., - ) + return Arrow.append( + recorder.filename, + (; + id=[id], + zip( + recorder.primal_variables, + [ + [MOI.get(model, MOI.VariablePrimal(), model[p])] for + p in recorder.primal_variables + ], + )..., + zip( + Symbol.("dual_" .* string.(recorder.dual_variables)), + [ + [MOI.get(model, MOI.ConstraintDual(), model[p])] for + p in recorder.dual_variables + ], + )..., + ), ) -end \ No newline at end of file +end diff --git a/src/csvrecorder.jl b/src/csvrecorder.jl index 75a3c0a..788f0d0 100644 --- a/src/csvrecorder.jl +++ b/src/csvrecorder.jl @@ -32,4 +32,4 @@ function record(recorder::Recorder{CSVFile}, model::JuMP.Model, id::Int64) end write(f, "\n") end -end \ No newline at end of file +end diff --git a/src/datasetgen.jl b/src/datasetgen.jl index fac2272..f3bcc8e 100644 --- a/src/datasetgen.jl +++ b/src/datasetgen.jl @@ -11,7 +11,12 @@ mutable struct Recorder{T<:RecorderFile} dual_variables::AbstractArray{Symbol} filterfn::Function - function Recorder{T}(filename::String; primal_variables=[], dual_variables=[], filterfn=(model)-> termination_status(model) == MOI.OPTIMAL) where T<:RecorderFile + function Recorder{T}( + filename::String; + primal_variables=[], + dual_variables=[], + filterfn=(model) -> termination_status(model) == MOI.OPTIMAL, + ) where {T<:RecorderFile} return new{T}(filename, primal_variables, dual_variables, filterfn) end end @@ -21,14 +26,16 @@ end Iterator for optimization problem instances. """ -struct ProblemIterator{T<:Real, Z<:Integer} +struct ProblemIterator{T<:Real,Z<:Integer} ids::Vector{Z} - pairs::Dict{VariableRef, Vector{T}} - function ProblemIterator(ids::Vector{Z}, pairs::Dict{VariableRef, Vector{T}}) where {T<:Real, Z<:Integer} + pairs::Dict{VariableRef,Vector{T}} + function ProblemIterator( + ids::Vector{Z}, pairs::Dict{VariableRef,Vector{T}} + ) where {T<:Real,Z<:Integer} for (p, val) in pairs @assert length(ids) == length(val) end - return new{T, Z}(ids, pairs) + return new{T,Z}(ids, pairs) end end @@ -38,7 +45,7 @@ end Update the value of a parameter in a JuMP model. """ function update_model!(model::JuMP.Model, p::VariableRef, val::T) where {T<:Real} - MOI.set(model, POI.ParameterValue(), p, val) + return MOI.set(model, POI.ParameterValue(), p, val) end """ @@ -46,8 +53,10 @@ end Update the value of a parameter in a JuMP model. """ -function update_model!(model::JuMP.Model, p::VariableRef, val::AbstractArray{T}) where {T<:Real} - MOI.set(model, POI.ParameterValue(), p, val) +function update_model!( + model::JuMP.Model, p::VariableRef, val::AbstractArray{T} +) where {T<:Real} + return MOI.set(model, POI.ParameterValue(), p, val) end """ @@ -66,7 +75,9 @@ end Solve an optimization problem and record the solution. """ -function solve_and_record(model::JuMP.Model, problem_iterator::ProblemIterator, recorder::Recorder, idx::Integer) +function solve_and_record( + model::JuMP.Model, problem_iterator::ProblemIterator, recorder::Recorder, idx::Integer +) update_model!(model, problem_iterator.pairs, idx) optimize!(model) if recorder.filterfn(model) @@ -81,6 +92,11 @@ end Solve a batch of optimization problems and record the solutions. """ -function solve_batch(model::JuMP.Model, problem_iterator::ProblemIterator, recorder::Recorder) - return sum(solve_and_record(model, problem_iterator, recorder, idx) for idx in 1:length(problem_iterator.ids)) / length(problem_iterator.ids) +function solve_batch( + model::JuMP.Model, problem_iterator::ProblemIterator, recorder::Recorder +) + return sum( + solve_and_record(model, problem_iterator, recorder, idx) for + idx in 1:length(problem_iterator.ids) + ) / length(problem_iterator.ids) end diff --git a/test/runtests.jl b/test/runtests.jl index b30cd30..17ce441 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -25,7 +25,7 @@ function testdataset_gen(path::String) # The recorder file = joinpath(path, "test.$(string(filetype))") # file path - recorder = Recorder{filetype}(file, primal_variables=[:x], dual_variables=[:cons]) + recorder = Recorder{filetype}(file; primal_variables=[:x], dual_variables=[:cons]) # Solve all problems and record solutions solve_batch(model, problem_iterator, recorder) @@ -34,7 +34,7 @@ function testdataset_gen(path::String) if filetype == CSVFile file1 = joinpath(path, "test.csv") @test isfile(file1) - @test length(readdlm(file1, ',')[:, 1]) == num_p+1 + @test length(readdlm(file1, ',')[:, 1]) == num_p + 1 @test length(readdlm(file1, ',')[1, :]) == 3 rm(file1) else @@ -55,7 +55,11 @@ end end # pglib @testset "pg_lib case" begin - include(joinpath(dirname(dirname(@__FILE__)), "examples", "powermodels", "pg_lib.jl")) + include( + joinpath( + dirname(dirname(@__FILE__)), "examples", "powermodels", "pg_lib.jl" + ), + ) end end end From 56823e48db4d531ad5082827759473564aa2818c Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Sat, 15 Jul 2023 14:19:23 -0300 Subject: [PATCH 5/9] fix path --- examples/powermodels/pg_lib.jl | 5 ++--- test/runtests.jl | 18 +++++++++--------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/examples/powermodels/pg_lib.jl b/examples/powermodels/pg_lib.jl index 71d52cb..1159ee8 100644 --- a/examples/powermodels/pg_lib.jl +++ b/examples/powermodels/pg_lib.jl @@ -78,7 +78,6 @@ function generate_dataset_pglib( end # Define test case from pglib -data_dir = joinpath(dirname(@__FILE__), "data") case_name = "pglib_opf_case5_pjm.m" # Define number of problems @@ -86,11 +85,11 @@ num_p = 10 # Generate dataset success_solves, number_generators = generate_dataset_pglib( - data_dir, case_name; download_files=true, filetype=CSVFile, num_p=num_p + path, case_name; download_files=true, filetype=CSVFile, num_p=num_p ) # Check if the number of successfull solves is equal to the number of problems saved -file = joinpath(data_dir, "test.csv") +file = joinpath(path, "test.csv") @test isfile(file) @test length(readdlm(file, ',')[:, 1]) == num_p * success_solves + 1 @test length(readdlm(file, ',')[1, :]) == number_generators + 1 diff --git a/test/runtests.jl b/test/runtests.jl index 17ce441..6826a7a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -49,17 +49,17 @@ end @testset "L2O.jl" begin @testset "Dataset Generation" begin - # Different filetypes mktempdir() do path + # Different filetypes testdataset_gen(path) - end - # pglib - @testset "pg_lib case" begin - include( - joinpath( - dirname(dirname(@__FILE__)), "examples", "powermodels", "pg_lib.jl" - ), - ) + # Pglib + @testset "pg_lib case" begin + include( + joinpath( + dirname(dirname(@__FILE__)), "examples", "powermodels", "pg_lib.jl" + ), + ) + end end end end From 2cbe4dae169620ee052d846cd415e6d5c12cee9a Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Sat, 15 Jul 2023 14:26:42 -0300 Subject: [PATCH 6/9] fix typo --- examples/powermodels/pg_lib.jl | 17 ----------------- test/runtests.jl | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/examples/powermodels/pg_lib.jl b/examples/powermodels/pg_lib.jl index 1159ee8..98f9b97 100644 --- a/examples/powermodels/pg_lib.jl +++ b/examples/powermodels/pg_lib.jl @@ -77,20 +77,3 @@ function generate_dataset_pglib( return solve_batch(model, problem_iterator, recorder), number_generators end -# Define test case from pglib -case_name = "pglib_opf_case5_pjm.m" - -# Define number of problems -num_p = 10 - -# Generate dataset -success_solves, number_generators = generate_dataset_pglib( - path, case_name; download_files=true, filetype=CSVFile, num_p=num_p -) - -# Check if the number of successfull solves is equal to the number of problems saved -file = joinpath(path, "test.csv") -@test isfile(file) -@test length(readdlm(file, ',')[:, 1]) == num_p * success_solves + 1 -@test length(readdlm(file, ',')[1, :]) == number_generators + 1 -rm(file) diff --git a/test/runtests.jl b/test/runtests.jl index 6826a7a..e5e0b29 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -59,6 +59,24 @@ end dirname(dirname(@__FILE__)), "examples", "powermodels", "pg_lib.jl" ), ) + + # Define test case from pglib + case_name = "pglib_opf_case5_pjm.m" + + # Define number of problems + num_p = 10 + + # Generate dataset + success_solves, number_generators = generate_dataset_pglib( + path, case_name; download_files=true, filetype=CSVFile, num_p=num_p + ) + + # Check if the number of successfull solves is equal to the number of problems saved + file = joinpath(path, "test.csv") + @test isfile(file) + @test length(readdlm(file, ',')[:, 1]) == num_p * success_solves + 1 + @test length(readdlm(file, ',')[1, :]) == number_generators + 1 + rm(file) end end end From c0e8ea36f7c535c5d0a7cfec3363596909bd3cd9 Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Sat, 15 Jul 2023 15:05:15 -0300 Subject: [PATCH 7/9] update code test --- examples/powermodels/pg_lib.jl | 14 +++++++------- test/runtests.jl | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/powermodels/pg_lib.jl b/examples/powermodels/pg_lib.jl index 98f9b97..a097570 100644 --- a/examples/powermodels/pg_lib.jl +++ b/examples/powermodels/pg_lib.jl @@ -19,19 +19,19 @@ function createvarrefs!(sp::JuMP.Model, pm::AbstractPowerModel) end """ - generate_dataset_pglib(data_dir::String, case_name::String; download_files::Bool=true, filetype::Type{RecorderFile}, - num_p=10 + generate_dataset_pglib(data_dir::AbstractString, case_name::AbstractString; download_files::Bool=true, filetype::Type{RecorderFile}, + num_p::Int=10 ) Generate dataset for pglib case_name with num_p problems and save it in data_dir """ function generate_dataset_pglib( - data_dir::String, - case_name::String; - download_files::Bool=true, - filetype::Type{T}, + data_dir, + case_name; + filetype=CSVFile, + download_files=true, num_p=10, -) where {T<:L2O.RecorderFile} +) case_file_path = joinpath(data_dir, case_name) if download_files && !isfile(case_file_path) Downloads.download( diff --git a/test/runtests.jl b/test/runtests.jl index e5e0b29..e825f89 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,6 +5,12 @@ using DelimitedFiles using JuMP, HiGHS import ParametricOptInterface as POI +include( + joinpath( + dirname(dirname(@__FILE__)), "examples", "powermodels", "pg_lib.jl" + ), +) + """ testdataset_gen(path::String) @@ -54,12 +60,6 @@ end testdataset_gen(path) # Pglib @testset "pg_lib case" begin - include( - joinpath( - dirname(dirname(@__FILE__)), "examples", "powermodels", "pg_lib.jl" - ), - ) - # Define test case from pglib case_name = "pglib_opf_case5_pjm.m" @@ -68,7 +68,7 @@ end # Generate dataset success_solves, number_generators = generate_dataset_pglib( - path, case_name; download_files=true, filetype=CSVFile, num_p=num_p + path, case_name; num_p=num_p ) # Check if the number of successfull solves is equal to the number of problems saved From db830268fca4ccbc2f49574abd090aef00fdb788 Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Sat, 15 Jul 2023 15:34:25 -0300 Subject: [PATCH 8/9] add call tests --- test/runtests.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index e825f89..1758bb9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -27,10 +27,14 @@ function testdataset_gen(path::String) # The problem iterator num_p = 10 + @test_throws AssertionError ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:3.0))) + @test_throws MethodError ProblemIterator(collect(1.0:3.0), Dict(p => collect(1.0:3.0))) problem_iterator = ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:num_p))) # The recorder file = joinpath(path, "test.$(string(filetype))") # file path + @test Recorder{filetype}(file; primal_variables=[:x]) isa Recorder{filetype} + @test Recorder{filetype}(file; dual_variables=[:cons]) isa Recorder{filetype} recorder = Recorder{filetype}(file; primal_variables=[:x], dual_variables=[:cons]) # Solve all problems and record solutions From 721d8b741f4f599f4f65aff942afdf55e07bfdcd Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Sat, 15 Jul 2023 15:56:41 -0300 Subject: [PATCH 9/9] fix code --- src/datasetgen.jl | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/datasetgen.jl b/src/datasetgen.jl index f3bcc8e..f10d871 100644 --- a/src/datasetgen.jl +++ b/src/datasetgen.jl @@ -44,18 +44,7 @@ end Update the value of a parameter in a JuMP model. """ -function update_model!(model::JuMP.Model, p::VariableRef, val::T) where {T<:Real} - return MOI.set(model, POI.ParameterValue(), p, val) -end - -""" - update_model!(model::JuMP.Model, p::VariableRef, val::AbstractArray{Real}) - -Update the value of a parameter in a JuMP model. -""" -function update_model!( - model::JuMP.Model, p::VariableRef, val::AbstractArray{T} -) where {T<:Real} +function update_model!(model::JuMP.Model, p::VariableRef, val) return MOI.set(model, POI.ParameterValue(), p, val) end