From 70ac97556ec7be0da1af1e585e0185d51380c24f Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Tue, 18 Jul 2023 17:33:26 -0300 Subject: [PATCH 1/3] change simulation ids to use UUIDs --- Project.toml | 1 + examples/powermodels/flux_forecaster.jl | 8 ++++++-- examples/powermodels/pglib_datagen.jl | 1 - src/L2O.jl | 1 + src/arrowrecorder.jl | 4 ++-- src/csvrecorder.jl | 4 ++-- src/datasetgen.jl | 24 +++++++++++++++++------- test/datasetgen.jl | 4 ++-- test/runtests.jl | 10 ++++++---- 9 files changed, 37 insertions(+), 20 deletions(-) diff --git a/Project.toml b/Project.toml index 916365f..bbc07b4 100644 --- a/Project.toml +++ b/Project.toml @@ -8,6 +8,7 @@ Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" ParametricOptInterface = "0ce4ce61-57bf-432b-a095-efac525d185e" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] Arrow = "2" diff --git a/examples/powermodels/flux_forecaster.jl b/examples/powermodels/flux_forecaster.jl index af7d0b9..5a0d7a7 100644 --- a/examples/powermodels/flux_forecaster.jl +++ b/examples/powermodels/flux_forecaster.jl @@ -10,8 +10,8 @@ function test_flux_forecaster(file_in::AbstractString, file_out::AbstractString) output_data = CSV.read(file_out, DataFrame) # Separate input and output variables - output_variables = output_data[:, 2:end] - input_features = input_data[output_data[:, 1], 2:end] # just use success solves + output_variables = output_data[!, Not(:id)] + input_features = innerjoin(input_data, output_data[!, [:id]], on = :id)[!, Not(:id)] # just use success solves # Define model model = Chain( @@ -38,5 +38,9 @@ function test_flux_forecaster(file_in::AbstractString, file_out::AbstractString) # Make predictions predictions = model(input_features) @test predictions isa Matrix + + # Delete the files + rm(file_in) + rm(file_out) end end diff --git a/examples/powermodels/pglib_datagen.jl b/examples/powermodels/pglib_datagen.jl index bc3e4c0..294b711 100644 --- a/examples/powermodels/pglib_datagen.jl +++ b/examples/powermodels/pglib_datagen.jl @@ -87,7 +87,6 @@ function generate_dataset_pglib( # The problem iterator problem_iterator = ProblemIterator( - collect(1:num_p), Dict( p .=> [ load_sampler(original_load[i], num_p) for diff --git a/src/L2O.jl b/src/L2O.jl index 54e7781..65f46ec 100644 --- a/src/L2O.jl +++ b/src/L2O.jl @@ -3,6 +3,7 @@ module L2O using Arrow using CSV using JuMP +using UUIDs import ParametricOptInterface as POI import Base: string diff --git a/src/arrowrecorder.jl b/src/arrowrecorder.jl index c05094f..816d1b9 100644 --- a/src/arrowrecorder.jl +++ b/src/arrowrecorder.jl @@ -3,11 +3,11 @@ abstract type ArrowFile <: RecorderFile end Base.string(::Type{ArrowFile}) = "arrow" """ - record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::T) + record(recorder::Recorder{ArrowFile}, id::UUID) Record optimization problem solution to an Arrow file. """ -function record(recorder::Recorder{ArrowFile}, model::JuMP.Model, id::T) where {T<:Integer} +function record(recorder::Recorder{ArrowFile}, id::UUID) return Arrow.append( recorder.filename, (; diff --git a/src/csvrecorder.jl b/src/csvrecorder.jl index a529927..ed98013 100644 --- a/src/csvrecorder.jl +++ b/src/csvrecorder.jl @@ -3,11 +3,11 @@ abstract type CSVFile <: RecorderFile end Base.string(::Type{CSVFile}) = "csv" """ - record(recorder::Recorder{CSVFile}, model::JuMP.Model, id::Int64) + record(recorder::Recorder{CSVFile}, id::UUID) Record optimization problem solution to a CSV file. """ -function record(recorder::Recorder{CSVFile}, model::JuMP.Model, id::Int64) +function record(recorder::Recorder{CSVFile}, id::UUID) if !isfile(recorder.filename) open(recorder.filename, "w") do f write(f, "id") diff --git a/src/datasetgen.jl b/src/datasetgen.jl index b503874..f1db811 100644 --- a/src/datasetgen.jl +++ b/src/datasetgen.jl @@ -22,23 +22,33 @@ mutable struct Recorder{T<:RecorderFile} end """ - ProblemIterator(ids::Vector{Integer}, pairs::Dict{VariableRef, Vector{Real}}) + ProblemIterator(ids::Vector{UUID}, pairs::Dict{VariableRef, Vector{Real}}) Iterator for optimization problem instances. """ -struct ProblemIterator{T<:Real,Z<:Integer} - ids::Vector{Z} +struct ProblemIterator{T<:Real} + ids::Vector{UUID} pairs::Dict{VariableRef,Vector{T}} function ProblemIterator( - ids::Vector{Z}, pairs::Dict{VariableRef,Vector{T}} - ) where {T<:Real,Z<:Integer} + ids::Vector{UUID}, pairs::Dict{VariableRef,Vector{T}} + ) where {T<:Real} for (p, val) in pairs @assert length(ids) == length(val) end - return new{T,Z}(ids, pairs) + return new{T}(ids, pairs) end end +function ProblemIterator(pairs::Dict{VariableRef,Vector{T}}) where {T<:Real} + ids = [uuid1() for _ in 1:length(first(values(pairs)))] + return ProblemIterator(ids, pairs) +end + +""" + save(problem_iterator::ProblemIterator, filename::String, file_type::Type{T}) + +Save optimization problem instances to a file. +""" function save( problem_iterator::ProblemIterator, filename::String, file_type::Type{T} ) where {T<:RecorderFile} @@ -85,7 +95,7 @@ function solve_and_record( update_model!(model, problem_iterator.pairs, idx) optimize!(model) if recorder.filterfn(model) - record(recorder, model, problem_iterator.ids[idx]) + record(recorder, problem_iterator.ids[idx]) return 1 end return 0 diff --git a/test/datasetgen.jl b/test/datasetgen.jl index 53c973b..27f764c 100644 --- a/test/datasetgen.jl +++ b/test/datasetgen.jl @@ -16,12 +16,12 @@ function testdataset_gen(path::AbstractString) # The problem iterator num_p = 10 @test_throws AssertionError ProblemIterator( - collect(1:num_p), Dict(p => collect(1.0:3.0)) + [uuid1() for _ in 1:num_p], Dict(p => collect(1.0:3.0)) ) @test_throws MethodError ProblemIterator( collect(1.0:3.0), Dict(p => collect(1.0:3.0)) ) - problem_iterator = ProblemIterator(collect(1:num_p), Dict(p => collect(1.0:num_p))) + problem_iterator = ProblemIterator(Dict(p => collect(1.0:num_p))) file_input = joinpath(path, "test_input.$(string(filetype))") # file path save(problem_iterator, file_input, filetype) @test isfile(file_input) diff --git a/test/runtests.jl b/test/runtests.jl index 6905f44..490cf54 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,10 +1,12 @@ -using L2O using Arrow -using Test using DelimitedFiles -using JuMP, HiGHS -import ParametricOptInterface as POI using Flux +using HiGHS +using JuMP +using L2O +import ParametricOptInterface as POI +using Test +using UUIDs const test_dir = dirname(@__FILE__) const examples_dir = joinpath(test_dir, "..", "examples") From 9feb44944681a19ae5f6f2ac65f6b224862b6a9a Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Tue, 18 Jul 2023 17:38:41 -0300 Subject: [PATCH 2/3] update readme --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 80fb780..3b99abf 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,11 @@ p = @variable(model, p in POI.Parameter(1.0)) # The parameter (defined using POI @constraint(model, cons, x + p >= 3) @objective(model, Min, 2x) -# The ids -problem_ids = collect(1:10) - # The parameter values parameter_values = Dict(p => collect(1.0:10.0)) # The iterator -problem_iterator = ProblemIterator(problem_ids, parameter_values) +problem_iterator = ProblemIterator(parameter_values) ``` The parameter values of the problem iterator can be saved by simply: @@ -52,6 +49,8 @@ Which creates the following CSV: | 9 | 9.0 | | 10 | 10.0| +ps.: For illustration purpose, I have represented the id's here as integers, but in reality they are generated as UUIDs. + ### The Recorder Then chose what values to record: @@ -79,6 +78,8 @@ Which creates the following CSV: | 9 | -6.0 | 2.0 | | 10 | -7.0 | 2.0 | +ps.: Ditto id's. + Similarly, there is also the option to save the database in arrow files: ```julia @@ -95,8 +96,8 @@ input_data = CSV.read("input_file.csv", DataFrame) output_data = CSV.read("output_file.csv", DataFrame) # Separate input and output variables -output_variables = output_data[:, 2:end] -input_features = input_data[output_data[:, 1], 2:end] # just use success solves +output_variables = output_data[!, Not(:id)] +input_features = innerjoin(input_data, output_data[!, [:id]], on = :id)[!, Not(:id)] # just use success solves # Define model model = Chain( From ae481a8d4e7d81e7fa06d67e1a47c1e152d48b0b Mon Sep 17 00:00:00 2001 From: andrewrosemberg Date: Tue, 18 Jul 2023 18:11:45 -0300 Subject: [PATCH 3/3] rm useless yml --- .travis.yml | 47 ----------------------------------------------- 1 file changed, 47 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8937c20..0000000 --- a/.travis.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Documentation: http://docs.travis-ci.com/user/languages/julia -language: julia -notifications: - email: false -julia: - - 1.0 - - 1.9 - - nightly -os: - - linux -arch: - - x64 - - x86 -cache: - directories: - - ~/.julia/artifacts -jobs: - fast_finish: true - allow_failures: - - julia: nightly - include: - - stage: Documentation - julia: 1 - script: - - | - julia --project=docs -e ' - using Pkg - Pkg.develop(PackageSpec(path=pwd())) - Pkg.instantiate()' - - | - julia --project=docs -e ' - using Pkg - Pkg.develop(PackageSpec(path=pwd())) - Pkg.instantiate() - using Documenter: DocMeta, doctest - using L2O - DocMeta.setdocmeta!(L2O, :DocTestSetup, :(using L2O); recursive=true) - doctest(L2O) - include("docs/make.jl")' - after_success: skip -after_success: - - | - julia -e ' - using Pkg - Pkg.add("Coverage") - using Coverage - Codecov.submit(process_folder())'