The option `postprocessing=true` crashes at Hessian preparation with DI.jl

[franckgaga]

Hello,

Following your comment, I tested postprocessing=true with AutoForwardDiff in MPC.jl and it crashes at preperation. Here's a MRE with MPC.jl:

using ModelPredictiveControl
using DifferentiationInterface, SparseConnectivityTracer, SparseMatrixColorings
import ForwardDiff
hessian = AutoSparse(
    AutoForwardDiff(); 
    sparsity_detector  = TracerSparsityDetector(), 
    coloring_algorithm = GreedyColoringAlgorithm(; postprocessing=true)
)
f(x,u,_,_) = 0.5*x + 0.5*u
h(x,_,_)   = x
model = NonLinModel(f, h, 1.0, 1, 1, 1, solver=nothing)
nmpc = NonLinMPC(model; Hp=5, direct=false, hessian)

giving:

ERROR: ArgumentError: `stack` on an empty collection is not allowed
Stacktrace:
  [1] _empty_stack(::Function, ::Vararg{Any})
    @ Base ./abstractarray.jl:2986
  [2] _typed_stack(::Colon, ::Type{…}, ::Type{…}, A::Base.Generator{…}, Aax::Tuple{…})
    @ Base ./abstractarray.jl:2901
  [3] _typed_stack
    @ ./abstractarray.jl:2928 [inlined]
  [4] _typed_stack
    @ ./abstractarray.jl:2922 [inlined]
  [5] _stack
    @ ./abstractarray.jl:2890 [inlined]
  [6] _stack
    @ ./abstractarray.jl:2882 [inlined]
  [7] stack
    @ ./abstractarray.jl:2879 [inlined]
  [8] _prepare_sparse_hessian_aux(::Val{…}, ::DifferentiationInterface.BatchSizeSettings{…}, ::SparseArrays.SparseMatrixCSC{…}, ::SparseMatrixColorings.StarSetColoringResult{…}, ::ModelPredictiveControl.var"#ℓ_gi#get_nonlincon_oracle##7"{…}, ::ADTypes.AutoSparse{…}, ::Vector{…}, ::DifferentiationInterface.Constant{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…})
    @ DifferentiationInterfaceSparseMatrixColoringsExt ~/.julia/packages/DifferentiationInterface/S6Hq1/ext/DifferentiationInterfaceSparseMatrixColoringsExt/hessian.jl:58
  [9] prepare_hessian_nokwarg(::Val{…}, ::ModelPredictiveControl.var"#ℓ_gi#get_nonlincon_oracle##7"{…}, ::ADTypes.AutoSparse{…}, ::Vector{…}, ::DifferentiationInterface.Constant{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…}, ::DifferentiationInterface.Cache{…})
    @ DifferentiationInterfaceSparseMatrixColoringsExt ~/.julia/packages/DifferentiationInterface/S6Hq1/ext/DifferentiationInterfaceSparseMatrixColoringsExt/hessian.jl:38
 [10] #prepare_hessian#57
    @ ~/.julia/packages/DifferentiationInterface/S6Hq1/src/second_order/hessian.jl:11 [inlined]
 [11] get_nonlincon_oracle(mpc::ModelPredictiveControl.NonLinMPC{…}, ::JuMP.Model)
    @ ModelPredictiveControl ~/.julia/dev/ModelPredictiveControl/src/controller/nonlinmpc.jl:791
 [12] init_optimization!(mpc::ModelPredictiveControl.NonLinMPC{…}, model::ModelPredictiveControl.NonLinModel{…}, optim::JuMP.Model)
    @ ModelPredictiveControl ~/.julia/dev/ModelPredictiveControl/src/controller/nonlinmpc.jl:567
 [13] (ModelPredictiveControl.NonLinMPC{…})(estim::ModelPredictiveControl.UnscentedKalmanFilter{…}, Hp::Int64, Hc::Int64, nb::Vector{…}, weights::ModelPredictiveControl.ControllerWeights{…}, JE::ModelPredictiveControl.var"#NonLinMPC##4#NonLinMPC##5", gc!::ModelPredictiveControl.var"#NonLinMPC##6#NonLinMPC##7", nc::Int64, p::Vector{…}, transcription::ModelPredictiveControl.SingleShooting, optim::JuMP.Model, gradient::ADTypes.AutoForwardDiff{…}, jacobian::ADTypes.AutoForwardDiff{…}, hessian::ADTypes.AutoSparse{…}, oracle::Bool)
    @ ModelPredictiveControl ~/.julia/dev/ModelPredictiveControl/src/controller/nonlinmpc.jl:140
 [14] ModelPredictiveControl.NonLinMPC(estim::ModelPredictiveControl.UnscentedKalmanFilter{…}; Hp::Int64, Hc::Int64, Mwt::Vector{…}, Nwt::Vector{…}, Lwt::Vector{…}, M_Hp::LinearAlgebra.Diagonal{…}, N_Hc::LinearAlgebra.Diagonal{…}, L_Hp::LinearAlgebra.Diagonal{…}, Cwt::Float64, Ewt::Float64, JE::ModelPredictiveControl.var"#NonLinMPC##4#NonLinMPC##5", gc!::ModelPredictiveControl.var"#NonLinMPC##14#NonLinMPC##15", gc::ModelPredictiveControl.var"#NonLinMPC##6#NonLinMPC##7", nc::Int64, p::Vector{…}, transcription::ModelPredictiveControl.SingleShooting, optim::JuMP.Model, gradient::ADTypes.AutoForwardDiff{…}, jacobian::ADTypes.AutoForwardDiff{…}, hessian::ADTypes.AutoSparse{…}, oracle::Bool)
    @ ModelPredictiveControl ~/.julia/dev/ModelPredictiveControl/src/controller/nonlinmpc.jl:413
 [15] NonLinMPC
    @ ~/.julia/dev/ModelPredictiveControl/src/controller/nonlinmpc.jl:375 [inlined]
 [16] #NonLinMPC#130
    @ ~/.julia/dev/ModelPredictiveControl/src/controller/nonlinmpc.jl:334 [inlined]
 [17] top-level scope
    @ ~/Dropbox/Programmation/Julia/TestMPC/src/debug_SMC.jl:12
Some type information was truncated. Use `show(err)` to see complete types.

using:

• SparseMatrixColorings v0.4.23
• SparseConnectivityTracer v1.1.2
• DifferentiationInterface v0.7.10
• ModelPredictiveControl v1.13.0
• JuMP v1.29.2
• MathOptInterface v1.46.0
• ForwardDiff v1.2.2

[gdalle]

Does the function in question have zero second derivatives?

[franckgaga]

Yes. Since that are no nonlinear inequality constraints here, the function in question:

  function ℓ_gi(Z̃, λi, ΔŨ, x̂0end, Ue, Ŷe, U0, Ŷ0, Û0, K0, X̂0, gc, geq, g, gi)
      update_predictions!(ΔŨ, x̂0end, Ue, Ŷe, U0, Ŷ0, Û0, K0, X̂0, gc, g, geq, mpc, Z̃)
      gi .= @views g[i_g]
      return dot(λi, gi)
  end

compute dot with λi and gi that are both Float64[], that is, i_g=Int64[].

Maybe I should add some logic to handle this specific case on my side. But the example above do work without postprocessing=true.

[gdalle]

Okay, it is the same error as JuliaDiff/DifferentiationInterface.jl#857, I'll extend the fix I did there to handle Hessians of linear functions and Jacobians of constant functions (which are empty sparse matrices, hence the edge case)

[gdalle]

Try this branch on for size?

JuliaDiff/DifferentiationInterface.jl#906