From 8c8426fadffce8023fcbe661ef7a498dc80f2f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1chym=20Barv=C3=ADnek?= Date: Sat, 17 Oct 2026 16:07:14 +0200 Subject: [PATCH 1/2] Update the dense BFGS inverse Hessian in O(n^2) BFGSUpdate_HInv::update formed (I - rho s y') H (I - rho y s') + rho s s' with two n x n matrix products. Expanded with u = H y it is the rank-two update H + (c s - rho u) s' - rho s u', c = rho^2 y'u + rho. --- src/stan/optimization/bfgs_update.hpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/stan/optimization/bfgs_update.hpp b/src/stan/optimization/bfgs_update.hpp index 9a559af9d06..2b23a79d7fe 100644 --- a/src/stan/optimization/bfgs_update.hpp +++ b/src/stan/optimization/bfgs_update.hpp @@ -25,21 +25,25 @@ class BFGSUpdate_HInv { inline Scalar update(const VectorT &yk, const VectorT &sk, bool reset = false) { Scalar rhok, skyk, B0fact; - HessianT Hupd; skyk = yk.dot(sk); rhok = 1.0 / skyk; - Hupd.noalias() - = HessianT::Identity(yk.size(), yk.size()) - rhok * sk * yk.transpose(); if (reset) { B0fact = yk.squaredNorm() / skyk; - _Hk.noalias() = ((1.0 / B0fact) * Hupd) * Hupd.transpose(); + _Hk.setIdentity(yk.size(), yk.size()); + _Hk /= B0fact; } else { B0fact = 1.0; - _Hk = Hupd * _Hk * Hupd.transpose(); } - _Hk.noalias() += rhok * sk * sk.transpose(); + // H <- (I - rho s y') H (I - rho y s') + rho s s' + // = H + (c s - rho u) s' - rho s u', u = H y, c = rho^2 y'u + rho + _u.noalias() = _Hk * yk; + const Scalar c = rhok * rhok * yk.dot(_u) + rhok; + _w = c * sk - rhok * _u; + _Hk.noalias() += _w * sk.transpose(); + _u *= rhok; + _Hk.noalias() -= sk * _u.transpose(); return B0fact; } @@ -57,6 +61,7 @@ class BFGSUpdate_HInv { private: HessianT _Hk; + VectorT _u, _w; }; } // namespace optimization } // namespace stan From 63a190e58a76e93f1af9a4233c6cc59b0ade0d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1chym=20Barv=C3=ADnek?= Date: Sat, 17 Oct 2026 16:07:15 +0200 Subject: [PATCH 2/2] Size the dense BFGS inverse Hessian before each update Before each update except at a reset, H is multiplied by tau = s'y / y'Hy when tau > 1 (Oren-Luenberger sizing restricted to enlarging H). Without it H kept the scale of the curvature at the last reset: started where the curvature is large, it underestimated the inverse curvature in the directions not explored since, and the relative gradient test stopped runs far from the optimum. --- src/stan/optimization/bfgs_update.hpp | 18 +++++++++- .../unit/optimization/bfgs_minimizer_test.cpp | 35 ++++++++++++++++++- .../unit/optimization/bfgs_update_test.cpp | 19 ++++++++++ src/test/unit/services/optimize/bfgs_test.cpp | 8 ++--- 4 files changed, 74 insertions(+), 6 deletions(-) diff --git a/src/stan/optimization/bfgs_update.hpp b/src/stan/optimization/bfgs_update.hpp index 2b23a79d7fe..d4a3d57456a 100644 --- a/src/stan/optimization/bfgs_update.hpp +++ b/src/stan/optimization/bfgs_update.hpp @@ -14,6 +14,15 @@ class BFGSUpdate_HInv { /** * Update the inverse Hessian approximation. * + * Except at a reset, the approximation H is multiplied by + * tau = s'y / y'Hy before the update if tau > 1 (Oren-Luenberger sizing + * restricted to enlarging H). Without it H keeps the scale set at the + * last reset, which far from the optimum can underestimate the inverse + * curvature by orders of magnitude in the directions not explored since. + * Scaling all of H can overestimate it along earlier steps instead; a + * later pair along such a direction has y'Hy > s'y, so it is not sized + * and the update corrects H there. + * * @param yk Difference between the current and previous gradient vector. * @param sk Difference between the current and previous state vector. * @param reset Whether to reset the approximation, forgetting about @@ -39,7 +48,14 @@ class BFGSUpdate_HInv { // H <- (I - rho s y') H (I - rho y s') + rho s s' // = H + (c s - rho u) s' - rho s u', u = H y, c = rho^2 y'u + rho _u.noalias() = _Hk * yk; - const Scalar c = rhok * rhok * yk.dot(_u) + rhok; + Scalar yu = yk.dot(_u); + if (!reset && 0 < yu && yu < skyk) { + const Scalar tau = skyk / yu; + _Hk *= tau; + _u *= tau; + yu = skyk; + } + const Scalar c = rhok * rhok * yu + rhok; _w = c * sk - rhok * _u; _Hk.noalias() += _w * sk.transpose(); _u *= rhok; diff --git a/src/test/unit/optimization/bfgs_minimizer_test.cpp b/src/test/unit/optimization/bfgs_minimizer_test.cpp index 66393b8b585..59d883d855a 100644 --- a/src/test/unit/optimization/bfgs_minimizer_test.cpp +++ b/src/test/unit/optimization/bfgs_minimizer_test.cpp @@ -339,6 +339,39 @@ TEST_F(OptimizationBfgsMinimizer, step) { bfgs.step(); } +namespace { +// f(x) = sum_i exp(i x_i) - i x_i, i = 1, ..., n, minimised at 0 with f = n +struct minimizer_exp_sum { + int operator()(const Eigen::VectorXd &x, double &f, Eigen::VectorXd &g) { + Eigen::ArrayXd c = Eigen::ArrayXd::LinSpaced(x.size(), 1, x.size()); + Eigen::ArrayXd e = (c * x.array()).exp(); + f = (e - c * x.array()).sum(); + g = (c * (e - 1)).matrix(); + return std::isfinite(f) && g.allFinite() ? 0 : 1; + } +}; +} // namespace + +TEST(OptimizationBfgsMinimizerExpSum, dense_bfgs_large_initial_curvature) { + // From x0 = (5, ..., 5) the curvature falls from 25 e^25 = 1.8e12 to at + // most 25 at the optimum. The dense inverse Hessian estimate used to keep + // the scale of the first steps in the directions not explored since, and + // with CmdStan's default tolerances the relative gradient test stopped the + // run at f - 5 = 7.6e3. + stan::optimization::BFGSMinimizer > + bfgs(minimizer_exp_sum{}); + bfgs._conv_opts.tolRelGrad = 1e7; + Eigen::VectorXd x0 = Eigen::VectorXd::Constant(5, 5); + bfgs.initialize(x0); + int ret = stan::optimization::TERM_SUCCESS; + while (ret == stan::optimization::TERM_SUCCESS) { + ret = bfgs.step(); + } + EXPECT_GT(ret, 0); + EXPECT_NEAR(5, bfgs.curr_f(), 1e-6); +} + TEST_F(OptimizationBfgsMinimizer, minimize) { stan::io::empty_var_context dummy_context; Model rb_model(dummy_context); @@ -347,5 +380,5 @@ TEST_F(OptimizationBfgsMinimizer, minimize) { EXPECT_EQ("", out.str()); Optimizer bfgs(_adaptor); - EXPECT_FLOAT_EQ(bfgs.minimize(cont_vector), 31); + EXPECT_FLOAT_EQ(bfgs.minimize(cont_vector), 20); } diff --git a/src/test/unit/optimization/bfgs_update_test.cpp b/src/test/unit/optimization/bfgs_update_test.cpp index 5b2dee161ef..d75b3c853b7 100644 --- a/src/test/unit/optimization/bfgs_update_test.cpp +++ b/src/test/unit/optimization/bfgs_update_test.cpp @@ -35,6 +35,25 @@ TEST(OptimizationBfgsUpdate, bfgs_update_secant) { } } +TEST(OptimizationBfgsUpdate, BFGSUpdate_HInv_enlarges_small_estimate) { + // The reset on a pair with curvature 1e4 along e1 sets H = 1e-4 I. The next + // pair has curvature 1 along e2, so H is first multiplied by 1e4, also + // along e3, which no pair has explored (H e3 stayed 1e-4 e3 before). A + // pair with curvature 100 along e3, where H is then 1, does not shrink H: + // H e1 stays e1, although the first pair measured curvature 1e4 along e1. + stan::optimization::BFGSUpdate_HInv<> bfgsUp; + Eigen::VectorXd s = Eigen::VectorXd::Unit(3, 0), p; + bfgsUp.update(1e4 * s, s, true); + s = Eigen::VectorXd::Unit(3, 1); + bfgsUp.update(s, s); + bfgsUp.search_direction(p, Eigen::VectorXd::Unit(3, 2)); + EXPECT_FLOAT_EQ(-1, p[2]); + s = Eigen::VectorXd::Unit(3, 2); + bfgsUp.update(100 * s, s); + bfgsUp.search_direction(p, Eigen::VectorXd::Unit(3, 0)); + EXPECT_FLOAT_EQ(-1, p[0]); +} + TEST(OptimizationBfgsUpdate, BFGSUpdate_HInv_update) { const int nDim = 10; diff --git a/src/test/unit/services/optimize/bfgs_test.cpp b/src/test/unit/services/optimize/bfgs_test.cpp index ac376fe1910..73601a6fb54 100644 --- a/src/test/unit/services/optimize/bfgs_test.cpp +++ b/src/test/unit/services/optimize/bfgs_test.cpp @@ -46,15 +46,15 @@ TEST_F(ServicesOptimize, rosenbrock) { EXPECT_EQ("x", parameter.names_[2]); EXPECT_EQ("y", parameter.names_[3]); - EXPECT_EQ(20, parameter.states_.size()); + EXPECT_EQ(22, parameter.states_.size()); EXPECT_FLOAT_EQ(0, parameter.states_.front()[2]) << "initial value should be (0, 0)"; EXPECT_FLOAT_EQ(0, parameter.states_.front()[3]) << "initial value should be (0, 0)"; - EXPECT_FLOAT_EQ(1, parameter.states_.back()[2]) + EXPECT_FLOAT_EQ(1.0000112, parameter.states_.back()[2]) << "optimal value should be (1, 1)"; - EXPECT_FLOAT_EQ(1, parameter.states_.back()[3]) + EXPECT_FLOAT_EQ(1.0000218, parameter.states_.back()[3]) << "optimal value should be (1, 1)"; EXPECT_FLOAT_EQ(return_code, 0); - EXPECT_EQ(19, interrupt.call_count()); + EXPECT_EQ(21, interrupt.call_count()); }