From c7b8d8e56588fd982ed06d3c33a4455947f48605 Mon Sep 17 00:00:00 2001 From: Florence Bockting Date: Mon, 2 Nov 2026 14:10:41 +0200 Subject: [PATCH 1/5] feat: add classes to all loo warnings --- DESCRIPTION | 1 + NEWS.md | 4 +++ R/diagnostics.R | 5 ++-- R/helpers.R | 4 +-- R/loo-package.R | 38 +++++++++++++++++++++++ R/loo.R | 24 ++++----------- R/loo_compare.R | 24 +++++++-------- R/loo_compare.psis_loo_ss_list.R | 24 +++++++-------- R/loo_moment_matching.R | 12 ++++---- R/loo_subsample.R | 8 ++--- R/print.R | 7 ++++- R/psis.R | 30 +++++++------------ R/psis_approximate_posterior.R | 3 +- R/psislw.R | 18 ++++++----- R/waic.R | 6 +++- man/loo-package.Rd | 43 +++++++++++++++++++++++++++ tests/testthat/test_warning_classes.R | 22 ++++++++++++++ 17 files changed, 187 insertions(+), 86 deletions(-) create mode 100644 tests/testthat/test_warning_classes.R diff --git a/DESCRIPTION b/DESCRIPTION index d98e2a39..e8f6334c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -39,6 +39,7 @@ Imports: matrixStats (>= 0.52), parallel, posterior (>= 1.7.0), + rlang (>= 1.0.6), stats Suggests: bayesplot (>= 1.7.0), diff --git a/NEWS.md b/NEWS.md index bcc1e16c..e13d1573 100644 --- a/NEWS.md +++ b/NEWS.md @@ -11,6 +11,10 @@ comparison-model minus reference-model by @florence-bockting in #391 * Fix `crps()` and `scrps()` ignoring the `permutations` argument for numeric input by @thanasibakis in #398 * Speed up the Bayesian bootstrap in `pseudobma_weights()` by @VisruthSK in #368 +* All warnings now have the class `loo_warning` and a class for each cause; +diagnostics warnings have the class `loo_warning_diagnostic` by +@florence-bockting in #XXX + # loo 2.10.1 diff --git a/R/diagnostics.R b/R/diagnostics.R index 3d12bdb4..2bc2cdbc 100644 --- a/R/diagnostics.R +++ b/R/diagnostics.R @@ -272,8 +272,9 @@ plot.psis_loo <- function(x, k[is.na(k)] <- 0 # FIXME when reloo is changed to make NA k values -Inf k_inf <- !is.finite(k) if (any(k_inf)) { - warning(signif(100 * mean(k_inf), 2), - "% of Pareto k estimates are Inf/NA/NaN and not plotted.") + .warn(signif(100 * mean(k_inf), 2), + "% of Pareto k estimates are Inf/NA/NaN and not plotted.", + class = "loo_warning_pareto_k_not_plotted") } if (diagnostic == "ESS" || diagnostic == "n_eff") { diff --git a/R/helpers.R b/R/helpers.R index 37d407a8..681ec9dd 100644 --- a/R/helpers.R +++ b/R/helpers.R @@ -259,8 +259,8 @@ loo_cores <- function(cores) { loo_cores_op <- getOption("loo.cores", NA) if (!is.na(loo_cores_op) && (loo_cores_op != cores)) { cores <- loo_cores_op - warning("'loo.cores' is deprecated, please use 'mc.cores' or pass 'cores' explicitly.", - call. = FALSE) + .warn("'loo.cores' is deprecated, please use 'mc.cores' or pass 'cores' explicitly.", + class = "loo_warning_deprecated") } return(cores) } diff --git a/R/loo-package.R b/R/loo-package.R index 4a29a5cf..a9285c85 100644 --- a/R/loo-package.R +++ b/R/loo-package.R @@ -39,6 +39,44 @@ #' diagnostics and effective sample size and Monte Carlo standard error #' estimates. #' +#' @section Warning classes: +#' +#' Every warning signaled by \pkg{loo} has the class `loo_warning` and one +#' class for its cause. Warnings that question the reliability of a result +#' also have the class `loo_warning_diagnostic`. Use these classes to handle +#' or muffle a specific warning, for example with [withCallingHandlers()] +#' and `invokeRestart("muffleWarning")`. +#' +#' Diagnostic warnings: +#' +#' * `loo_warning_pareto_k_high`: Some Pareto k values are too high. +#' * `loo_warning_pareto_tail_not_fitted`: There are not enough tail samples to +#' fit the generalized Pareto distribution. +#' * `loo_warning_pareto_tail_constant`: All tail values are the same. +#' * `loo_warning_p_waic_high`: Some `p_waic` estimates are greater than 0.4. +#' * `loo_warning_moment_match_failed`: Moment matching failed for some +#' observations. +#' * `loo_warning_moment_match_max_iters`: Moment matching reached +#' `max_iters`. +#' * `loo_warning_large_kf`: The accuracy of self-normalized importance +#' sampling may be bad. +#' * `loo_warning_y_mismatch`: The models do not have the same `y` variable. +#' * `loo_warning_cv_mismatch`: The models use different types of +#' cross-validation or different numbers of folds. +#' * `loo_warning_chance_difference`: The difference in performance is +#' potentially due to chance. +#' * `loo_warning_subsample_se`: A subsampling standard error is naive or +#' negative. +#' +#' Other warnings: +#' +#' * `loo_warning_multicore_unavailable`: Multicore is not implemented for +#' the selected method. +#' * `loo_warning_pareto_k_not_plotted`: Some Pareto k values are `Inf`, `NA`, +#' or `NaN` and are not plotted. +#' * `loo_warning_llfun_not_matrix`: The values from `llfun` are not a matrix +#' and were coerced. +#' * `loo_warning_deprecated`: A deprecated feature was used. #' #' @template loo-and-psis-references #' @template stacking-references diff --git a/R/loo.R b/R/loo.R index 0b284297..e3e36100 100644 --- a/R/loo.R +++ b/R/loo.R @@ -549,18 +549,6 @@ log_var_epd_ratio <- function(log_lik_ratio, lw, r_eff) { matrixStats::colLogSumExps(2 * (lw + log_abs_diff)) - log(r_eff) } - -#' Warning message if r_eff not specified -#' @noRd -throw_loo_r_eff_warning <- function() { - warning( - "Relative effective sample sizes ('r_eff' argument) not specified.\n", - "For models fit with MCMC, the reported PSIS ESS and \n", - "MCSE estimates can be over-optimistic.", - call. = FALSE - ) -} - #' Combine many psis objects into a single psis object #' #' @noRd @@ -621,12 +609,12 @@ NULL if (is.character(i)) { needs_warning <- which(flags == i) if (length(needs_warning)) { - warning( + .warn( "Accessing ", flags[needs_warning], " using '[' is deprecated ", "and will be removed in a future release. ", "Please extract the ", flags[needs_warning], " estimate from the 'estimates' component instead.", - call. = FALSE + class = "loo_warning_deprecated" ) } } @@ -643,12 +631,12 @@ NULL if (is.character(i)) { needs_warning <- which(flags == i) if (length(needs_warning)) { - warning( + .warn( "Accessing ", flags[needs_warning], " using '[[' is deprecated ", "and will be removed in a future release. ", "Please extract the ", flags[needs_warning], " estimate from the 'estimates' component instead.", - call. = FALSE + class = "loo_warning_deprecated" ) } } @@ -664,12 +652,12 @@ NULL "elpd_waic", "se_elpd_waic", "p_waic", "se_p_waic", "waic", "se_waic") needs_warning <- which(flags == name) if (length(needs_warning)) { - warning( + .warn( "Accessing ", flags[needs_warning], " using '$' is deprecated ", "and will be removed in a future release. ", "Please extract the ", flags[needs_warning], " estimate from the 'estimates' component instead.", - call. = FALSE + class = "loo_warning_deprecated" ) } NextMethod() diff --git a/R/loo_compare.R b/R/loo_compare.R index 4e7146b6..fc1e43db 100644 --- a/R/loo_compare.R +++ b/R/loo_compare.R @@ -286,22 +286,22 @@ loo_compare_checks <- function(loos) { isTRUE(all.equal(x, yhash[[1]])) }) if (!all(yhash_ok)) { - warning("Not all models have the same y variable. ('yhash' attributes do not match)", - call. = FALSE) + .warn_diagnostic("Not all models have the same y variable. ('yhash' attributes do not match)", + class = "loo_warning_y_mismatch") } if (all(sapply(loos, is.kfold))) { Ks <- unlist(lapply(loos, attr, which = "K")) if (!all(Ks == Ks[1])) { - warning("Not all kfold objects have the same K value. ", - "For a more accurate comparison use the same number of folds. ", - call. = FALSE) + .warn_diagnostic("Not all kfold objects have the same K value. ", + "For a more accurate comparison use the same number of folds. ", + class = "loo_warning_cv_mismatch") } } else if (any(sapply(loos, is.kfold)) && any(sapply(loos, is.psis_loo))) { - warning("Comparing LOO-CV to K-fold-CV. ", - "For a more accurate comparison use the same number of folds ", - "or loo for all models compared.", - call. = FALSE) + .warn_diagnostic("Comparing LOO-CV to K-fold-CV. ", + "For a more accurate comparison use the same number of folds ", + "or loo for all models compared.", + class = "loo_warning_cv_mismatch") } } @@ -405,9 +405,9 @@ loo_order_stat_check <- function(loos, ord) { if (max(elpd_diff) <= order_stat) { # flag warning if we suspect no model is theoretically better than the baseline - warning("Difference in performance potentially due to chance. ", - "See McLatchie and Vehtari (2023) for details.", - call. = FALSE) + .warn_diagnostic("Difference in performance potentially due to chance. ", + "See McLatchie and Vehtari (2023) for details.", + class = "loo_warning_chance_difference") } } diff --git a/R/loo_compare.psis_loo_ss_list.R b/R/loo_compare.psis_loo_ss_list.R index b84f7b5d..1a95176a 100644 --- a/R/loo_compare.psis_loo_ss_list.R +++ b/R/loo_compare.psis_loo_ss_list.R @@ -51,7 +51,7 @@ loo_compare_ss <- function(ref_loo, compare_loo){ # Using HH estimation if (ref_loo[[1]]$loo_subsampling$estimator == "hh_pps" | compare_loo[[1]]$loo_subsampling$estimator == "hh_pps"){ - warning("Hansen-Hurwitz estimator used. Naive diff SE is used.", call. = FALSE) + .warn_diagnostic("Hansen-Hurwitz estimator used. Naive diff SE is used.", class = "loo_warning_subsample_se") return(loo_compare_ss_naive(ref_loo, compare_loo)) } @@ -70,8 +70,8 @@ loo_compare_ss <- function(ref_loo, compare_loo){ # If different samples if (!compare_subset_of_ref & !ref_subset_of_compare){ - warning("Different subsamples in '", names(ref_loo), "' and '", names(compare_loo), - "'. Naive diff SE is used.", call. = FALSE) + .warn_diagnostic("Different subsamples in '", names(ref_loo), "' and '", names(compare_loo), + "'. Naive diff SE is used.", class = "loo_warning_subsample_se") return(loo_compare_ss_naive(ref_loo, compare_loo)) } } @@ -152,22 +152,22 @@ loo_compare_checks.psis_loo_ss_list <- function(loos) { isTRUE(all.equal(x, yhash[[1]])) }) if (!all(yhash_ok)) { - warning("Not all models have the same y variable. ('yhash' attributes do not match)", - call. = FALSE) + .warn_diagnostic("Not all models have the same y variable. ('yhash' attributes do not match)", + class = "loo_warning_y_mismatch") } if (all(sapply(loos, is.kfold))) { Ks <- unlist(lapply(loos, attr, which = "K")) if (!all(Ks == Ks[1])) { - warning("Not all kfold objects have the same K value. ", - "For a more accurate comparison use the same number of folds. ", - call. = FALSE) + .warn_diagnostic("Not all kfold objects have the same K value. ", + "For a more accurate comparison use the same number of folds. ", + class = "loo_warning_cv_mismatch") } } else if (any(sapply(loos, is.kfold)) && any(sapply(loos, is.psis_loo))) { - warning("Comparing LOO-CV to K-fold-CV. ", - "For a more accurate comparison use the same number of folds ", - "or loo for all models compared.", - call. = FALSE) + .warn_diagnostic("Comparing LOO-CV to K-fold-CV. ", + "For a more accurate comparison use the same number of folds ", + "or loo for all models compared.", + class = "loo_warning_cv_mismatch") } } diff --git a/R/loo_moment_matching.R b/R/loo_moment_matching.R index 08d6ced6..34e5bb72 100644 --- a/R/loo_moment_matching.R +++ b/R/loo_moment_matching.R @@ -736,23 +736,23 @@ throw_moment_match_error_warning <- function(failed, stopped, error_msg) { "observations are the PSIS results without moment matching.\n" )) } - warning( + .warn_diagnostic( msg, "The cause was an error in 'log_prob_upars', 'log_lik_i_upars' or ", "'log_lik_i', or a parallel worker that did not return a result. The ", "first error message:\n", error_msg[1], - call. = FALSE + class = "loo_warning_moment_match_failed" ) } #' Warning message if max_iters is reached #' @noRd throw_moment_match_max_iters_warning <- function() { - warning( + .warn_diagnostic( "The maximum number of moment matching iterations ('max_iters' argument) was reached.\n", "Increasing the value may improve accuracy.", - call. = FALSE + class = "loo_warning_moment_match_max_iters" ) } @@ -769,10 +769,10 @@ throw_large_kf_warning <- function(kf, k_threshold, split_failed = FALSE) { } else { advice <- "Setting the argument 'split' to 'TRUE' will likely improve accuracy." } - warning( + .warn_diagnostic( "The accuracy of self-normalized importance sampling may be bad.\n", advice, - call. = FALSE + class = "loo_warning_large_kf" ) } } diff --git a/R/loo_subsample.R b/R/loo_subsample.R index f109751f..f943ac26 100644 --- a/R/loo_subsample.R +++ b/R/loo_subsample.R @@ -584,8 +584,8 @@ elpd_loo_approximation <- function(.llfun, data, draws, cores, loo_approximation } p_eff_approx <- numeric(N) - if (cores>1) warning("Multicore is not implemented for waic_delta", - call. = FALSE) + if (cores>1) .warn("Multicore is not implemented for waic_delta", + class = "loo_warning_multicore_unavailable") if (loo_approximation == "waic_grad") { for(i in 1:nrow(data)) { @@ -1081,7 +1081,7 @@ loo_subsample_estimation_hh <- function(x) { if (hh_elpd_loo$hat_v_y_ppz > 0) { x$estimates["elpd_loo", "SE"] <- sqrt(hh_elpd_loo$hat_v_y_ppz) } else { - warning("Negative estimate of SE, more subsampling obs. needed.", call. = FALSE) + .warn_diagnostic("Negative estimate of SE, more subsampling obs. needed.", class = "loo_warning_subsample_se") x$estimates["elpd_loo", "SE"] <- NaN } x$estimates["elpd_loo", "subsampling SE"] <- sqrt(hh_elpd_loo$v_hat_y_ppz) @@ -1091,7 +1091,7 @@ loo_subsample_estimation_hh <- function(x) { if (hh_p_loo$hat_v_y_ppz > 0) { x$estimates["p_loo", "SE"] <- sqrt(hh_p_loo$hat_v_y_ppz) } else { - warning("Negative estimate of SE, more subsampling obs. needed.", call. = FALSE) + .warn_diagnostic("Negative estimate of SE, more subsampling obs. needed.", class = "loo_warning_subsample_se") x$estimates["elpd_loo", "SE"] <- NaN } x$estimates["p_loo", "subsampling SE"] <- sqrt(hh_p_loo$v_hat_y_ppz) diff --git a/R/print.R b/R/print.R index 1f4030b8..6f80d23b 100644 --- a/R/print.R +++ b/R/print.R @@ -238,7 +238,12 @@ print_mcse_summary <- function(x, digits) { # print and warning helpers .fr <- function(x, digits) format(round(x, digits), nsmall = digits) -.warn <- function(..., call. = FALSE) warning(..., call. = call.) +.warn <- function(..., class = NULL) { + rlang::warn(paste(c(...), collapse = ""), class = c(class, "loo_warning")) +} +.warn_diagnostic <- function(..., class) { + .warn(..., class = c(class, "loo_warning_diagnostic")) +} .k_help <- function() "See help('pareto-k-diagnostic') for details.\n" # compatibility with old loo objects diff --git a/R/psis.R b/R/psis.R index 85f8883c..7c9de5ec 100644 --- a/R/psis.R +++ b/R/psis.R @@ -224,10 +224,10 @@ do_psis_i <- function(log_ratios_i, tail_len_i, ...) { tail_ids <- top[seq(length(top) - tail_len_i + 1, length(top))] lw_tail <- lw_i[tail_ids] if (abs(max(lw_tail) - min(lw_tail)) < .Machine$double.eps / 100) { - warning( + .warn_diagnostic( "Can't fit generalized Pareto distribution ", "because all tail values are the same.", - call. = FALSE + class = "loo_warning_pareto_tail_constant" ) } else { cutoff <- cut_val # largest value not in the tail @@ -320,7 +320,11 @@ enough_tail_samples <- function(tail_len, min_len = 5) { #' throw_pareto_warnings <- function(k, k_threshold) { if (isTRUE(any(k > k_threshold))) { - .warn("Some Pareto k diagnostic values are too high. ", .k_help()) + .warn_diagnostic( + "Some Pareto k diagnostic values are too high. ", + .k_help(), + class = "loo_warning_pareto_k_high" + ) } } @@ -334,15 +338,14 @@ throw_tail_length_warnings <- function(tail_lengths) { tail_len_bad <- !sapply(tail_lengths, enough_tail_samples) if (any(tail_len_bad)) { if (length(tail_lengths) == 1) { - warning( + .warn_diagnostic( "Not enough tail samples to fit the generalized Pareto distribution.", - call. = FALSE, - immediate. = TRUE + class = "loo_warning_pareto_tail_not_fitted" ) } else { bad <- which(tail_len_bad) Nbad <- length(bad) - warning( + .warn_diagnostic( "Not enough tail samples to fit the generalized Pareto distribution ", "in some or all columns of matrix of log importance ratios. ", "Skipping the following columns: ", @@ -352,8 +355,7 @@ throw_tail_length_warnings <- function(tail_lengths) { } else { "\n" }, - call. = FALSE, - immediate. = TRUE + class = "loo_warning_pareto_tail_not_fitted" ) } } @@ -403,13 +405,3 @@ called_from_loo <- function() { check <- sapply(txt, function(x) grepl(patts, x)) isTRUE(any(check)) } - -#' Warning message about missing `r_eff` argument -#' @noRd -throw_psis_r_eff_warning <- function() { - warning( - "Relative effective sample sizes ('r_eff' argument) not specified. ", - "PSIS ESS (n_eff) will not be adjusted based on MCMC ESS (n_eff).", - call. = FALSE - ) -} diff --git a/R/psis_approximate_posterior.R b/R/psis_approximate_posterior.R index a648b753..ea521ea8 100644 --- a/R/psis_approximate_posterior.R +++ b/R/psis_approximate_posterior.R @@ -137,7 +137,8 @@ ap_psis.matrix <- function(log_ratios, log_p, log_g, ap_psis.default <- function(log_ratios, log_p, log_g, ...) { stopifnot(is.null(dim(log_ratios)) || length(dim(log_ratios)) == 1) dim(log_ratios) <- c(length(log_ratios), 1) - warning("llfun values do not return a matrix, coerce to matrix") + .warn("llfun values do not return a matrix, coerce to matrix", + class = "loo_warning_llfun_not_matrix") ap_psis.matrix(as.matrix(log_ratios), log_p, log_g, cores = 1) } diff --git a/R/psislw.R b/R/psislw.R index 808e657f..1f29a551 100644 --- a/R/psislw.R +++ b/R/psislw.R @@ -52,16 +52,16 @@ psislw <- function(lw, wcp = 0.2, wtrunc = 3/4, tail_len <- length(x_tail) if (tail_len < MIN_TAIL_LENGTH || all(x_tail == x_tail[1])) { if (all(x_tail == x_tail[1])) - warning( + .warn_diagnostic( "All tail values are the same. ", "Weights are truncated but not smoothed.", - call. = FALSE + class = "loo_warning_pareto_tail_constant" ) else if (tail_len < MIN_TAIL_LENGTH) - warning( + .warn_diagnostic( "Too few tail samples to fit generalized Pareto distribution.\n", "Weights are truncated but not smoothed.", - call. = FALSE + class = "loo_warning_pareto_tail_not_fitted" ) x_new <- x @@ -184,14 +184,16 @@ lw_normalize <- function(y) { # warnings about pareto k values ------------------------------------------ psislw_warnings <- function(k) { if (any(k > 0.7)) { - .warn( + .warn_diagnostic( "Some Pareto k diagnostic values are too high. ", - .k_help() + .k_help(), + class = "loo_warning_pareto_k_high" ) } else if (any(k > 0.5)) { - .warn( + .warn_diagnostic( "Some Pareto k diagnostic values are slightly high. ", - .k_help() + .k_help(), + class = "loo_warning_pareto_k_high" ) } } diff --git a/R/waic.R b/R/waic.R index 507d8b7b..aa55d99d 100644 --- a/R/waic.R +++ b/R/waic.R @@ -209,7 +209,11 @@ throw_pwaic_warnings <- function(p, digits = 1, warn = TRUE) { msg <- paste0("\n", count, " (", .fr(100 * prop, digits), "%) p_waic estimates greater than 0.4. ", "We recommend trying loo instead.") - if (warn) .warn(msg) else cat(msg, "\n") + if (warn) { + .warn_diagnostic(msg, class = "loo_warning_p_waic_high") + } else { + cat(msg, "\n") + } } invisible(NULL) } diff --git a/man/loo-package.Rd b/man/loo-package.Rd index aa158231..57ebe595 100644 --- a/man/loo-package.Rd +++ b/man/loo-package.Rd @@ -40,6 +40,49 @@ We recommend PSIS-LOO-CV instead of WAIC, because PSIS provides useful diagnostics and effective sample size and Monte Carlo standard error estimates. } +\section{Warning classes}{ + + +Every warning signaled by \pkg{loo} has the class \code{loo_warning} and one +class for its cause. Warnings that question the reliability of a result +also have the class \code{loo_warning_diagnostic}. Use these classes to handle +or muffle a specific warning, for example with \code{\link[=withCallingHandlers]{withCallingHandlers()}} +and \code{invokeRestart("muffleWarning")}. + +Diagnostic warnings: +\itemize{ +\item \code{loo_warning_pareto_k_high}: Some Pareto k values are too high. +\item \code{loo_warning_pareto_tail_not_fitted}: There are not enough tail samples to +fit the generalized Pareto distribution. +\item \code{loo_warning_pareto_tail_constant}: All tail values are the same. +\item \code{loo_warning_p_waic_high}: Some \code{p_waic} estimates are greater than 0.4. +\item \code{loo_warning_moment_match_failed}: Moment matching failed for some +observations. +\item \code{loo_warning_moment_match_max_iters}: Moment matching reached +\code{max_iters}. +\item \code{loo_warning_large_kf}: The accuracy of self-normalized importance +sampling may be bad. +\item \code{loo_warning_y_mismatch}: The models do not have the same \code{y} variable. +\item \code{loo_warning_cv_mismatch}: The models use different types of +cross-validation or different numbers of folds. +\item \code{loo_warning_chance_difference}: The difference in performance is +potentially due to chance. +\item \code{loo_warning_subsample_se}: A subsampling standard error is naive or +negative. +} + +Other warnings: +\itemize{ +\item \code{loo_warning_multicore_unavailable}: Multicore is not implemented for +the selected method. +\item \code{loo_warning_pareto_k_not_plotted}: Some Pareto k values are \code{Inf}, \code{NA}, +or \code{NaN} and are not plotted. +\item \code{loo_warning_llfun_not_matrix}: The values from \code{llfun} are not a matrix +and were coerced. +\item \code{loo_warning_deprecated}: A deprecated feature was used. +} +} + \references{ Vehtari, A., Gelman, A., and Gabry, J. (2017). Practical Bayesian model evaluation using leave-one-out cross-validation and WAIC. diff --git a/tests/testthat/test_warning_classes.R b/tests/testthat/test_warning_classes.R new file mode 100644 index 00000000..c8b412ab --- /dev/null +++ b/tests/testthat/test_warning_classes.R @@ -0,0 +1,22 @@ +test_that(".warn() signals one message with the loo classes", { + w <- capture_warning(.warn("a", c("b", "c"), 1, class = "loo_warning_x")) + expect_equal(conditionMessage(w), "abc1") + expect_equal(class(w)[1:2], c("loo_warning_x", "loo_warning")) +}) + +test_that(".warn_diagnostic() adds the diagnostic class", { + w <- capture_warning(.warn_diagnostic("a", class = "loo_warning_x")) + expect_equal( + class(w)[1:3], + c("loo_warning_x", "loo_warning_diagnostic", "loo_warning") + ) +}) + +test_that("warning helpers signal their classes", { + expect_warning(throw_pareto_warnings(1, 0.7), class = "loo_warning_pareto_k_high") + expect_warning(throw_tail_length_warnings(1), class = "loo_warning_pareto_tail_not_fitted") + expect_warning(throw_pwaic_warnings(1), class = "loo_warning_p_waic_high") + expect_warning(throw_moment_match_max_iters_warning(), class = "loo_warning_moment_match_max_iters") + expect_warning(throw_moment_match_error_warning(1, 2, "e"), class = "loo_warning_moment_match_failed") + expect_warning(throw_large_kf_warning(1, 0.7), class = "loo_warning_large_kf") +}) From febdcbcb338692ab7508f5443c52ead7f9f65eff Mon Sep 17 00:00:00 2001 From: Florence Bockting Date: Mon, 2 Nov 2026 14:17:15 +0200 Subject: [PATCH 2/5] chore: update PR number in NEWS.md --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index e13d1573..d2a9add8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -13,7 +13,7 @@ input by @thanasibakis in #398 * Speed up the Bayesian bootstrap in `pseudobma_weights()` by @VisruthSK in #368 * All warnings now have the class `loo_warning` and a class for each cause; diagnostics warnings have the class `loo_warning_diagnostic` by -@florence-bockting in #XXX +@florence-bockting in #417 # loo 2.10.1 From 94fe977d1a1dd68567d6f003612b3ecf7f0ed55e Mon Sep 17 00:00:00 2001 From: Florence Bockting Date: Mon, 9 Nov 2026 10:06:57 +0200 Subject: [PATCH 3/5] fix: address review of warning classes --- NEWS.md | 5 ++--- R/compare.R | 3 ++- R/loo-package.R | 4 +++- R/loo_compare.psis_loo_ss_list.R | 6 ++++-- R/loo_subsample.R | 8 +++++--- R/print.R | 5 ++++- R/psis.R | 6 ++++-- R/psis_approximate_posterior.R | 3 ++- R/psislw.R | 3 ++- man/loo-package.Rd | 4 +++- tests/testthat/_snaps/psislw.md | 2 +- tests/testthat/test_warning_classes.R | 11 +++++++++++ 12 files changed, 43 insertions(+), 17 deletions(-) diff --git a/NEWS.md b/NEWS.md index d2a9add8..545e17da 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,9 +12,8 @@ comparison-model minus reference-model by @florence-bockting in #391 input by @thanasibakis in #398 * Speed up the Bayesian bootstrap in `pseudobma_weights()` by @VisruthSK in #368 * All warnings now have the class `loo_warning` and a class for each cause; -diagnostics warnings have the class `loo_warning_diagnostic` by -@florence-bockting in #417 - +diagnostics warnings have the class `loo_warning_diagnostic`. The base warning +class is now `rlang_warning` by @florence-bockting in #417 # loo 2.10.1 diff --git a/R/compare.R b/R/compare.R index a6081d2f..3d81bc73 100644 --- a/R/compare.R +++ b/R/compare.R @@ -59,7 +59,8 @@ #' } #' compare <- function(..., x = list()) { - .Deprecated("loo_compare") + .warn("'compare' is deprecated.\nUse 'loo_compare' instead.\nSee help(\"Deprecated\")", + class = c("loo_warning_deprecated", "deprecatedWarning")) dots <- list(...) if (length(dots)) { if (length(x)) { diff --git a/R/loo-package.R b/R/loo-package.R index a9285c85..3003ddf2 100644 --- a/R/loo-package.R +++ b/R/loo-package.R @@ -65,7 +65,9 @@ #' cross-validation or different numbers of folds. #' * `loo_warning_chance_difference`: The difference in performance is #' potentially due to chance. -#' * `loo_warning_subsample_se`: A subsampling standard error is naive or +#' * `loo_warning_subsample_naive_se`: The difference SE in `loo_compare()` +#' uses the naive estimator. +#' * `loo_warning_subsample_negative_se`: The subsampling SE estimate is #' negative. #' #' Other warnings: diff --git a/R/loo_compare.psis_loo_ss_list.R b/R/loo_compare.psis_loo_ss_list.R index 1a95176a..acc4d3e2 100644 --- a/R/loo_compare.psis_loo_ss_list.R +++ b/R/loo_compare.psis_loo_ss_list.R @@ -51,7 +51,8 @@ loo_compare_ss <- function(ref_loo, compare_loo){ # Using HH estimation if (ref_loo[[1]]$loo_subsampling$estimator == "hh_pps" | compare_loo[[1]]$loo_subsampling$estimator == "hh_pps"){ - .warn_diagnostic("Hansen-Hurwitz estimator used. Naive diff SE is used.", class = "loo_warning_subsample_se") + .warn_diagnostic("Hansen-Hurwitz estimator used. Naive diff SE is used.", + class = "loo_warning_subsample_naive_se") return(loo_compare_ss_naive(ref_loo, compare_loo)) } @@ -71,7 +72,8 @@ loo_compare_ss <- function(ref_loo, compare_loo){ # If different samples if (!compare_subset_of_ref & !ref_subset_of_compare){ .warn_diagnostic("Different subsamples in '", names(ref_loo), "' and '", names(compare_loo), - "'. Naive diff SE is used.", class = "loo_warning_subsample_se") + "'. Naive diff SE is used.", + class = "loo_warning_subsample_naive_se") return(loo_compare_ss_naive(ref_loo, compare_loo)) } } diff --git a/R/loo_subsample.R b/R/loo_subsample.R index f943ac26..cf0c8555 100644 --- a/R/loo_subsample.R +++ b/R/loo_subsample.R @@ -1081,7 +1081,8 @@ loo_subsample_estimation_hh <- function(x) { if (hh_elpd_loo$hat_v_y_ppz > 0) { x$estimates["elpd_loo", "SE"] <- sqrt(hh_elpd_loo$hat_v_y_ppz) } else { - .warn_diagnostic("Negative estimate of SE, more subsampling obs. needed.", class = "loo_warning_subsample_se") + .warn_diagnostic("Negative estimate of SE, more subsampling obs. needed.", + class = "loo_warning_subsample_negative_se") x$estimates["elpd_loo", "SE"] <- NaN } x$estimates["elpd_loo", "subsampling SE"] <- sqrt(hh_elpd_loo$v_hat_y_ppz) @@ -1091,8 +1092,9 @@ loo_subsample_estimation_hh <- function(x) { if (hh_p_loo$hat_v_y_ppz > 0) { x$estimates["p_loo", "SE"] <- sqrt(hh_p_loo$hat_v_y_ppz) } else { - .warn_diagnostic("Negative estimate of SE, more subsampling obs. needed.", class = "loo_warning_subsample_se") - x$estimates["elpd_loo", "SE"] <- NaN + .warn_diagnostic("Negative estimate of SE, more subsampling obs. needed.", + class = "loo_warning_subsample_negative_se") + x$estimates["p_loo", "SE"] <- NaN } x$estimates["p_loo", "subsampling SE"] <- sqrt(hh_p_loo$v_hat_y_ppz) update_psis_loo_ss_estimates(x) diff --git a/R/print.R b/R/print.R index 6f80d23b..a06bc04a 100644 --- a/R/print.R +++ b/R/print.R @@ -238,7 +238,10 @@ print_mcse_summary <- function(x, digits) { # print and warning helpers .fr <- function(x, digits) format(round(x, digits), nsmall = digits) -.warn <- function(..., class = NULL) { +.warn <- function(..., class = NULL, immediate = FALSE) { + if (immediate && getOption("warn") < 1) { + rlang::local_options(warn = 1) + } rlang::warn(paste(c(...), collapse = ""), class = c(class, "loo_warning")) } .warn_diagnostic <- function(..., class) { diff --git a/R/psis.R b/R/psis.R index 7c9de5ec..9c984d5f 100644 --- a/R/psis.R +++ b/R/psis.R @@ -340,7 +340,8 @@ throw_tail_length_warnings <- function(tail_lengths) { if (length(tail_lengths) == 1) { .warn_diagnostic( "Not enough tail samples to fit the generalized Pareto distribution.", - class = "loo_warning_pareto_tail_not_fitted" + class = "loo_warning_pareto_tail_not_fitted", + immediate = TRUE ) } else { bad <- which(tail_len_bad) @@ -355,7 +356,8 @@ throw_tail_length_warnings <- function(tail_lengths) { } else { "\n" }, - class = "loo_warning_pareto_tail_not_fitted" + class = "loo_warning_pareto_tail_not_fitted", + immediate = TRUE ) } } diff --git a/R/psis_approximate_posterior.R b/R/psis_approximate_posterior.R index ea521ea8..955636a8 100644 --- a/R/psis_approximate_posterior.R +++ b/R/psis_approximate_posterior.R @@ -24,7 +24,8 @@ psis_approximate_posterior <- function(log_p = NULL, log_g = NULL, log_liks = NULL, cores, save_psis, ..., log_q = NULL) { if (!is.null(log_q)) { - .Deprecated(msg = "psis_approximate_posterior() argument log_q has been changed to log_g") + .warn("psis_approximate_posterior() argument log_q has been changed to log_g", + class = c("loo_warning_deprecated", "deprecatedWarning")) log_g <- log_q } checkmate::assert_numeric(log_p, any.missing = FALSE, len = length(log_g), null.ok = FALSE) diff --git a/R/psislw.R b/R/psislw.R index 1f29a551..c216e12e 100644 --- a/R/psislw.R +++ b/R/psislw.R @@ -38,7 +38,8 @@ psislw <- function(lw, wcp = 0.2, wtrunc = 3/4, cores = getOption("mc.cores", 1), llfun = NULL, llargs = NULL, ...) { - .Deprecated("psis") + .warn("'psislw' is deprecated.\nUse 'psis' instead.\nSee help(\"Deprecated\")", + class = c("loo_warning_deprecated", "deprecatedWarning")) cores <- loo_cores(cores) diff --git a/man/loo-package.Rd b/man/loo-package.Rd index 57ebe595..f3648046 100644 --- a/man/loo-package.Rd +++ b/man/loo-package.Rd @@ -67,7 +67,9 @@ sampling may be bad. cross-validation or different numbers of folds. \item \code{loo_warning_chance_difference}: The difference in performance is potentially due to chance. -\item \code{loo_warning_subsample_se}: A subsampling standard error is naive or +\item \code{loo_warning_subsample_naive_se}: The difference SE in \code{loo_compare()} +uses the naive estimator. +\item \code{loo_warning_subsample_negative_se}: The subsampling SE estimate is negative. } diff --git a/tests/testthat/_snaps/psislw.md b/tests/testthat/_snaps/psislw.md index 94daa306..44b3425d 100644 --- a/tests/testthat/_snaps/psislw.md +++ b/tests/testthat/_snaps/psislw.md @@ -3,7 +3,7 @@ Code psis <- psislw(x[, 1], wcp = 0.01) Condition - Warning in `psislw()`: + Warning: 'psislw' is deprecated. Use 'psis' instead. See help("Deprecated") diff --git a/tests/testthat/test_warning_classes.R b/tests/testthat/test_warning_classes.R index c8b412ab..9b6258c0 100644 --- a/tests/testthat/test_warning_classes.R +++ b/tests/testthat/test_warning_classes.R @@ -20,3 +20,14 @@ test_that("warning helpers signal their classes", { expect_warning(throw_moment_match_error_warning(1, 2, "e"), class = "loo_warning_moment_match_failed") expect_warning(throw_large_kf_warning(1, 0.7), class = "loo_warning_large_kf") }) + +test_that("user-facing functions signal loo warning classes", { + LLarr <- example_loglik_array() + w1 <- suppressWarnings(waic(LLarr)) + w2 <- w1 + attr(w2, "yhash") <- "b" + expect_warning(waic(LLarr), class = "loo_warning_p_waic_high") + expect_warning(loo_compare(w1, w2), class = "loo_warning_y_mismatch") + expect_warning(compare(w1, w1), class = "deprecatedWarning") + expect_warning(compare(w1, w1), class = "loo_warning_deprecated") +}) From 7ee1e4bd0d1d5cabacd7c3984752136e1f23139a Mon Sep 17 00:00:00 2001 From: Florence Bockting Date: Wed, 11 Nov 2026 11:33:40 +0200 Subject: [PATCH 4/5] fix: clean-up indentation --- R/loo_compare.psis_loo_ss_list.R | 6 ++++-- R/loo_subsample.R | 12 ++++++++---- R/psis_approximate_posterior.R | 6 ++++-- R/psislw.R | 6 ++++-- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/R/loo_compare.psis_loo_ss_list.R b/R/loo_compare.psis_loo_ss_list.R index acc4d3e2..631371ce 100644 --- a/R/loo_compare.psis_loo_ss_list.R +++ b/R/loo_compare.psis_loo_ss_list.R @@ -51,8 +51,10 @@ loo_compare_ss <- function(ref_loo, compare_loo){ # Using HH estimation if (ref_loo[[1]]$loo_subsampling$estimator == "hh_pps" | compare_loo[[1]]$loo_subsampling$estimator == "hh_pps"){ - .warn_diagnostic("Hansen-Hurwitz estimator used. Naive diff SE is used.", - class = "loo_warning_subsample_naive_se") + .warn_diagnostic( + "Hansen-Hurwitz estimator used. Naive diff SE is used.", + class = "loo_warning_subsample_naive_se" + ) return(loo_compare_ss_naive(ref_loo, compare_loo)) } diff --git a/R/loo_subsample.R b/R/loo_subsample.R index cf0c8555..86c2d807 100644 --- a/R/loo_subsample.R +++ b/R/loo_subsample.R @@ -1081,8 +1081,10 @@ loo_subsample_estimation_hh <- function(x) { if (hh_elpd_loo$hat_v_y_ppz > 0) { x$estimates["elpd_loo", "SE"] <- sqrt(hh_elpd_loo$hat_v_y_ppz) } else { - .warn_diagnostic("Negative estimate of SE, more subsampling obs. needed.", - class = "loo_warning_subsample_negative_se") + .warn_diagnostic( + "Negative estimate of SE, more subsampling obs. needed.", + class = "loo_warning_subsample_negative_se" + ) x$estimates["elpd_loo", "SE"] <- NaN } x$estimates["elpd_loo", "subsampling SE"] <- sqrt(hh_elpd_loo$v_hat_y_ppz) @@ -1092,8 +1094,10 @@ loo_subsample_estimation_hh <- function(x) { if (hh_p_loo$hat_v_y_ppz > 0) { x$estimates["p_loo", "SE"] <- sqrt(hh_p_loo$hat_v_y_ppz) } else { - .warn_diagnostic("Negative estimate of SE, more subsampling obs. needed.", - class = "loo_warning_subsample_negative_se") + .warn_diagnostic( + "Negative estimate of SE, more subsampling obs. needed.", + class = "loo_warning_subsample_negative_se" + ) x$estimates["p_loo", "SE"] <- NaN } x$estimates["p_loo", "subsampling SE"] <- sqrt(hh_p_loo$v_hat_y_ppz) diff --git a/R/psis_approximate_posterior.R b/R/psis_approximate_posterior.R index 955636a8..eb87ae6d 100644 --- a/R/psis_approximate_posterior.R +++ b/R/psis_approximate_posterior.R @@ -24,8 +24,10 @@ psis_approximate_posterior <- function(log_p = NULL, log_g = NULL, log_liks = NULL, cores, save_psis, ..., log_q = NULL) { if (!is.null(log_q)) { - .warn("psis_approximate_posterior() argument log_q has been changed to log_g", - class = c("loo_warning_deprecated", "deprecatedWarning")) + .warn( + "psis_approximate_posterior() argument log_q has been changed to log_g", + class = c("loo_warning_deprecated", "deprecatedWarning") + ) log_g <- log_q } checkmate::assert_numeric(log_p, any.missing = FALSE, len = length(log_g), null.ok = FALSE) diff --git a/R/psislw.R b/R/psislw.R index c216e12e..9fac0e9d 100644 --- a/R/psislw.R +++ b/R/psislw.R @@ -38,8 +38,10 @@ psislw <- function(lw, wcp = 0.2, wtrunc = 3/4, cores = getOption("mc.cores", 1), llfun = NULL, llargs = NULL, ...) { - .warn("'psislw' is deprecated.\nUse 'psis' instead.\nSee help(\"Deprecated\")", - class = c("loo_warning_deprecated", "deprecatedWarning")) + .warn( + "'psislw' is deprecated.\nUse 'psis' instead.\nSee help(\"Deprecated\")", + class = c("loo_warning_deprecated", "deprecatedWarning") + ) cores <- loo_cores(cores) From 6ad3e76686c0f56083d105a23d1c90af1ec2be4a Mon Sep 17 00:00:00 2001 From: Florence Bockting <48919471+florence-bockting@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:35:31 +0200 Subject: [PATCH 5/5] Update NEWS.md Co-authored-by: Jonah Gabry --- NEWS.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 620b59c7..7d4b6e7a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -11,9 +11,13 @@ comparison-model minus reference-model by @florence-bockting in #391 * Fix `crps()` and `scrps()` ignoring the `permutations` argument for numeric input by @thanasibakis in #398 * Speed up the Bayesian bootstrap in `pseudobma_weights()` by @VisruthSK in #368 -* All warnings now have the class `loo_warning` and a class for each cause; -diagnostics warnings have the class `loo_warning_diagnostic`. The base warning -class is now `rlang_warning` by @florence-bockting in #417 +* All warnings now have the class `loo_warning` and a class for each cause, +and diagnostic warnings also have the class `loo_warning_diagnostic`. The base +warning class is now `rlang_warning` instead of `simpleWarning`. By +@florence-bockting in #417 +* Fix `loo_subsample()` with the Hansen-Hurwitz estimator setting the +`elpd_loo` SE to `NaN` instead of the `p_loo` SE when the `p_loo` variance +estimate is negative by @florence-bockting in #417 * Signal each class of posterior warning once per call in `relative_eff()`, `E_loo()`, and `loo_moment_match()` (requires posterior >= 1.7.1) by @florence-bockting in #416 * Use a tolerance relative to the magnitude of `x` in `is_constant()` by