diff --git a/DESCRIPTION b/DESCRIPTION index daa0db08..9d7e12f1 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -39,6 +39,7 @@ Imports: matrixStats (>= 0.52), parallel, posterior (>= 1.7.0), + rlang (>= 1.0.6), stats Suggests: bayesplot (>= 1.7.0), diff --git a/NEWS.md b/NEWS.md index 9f6f97cb..7d4b6e7a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -11,6 +11,13 @@ comparison-model minus reference-model by @florence-bockting in #391 * Fix `crps()` and `scrps()` ignoring the `permutations` argument for numeric input by @thanasibakis in #398 * Speed up the Bayesian bootstrap in `pseudobma_weights()` by @VisruthSK in #368 +* All warnings now have the class `loo_warning` and a class for each cause, +and diagnostic warnings also have the class `loo_warning_diagnostic`. The base +warning class is now `rlang_warning` instead of `simpleWarning`. By +@florence-bockting in #417 +* Fix `loo_subsample()` with the Hansen-Hurwitz estimator setting the +`elpd_loo` SE to `NaN` instead of the `p_loo` SE when the `p_loo` variance +estimate is negative by @florence-bockting in #417 * Signal each class of posterior warning once per call in `relative_eff()`, `E_loo()`, and `loo_moment_match()` (requires posterior >= 1.7.1) by @florence-bockting in #416 * Use a tolerance relative to the magnitude of `x` in `is_constant()` by diff --git a/R/compare.R b/R/compare.R index a6081d2f..3d81bc73 100644 --- a/R/compare.R +++ b/R/compare.R @@ -59,7 +59,8 @@ #' } #' compare <- function(..., x = list()) { - .Deprecated("loo_compare") + .warn("'compare' is deprecated.\nUse 'loo_compare' instead.\nSee help(\"Deprecated\")", + class = c("loo_warning_deprecated", "deprecatedWarning")) dots <- list(...) if (length(dots)) { if (length(x)) { diff --git a/R/diagnostics.R b/R/diagnostics.R index 3d12bdb4..2bc2cdbc 100644 --- a/R/diagnostics.R +++ b/R/diagnostics.R @@ -272,8 +272,9 @@ plot.psis_loo <- function(x, k[is.na(k)] <- 0 # FIXME when reloo is changed to make NA k values -Inf k_inf <- !is.finite(k) if (any(k_inf)) { - warning(signif(100 * mean(k_inf), 2), - "% of Pareto k estimates are Inf/NA/NaN and not plotted.") + .warn(signif(100 * mean(k_inf), 2), + "% of Pareto k estimates are Inf/NA/NaN and not plotted.", + class = "loo_warning_pareto_k_not_plotted") } if (diagnostic == "ESS" || diagnostic == "n_eff") { diff --git a/R/helpers.R b/R/helpers.R index 3a653f60..4b91c420 100644 --- a/R/helpers.R +++ b/R/helpers.R @@ -259,8 +259,8 @@ loo_cores <- function(cores) { loo_cores_op <- getOption("loo.cores", NA) if (!is.na(loo_cores_op) && (loo_cores_op != cores)) { cores <- loo_cores_op - warning("'loo.cores' is deprecated, please use 'mc.cores' or pass 'cores' explicitly.", - call. = FALSE) + .warn("'loo.cores' is deprecated, please use 'mc.cores' or pass 'cores' explicitly.", + class = "loo_warning_deprecated") } return(cores) } diff --git a/R/loo-package.R b/R/loo-package.R index 4a29a5cf..3003ddf2 100644 --- a/R/loo-package.R +++ b/R/loo-package.R @@ -39,6 +39,46 @@ #' diagnostics and effective sample size and Monte Carlo standard error #' estimates. #' +#' @section Warning classes: +#' +#' Every warning signaled by \pkg{loo} has the class `loo_warning` and one +#' class for its cause. Warnings that question the reliability of a result +#' also have the class `loo_warning_diagnostic`. Use these classes to handle +#' or muffle a specific warning, for example with [withCallingHandlers()] +#' and `invokeRestart("muffleWarning")`. +#' +#' Diagnostic warnings: +#' +#' * `loo_warning_pareto_k_high`: Some Pareto k values are too high. +#' * `loo_warning_pareto_tail_not_fitted`: There are not enough tail samples to +#' fit the generalized Pareto distribution. +#' * `loo_warning_pareto_tail_constant`: All tail values are the same. +#' * `loo_warning_p_waic_high`: Some `p_waic` estimates are greater than 0.4. +#' * `loo_warning_moment_match_failed`: Moment matching failed for some +#' observations. +#' * `loo_warning_moment_match_max_iters`: Moment matching reached +#' `max_iters`. +#' * `loo_warning_large_kf`: The accuracy of self-normalized importance +#' sampling may be bad. +#' * `loo_warning_y_mismatch`: The models do not have the same `y` variable. +#' * `loo_warning_cv_mismatch`: The models use different types of +#' cross-validation or different numbers of folds. +#' * `loo_warning_chance_difference`: The difference in performance is +#' potentially due to chance. +#' * `loo_warning_subsample_naive_se`: The difference SE in `loo_compare()` +#' uses the naive estimator. +#' * `loo_warning_subsample_negative_se`: The subsampling SE estimate is +#' negative. +#' +#' Other warnings: +#' +#' * `loo_warning_multicore_unavailable`: Multicore is not implemented for +#' the selected method. +#' * `loo_warning_pareto_k_not_plotted`: Some Pareto k values are `Inf`, `NA`, +#' or `NaN` and are not plotted. +#' * `loo_warning_llfun_not_matrix`: The values from `llfun` are not a matrix +#' and were coerced. +#' * `loo_warning_deprecated`: A deprecated feature was used. #' #' @template loo-and-psis-references #' @template stacking-references diff --git a/R/loo.R b/R/loo.R index 0b284297..e3e36100 100644 --- a/R/loo.R +++ b/R/loo.R @@ -549,18 +549,6 @@ log_var_epd_ratio <- function(log_lik_ratio, lw, r_eff) { matrixStats::colLogSumExps(2 * (lw + log_abs_diff)) - log(r_eff) } - -#' Warning message if r_eff not specified -#' @noRd -throw_loo_r_eff_warning <- function() { - warning( - "Relative effective sample sizes ('r_eff' argument) not specified.\n", - "For models fit with MCMC, the reported PSIS ESS and \n", - "MCSE estimates can be over-optimistic.", - call. = FALSE - ) -} - #' Combine many psis objects into a single psis object #' #' @noRd @@ -621,12 +609,12 @@ NULL if (is.character(i)) { needs_warning <- which(flags == i) if (length(needs_warning)) { - warning( + .warn( "Accessing ", flags[needs_warning], " using '[' is deprecated ", "and will be removed in a future release. ", "Please extract the ", flags[needs_warning], " estimate from the 'estimates' component instead.", - call. = FALSE + class = "loo_warning_deprecated" ) } } @@ -643,12 +631,12 @@ NULL if (is.character(i)) { needs_warning <- which(flags == i) if (length(needs_warning)) { - warning( + .warn( "Accessing ", flags[needs_warning], " using '[[' is deprecated ", "and will be removed in a future release. ", "Please extract the ", flags[needs_warning], " estimate from the 'estimates' component instead.", - call. = FALSE + class = "loo_warning_deprecated" ) } } @@ -664,12 +652,12 @@ NULL "elpd_waic", "se_elpd_waic", "p_waic", "se_p_waic", "waic", "se_waic") needs_warning <- which(flags == name) if (length(needs_warning)) { - warning( + .warn( "Accessing ", flags[needs_warning], " using '$' is deprecated ", "and will be removed in a future release. ", "Please extract the ", flags[needs_warning], " estimate from the 'estimates' component instead.", - call. = FALSE + class = "loo_warning_deprecated" ) } NextMethod() diff --git a/R/loo_compare.R b/R/loo_compare.R index 4e7146b6..fc1e43db 100644 --- a/R/loo_compare.R +++ b/R/loo_compare.R @@ -286,22 +286,22 @@ loo_compare_checks <- function(loos) { isTRUE(all.equal(x, yhash[[1]])) }) if (!all(yhash_ok)) { - warning("Not all models have the same y variable. ('yhash' attributes do not match)", - call. = FALSE) + .warn_diagnostic("Not all models have the same y variable. ('yhash' attributes do not match)", + class = "loo_warning_y_mismatch") } if (all(sapply(loos, is.kfold))) { Ks <- unlist(lapply(loos, attr, which = "K")) if (!all(Ks == Ks[1])) { - warning("Not all kfold objects have the same K value. ", - "For a more accurate comparison use the same number of folds. ", - call. = FALSE) + .warn_diagnostic("Not all kfold objects have the same K value. ", + "For a more accurate comparison use the same number of folds. ", + class = "loo_warning_cv_mismatch") } } else if (any(sapply(loos, is.kfold)) && any(sapply(loos, is.psis_loo))) { - warning("Comparing LOO-CV to K-fold-CV. ", - "For a more accurate comparison use the same number of folds ", - "or loo for all models compared.", - call. = FALSE) + .warn_diagnostic("Comparing LOO-CV to K-fold-CV. ", + "For a more accurate comparison use the same number of folds ", + "or loo for all models compared.", + class = "loo_warning_cv_mismatch") } } @@ -405,9 +405,9 @@ loo_order_stat_check <- function(loos, ord) { if (max(elpd_diff) <= order_stat) { # flag warning if we suspect no model is theoretically better than the baseline - warning("Difference in performance potentially due to chance. ", - "See McLatchie and Vehtari (2023) for details.", - call. = FALSE) + .warn_diagnostic("Difference in performance potentially due to chance. ", + "See McLatchie and Vehtari (2023) for details.", + class = "loo_warning_chance_difference") } } diff --git a/R/loo_compare.psis_loo_ss_list.R b/R/loo_compare.psis_loo_ss_list.R index b84f7b5d..631371ce 100644 --- a/R/loo_compare.psis_loo_ss_list.R +++ b/R/loo_compare.psis_loo_ss_list.R @@ -51,7 +51,10 @@ loo_compare_ss <- function(ref_loo, compare_loo){ # Using HH estimation if (ref_loo[[1]]$loo_subsampling$estimator == "hh_pps" | compare_loo[[1]]$loo_subsampling$estimator == "hh_pps"){ - warning("Hansen-Hurwitz estimator used. Naive diff SE is used.", call. = FALSE) + .warn_diagnostic( + "Hansen-Hurwitz estimator used. Naive diff SE is used.", + class = "loo_warning_subsample_naive_se" + ) return(loo_compare_ss_naive(ref_loo, compare_loo)) } @@ -70,8 +73,9 @@ loo_compare_ss <- function(ref_loo, compare_loo){ # If different samples if (!compare_subset_of_ref & !ref_subset_of_compare){ - warning("Different subsamples in '", names(ref_loo), "' and '", names(compare_loo), - "'. Naive diff SE is used.", call. = FALSE) + .warn_diagnostic("Different subsamples in '", names(ref_loo), "' and '", names(compare_loo), + "'. Naive diff SE is used.", + class = "loo_warning_subsample_naive_se") return(loo_compare_ss_naive(ref_loo, compare_loo)) } } @@ -152,22 +156,22 @@ loo_compare_checks.psis_loo_ss_list <- function(loos) { isTRUE(all.equal(x, yhash[[1]])) }) if (!all(yhash_ok)) { - warning("Not all models have the same y variable. ('yhash' attributes do not match)", - call. = FALSE) + .warn_diagnostic("Not all models have the same y variable. ('yhash' attributes do not match)", + class = "loo_warning_y_mismatch") } if (all(sapply(loos, is.kfold))) { Ks <- unlist(lapply(loos, attr, which = "K")) if (!all(Ks == Ks[1])) { - warning("Not all kfold objects have the same K value. ", - "For a more accurate comparison use the same number of folds. ", - call. = FALSE) + .warn_diagnostic("Not all kfold objects have the same K value. ", + "For a more accurate comparison use the same number of folds. ", + class = "loo_warning_cv_mismatch") } } else if (any(sapply(loos, is.kfold)) && any(sapply(loos, is.psis_loo))) { - warning("Comparing LOO-CV to K-fold-CV. ", - "For a more accurate comparison use the same number of folds ", - "or loo for all models compared.", - call. = FALSE) + .warn_diagnostic("Comparing LOO-CV to K-fold-CV. ", + "For a more accurate comparison use the same number of folds ", + "or loo for all models compared.", + class = "loo_warning_cv_mismatch") } } diff --git a/R/loo_moment_matching.R b/R/loo_moment_matching.R index 4ac89103..421b764a 100644 --- a/R/loo_moment_matching.R +++ b/R/loo_moment_matching.R @@ -738,23 +738,23 @@ throw_moment_match_error_warning <- function(failed, stopped, error_msg) { "observations are the PSIS results without moment matching.\n" )) } - warning( + .warn_diagnostic( msg, "The cause was an error in 'log_prob_upars', 'log_lik_i_upars' or ", "'log_lik_i', or a parallel worker that did not return a result. The ", "first error message:\n", error_msg[1], - call. = FALSE + class = "loo_warning_moment_match_failed" ) } #' Warning message if max_iters is reached #' @noRd throw_moment_match_max_iters_warning <- function() { - warning( + .warn_diagnostic( "The maximum number of moment matching iterations ('max_iters' argument) was reached.\n", "Increasing the value may improve accuracy.", - call. = FALSE + class = "loo_warning_moment_match_max_iters" ) } @@ -771,10 +771,10 @@ throw_large_kf_warning <- function(kf, k_threshold, split_failed = FALSE) { } else { advice <- "Setting the argument 'split' to 'TRUE' will likely improve accuracy." } - warning( + .warn_diagnostic( "The accuracy of self-normalized importance sampling may be bad.\n", advice, - call. = FALSE + class = "loo_warning_large_kf" ) } } diff --git a/R/loo_subsample.R b/R/loo_subsample.R index f109751f..86c2d807 100644 --- a/R/loo_subsample.R +++ b/R/loo_subsample.R @@ -584,8 +584,8 @@ elpd_loo_approximation <- function(.llfun, data, draws, cores, loo_approximation } p_eff_approx <- numeric(N) - if (cores>1) warning("Multicore is not implemented for waic_delta", - call. = FALSE) + if (cores>1) .warn("Multicore is not implemented for waic_delta", + class = "loo_warning_multicore_unavailable") if (loo_approximation == "waic_grad") { for(i in 1:nrow(data)) { @@ -1081,7 +1081,10 @@ loo_subsample_estimation_hh <- function(x) { if (hh_elpd_loo$hat_v_y_ppz > 0) { x$estimates["elpd_loo", "SE"] <- sqrt(hh_elpd_loo$hat_v_y_ppz) } else { - warning("Negative estimate of SE, more subsampling obs. needed.", call. = FALSE) + .warn_diagnostic( + "Negative estimate of SE, more subsampling obs. needed.", + class = "loo_warning_subsample_negative_se" + ) x$estimates["elpd_loo", "SE"] <- NaN } x$estimates["elpd_loo", "subsampling SE"] <- sqrt(hh_elpd_loo$v_hat_y_ppz) @@ -1091,8 +1094,11 @@ loo_subsample_estimation_hh <- function(x) { if (hh_p_loo$hat_v_y_ppz > 0) { x$estimates["p_loo", "SE"] <- sqrt(hh_p_loo$hat_v_y_ppz) } else { - warning("Negative estimate of SE, more subsampling obs. needed.", call. = FALSE) - x$estimates["elpd_loo", "SE"] <- NaN + .warn_diagnostic( + "Negative estimate of SE, more subsampling obs. needed.", + class = "loo_warning_subsample_negative_se" + ) + x$estimates["p_loo", "SE"] <- NaN } x$estimates["p_loo", "subsampling SE"] <- sqrt(hh_p_loo$v_hat_y_ppz) update_psis_loo_ss_estimates(x) diff --git a/R/print.R b/R/print.R index 1f4030b8..a06bc04a 100644 --- a/R/print.R +++ b/R/print.R @@ -238,7 +238,15 @@ print_mcse_summary <- function(x, digits) { # print and warning helpers .fr <- function(x, digits) format(round(x, digits), nsmall = digits) -.warn <- function(..., call. = FALSE) warning(..., call. = call.) +.warn <- function(..., class = NULL, immediate = FALSE) { + if (immediate && getOption("warn") < 1) { + rlang::local_options(warn = 1) + } + rlang::warn(paste(c(...), collapse = ""), class = c(class, "loo_warning")) +} +.warn_diagnostic <- function(..., class) { + .warn(..., class = c(class, "loo_warning_diagnostic")) +} .k_help <- function() "See help('pareto-k-diagnostic') for details.\n" # compatibility with old loo objects diff --git a/R/psis.R b/R/psis.R index 85f8883c..9c984d5f 100644 --- a/R/psis.R +++ b/R/psis.R @@ -224,10 +224,10 @@ do_psis_i <- function(log_ratios_i, tail_len_i, ...) { tail_ids <- top[seq(length(top) - tail_len_i + 1, length(top))] lw_tail <- lw_i[tail_ids] if (abs(max(lw_tail) - min(lw_tail)) < .Machine$double.eps / 100) { - warning( + .warn_diagnostic( "Can't fit generalized Pareto distribution ", "because all tail values are the same.", - call. = FALSE + class = "loo_warning_pareto_tail_constant" ) } else { cutoff <- cut_val # largest value not in the tail @@ -320,7 +320,11 @@ enough_tail_samples <- function(tail_len, min_len = 5) { #' throw_pareto_warnings <- function(k, k_threshold) { if (isTRUE(any(k > k_threshold))) { - .warn("Some Pareto k diagnostic values are too high. ", .k_help()) + .warn_diagnostic( + "Some Pareto k diagnostic values are too high. ", + .k_help(), + class = "loo_warning_pareto_k_high" + ) } } @@ -334,15 +338,15 @@ throw_tail_length_warnings <- function(tail_lengths) { tail_len_bad <- !sapply(tail_lengths, enough_tail_samples) if (any(tail_len_bad)) { if (length(tail_lengths) == 1) { - warning( + .warn_diagnostic( "Not enough tail samples to fit the generalized Pareto distribution.", - call. = FALSE, - immediate. = TRUE + class = "loo_warning_pareto_tail_not_fitted", + immediate = TRUE ) } else { bad <- which(tail_len_bad) Nbad <- length(bad) - warning( + .warn_diagnostic( "Not enough tail samples to fit the generalized Pareto distribution ", "in some or all columns of matrix of log importance ratios. ", "Skipping the following columns: ", @@ -352,8 +356,8 @@ throw_tail_length_warnings <- function(tail_lengths) { } else { "\n" }, - call. = FALSE, - immediate. = TRUE + class = "loo_warning_pareto_tail_not_fitted", + immediate = TRUE ) } } @@ -403,13 +407,3 @@ called_from_loo <- function() { check <- sapply(txt, function(x) grepl(patts, x)) isTRUE(any(check)) } - -#' Warning message about missing `r_eff` argument -#' @noRd -throw_psis_r_eff_warning <- function() { - warning( - "Relative effective sample sizes ('r_eff' argument) not specified. ", - "PSIS ESS (n_eff) will not be adjusted based on MCMC ESS (n_eff).", - call. = FALSE - ) -} diff --git a/R/psis_approximate_posterior.R b/R/psis_approximate_posterior.R index a648b753..eb87ae6d 100644 --- a/R/psis_approximate_posterior.R +++ b/R/psis_approximate_posterior.R @@ -24,7 +24,10 @@ psis_approximate_posterior <- function(log_p = NULL, log_g = NULL, log_liks = NULL, cores, save_psis, ..., log_q = NULL) { if (!is.null(log_q)) { - .Deprecated(msg = "psis_approximate_posterior() argument log_q has been changed to log_g") + .warn( + "psis_approximate_posterior() argument log_q has been changed to log_g", + class = c("loo_warning_deprecated", "deprecatedWarning") + ) log_g <- log_q } checkmate::assert_numeric(log_p, any.missing = FALSE, len = length(log_g), null.ok = FALSE) @@ -137,7 +140,8 @@ ap_psis.matrix <- function(log_ratios, log_p, log_g, ap_psis.default <- function(log_ratios, log_p, log_g, ...) { stopifnot(is.null(dim(log_ratios)) || length(dim(log_ratios)) == 1) dim(log_ratios) <- c(length(log_ratios), 1) - warning("llfun values do not return a matrix, coerce to matrix") + .warn("llfun values do not return a matrix, coerce to matrix", + class = "loo_warning_llfun_not_matrix") ap_psis.matrix(as.matrix(log_ratios), log_p, log_g, cores = 1) } diff --git a/R/psislw.R b/R/psislw.R index 808e657f..9fac0e9d 100644 --- a/R/psislw.R +++ b/R/psislw.R @@ -38,7 +38,10 @@ psislw <- function(lw, wcp = 0.2, wtrunc = 3/4, cores = getOption("mc.cores", 1), llfun = NULL, llargs = NULL, ...) { - .Deprecated("psis") + .warn( + "'psislw' is deprecated.\nUse 'psis' instead.\nSee help(\"Deprecated\")", + class = c("loo_warning_deprecated", "deprecatedWarning") + ) cores <- loo_cores(cores) @@ -52,16 +55,16 @@ psislw <- function(lw, wcp = 0.2, wtrunc = 3/4, tail_len <- length(x_tail) if (tail_len < MIN_TAIL_LENGTH || all(x_tail == x_tail[1])) { if (all(x_tail == x_tail[1])) - warning( + .warn_diagnostic( "All tail values are the same. ", "Weights are truncated but not smoothed.", - call. = FALSE + class = "loo_warning_pareto_tail_constant" ) else if (tail_len < MIN_TAIL_LENGTH) - warning( + .warn_diagnostic( "Too few tail samples to fit generalized Pareto distribution.\n", "Weights are truncated but not smoothed.", - call. = FALSE + class = "loo_warning_pareto_tail_not_fitted" ) x_new <- x @@ -184,14 +187,16 @@ lw_normalize <- function(y) { # warnings about pareto k values ------------------------------------------ psislw_warnings <- function(k) { if (any(k > 0.7)) { - .warn( + .warn_diagnostic( "Some Pareto k diagnostic values are too high. ", - .k_help() + .k_help(), + class = "loo_warning_pareto_k_high" ) } else if (any(k > 0.5)) { - .warn( + .warn_diagnostic( "Some Pareto k diagnostic values are slightly high. ", - .k_help() + .k_help(), + class = "loo_warning_pareto_k_high" ) } } diff --git a/R/waic.R b/R/waic.R index 507d8b7b..aa55d99d 100644 --- a/R/waic.R +++ b/R/waic.R @@ -209,7 +209,11 @@ throw_pwaic_warnings <- function(p, digits = 1, warn = TRUE) { msg <- paste0("\n", count, " (", .fr(100 * prop, digits), "%) p_waic estimates greater than 0.4. ", "We recommend trying loo instead.") - if (warn) .warn(msg) else cat(msg, "\n") + if (warn) { + .warn_diagnostic(msg, class = "loo_warning_p_waic_high") + } else { + cat(msg, "\n") + } } invisible(NULL) } diff --git a/man/loo-package.Rd b/man/loo-package.Rd index aa158231..f3648046 100644 --- a/man/loo-package.Rd +++ b/man/loo-package.Rd @@ -40,6 +40,51 @@ We recommend PSIS-LOO-CV instead of WAIC, because PSIS provides useful diagnostics and effective sample size and Monte Carlo standard error estimates. } +\section{Warning classes}{ + + +Every warning signaled by \pkg{loo} has the class \code{loo_warning} and one +class for its cause. Warnings that question the reliability of a result +also have the class \code{loo_warning_diagnostic}. Use these classes to handle +or muffle a specific warning, for example with \code{\link[=withCallingHandlers]{withCallingHandlers()}} +and \code{invokeRestart("muffleWarning")}. + +Diagnostic warnings: +\itemize{ +\item \code{loo_warning_pareto_k_high}: Some Pareto k values are too high. +\item \code{loo_warning_pareto_tail_not_fitted}: There are not enough tail samples to +fit the generalized Pareto distribution. +\item \code{loo_warning_pareto_tail_constant}: All tail values are the same. +\item \code{loo_warning_p_waic_high}: Some \code{p_waic} estimates are greater than 0.4. +\item \code{loo_warning_moment_match_failed}: Moment matching failed for some +observations. +\item \code{loo_warning_moment_match_max_iters}: Moment matching reached +\code{max_iters}. +\item \code{loo_warning_large_kf}: The accuracy of self-normalized importance +sampling may be bad. +\item \code{loo_warning_y_mismatch}: The models do not have the same \code{y} variable. +\item \code{loo_warning_cv_mismatch}: The models use different types of +cross-validation or different numbers of folds. +\item \code{loo_warning_chance_difference}: The difference in performance is +potentially due to chance. +\item \code{loo_warning_subsample_naive_se}: The difference SE in \code{loo_compare()} +uses the naive estimator. +\item \code{loo_warning_subsample_negative_se}: The subsampling SE estimate is +negative. +} + +Other warnings: +\itemize{ +\item \code{loo_warning_multicore_unavailable}: Multicore is not implemented for +the selected method. +\item \code{loo_warning_pareto_k_not_plotted}: Some Pareto k values are \code{Inf}, \code{NA}, +or \code{NaN} and are not plotted. +\item \code{loo_warning_llfun_not_matrix}: The values from \code{llfun} are not a matrix +and were coerced. +\item \code{loo_warning_deprecated}: A deprecated feature was used. +} +} + \references{ Vehtari, A., Gelman, A., and Gabry, J. (2017). Practical Bayesian model evaluation using leave-one-out cross-validation and WAIC. diff --git a/tests/testthat/_snaps/psislw.md b/tests/testthat/_snaps/psislw.md index 94daa306..44b3425d 100644 --- a/tests/testthat/_snaps/psislw.md +++ b/tests/testthat/_snaps/psislw.md @@ -3,7 +3,7 @@ Code psis <- psislw(x[, 1], wcp = 0.01) Condition - Warning in `psislw()`: + Warning: 'psislw' is deprecated. Use 'psis' instead. See help("Deprecated") diff --git a/tests/testthat/test_warning_classes.R b/tests/testthat/test_warning_classes.R new file mode 100644 index 00000000..9b6258c0 --- /dev/null +++ b/tests/testthat/test_warning_classes.R @@ -0,0 +1,33 @@ +test_that(".warn() signals one message with the loo classes", { + w <- capture_warning(.warn("a", c("b", "c"), 1, class = "loo_warning_x")) + expect_equal(conditionMessage(w), "abc1") + expect_equal(class(w)[1:2], c("loo_warning_x", "loo_warning")) +}) + +test_that(".warn_diagnostic() adds the diagnostic class", { + w <- capture_warning(.warn_diagnostic("a", class = "loo_warning_x")) + expect_equal( + class(w)[1:3], + c("loo_warning_x", "loo_warning_diagnostic", "loo_warning") + ) +}) + +test_that("warning helpers signal their classes", { + expect_warning(throw_pareto_warnings(1, 0.7), class = "loo_warning_pareto_k_high") + expect_warning(throw_tail_length_warnings(1), class = "loo_warning_pareto_tail_not_fitted") + expect_warning(throw_pwaic_warnings(1), class = "loo_warning_p_waic_high") + expect_warning(throw_moment_match_max_iters_warning(), class = "loo_warning_moment_match_max_iters") + expect_warning(throw_moment_match_error_warning(1, 2, "e"), class = "loo_warning_moment_match_failed") + expect_warning(throw_large_kf_warning(1, 0.7), class = "loo_warning_large_kf") +}) + +test_that("user-facing functions signal loo warning classes", { + LLarr <- example_loglik_array() + w1 <- suppressWarnings(waic(LLarr)) + w2 <- w1 + attr(w2, "yhash") <- "b" + expect_warning(waic(LLarr), class = "loo_warning_p_waic_high") + expect_warning(loo_compare(w1, w2), class = "loo_warning_y_mismatch") + expect_warning(compare(w1, w1), class = "deprecatedWarning") + expect_warning(compare(w1, w1), class = "loo_warning_deprecated") +})