From a5c8750fd796a6b8d4f4c64e0fc68a963639a472 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 3 Jan 2024 15:35:50 +0100 Subject: [PATCH 1/4] Replace instances of "coverage" with either "interval coverage" or "quantile coverage" --- R/add_coverage.R | 16 ++++-- R/metrics-quantile.R | 82 ++++++++++++++------------- R/metrics-sample.R | 9 +-- R/pit.R | 8 +-- man/add_coverage.Rd | 16 ++++-- man/interval_coverage.Rd | 11 ++-- man/interval_coverage_dev_quantile.Rd | 51 +++++++---------- 7 files changed, 99 insertions(+), 94 deletions(-) diff --git a/R/add_coverage.R b/R/add_coverage.R index e7e251614..398d06564 100644 --- a/R/add_coverage.R +++ b/R/add_coverage.R @@ -7,7 +7,7 @@ #' #' **Interval coverage** #' -#' Coverage for a given interval range is defined as the proportion of +#' Interval coverage for a given interval range is defined as the proportion of #' observations that fall within the corresponding central prediction intervals. #' Central prediction intervals are symmetric around the median and and formed #' by two quantiles that denote the lower and upper bound. For example, the 50% @@ -15,10 +15,11 @@ #' quantiles of the predictive distribution. #' #' The function `add_coverage()` computes the coverage per central prediction -#' interval, so the coverage will always be either `TRUE` (observed value falls -#' within the interval) or `FALSE` (observed value falls outside the interval). -#' You can summarise the coverage values to get the proportion of observations -#' that fall within the central prediction intervals. +#' interval, so the interval coverage will always be either `TRUE` +#' (observed value falls within the interval) or `FALSE` (observed value falls +#' outside the interval). You can summarise the interval coverage values to get +#' the proportion of observations that fall within the central prediction +#' intervals. #' #' **Quantile coverage** #' @@ -26,10 +27,13 @@ #' observed values that are smaller than the corresponding predictive quantile. #' For example, the 0.5 quantile coverage is the proportion of observed values #' that are smaller than the 0.5 quantile of the predictive distribution. +#' Just as above, for a single observation and the quantile of a single +#' predictive distribution, the value will either be `TRUE` or `FALSE`. #' #' **Coverage deviation** #' -#' The coverage deviation is the difference between the desired coverage and the +#' The coverage deviation is the difference between the desired coverage +#' (can be either interval or quantile coverage) and the #' actual coverage. For example, if the desired coverage is 90% and the actual #' coverage is 80%, the coverage deviation is -0.1. #' diff --git a/R/metrics-quantile.R b/R/metrics-quantile.R index 14def7b18..2f2321576 100644 --- a/R/metrics-quantile.R +++ b/R/metrics-quantile.R @@ -217,10 +217,11 @@ underprediction <- function(observed, predicted, quantile, ...) { #' @inheritParams wis #' @param range A single number with the range of the prediction interval in #' percent (e.g. 50 for a 50% prediction interval) for which you want to compute -#' coverage. +#' interval coverage. #' @importFrom checkmate assert_number -#' @return A vector of length n with TRUE if the observed value is within the -#' corresponding prediction interval and FALSE otherwise. +#' @return A vector of length n with elements either TRUE, +#' if the observed value is within the corresponding prediction interval, and +#' FALSE otherwise. #' @name interval_coverage #' @export #' @keywords metric @@ -239,16 +240,16 @@ interval_coverage_quantile <- function(observed, predicted, quantile, range = 50 necessary_quantiles <- c((100 - range) / 2, 100 - (100 - range) / 2) / 100 if (!all(necessary_quantiles %in% quantile)) { warning( - "To compute the coverage for a range of ", range, "%, the quantiles ", - necessary_quantiles, " are required. Returning `NA`." + "To compute the interval coverage for a range of ", range, + "%, the quantiles ", necessary_quantiles, " are required. Returning `NA`." ) return(NA) } r <- range reformatted <- quantile_to_interval(observed, predicted, quantile) reformatted <- reformatted[range %in% r] - reformatted[, coverage := (observed >= lower) & (observed <= upper)] - return(reformatted$coverage) + reformatted[, interval_coverage := (observed >= lower) & (observed <= upper)] + return(reformatted$interval_coverage) } @@ -257,9 +258,9 @@ interval_coverage_quantile <- function(observed, predicted, quantile, range = 50 #' of a forecast. #' #' The function is similar to [interval_coverage_quantile()], -#' but looks at all provided prediction intervals instead of only one. It -#' compares nominal coverage (i.e. the desired coverage) with the actual -#' observed coverage. +#' but takes all provided prediction intervals into account and +#' compares nominal interval coverage (i.e. the desired interval coverage) with +#' the actual observed interval coverage. #' #' A central symmetric prediction interval is defined by a lower and an #' upper bound formed by a pair of predictive quantiles. For example, a 50% @@ -269,43 +270,44 @@ interval_coverage_quantile <- function(observed, predicted, quantile, range = 50 #' observed values with their 90% prediction intervals, and so on. #' #' For every prediction interval, the deviation is computed as the difference -#' between the observed coverage and the nominal coverage -#' For a single observed value and a single prediction interval, -#' coverage is always either 0 or 1. This is not the case for a single observed -#' value and multiple prediction intervals, but it still doesn't make that much +#' between the observed interval coverage and the nominal interval coverage +#' For a single observed value and a single prediction interval, coverage is +#' always either 0 or 1 (`FALSE` or `TRUE`). This is not the case for a single +#' observed value and multiple prediction intervals, +#' but it still doesn't make that much #' sense to compare nominal (desired) coverage and actual coverage for a single #' observation. In that sense coverage deviation only really starts to make #' sense as a metric when averaged across multiple observations). #' -#' Positive values of coverage deviation are an indication for underconfidence, -#' i.e. the forecaster could likely have issued a narrower forecast. Negative -#' values are an indication for overconfidence, i.e. the forecasts were too -#' narrow. +#' Positive values of interval coverage deviation are an indication for +#' underconfidence, i.e. the forecaster could likely have issued a narrower +#' forecast. Negative values are an indication for overconfidence, i.e. the +#' forecasts were too narrow. #' #' \deqn{ -#' \textrm{coverage deviation} = -#' \mathbf{1}(\textrm{observed value falls within interval} - -#' \textrm{nominal coverage}) +#' \textrm{interval coverage deviation} = +#' \mathbf{1}(\textrm{observed value falls within interval}) - +#' \textrm{nominal interval coverage} #' }{ -#' coverage deviation = -#' 1(observed value falls within interval) - nominal coverage +#' interval coverage deviation = +#' 1(observed value falls within interval) - nominal interval coverage #' } -#' The coverage deviation is then averaged across all prediction intervals. -#' The median is ignored when computing coverage deviation. +#' The interval coverage deviation is then averaged across all prediction +#' intervals. The median is ignored when computing coverage deviation. #' @inheritParams wis -#' @return A numeric vector of length n with the coverage deviation for each -#' forecast (comprising one or multiple prediction intervals). +#' @return A numeric vector of length n with the interval coverage deviation +#' for each forecast (comprising one or multiple prediction intervals). #' @export #' @keywords metric #' @examples -#' observed <- c(1, -15, 22) -#' predicted <- rbind( -#' c(-1, 0, 1, 2, 3), -#' c(-2, 1, 2, 2, 4), -#' c(-2, 0, 3, 3, 4) -#' ) -#' quantile <- c(0.1, 0.25, 0.5, 0.75, 0.9) -#' interval_coverage_dev_quantile(observed, predicted, quantile) +# observed <- c(1, -15, 22) +# predicted <- rbind( +# c(-1, 0, 1, 2, 3), +# c(-2, 1, 2, 2, 4), +# c(-2, 0, 3, 3, 4) +# ) +# quantile <- c(0.1, 0.25, 0.5, 0.75, 0.9) +# interval_coverage_dev_quantile(observed, predicted, quantile) interval_coverage_dev_quantile <- function(observed, predicted, quantile) { assert_input_quantile(observed, predicted, quantile) @@ -319,7 +321,7 @@ interval_coverage_dev_quantile <- function(observed, predicted, quantile) { if (!all(necessary_quantiles %in% quantile)) { missing <- necessary_quantiles[!necessary_quantiles %in% quantile] warning( - "To compute coverage deviation, all quantiles must form central ", + "To compute inteval coverage deviation, all quantiles must form central ", "symmetric prediction intervals. Missing quantiles: ", toString(missing), ". Returning `NA`." ) @@ -327,11 +329,11 @@ interval_coverage_dev_quantile <- function(observed, predicted, quantile) { } reformatted <- quantile_to_interval(observed, predicted, quantile)[range != 0] - reformatted[, coverage := (observed >= lower) & (observed <= upper)] - reformatted[, coverage_deviation := coverage - range / 100] - out <- reformatted[, .(coverage_deviation = mean(coverage_deviation)), + reformatted[, interval_coverage := (observed >= lower) & (observed <= upper)] + reformatted[, interval_coverage_dev := interval_coverage - range / 100] + out <- reformatted[, .(interval_coverage_dev = mean(interval_coverage_dev)), by = "forecast_id"] - return(out$coverage_deviation) + return(out$interval_coverage_dev) } diff --git a/R/metrics-sample.R b/R/metrics-sample.R index 81d62e8fc..335292b18 100644 --- a/R/metrics-sample.R +++ b/R/metrics-sample.R @@ -275,7 +275,6 @@ crps_sample <- function(observed, predicted, ...) { #' predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) #' mad_sample(predicted = predicted) #' @keywords metric - mad_sample <- function(observed = NULL, predicted, ...) { assert_input_sample(rep(NA_real_, nrow(predicted)), predicted) @@ -286,13 +285,15 @@ mad_sample <- function(observed = NULL, predicted, ...) { #' @title Interval Coverage -#' @description To compute coverage for sample-based forecasts, +#' @description To compute interval coverage for sample-based forecasts, #' predictive samples are converted first into predictive quantiles using the #' sample quantiles. #' @importFrom checkmate assert_number #' @rdname interval_coverage #' @export #' @examples +#' observed <- rpois(30, lambda = 1:30) +#' predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) #' interval_coverage_sample(observed, predicted) interval_coverage_sample <- function(observed, predicted, range = 50) { assert_input_sample(observed, predicted) @@ -313,7 +314,7 @@ interval_coverage_sample <- function(observed, predicted, range = 50) { # this could call interval_coverage_quantile instead # ========================================================== interval_dt <- quantile_to_interval(quantile_dt, format = "wide") - interval_dt[, coverage := (observed >= lower) & (observed <= upper)] + interval_dt[, interval_coverage := (observed >= lower) & (observed <= upper)] # ========================================================== - return(interval_dt$coverage) + return(interval_dt$interval_coverage) } diff --git a/R/pit.R b/R/pit.R index fd8f4819d..e1783ff34 100644 --- a/R/pit.R +++ b/R/pit.R @@ -191,16 +191,16 @@ pit <- function(data, if (forecast_type == "quantile") { data[, quantile_coverage := (observed <= predicted)] - coverage <- data[, .(quantile_coverage = mean(quantile_coverage)), + quantile_coverage <- data[, .(quantile_coverage = mean(quantile_coverage)), by = c(unique(c(by, "quantile")))] - coverage <- coverage[order(quantile), + quantile_coverage <- quantile_coverage[order(quantile), .( quantile = c(quantile, 1), pit_value = diff(c(0, quantile_coverage, 1)) ), - by = c(get_forecast_unit(coverage)) + by = c(get_forecast_unit(quantile_coverage)) ] - return(coverage[]) + return(quantile_coverage[]) } # if prediction type is not quantile, calculate PIT values based on samples diff --git a/man/add_coverage.Rd b/man/add_coverage.Rd index d6c82d467..addd1f72d 100644 --- a/man/add_coverage.Rd +++ b/man/add_coverage.Rd @@ -22,7 +22,7 @@ quantile format (following the input requirements of \code{score()}). \strong{Interval coverage} -Coverage for a given interval range is defined as the proportion of +Interval coverage for a given interval range is defined as the proportion of observations that fall within the corresponding central prediction intervals. Central prediction intervals are symmetric around the median and and formed by two quantiles that denote the lower and upper bound. For example, the 50\% @@ -30,10 +30,11 @@ central prediction interval is the interval between the 0.25 and 0.75 quantiles of the predictive distribution. The function \code{add_coverage()} computes the coverage per central prediction -interval, so the coverage will always be either \code{TRUE} (observed value falls -within the interval) or \code{FALSE} (observed value falls outside the interval). -You can summarise the coverage values to get the proportion of observations -that fall within the central prediction intervals. +interval, so the interval coverage will always be either \code{TRUE} +(observed value falls within the interval) or \code{FALSE} (observed value falls +outside the interval). You can summarise the interval coverage values to get +the proportion of observations that fall within the central prediction +intervals. \strong{Quantile coverage} @@ -41,10 +42,13 @@ Quantile coverage for a given quantile is defined as the proportion of observed values that are smaller than the corresponding predictive quantile. For example, the 0.5 quantile coverage is the proportion of observed values that are smaller than the 0.5 quantile of the predictive distribution. +Just as above, for a single observation and the quantile of a single +predictive distribution, the value will either be \code{TRUE} or \code{FALSE}. \strong{Coverage deviation} -The coverage deviation is the difference between the desired coverage and the +The coverage deviation is the difference between the desired coverage +(can be either interval or quantile coverage) and the actual coverage. For example, if the desired coverage is 90\% and the actual coverage is 80\%, the coverage deviation is -0.1. } diff --git a/man/interval_coverage.Rd b/man/interval_coverage.Rd index 3da7973cc..75fa70218 100644 --- a/man/interval_coverage.Rd +++ b/man/interval_coverage.Rd @@ -24,11 +24,12 @@ vector of size N.} \item{range}{A single number with the range of the prediction interval in percent (e.g. 50 for a 50\% prediction interval) for which you want to compute -coverage.} +interval coverage.} } \value{ -A vector of length n with TRUE if the observed value is within the -corresponding prediction interval and FALSE otherwise. +A vector of length n with elements either TRUE, +if the observed value is within the corresponding prediction interval, and +FALSE otherwise. } \description{ Check whether the observed value is within a given central @@ -37,7 +38,7 @@ upper bound formed by a pair of predictive quantiles. For example, a 50\% prediction interval is formed by the 0.25 and 0.75 quantiles of the predictive distribution. -To compute coverage for sample-based forecasts, +To compute interval coverage for sample-based forecasts, predictive samples are converted first into predictive quantiles using the sample quantiles. } @@ -50,6 +51,8 @@ predicted <- rbind( ) quantile <- c(0.1, 0.25, 0.5, 0.75, 0.9) interval_coverage_quantile(observed, predicted, quantile) +observed <- rpois(30, lambda = 1:30) +predicted <- replicate(200, rpois(n = 30, lambda = 1:30)) interval_coverage_sample(observed, predicted) } \keyword{metric} diff --git a/man/interval_coverage_dev_quantile.Rd b/man/interval_coverage_dev_quantile.Rd index 181c45c6b..3d54a8514 100644 --- a/man/interval_coverage_dev_quantile.Rd +++ b/man/interval_coverage_dev_quantile.Rd @@ -19,17 +19,17 @@ vector of size N.} \item{quantile}{vector with quantile levels of size N} } \value{ -A numeric vector of length n with the coverage deviation for each -forecast (comprising one or multiple prediction intervals). +A numeric vector of length n with the interval coverage deviation +for each forecast (comprising one or multiple prediction intervals). } \description{ Check the agreement between desired and actual interval coverage of a forecast. The function is similar to \code{\link[=interval_coverage_quantile]{interval_coverage_quantile()}}, -but looks at all provided prediction intervals instead of only one. It -compares nominal coverage (i.e. the desired coverage) with the actual -observed coverage. +but takes all provided prediction intervals into account and +compares nominal interval coverage (i.e. the desired interval coverage) with +the actual observed interval coverage. A central symmetric prediction interval is defined by a lower and an upper bound formed by a pair of predictive quantiles. For example, a 50\% @@ -39,38 +39,29 @@ predictive distribution. Ideally, a forecaster should aim to cover about observed values with their 90\% prediction intervals, and so on. For every prediction interval, the deviation is computed as the difference -between the observed coverage and the nominal coverage -For a single observed value and a single prediction interval, -coverage is always either 0 or 1. This is not the case for a single observed -value and multiple prediction intervals, but it still doesn't make that much +between the observed interval coverage and the nominal interval coverage +For a single observed value and a single prediction interval, coverage is +always either 0 or 1 (\code{FALSE} or \code{TRUE}). This is not the case for a single +observed value and multiple prediction intervals, +but it still doesn't make that much sense to compare nominal (desired) coverage and actual coverage for a single observation. In that sense coverage deviation only really starts to make sense as a metric when averaged across multiple observations). -Positive values of coverage deviation are an indication for underconfidence, -i.e. the forecaster could likely have issued a narrower forecast. Negative -values are an indication for overconfidence, i.e. the forecasts were too -narrow. +Positive values of interval coverage deviation are an indication for +underconfidence, i.e. the forecaster could likely have issued a narrower +forecast. Negative values are an indication for overconfidence, i.e. the +forecasts were too narrow. \deqn{ -\textrm{coverage deviation} = -\mathbf{1}(\textrm{observed value falls within interval} - -\textrm{nominal coverage}) +\textrm{interval coverage deviation} = +\mathbf{1}(\textrm{observed value falls within interval}) - +\textrm{nominal interval coverage} }{ -coverage deviation = -1(observed value falls within interval) - nominal coverage +interval coverage deviation = +1(observed value falls within interval) - nominal interval coverage } -The coverage deviation is then averaged across all prediction intervals. -The median is ignored when computing coverage deviation. -} -\examples{ -observed <- c(1, -15, 22) -predicted <- rbind( - c(-1, 0, 1, 2, 3), - c(-2, 1, 2, 2, 4), - c(-2, 0, 3, 3, 4) -) -quantile <- c(0.1, 0.25, 0.5, 0.75, 0.9) -interval_coverage_dev_quantile(observed, predicted, quantile) +The interval coverage deviation is then averaged across all prediction +intervals. The median is ignored when computing coverage deviation. } \keyword{metric} From 8ddf667b74e5117b9b0f014b4327ed4f6c53d31c Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 3 Jan 2024 15:40:13 +0100 Subject: [PATCH 2/4] Fix linting issues --- R/metrics-quantile.R | 23 ++++++++++++----------- R/pit.R | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/R/metrics-quantile.R b/R/metrics-quantile.R index 2f2321576..ce14a37b5 100644 --- a/R/metrics-quantile.R +++ b/R/metrics-quantile.R @@ -300,14 +300,14 @@ interval_coverage_quantile <- function(observed, predicted, quantile, range = 50 #' @export #' @keywords metric #' @examples -# observed <- c(1, -15, 22) -# predicted <- rbind( -# c(-1, 0, 1, 2, 3), -# c(-2, 1, 2, 2, 4), -# c(-2, 0, 3, 3, 4) -# ) -# quantile <- c(0.1, 0.25, 0.5, 0.75, 0.9) -# interval_coverage_dev_quantile(observed, predicted, quantile) +#' observed <- c(1, -15, 22) +#' predicted <- rbind( +#' c(-1, 0, 1, 2, 3), +#' c(-2, 1, 2, 2, 4), +#' c(-2, 0, 3, 3, 4) +#' ) +#' quantile <- c(0.1, 0.25, 0.5, 0.75, 0.9) +#' interval_coverage_dev_quantile(observed, predicted, quantile) interval_coverage_dev_quantile <- function(observed, predicted, quantile) { assert_input_quantile(observed, predicted, quantile) @@ -330,9 +330,10 @@ interval_coverage_dev_quantile <- function(observed, predicted, quantile) { reformatted <- quantile_to_interval(observed, predicted, quantile)[range != 0] reformatted[, interval_coverage := (observed >= lower) & (observed <= upper)] - reformatted[, interval_coverage_dev := interval_coverage - range / 100] - out <- reformatted[, .(interval_coverage_dev = mean(interval_coverage_dev)), - by = "forecast_id"] + reformatted[, interval_coverage_deviation := interval_coverage - range / 100] + out <- reformatted[, .( + interval_coverage_deviation = mean(interval_coverage_deviation) + ), by = "forecast_id"] return(out$interval_coverage_dev) } diff --git a/R/pit.R b/R/pit.R index e1783ff34..9d2793a2b 100644 --- a/R/pit.R +++ b/R/pit.R @@ -192,7 +192,7 @@ pit <- function(data, if (forecast_type == "quantile") { data[, quantile_coverage := (observed <= predicted)] quantile_coverage <- data[, .(quantile_coverage = mean(quantile_coverage)), - by = c(unique(c(by, "quantile")))] + by = c(unique(c(by, "quantile")))] quantile_coverage <- quantile_coverage[order(quantile), .( quantile = c(quantile, 1), From 654ce20d37638086e928c1468783465750f35477 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 3 Jan 2024 22:09:32 +0100 Subject: [PATCH 3/4] rename coverage to interval_coverage or quantile_coverage --- R/default-scoring-rules.R | 12 ++++++------ R/plot.R | 2 +- R/utils.R | 3 ++- man/interval_coverage_dev_quantile.Rd | 10 ++++++++++ man/rules_quantile.Rd | 6 +++--- 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/R/default-scoring-rules.R b/R/default-scoring-rules.R index 9dd6fd24d..ac02137e1 100644 --- a/R/default-scoring-rules.R +++ b/R/default-scoring-rules.R @@ -130,11 +130,11 @@ rules_sample <- function(select = NULL, exclude = NULL) { #' - "underprediction" = [underprediction()] #' - "dispersion" = [dispersion()] #' - "bias" = [bias_quantile()] -#' - "coverage_50" = [interval_coverage_quantile()] -#' - "coverage_90" = function(...) \{ +#' - "interval_coverage_50" = [interval_coverage_quantile()] +#' - "interval_coverage_90" = function(...) \{ #' run_safely(..., range = 90, fun = [interval_coverage_quantile]) #' \} -#' - "coverage_deviation" = [interval_coverage_dev_quantile()], +#' - "interval_coverage_deviation" = [interval_coverage_dev_quantile()], #' - "ae_median" = [ae_median_quantile()] #' #' Note: The `coverage_90` scoring rule is created as a wrapper around @@ -157,11 +157,11 @@ rules_quantile <- function(select = NULL, exclude = NULL) { underprediction = underprediction, dispersion = dispersion, bias = bias_quantile, - coverage_50 = interval_coverage_quantile, - coverage_90 = function(...) { + interval_coverage_50 = interval_coverage_quantile, + interval_coverage_90 = function(...) { run_safely(..., range = 90, fun = interval_coverage_quantile) }, - coverage_deviation = interval_coverage_dev_quantile, + interval_coverage_deviation = interval_coverage_dev_quantile, ae_median = ae_median_quantile ) selected <- select_rules(all, select, exclude) diff --git a/R/plot.R b/R/plot.R index 3299a3dfe..d93f36f1e 100644 --- a/R/plot.R +++ b/R/plot.R @@ -57,7 +57,7 @@ plot_score_table <- function(scores, # define which metrics are scaled using min (larger is worse) and # which not (metrics like bias where deviations in both directions are bad) - metrics_zero_good <- c("bias", "coverage_deviation") + metrics_zero_good <- c("bias", "interval_coverage_deviation") metrics_no_color <- "coverage" metrics_min_good <- setdiff(metrics, c( diff --git a/R/utils.R b/R/utils.R index 48876340a..bcf234e0b 100644 --- a/R/utils.R +++ b/R/utils.R @@ -5,7 +5,8 @@ #' @keywords info available_metrics <- function() { return(unique(c(scoringutils::metrics$Name, - "wis", "coverage_50", "coverage_90"))) + "wis", "interval_coverage_50", "interval_coverage_90", + "interval_coverage_deviation"))) } diff --git a/man/interval_coverage_dev_quantile.Rd b/man/interval_coverage_dev_quantile.Rd index 3d54a8514..f6b467523 100644 --- a/man/interval_coverage_dev_quantile.Rd +++ b/man/interval_coverage_dev_quantile.Rd @@ -64,4 +64,14 @@ interval coverage deviation = The interval coverage deviation is then averaged across all prediction intervals. The median is ignored when computing coverage deviation. } +\examples{ +observed <- c(1, -15, 22) +predicted <- rbind( + c(-1, 0, 1, 2, 3), + c(-2, 1, 2, 2, 4), + c(-2, 0, 3, 3, 4) +) +quantile <- c(0.1, 0.25, 0.5, 0.75, 0.9) +interval_coverage_dev_quantile(observed, predicted, quantile) +} \keyword{metric} diff --git a/man/rules_quantile.Rd b/man/rules_quantile.Rd index 78b24342b..dfe9dd891 100644 --- a/man/rules_quantile.Rd +++ b/man/rules_quantile.Rd @@ -27,11 +27,11 @@ The default scoring rules are: \item "underprediction" = \code{\link[=underprediction]{underprediction()}} \item "dispersion" = \code{\link[=dispersion]{dispersion()}} \item "bias" = \code{\link[=bias_quantile]{bias_quantile()}} -\item "coverage_50" = \code{\link[=interval_coverage_quantile]{interval_coverage_quantile()}} -\item "coverage_90" = function(...) \{ +\item "interval_coverage_50" = \code{\link[=interval_coverage_quantile]{interval_coverage_quantile()}} +\item "interval_coverage_90" = function(...) \{ run_safely(..., range = 90, fun = \link{interval_coverage_quantile}) \} -\item "coverage_deviation" = \code{\link[=interval_coverage_dev_quantile]{interval_coverage_dev_quantile()}}, +\item "interval_coverage_deviation" = \code{\link[=interval_coverage_dev_quantile]{interval_coverage_dev_quantile()}}, \item "ae_median" = \code{\link[=ae_median_quantile]{ae_median_quantile()}} } From 47207e85e21d6abd7d8ff3594e96072c3e86d69d Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 3 Jan 2024 22:09:42 +0100 Subject: [PATCH 4/4] Update tests --- .../plot_correlation/plot-correlation.svg | 286 +++++++++--------- .../plot_score_table/plot-score-table.svg | 206 ++++++------- tests/testthat/setup.R | 6 +- 3 files changed, 250 insertions(+), 248 deletions(-) diff --git a/tests/testthat/_snaps/plot_correlation/plot-correlation.svg b/tests/testthat/_snaps/plot_correlation/plot-correlation.svg index a33e8a207..d51d9d2b0 100644 --- a/tests/testthat/_snaps/plot_correlation/plot-correlation.svg +++ b/tests/testthat/_snaps/plot_correlation/plot-correlation.svg @@ -20,152 +20,152 @@ - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1 -0.94 -1 -0.28 --0.03 -1 -0.46 -0.32 -0.15 -1 -0.11 -0.22 --0.35 -0.11 -1 --0.21 --0.15 --0.21 --0.09 -0.01 -1 --0.41 --0.32 --0.36 --0.09 -0.1 -0.37 -1 --0.34 --0.25 --0.33 --0.12 -0.06 -0.85 -0.64 -1 -0.99 -0.9 -0.34 -0.54 -0.1 --0.25 --0.41 --0.38 -1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +1 +0.94 +1 +0.28 +-0.03 +1 +0.46 +0.32 +0.15 +1 +0.11 +0.22 +-0.35 +0.11 +1 +-0.21 +-0.15 +-0.21 +-0.09 +0.01 +1 +-0.41 +-0.32 +-0.36 +-0.09 +0.1 +0.37 +1 +-0.34 +-0.25 +-0.33 +-0.12 +0.06 +0.85 +0.64 +1 +0.99 +0.9 +0.34 +0.54 +0.1 +-0.25 +-0.41 +-0.38 +1 - -wis -overprediction -underprediction -dispersion -bias -coverage_50 -coverage_90 -coverage_deviation -ae_median - - - - - - - - - - - - - - - - - - - -ae_median -coverage_deviation -coverage_90 -coverage_50 -bias -dispersion -underprediction -overprediction -wis - -0.0 -0.5 -1.0 -Correlation - - - - - - -plot__correlation + +wis +overprediction +underprediction +dispersion +bias +interval_coverage_50 +interval_coverage_90 +interval_coverage_deviation +ae_median + + + + + + + + + + + + + + + + + + + +ae_median +interval_coverage_deviation +interval_coverage_90 +interval_coverage_50 +bias +dispersion +underprediction +overprediction +wis + +0.0 +0.5 +1.0 +Correlation + + + + + + +plot__correlation diff --git a/tests/testthat/_snaps/plot_score_table/plot-score-table.svg b/tests/testthat/_snaps/plot_score_table/plot-score-table.svg index 25b15d296..ead05c2a1 100644 --- a/tests/testthat/_snaps/plot_score_table/plot-score-table.svg +++ b/tests/testthat/_snaps/plot_score_table/plot-score-table.svg @@ -20,113 +20,113 @@ - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -9000 -10000 -10000 -50 -5000 -7000 -6000 -9 -2000 -5000 -2000 -20 -2000 -2000 -3000 -30 -0.008 -0.2 --0.04 --0.02 -0.6 -0.5 -0.4 -0.5 -0.9 -0.9 -0.8 -0.9 -0.05 -0.002 --0.06 --0.02 -10000 -20000 -10000 -80 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +9000 +10000 +10000 +50 +5000 +7000 +6000 +9 +2000 +5000 +2000 +20 +2000 +2000 +3000 +30 +0.008 +0.2 +-0.04 +-0.02 +0.6 +0.5 +0.4 +0.5 +0.9 +0.9 +0.8 +0.9 +0.05 +0.002 +-0.06 +-0.02 +10000 +20000 +10000 +80 - -EuroCOVIDhub-baseline -EuroCOVIDhub-ensemble -UMass-MechBayes -epiforecasts-EpiNow2 - - - - - - - - - - - - - - -wis -overprediction -underprediction -dispersion -bias -coverage_50 -coverage_90 -coverage_deviation -ae_median + +EuroCOVIDhub-baseline +EuroCOVIDhub-ensemble +UMass-MechBayes +epiforecasts-EpiNow2 + + + + + + + + + + + + + + +wis +overprediction +underprediction +dispersion +bias +interval_coverage_50 +interval_coverage_90 +interval_coverage_deviation +ae_median plot_score_table diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index 362f52adb..4fadd0921 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -4,10 +4,12 @@ library(data.table) suppressMessages(library(magrittr)) metrics_no_cov <- rules_quantile( - exclude = c("coverage_50", "coverage_90", "coverage_deviation") + exclude = c("interval_coverage_50", "interval_coverage_90", + "interval_coverage_deviation") ) metrics_no_cov_no_ae <- rules_quantile( - exclude = c("coverage_50", "coverage_90", "coverage_deviation", "ae_median") + exclude = c("interval_coverage_50", "interval_coverage_90", + "interval_coverage_deviation", "ae_median") )