Issue #436 Rename instances of "coverage" to either "interval coverage" or "quantile coverage"

R/add_coverage.R

@@ -7,29 +7,33 @@
 #'
 #' **Interval coverage**
 #'
-#' Coverage for a given interval range is defined as the proportion of
+#' Interval coverage for a given interval range is defined as the proportion of
 #' observations that fall within the corresponding central prediction intervals.
 #' Central prediction intervals are symmetric around the median and and formed
 #' by two quantiles that denote the lower and upper bound. For example, the 50%
 #' central prediction interval is the interval between the 0.25 and 0.75
 #' quantiles of the predictive distribution.
 #'
 #' The function `add_coverage()` computes the coverage per central prediction
-#' interval, so the coverage will always be either `TRUE` (observed value falls
-#' within the interval) or `FALSE` (observed value falls outside the interval).
-#' You can summarise the coverage values to get the proportion of observations
-#' that fall within the central prediction intervals.
+#' interval, so the interval coverage will always be either `TRUE`
+#' (observed value falls within the interval) or `FALSE`  (observed value falls
+#' outside the interval). You can summarise the interval coverage values to get
+#' the proportion of observations that fall within the central prediction
+#' intervals.
 #'
 #' **Quantile coverage**
 #'
 #' Quantile coverage for a given quantile is defined as the proportion of
 #' observed values that are smaller than the corresponding predictive quantile.
 #' For example, the 0.5 quantile coverage is the proportion of observed values
 #' that are smaller than the 0.5 quantile of the predictive distribution.
+#' Just as above, for a single observation and the quantile of a single
+#' predictive distribution, the value will either be `TRUE` or `FALSE`.
 #'
 #' **Coverage deviation**
 #'
-#' The coverage deviation is the difference between the desired coverage and the
+#' The coverage deviation is the difference between the desired coverage
+#' (can be either interval or quantile coverage) and the
 #' actual coverage. For example, if the desired coverage is 90% and the actual
 #' coverage is 80%, the coverage deviation is -0.1.
 #'

R/default-scoring-rules.R

@@ -130,11 +130,11 @@ rules_sample <- function(select = NULL, exclude = NULL) {
 #' - "underprediction" = [underprediction()]
 #' - "dispersion" = [dispersion()]
 #' - "bias" = [bias_quantile()]
-#' - "coverage_50" = [interval_coverage_quantile()]
-#' - "coverage_90" = function(...) \{
+#' - "interval_coverage_50" = [interval_coverage_quantile()]
+#' - "interval_coverage_90" = function(...) \{
 #'      run_safely(..., range = 90, fun = [interval_coverage_quantile])
 #'   \}
-#' - "coverage_deviation" = [interval_coverage_dev_quantile()],
+#' - "interval_coverage_deviation" = [interval_coverage_dev_quantile()],
 #' - "ae_median" = [ae_median_quantile()]
 #'
 #' Note: The `coverage_90` scoring rule is created as a wrapper around
@@ -157,11 +157,11 @@ rules_quantile <- function(select = NULL, exclude = NULL) {
     underprediction = underprediction,
     dispersion = dispersion,
     bias = bias_quantile,
-    coverage_50 = interval_coverage_quantile,
-    coverage_90 = function(...) {
+    interval_coverage_50 = interval_coverage_quantile,
+    interval_coverage_90 = function(...) {
       run_safely(..., range = 90, fun = interval_coverage_quantile)
     },
-    coverage_deviation = interval_coverage_dev_quantile,
+    interval_coverage_deviation = interval_coverage_dev_quantile,
     ae_median = ae_median_quantile
   )
   selected <- select_rules(all, select, exclude)

R/metrics-quantile.R

@@ -217,10 +217,11 @@ underprediction <- function(observed, predicted, quantile, ...) {
 #' @inheritParams wis
 #' @param range A single number with the range of the prediction interval in
 #' percent (e.g. 50 for a 50% prediction interval) for which you want to compute
-#' coverage.
+#' interval coverage.
 #' @importFrom checkmate assert_number
-#' @return A vector of length n with TRUE if the observed value is within the
-#' corresponding prediction interval and FALSE otherwise.
+#' @return A vector of length n with elements either TRUE,
+#' if the observed value is within the corresponding prediction interval, and
+#' FALSE otherwise.
 #' @name interval_coverage
 #' @export
 #' @keywords metric
@@ -239,16 +240,16 @@ interval_coverage_quantile <- function(observed, predicted, quantile, range = 50
   necessary_quantiles <- c((100 - range) / 2, 100 - (100 - range) / 2) / 100
   if (!all(necessary_quantiles %in% quantile)) {
     warning(
-      "To compute the coverage for a range of ", range, "%, the quantiles ",
-      necessary_quantiles, " are required. Returning `NA`."
+      "To compute the interval coverage for a range of ", range,
+      "%, the quantiles ", necessary_quantiles, " are required. Returning `NA`."
     )
     return(NA)
   }
   r <- range
   reformatted <- quantile_to_interval(observed, predicted, quantile)
   reformatted <- reformatted[range %in% r]
-  reformatted[, coverage := (observed >= lower) & (observed <= upper)]
-  return(reformatted$coverage)
+  reformatted[, interval_coverage := (observed >= lower) & (observed <= upper)]
+  return(reformatted$interval_coverage)
 }
 
 
@@ -257,9 +258,9 @@ interval_coverage_quantile <- function(observed, predicted, quantile, range = 50
 #' of a forecast.
 #'
 #' The function is similar to [interval_coverage_quantile()],
-#' but looks at all provided prediction intervals instead of only one. It
-#' compares nominal coverage (i.e. the desired coverage) with the actual
-#' observed coverage.
+#' but takes all provided prediction intervals into account and
+#' compares nominal interval coverage (i.e. the desired interval coverage) with
+#' the actual observed interval coverage.
 #'
 #' A central symmetric prediction interval is defined by a lower and an
 #' upper bound formed by a pair of predictive quantiles. For example, a 50%
@@ -269,40 +270,41 @@ interval_coverage_quantile <- function(observed, predicted, quantile, range = 50
 #' observed values with their 90% prediction intervals, and so on.
 #'
 #' For every prediction interval, the deviation is computed as the difference
-#' between the observed coverage and the nominal coverage
-#' For a single observed value and a single prediction interval,
-#' coverage is always either 0 or 1. This is not the case for a single observed
-#' value and multiple prediction intervals, but it still doesn't make that much
+#' between the observed interval coverage and the nominal interval coverage
+#' For a single observed value and a single prediction interval, coverage is
+#' always either 0 or 1 (`FALSE` or `TRUE`). This is not the case for a single
+#' observed value and multiple prediction intervals,
+#' but it still doesn't make that much
 #' sense to compare nominal (desired) coverage and actual coverage for a single
 #' observation. In that sense coverage deviation only really starts to make
 #' sense as a metric when averaged across multiple observations).
 #'
-#' Positive values of coverage deviation are an indication for underconfidence,
-#' i.e. the forecaster could likely have issued a narrower forecast. Negative
-#' values are an indication for overconfidence, i.e. the forecasts were too
-#' narrow.
+#' Positive values of interval coverage deviation are an indication for
+#' underconfidence, i.e. the forecaster could likely have issued a narrower
+#' forecast. Negative values are an indication for overconfidence, i.e. the
+#' forecasts were too narrow.
 #'
 #' \deqn{
-#' \textrm{coverage deviation} =
-#' \mathbf{1}(\textrm{observed value falls within interval} -
-#' \textrm{nominal coverage})
+#' \textrm{interval coverage deviation} =
+#' \mathbf{1}(\textrm{observed value falls within interval}) -
+#' \textrm{nominal interval coverage}
 #' }{
-#' coverage deviation =
-#' 1(observed value falls within interval) - nominal coverage
+#' interval coverage deviation =
+#' 1(observed value falls within interval) - nominal interval coverage
 #' }
-#' The coverage deviation is then averaged across all prediction intervals.
-#' The median is ignored when computing coverage deviation.
+#' The interval coverage deviation is then averaged across all prediction
+#' intervals. The median is ignored when computing coverage deviation.
 #' @inheritParams wis
-#' @return A numeric vector of length n with the coverage deviation for each
-#' forecast (comprising one or multiple prediction intervals).
+#' @return A numeric vector of length n with the interval coverage deviation
+#' for each forecast (comprising one or multiple prediction intervals).
 #' @export
 #' @keywords metric
 #' @examples
 #' observed <- c(1, -15, 22)
 #' predicted <- rbind(
 #'   c(-1, 0, 1, 2, 3),
 #'   c(-2, 1, 2, 2, 4),
-#'    c(-2, 0, 3, 3, 4)
+#'   c(-2, 0, 3, 3, 4)
 #' )
 #' quantile <- c(0.1, 0.25, 0.5, 0.75, 0.9)
 #' interval_coverage_dev_quantile(observed, predicted, quantile)
@@ -319,19 +321,20 @@ interval_coverage_dev_quantile <- function(observed, predicted, quantile) {
   if (!all(necessary_quantiles %in% quantile)) {
     missing <- necessary_quantiles[!necessary_quantiles %in% quantile]
     warning(
-      "To compute coverage deviation, all quantiles must form central ",
+      "To compute inteval coverage deviation, all quantiles must form central ",
       "symmetric prediction intervals. Missing quantiles: ",
       toString(missing), ". Returning `NA`."
     )
     return(NA)
   }
 
   reformatted <- quantile_to_interval(observed, predicted, quantile)[range != 0]
-  reformatted[, coverage := (observed >= lower) & (observed <= upper)]
-  reformatted[, coverage_deviation := coverage - range / 100]
-  out <- reformatted[, .(coverage_deviation = mean(coverage_deviation)),
-                     by = "forecast_id"]
-  return(out$coverage_deviation)
+  reformatted[, interval_coverage := (observed >= lower) & (observed <= upper)]
+  reformatted[, interval_coverage_deviation := interval_coverage - range / 100]
+  out <- reformatted[, .(
+    interval_coverage_deviation = mean(interval_coverage_deviation)
+  ), by = "forecast_id"]
+  return(out$interval_coverage_dev)
 }
 
 

R/metrics-sample.R

@@ -275,7 +275,6 @@ crps_sample <- function(observed, predicted, ...) {
 #' predicted <- replicate(200, rpois(n = 30, lambda = 1:30))
 #' mad_sample(predicted = predicted)
 #' @keywords metric
-
 mad_sample <- function(observed = NULL, predicted, ...) {
 
   assert_input_sample(rep(NA_real_, nrow(predicted)), predicted)
@@ -286,13 +285,15 @@ mad_sample <- function(observed = NULL, predicted, ...) {
 
 
 #' @title Interval Coverage
-#' @description To compute coverage for sample-based forecasts,
+#' @description To compute interval coverage for sample-based forecasts,
 #' predictive samples are converted first into predictive quantiles using the
 #' sample quantiles.
 #' @importFrom checkmate assert_number
 #' @rdname interval_coverage
 #' @export
 #' @examples
+#' observed <- rpois(30, lambda = 1:30)
+#' predicted <- replicate(200, rpois(n = 30, lambda = 1:30))
 #' interval_coverage_sample(observed, predicted)
 interval_coverage_sample <- function(observed, predicted, range = 50) {
   assert_input_sample(observed, predicted)
@@ -313,7 +314,7 @@ interval_coverage_sample <- function(observed, predicted, range = 50) {
   # this could call interval_coverage_quantile instead
   # ==========================================================
   interval_dt <- quantile_to_interval(quantile_dt, format = "wide")
-  interval_dt[, coverage := (observed >= lower) & (observed <= upper)]
+  interval_dt[, interval_coverage := (observed >= lower) & (observed <= upper)]
   # ==========================================================
-  return(interval_dt$coverage)
+  return(interval_dt$interval_coverage)
 }

R/pit.R

@@ -191,16 +191,16 @@ pit <- function(data,
 
   if (forecast_type == "quantile") {
     data[, quantile_coverage := (observed <= predicted)]
-    coverage <- data[, .(quantile_coverage = mean(quantile_coverage)),
-                     by = c(unique(c(by, "quantile")))]
-    coverage <- coverage[order(quantile),
+    quantile_coverage <- data[, .(quantile_coverage = mean(quantile_coverage)),
+                              by = c(unique(c(by, "quantile")))]
+    quantile_coverage <- quantile_coverage[order(quantile),
       .(
         quantile = c(quantile, 1),
         pit_value = diff(c(0, quantile_coverage, 1))
       ),
-      by = c(get_forecast_unit(coverage))
+      by = c(get_forecast_unit(quantile_coverage))
     ]
-    return(coverage[])
+    return(quantile_coverage[])
   }
 
   # if prediction type is not quantile, calculate PIT values based on samples

R/plot.R

@@ -57,7 +57,7 @@ plot_score_table <- function(scores,
 
   # define which metrics are scaled using min (larger is worse) and
   # which not (metrics like bias where deviations in both directions are bad)
-  metrics_zero_good <- c("bias", "coverage_deviation")
+  metrics_zero_good <- c("bias", "interval_coverage_deviation")
   metrics_no_color <- "coverage"
 
   metrics_min_good <- setdiff(metrics, c(

R/utils.R

@@ -5,7 +5,8 @@
 #' @keywords info
 available_metrics <- function() {
   return(unique(c(scoringutils::metrics$Name,
-                  "wis", "coverage_50", "coverage_90")))
+                  "wis", "interval_coverage_50", "interval_coverage_90",
+                  "interval_coverage_deviation")))
 }