Issue 555: Rename interval_coverage_ family and remove sample version

NAMESPACE

@@ -30,9 +30,8 @@ export(get_duplicate_forecasts)
 export(get_forecast_counts)
 export(get_forecast_type)
 export(get_forecast_unit)
-export(interval_coverage_dev_quantile)
-export(interval_coverage_quantile)
-export(interval_coverage_sample)
+export(interval_coverage)
+export(interval_coverage_deviation)
 export(log_shift)
 export(logs_binary)
 export(logs_sample)

NEWS.md

@@ -38,7 +38,8 @@ The update introduces breaking changes. If you want to keep using the older vers
 - Files ending in ".Rda" were renamed to ".rds" where appropriate when used together with `saveRDS()` or `readRDS()`.
 - `score()` now calls `na.omit()` on the data, instead of only removing rows with missing values in the columns `observed` and `predicted`. This is because `NA` values in other columns can also mess up e.g. grouping of forecasts according to the unit of a single forecast.
 - added documentation for the return value of `summarise_scores()`.
-- Removed abs_error and squared_error from the package in favour of `Metrics::ae` and `Metrics::se`. 
+- Removed abs_error and squared_error from the package in favour of `Metrics::ae` and `Metrics::se`.
+- Renamed `interval_coverage_quantile()` and `interval_coverage_dev_quantile()` to `interval_coverage()` and `interval_coverage_deviation()`, respectively. Removed `interval_coverage_sample()` as users are now expected to convert to a quantile format first before scoring.
 - Added unit tests for `interval_coverage_quantile()` and `interval_coverage_dev_quantile()` in order to make sure that the functions provide the correct warnings when insufficient quantiles are provided.
 - Documentation pkgdown pages are now created both for the stable and dev versions.
 
@@ -50,7 +51,7 @@ The update introduces breaking changes. If you want to keep using the older vers
 
 ## Bug fixes
 - Fixes a bug with `set_forecast_unit()` where the function only workded with a data.table, but not a data.frame as an input. 
-- The metrics table in the vignette [Details on the metrics implemented in `scoringutils`](https://epiforecasts.io/scoringutils/articles/metric-details.html) had duplicated entries. This was fixed by removing the duplicated rows. 
+- The metrics table in the vignette [Details on the metrics implemented in `scoringutils`](https://epiforecasts.io/scoringutils/articles/metric-details.html) had duplicated entries. This was fixed by removing the duplicated rows.
 
 # scoringutils 1.2.1
 

R/default-scoring-rules.R

@@ -130,17 +130,17 @@ rules_sample <- function(select = NULL, exclude = NULL) {
 #' - "underprediction" = [underprediction()]
 #' - "dispersion" = [dispersion()]
 #' - "bias" = [bias_quantile()]
-#' - "interval_coverage_50" = [interval_coverage_quantile()]
+#' - "interval_coverage_50" = [interval_coverage()]
 #' - "interval_coverage_90" = function(...) \{
-#'      run_safely(..., range = 90, fun = [interval_coverage_quantile])
+#'      run_safely(..., range = 90, fun = [interval_coverage])
 #'   \}
-#' - "interval_coverage_deviation" = [interval_coverage_dev_quantile()],
+#' - "interval_coverage_deviation" = [interval_coverage_deviation()],
 #' - "ae_median" = [ae_median_quantile()]
 #'
 #' Note: The `coverage_90` scoring rule is created as a wrapper around
-#' [interval_coverage_quantile()], making use of the function [run_safely()].
+#' [interval_coverage()], making use of the function [run_safely()].
 #' This construct allows the function to deal with arbitrary arguments in `...`,
-#' while making sure that only those that [interval_coverage_quantile()] can
+#' while making sure that only those that [interval_coverage()] can
 #' accept get passed on to it. `range = 90` is set in the function definition,
 #' as passing an argument `range = 90` to [score()] would mean it would also
 #' get passed to `coverage_50`.
@@ -157,11 +157,11 @@ rules_quantile <- function(select = NULL, exclude = NULL) {
     underprediction = underprediction,
     dispersion = dispersion,
     bias = bias_quantile,
-    interval_coverage_50 = interval_coverage_quantile,
+    interval_coverage_50 = interval_coverage,
     interval_coverage_90 = function(...) {
-      run_safely(..., range = 90, fun = interval_coverage_quantile)
+      run_safely(..., range = 90, fun = interval_coverage)
     },
-    interval_coverage_deviation = interval_coverage_dev_quantile,
+    interval_coverage_deviation = interval_coverage_deviation,
     ae_median = ae_median_quantile
   )
   selected <- select_rules(all, select, exclude)

R/metrics-quantile.R

@@ -233,8 +233,8 @@ underprediction <- function(observed, predicted, quantile, ...) {
 #'    c(-2, 0, 3, 3, 4)
 #' )
 #' quantile <- c(0.1, 0.25, 0.5, 0.75, 0.9)
-#' interval_coverage_quantile(observed, predicted, quantile)
-interval_coverage_quantile <- function(observed, predicted, quantile, range = 50) {
+#' interval_coverage(observed, predicted, quantile)
+interval_coverage <- function(observed, predicted, quantile, range = 50) {
   assert_input_quantile(observed, predicted, quantile)
   assert_number(range)
   necessary_quantiles <- c((100 - range) / 2, 100 - (100 - range) / 2) / 100
@@ -258,7 +258,7 @@ interval_coverage_quantile <- function(observed, predicted, quantile, range = 50
 #' @description Check the agreement between desired and actual interval coverage
 #' of a forecast.
 #'
-#' The function is similar to [interval_coverage_quantile()],
+#' The function is similar to [interval_coverage()],
 #' but takes all provided prediction intervals into account and
 #' compares nominal interval coverage (i.e. the desired interval coverage) with
 #' the actual observed interval coverage.
@@ -308,8 +308,8 @@ interval_coverage_quantile <- function(observed, predicted, quantile, range = 50
 #'   c(-2, 0, 3, 3, 4)
 #' )
 #' quantile <- c(0.1, 0.25, 0.5, 0.75, 0.9)
-#' interval_coverage_dev_quantile(observed, predicted, quantile)
-interval_coverage_dev_quantile <- function(observed, predicted, quantile) {
+#' interval_coverage_deviation(observed, predicted, quantile)
+interval_coverage_deviation <- function(observed, predicted, quantile) {
   assert_input_quantile(observed, predicted, quantile)
 
   # transform available quantiles into central interval ranges

R/metrics-sample.R

@@ -281,39 +281,3 @@ mad_sample <- function(observed = NULL, predicted, ...) {
   sharpness <- apply(predicted, MARGIN = 1, mad, ...)
   return(sharpness)
 }
-
-
-#' @title Interval Coverage
-#' @description To compute interval coverage for sample-based forecasts,
-#' predictive samples are converted first into predictive quantiles using the
-#' sample quantiles.
-#' @importFrom checkmate assert_number
-#' @rdname interval_coverage
-#' @export
-#' @examples
-#' observed <- rpois(30, lambda = 1:30)
-#' predicted <- replicate(200, rpois(n = 30, lambda = 1:30))
-#' interval_coverage_sample(observed, predicted)
-interval_coverage_sample <- function(observed, predicted, range = 50) {
-  assert_input_sample(observed, predicted)
-  assert_number(range)
-  necessary_quantiles <- c((100 - range) / 2, 100 - (100 - range) / 2) / 100
-
-  # this could be its own function, sample_to_quantile.numeric
-  # ==========================================================
-  n <- length(observed)
-  N <- length(predicted) / n
-  dt <- data.table(
-    observed = rep(observed, each = N),
-    predicted = as.vector(t(predicted))
-  )
-  quantile_dt <- sample_to_quantile(dt, necessary_quantiles)
-  # ==========================================================
-
-  # this could call interval_coverage_quantile instead
-  # ==========================================================
-  interval_dt <- quantile_to_interval(quantile_dt, format = "wide")
-  interval_dt[, interval_coverage := (observed >= lower) & (observed <= upper)]
-  # ==========================================================
-  return(interval_dt$interval_coverage)
-}

tests/testthat/test-metrics-quantile.R

@@ -585,18 +585,18 @@ test_that("wis is the sum of overprediction, underprediction, dispersion", {
 
 
 # ============================================================================ #
-# `interval_coverage_quantile` =============================================== #
+# `interval_coverage` =============================================== #
 # ============================================================================ #
-test_that("interval_coverage_quantile works", {
+test_that("interval_coverage works", {
   expect_equal(
-    interval_coverage_quantile(observed, predicted, quantile, range = 50),
+    interval_coverage(observed, predicted, quantile, range = 50),
     c(TRUE, FALSE, FALSE)
   )
 })
 
-test_that("interval_coverage_quantile rejects wrong inputs", {
+test_that("interval_coverage rejects wrong inputs", {
   expect_error(
-    interval_coverage_quantile(observed, predicted, quantile, range = c(50, 0)),
+    interval_coverage(observed, predicted, quantile, range = c(50, 0)),
     "Assertion on 'range' failed: Must have length 1."
   )
 })
@@ -605,7 +605,7 @@ test_that("interval_coverage_quantile throws a warning when a required quantile
   dropped_quantile_pred <- predicted[, -4]
   dropped_quantiles <- quantile[-4]
   expect_warning(
-    interval_coverage_quantile(
+    interval_coverage(
       observed, dropped_quantile_pred, dropped_quantiles, range = 50
     ),
     "To compute the interval coverage for a range of 50%, the quantiles `0.25, 0.75` are required. Returning `NA`"
@@ -614,22 +614,22 @@ test_that("interval_coverage_quantile throws a warning when a required quantile
 
 
 # ============================================================================ #
-# `interval_coverage_dev_quantile` ===================================== #
+# `interval_coverage_deviation` ===================================== #
 # ============================================================================ #
-test_that("interval_coverage_dev_quantile works", {
+test_that("interval_coverage_deviation works", {
   existing_ranges <- unique(get_range_from_quantile(quantile))
   expect_equal(existing_ranges, c(80, 50, 0))
 
-  cov_50 <- interval_coverage_quantile(observed, predicted, quantile, range = c(50))
-  cov_80 <- interval_coverage_quantile(observed, predicted, quantile, range = c(80))
+  cov_50 <- interval_coverage(observed, predicted, quantile, range = c(50))
+  cov_80 <- interval_coverage(observed, predicted, quantile, range = c(80))
   manual <- 0.5 * (cov_50 - 0.5) + 0.5 * (cov_80 - 0.8)
 
   expect_equal(
-    interval_coverage_dev_quantile(observed, predicted, quantile),
+    interval_coverage_deviation(observed, predicted, quantile),
     manual
   )
   expect_warning(
-    interval_coverage_dev_quantile(
+    interval_coverage_deviation(
       observed, predicted, c(quantile[-4], 0.76)
     ),
     "To compute inteval coverage deviation, all quantiles must form central symmetric prediction intervals. Missing quantiles: 0.24, 0.75. Returning `NA`."