From be165db68371141f57902fc702dfad6615b86bec Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sat, 9 Jul 2022 17:44:37 -0400 Subject: [PATCH 1/4] fix the way separate_results gets passed to score_quantile --- R/score_quantile.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/score_quantile.R b/R/score_quantile.R index 5616eb095..b0dda73f5 100644 --- a/R/score_quantile.R +++ b/R/score_quantile.R @@ -71,7 +71,7 @@ score_quantile <- function(data, list(true_value, lower, upper, range, weigh, - separate_results = TRUE + separate_results = separate_results ) )] } From 62a5e30b99e196efc8fcb10e70ad167661a5e67f Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sat, 9 Jul 2022 17:50:56 -0400 Subject: [PATCH 2/4] update roxygen version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index a342bd677..9035373e3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -64,7 +64,7 @@ Config/Needs/website: r-lib/pkgdown, amirmasoudabdol/preferably Config/testthat/edition: 3 -RoxygenNote: 7.1.2 +RoxygenNote: 7.2.0 URL: https://epiforecasts.io/scoringutils/, https://github.com/epiforecasts/scoringutils BugReports: https://github.com/epiforecasts/scoringutils/issues VignetteBuilder: knitr From 78b57031f2bcc8c0fa6ee42bf0ff40c92e210680 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sat, 9 Jul 2022 17:51:31 -0400 Subject: [PATCH 3/4] add unit test for quantile_score fix --- tests/testthat/test-score.R | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/testthat/test-score.R b/tests/testthat/test-score.R index 8f647248b..3b42c8e63 100644 --- a/tests/testthat/test-score.R +++ b/tests/testthat/test-score.R @@ -60,6 +60,22 @@ test_that("function produces output for a quantile format case", { ) }) +test_that("score_quantile correctly handles separate results = FALSE", { + quantile_example <- data.table::setDT(scoringutils::example_quantile) + eval <- suppressMessages( + score( + quantile_example[!is.na(prediction)], + separate_results = FALSE + ) + ) + + expect_equal( + nrow(eval) > 1, + TRUE + ) +}) + + test_that("score() quantile produces desired metrics", { data <- data.frame( true_value = rep(1:10, each = 2), From 6cfd5330cfa39c43f29c89755a213692f1906988 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Sat, 9 Jul 2022 17:51:41 -0400 Subject: [PATCH 4/4] update documentation --- man/scoringutils-package.Rd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/scoringutils-package.Rd b/man/scoringutils-package.Rd index ef49fa8d4..540e57a3f 100644 --- a/man/scoringutils-package.Rd +++ b/man/scoringutils-package.Rd @@ -6,7 +6,7 @@ \alias{scoringutils-package} \title{scoringutils: Utilities for Scoring and Assessing Predictions} \description{ -Provides a collection of metrics and proper scoring rules (Tilmann Gneiting & Adrian E Raftery (2007) , Jordan, A., Krüger, F., & Lerch, S. (2019) ) within a consistent framework for evaluation, comparison and visualisation of forecasts. In addition to proper scoring rules, functions are provided to assess bias, sharpness and calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) ) of forecasts. Several types of predictions (e.g. binary, discrete, continuous) which may come in different formats (e.g. forecasts represented by predictive samples or by quantiles of the predictive distribution) can be evaluated. Scoring metrics can be used either through a convenient data.frame format, or can be applied as individual functions in a vector / matrix format. All functionality has been implemented with a focus on performance and is robustly tested. +Provides a collection of metrics and proper scoring rules (Tilmann Gneiting & Adrian E Raftery (2007) \doi{10.1198/016214506000001437}, Jordan, A., Krüger, F., & Lerch, S. (2019) \doi{10.18637/jss.v090.i12}) within a consistent framework for evaluation, comparison and visualisation of forecasts. In addition to proper scoring rules, functions are provided to assess bias, sharpness and calibration (Sebastian Funk, Anton Camacho, Adam J. Kucharski, Rachel Lowe, Rosalind M. Eggo, W. John Edmunds (2019) \doi{10.1371/journal.pcbi.1006785}) of forecasts. Several types of predictions (e.g. binary, discrete, continuous) which may come in different formats (e.g. forecasts represented by predictive samples or by quantiles of the predictive distribution) can be evaluated. Scoring metrics can be used either through a convenient data.frame format, or can be applied as individual functions in a vector / matrix format. All functionality has been implemented with a focus on performance and is robustly tested. } \seealso{ Useful links: