From f7faff2f3d9e4ff8701499b94d8016737f1dee70 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Tue, 14 Nov 2023 13:50:17 +0100 Subject: [PATCH 1/6] horrible, but working version of a refactoring of score.scoringutils_quantile --- R/score.R | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/R/score.R b/R/score.R index 7ec7763e3..67efd5126 100644 --- a/R/score.R +++ b/R/score.R @@ -263,18 +263,21 @@ score.scoringutils_quantile <- function(data, metrics = metrics_quantile, ...) { quantile <- unlist(unique(data$quantile)) data[, c("observed", "predicted", "quantile", "scoringutils_quantile") := NULL] - # for each metric, compute score - lapply(seq_along(metrics), function(i, ...) { - metric_name <- names(metrics[i]) - fun <- metrics[[i]] - matching_args <- filter_function_args(fun, args) - - data[, eval(metric_name) := do.call( - fun, c(list(observed), list(predicted), list(quantile), matching_args) + expr <- expression( + data[, (metric_name) := do.call( + fun, c(list(args$internal_first_arg, + args$internal_second_arg, + args$interal_third_arg), + matching_args) )] - return() - }, - ...) + ) + + data <- apply_metrics( + data, metrics, expr, + internal_first_arg = observed, + internal_second_arg = predicted, + interal_third_arg = quantile, + ...) return(data) }) @@ -283,3 +286,20 @@ score.scoringutils_quantile <- function(data, metrics = metrics_quantile, ...) { return(data[]) } + +apply_metrics <- function(data, metrics, expr, ...) { + args <- list(...) + lapply(seq_along(metrics), function(i, data, args) { + + metric_name <- names(metrics[i]) + fun <- metrics[[i]] + matching_args <- filter_function_args(fun, args) + + eval(expr) + + }, data, args) + return(data) +} + + + From cb9bcbe0f95b353943f6a7a59849053698f4ea7c Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Tue, 14 Nov 2023 14:18:51 +0100 Subject: [PATCH 2/6] More elegant version, currently failing for sample-based forecasts because we can't pass the `forecast_unit` arg to apply_metrics --- R/score.R | 75 +++++++++++++++++++------------------------------------ 1 file changed, 26 insertions(+), 49 deletions(-) diff --git a/R/score.R b/R/score.R index 67efd5126..d31800ba0 100644 --- a/R/score.R +++ b/R/score.R @@ -152,18 +152,15 @@ score.scoringutils_binary <- function(data, metrics = metrics_binary, ...) { data <- remove_na_observed_predicted(data) metrics <- validate_metrics(metrics) - # Extract the arguments passed in ... - args <- list(...) - lapply(seq_along(metrics), function(i, ...) { - metric_name <- names(metrics[i]) - fun <- metrics[[i]] - matching_args <- filter_function_args(fun, args) - + expr <- expression( data[, (metric_name) := do.call( - fun, c(list(observed, predicted), matching_args) + run_safely, list(observed, predicted, ..., fun = fun) )] - return() - }, ...) + ) + data <- apply_metrics( + data, metrics, expr, + ... + ) setattr(data, "metric_names", names(metrics)) @@ -180,18 +177,15 @@ score.scoringutils_point <- function(data, metrics = metrics_point, ...) { data <- remove_na_observed_predicted(data) metrics <- validate_metrics(metrics) - # Extract the arguments passed in ... - args <- list(...) - lapply(seq_along(metrics), function(i, ...) { - metric_name <- names(metrics[i]) - fun <- metrics[[i]] - matching_args <- filter_function_args(fun, args) - + expr <- expression( data[, (metric_name) := do.call( - fun, c(list(observed, predicted), matching_args) + run_safely, list(observed, predicted, ..., fun = fun) )] - return() - }, ...) + ) + data <- apply_metrics( + data, metrics, expr, + ... + ) setattr(data, "metric_names", names(metrics)) @@ -206,19 +200,15 @@ score.scoringutils_sample <- function(data, metrics = metrics_sample, ...) { forecast_unit <- attr(data, "forecast_unit") metrics <- validate_metrics(metrics) - # Extract the arguments passed in ... - args <- list(...) - lapply(seq_along(metrics), function(i, ...) { - metric_name <- names(metrics[i]) - fun <- metrics[[i]] - matching_args <- filter_function_args(fun, args) - + expr <- expression( data[, (metric_name) := do.call( - fun, c(list(unique(observed), t(predicted)), matching_args) + run_safely, list(observed, predicted, ..., fun = fun) ), by = forecast_unit] - return() - }, - ...) + ) + data <- apply_metrics( + data, metrics, expr, + ... + ) data <- data[ , lapply(.SD, unique), @@ -264,20 +254,12 @@ score.scoringutils_quantile <- function(data, metrics = metrics_quantile, ...) { data[, c("observed", "predicted", "quantile", "scoringutils_quantile") := NULL] expr <- expression( - data[, (metric_name) := do.call( - fun, c(list(args$internal_first_arg, - args$internal_second_arg, - args$interal_third_arg), - matching_args) - )] + data[, (metric_name) := do.call(run_safely, list(..., fun = fun))] ) - data <- apply_metrics( data, metrics, expr, - internal_first_arg = observed, - internal_second_arg = predicted, - interal_third_arg = quantile, - ...) + observed, predicted, quantile, ... + ) return(data) }) @@ -288,16 +270,11 @@ score.scoringutils_quantile <- function(data, metrics = metrics_quantile, ...) { } apply_metrics <- function(data, metrics, expr, ...) { - args <- list(...) - lapply(seq_along(metrics), function(i, data, args) { - + lapply(seq_along(metrics), function(i, data, ...) { metric_name <- names(metrics[i]) fun <- metrics[[i]] - matching_args <- filter_function_args(fun, args) - eval(expr) - - }, data, args) + }, data, ...) return(data) } From 102370d11303dc3e3b9133dc52de58fcdf2f62c3 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Tue, 14 Nov 2023 14:29:44 +0100 Subject: [PATCH 3/6] Fix score.scoringutils_sample by using matrices --- R/score.R | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/R/score.R b/R/score.R index d31800ba0..b0f601d8d 100644 --- a/R/score.R +++ b/R/score.R @@ -200,21 +200,31 @@ score.scoringutils_sample <- function(data, metrics = metrics_sample, ...) { forecast_unit <- attr(data, "forecast_unit") metrics <- validate_metrics(metrics) - expr <- expression( - data[, (metric_name) := do.call( - run_safely, list(observed, predicted, ..., fun = fun) - ), by = forecast_unit] - ) - data <- apply_metrics( - data, metrics, expr, - ... - ) + # transpose the forecasts that belong to the same forecast unit + d_transposed <- data[, .(predicted = list(predicted), + observed = unique(observed), + scoringutils_N = length(list(sample_id))), + by = forecast_unit] - data <- data[ - , lapply(.SD, unique), - .SDcols = colnames(data) %like% paste(names(metrics), collapse = "|"), - by = forecast_unit - ] + # split according to number of samples and do calculations for different + # sample lengths separately + d_split <- split(d_transposed, d_transposed$scoringutils_N) + + split_result <- lapply(d_split, function(data) { + # create a matrix + observed <- data$observed + predicted <- do.call(rbind, data$predicted) + data[, c("observed", "predicted", "scoringutils_N") := NULL] + + expr <- expression( + data[, (metric_name) := do.call(run_safely, list(..., fun = fun))] + ) + data <- apply_metrics( + data, metrics, expr, + observed, predicted, ... + ) + return(data) + }) setattr(data, "metric_names", names(metrics)) @@ -230,9 +240,6 @@ score.scoringutils_quantile <- function(data, metrics = metrics_quantile, ...) { forecast_unit <- attr(data, "forecast_unit") metrics <- validate_metrics(metrics) - # Extract the arguments passed in ... - args <- list(...) - # transpose the forecasts that belong to the same forecast unit # make sure the quantiles and predictions are ordered in the same way d_transposed <- data[, .(predicted = list(predicted[order(quantile)]), From 004d2b852d1d03f42110d0b40ab224baca1ce55c Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Tue, 14 Nov 2023 14:52:30 +0100 Subject: [PATCH 4/6] move the expression into `apply_metrics()` --- R/score.R | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/R/score.R b/R/score.R index b0f601d8d..29230e6e0 100644 --- a/R/score.R +++ b/R/score.R @@ -152,14 +152,9 @@ score.scoringutils_binary <- function(data, metrics = metrics_binary, ...) { data <- remove_na_observed_predicted(data) metrics <- validate_metrics(metrics) - expr <- expression( - data[, (metric_name) := do.call( - run_safely, list(observed, predicted, ..., fun = fun) - )] - ) data <- apply_metrics( - data, metrics, expr, - ... + data, metrics, + data$observed, data$predicted, ... ) setattr(data, "metric_names", names(metrics)) @@ -177,14 +172,9 @@ score.scoringutils_point <- function(data, metrics = metrics_point, ...) { data <- remove_na_observed_predicted(data) metrics <- validate_metrics(metrics) - expr <- expression( - data[, (metric_name) := do.call( - run_safely, list(observed, predicted, ..., fun = fun) - )] - ) data <- apply_metrics( - data, metrics, expr, - ... + data, metrics, + data$observed, data$predicted, ... ) setattr(data, "metric_names", names(metrics)) @@ -216,16 +206,13 @@ score.scoringutils_sample <- function(data, metrics = metrics_sample, ...) { predicted <- do.call(rbind, data$predicted) data[, c("observed", "predicted", "scoringutils_N") := NULL] - expr <- expression( - data[, (metric_name) := do.call(run_safely, list(..., fun = fun))] - ) data <- apply_metrics( - data, metrics, expr, + data, metrics, observed, predicted, ... ) return(data) }) - + data <- rbindlist(split_result) setattr(data, "metric_names", names(metrics)) return(data[]) @@ -260,11 +247,8 @@ score.scoringutils_quantile <- function(data, metrics = metrics_quantile, ...) { quantile <- unlist(unique(data$quantile)) data[, c("observed", "predicted", "quantile", "scoringutils_quantile") := NULL] - expr <- expression( - data[, (metric_name) := do.call(run_safely, list(..., fun = fun))] - ) data <- apply_metrics( - data, metrics, expr, + data, metrics, observed, predicted, quantile, ... ) return(data) @@ -276,7 +260,10 @@ score.scoringutils_quantile <- function(data, metrics = metrics_quantile, ...) { return(data[]) } -apply_metrics <- function(data, metrics, expr, ...) { +apply_metrics <- function(data, metrics, ...) { + expr <- expression( + data[, (metric_name) := do.call(run_safely, list(..., fun = fun))] + ) lapply(seq_along(metrics), function(i, data, ...) { metric_name <- names(metrics[i]) fun <- metrics[[i]] From 7eb7c96300f0344c70d39c939418abb3852c4d80 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Tue, 14 Nov 2023 16:09:50 +0100 Subject: [PATCH 5/6] Update metrics_quantile (improved error handling + getting rid of `run_safely()` --- NAMESPACE | 1 + R/metrics-quantile.R | 6 ++++++ data/metrics_quantile.rda | Bin 13133 -> 13024 bytes inst/create-list-available-forecasts.R | 4 ++-- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index bc2a6b542..d3b6ffe22 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -82,6 +82,7 @@ importFrom(checkmate,assert_data_frame) importFrom(checkmate,assert_data_table) importFrom(checkmate,assert_factor) importFrom(checkmate,assert_list) +importFrom(checkmate,assert_logical) importFrom(checkmate,assert_number) importFrom(checkmate,assert_numeric) importFrom(checkmate,check_atomic_vector) diff --git a/R/metrics-quantile.R b/R/metrics-quantile.R index db187d376..985b00002 100644 --- a/R/metrics-quantile.R +++ b/R/metrics-quantile.R @@ -91,6 +91,7 @@ #' @param count_median_twice if TRUE, count the median twice in the score #' @param na.rm if TRUE, ignore NA values when computing the score #' @importFrom stats weighted.mean +#' @importFrom checkmate assert_logical #' @return #' `wis()`: a numeric vector with WIS values of size n (one per observation), #' or a list with separate entries if `separate_results` is `TRUE`. @@ -105,6 +106,11 @@ wis <- function(observed, assert_input_quantile(observed, predicted, quantile) reformatted <- quantile_to_interval(observed, predicted, quantile) + assert_logical(separate_results, len = 1) + assert_logical(weigh, len = 1) + assert_logical(count_median_twice, len = 1) + assert_logical(na.rm, len = 1) + if (separate_results) { cols <- c("wis", "dispersion", "underprediction", "overprediction") } else { diff --git a/data/metrics_quantile.rda b/data/metrics_quantile.rda index 70a00a9329468320b53b2c7e803ec070df6b33a9..b14a8321c408c6e6d9010c2262f0575fa845a9ea 100644 GIT binary patch literal 13024 zcmV<6G9S%CT4*^jL0KkKS;tuSi~!s@|NsC0|NsC0|NsC0|NsC0|NsC0`|!&WF>;(YUfB*mmtXjZPW-oB=-q@f5 zzE71_t4^8vIaLFKsngqnqPe1}@L|_Frnqx_( zLTxk*g*It2YI!q3Fs9Q1Cyf;J!kM%RZ$xTgHj(5fqZ1QTA*R&z8h)v=O{mjPNMIUG z>Vi=a0Wg{aK$tr%6B8zh>S3v-Pez!K)6!^qo~D{? ziHPw=(^G0R(@#+JnrWs@ig<~L$bP1XdLu&u44PyNj88~upc*s{GBRjrGf*N)2nmpw z0Srlrl=Ng_G{~A$^qx&q)6t4;G%4iGQ_~Ts@F)){^&X%!*&{#z000M~0p%Eb$7Y}g0+2yUDXKt2VgINV3-OydX5CCaI>74NDz;w% z%x5Mf9eMKhX>pN*Dy_n z2G5sSL{E)EZOZtVn*oZbWJ2hHkKK{tisnVW!vJUQM+Jlu-PB-Odjc-G~sSARL0 z0W9vnmS>?RLCo+6enk(=n@GKWhSwetL|$&aBFTHt4u3PjnM{Fzz`&JJT8%(5_7v>D zPx8N&{Qqmi#o<8uZ7+5&NFqoC5&(Vz!9f90FwSr}x4X1thX{az=~FM6n&S+#hhxi> znuiAiY1{c&;m?(cXKYh|#~>d3X-VJf()s)u6C zOLV))uVw-M3!5&lG0$SX1@a%5z=y+DcS-sxQNuRFn!DCBdf*4h$uR^O&SV{vBdR+BP zcNSteG}1+9yWtcR(FQP9i^$xP z(LA-YnYPVsm86A9MC*#%8Ochbvo=~G2n<2--PAYH2#hz4tsL(KHFYE;k{WnX8IeE` zzl2Zs~^6{&w|Mo}ScbNbIT#R04XXv<*$Z&rOzJQZy}S@%Ad*NSb3Gyv1P}@= zO6unHSf-ZJ`l6OMd@U=jj;fPK$}qw=pxV-j>C8~HIUC6Zu?TV2XfoOFdnK7$Z6aEcA`&3RB9aI)2_PuD{(G9; zcD=t7&f=fn7YzM>w-@|sz~bLlF_Gm<1y;F|mEO!lwG$BIC6KaGQcjbToj@im0A5r9 zI>%CwLAy$L9aI`el`(|Qys&cmgMeGQ_hB+6{OQzd0M(B(tu5DV3z-Mq`AT^@@1+mE zow6QUWpf8qp>c<;jxhJnxEJ3c&H>B21Cs}5v;%(VO{-f_6MWUYWDr{GT&{ptJ@$3L6o z50d{v9Hae|^3U>lq2~K25%SRU!_Pl*2cG%i^G{r!v^=ND4;eopJW%=}*c*iE&gkk6 z>^{lnCSdl*U~i}THlwa&LDmpABv+tCMfU`RU<8s35JjzKs&xFNmSb#l7G^#za2NO# z)|?X2mtI*tI9>~QI9`-ON~-50?nglhjtKfPv`ric) z8_;7wPm*Om73TjyJ1+<(1k8iLC@*8;^17x)GLZx%krScOPCeZBpK$;18G_f~MjJhR zZ4f=Vx`pNt=~LaT0P=Blx`k(+xF})Q-b}tTyc>7{7r)& zN#d%YdK-<)8pC&_wp@xzdJeLVbMDf6MYgErEP-L^_ddRU|68zph=(S7eMgbhejfG9 zcbg}B)x~PPPRhj_kKR}`3^u$yLBhmPs;!emIG}fE({Da)qmQ2iI$&hl+Fzq2v}Li9 zZFWr7`Pt{$r)zIMo$T;!Jss=2W$W=-*=%;%CZ?>^U&iH8%_cm$9>+!6C(K*i#S|41 z>)~e3zZlhi%+I9N8yhT@ac(n2PFmDsKt%z=Ilc&)5gxX@;kLJ z3ufjwDzB8y7&ev*)$Q0_eTukOa%7Tx?3&Ic)YiY^WPVr~rtEHG+kpAbqKc}b%>_i7 z$f&xV)3p9>p5v-SGia9~2yqZTX9oFV8wDUX0i~*NM0$X_$sCbBaiXSzgUOI06~KO^ z5~f&GFpw*j0D42*+OB%%#Qq5;i=b7~X8n)sdVt?R_N0%(IF zyQi#9kuJw8xw(*cpcalR+D+*0L_TQ}L;&&ND(KvhDFI9K+B?X%0PfB`KY7+Wy3OO| zsgx3GR%%f=pe`zLC6-NHXSc<*hNd9VyGZ6ag8IK_?|PrOwbu0WovoxieucLR^1heS z`d9Zc3$ErJ;v9Sdy)Gw&M^ag3t|coa3&@4^34~!Q;ltELQrZPy79+CVRE8ZOic+j#MHbkXr+@Zq{?)+$ueGN6jO;?k0URc*YV81 zuJ`|)f3Sgcy6=@^E?UDNJ14{SL7j@D*C%|0Od^af-Ph^IWQido8Y5zuJpPIJ{f!*N zjn`toLqW7fRj5-OW5KxbIGbH_o0Q~vEN2VDu@X>viQZ-OKTpB0`frPGF%)Kc6kOsc zHJ1(-a@DjtHu|VMT^=7DtG8vHwi~NyOO?;ZFZXO79Hj27oEjJ+RaLU6i&koErJZ?) zHg1fQlTCrjg_i$&Glc3@X?u<a=-gJ73P+CBfj4{M>||K6MHSuX99WCENoIC9 zW_9ym&0a`x0^?hYhSv$K!9B_{yS({&)vr1Chj#X{9&uKgf?B{1-^@++CGvIEKp)`@-_y@Qt_D69J@WzINBqI)fdbeiT;Ryu33zxkN zmH_W%-9Hog-qXy%{eu@*dY-J|{qFs}r&wu`>6ZsA4WTw1<_~UzrGXJ$6`^^ToK`@n zBfWr1Ed?B3`-g^`G=I?7(}y0j2W9ScZ;dWfx^MSDwoM{A-CW7N#Lsj`V9}M0ej?3t{_a@+$4O!@7svC*} zPC|@89bNUf(?!w{?>EO!F&lf!NsyO_`0=)=Vu1AiKIIDWwv9CGu^c}}l$7~WrKGeP z=H@J2q)%o*+D^>}Vg@<6iP*b?Bll4Pp(Kd#!+Hgv;wFel9Q1a=P08mUuRqz37S4_^ z1?%acuCY}Bgf}sJ~TvZFkWa-A3 zlb3Z09wZYNGan_~=wscY`0-|-ds5@h^nX8dg9JArxsHb18$H`6MlQ@a;wS8(w)(DC zY9sHZ{zj*x%TIlz$+^={b;^j=L0Da01v^Es887vG`<6ByRMy1eZLT#ismra15j!hW zVMK{Qad@lithvN&8$6xUeCwTtd=ur?l-o>E?~*!`co)~QVCfu;cz)vxc0(7%3>@`W|JDwdays0JOjA;O3sTk79q zsSEMYYyNNJY4*o|pFZi3-F4lhha~FLrhL9ScOn}w6R!g|dr#!W;F)}iDjgZg*Trpc z?CX#h1}5qruY-4_+R<@xw-jw(!5$^n4aQji*^9!wZXhVR2(%3{sL7 zX;?^D#WhD5Wzxp-yjL}cI2Co6CoKPiyx|6Ocn;R_M>9e*Yg$vMX7e*6HS7BOit2HW zAk`6-Ak%`mpdN9pA!Dka82GR_*Li9 z?M)T3OR>%b`b zs|`b^lF8EUy!gVxeMH}5AuVKtLlBfAsT3BIK?w#dY4jp|Rb(%N$!U!A!a8sEfa*MV>73&SifAU!92#|Xd3x2h`wl=s#C@u$L_=vt z6_fC!w%K1BLfj~|1}GC!i?ANBCTE1e6KvfJBoO6P5b}ult?QI2e-As@Qt`%%uIw>; z`?gYpqY-F!M~jJY9U3{MKH}jD-sUhTP(uEO6=7gs|EYpt*rcQ)bGGYqT$M#rYi_Bn zyPv#qrv0IWyk=nby?!kywTL{a5we!f2At3oEm#jNA7jPv-+P$C=WCHUi^nG0({+0R z*1O5b{>!;%W4jsAlhWW%D8-#{uqKgpFW|dbqynt~#;Y_I)cJ^&Ht;2=>}{^Ic?+vN zufKTQSx6y9R0BO0Y=VoSLg6rnc8@w4@|SjbmEFpKrc8l|!ji)l=eiDYQcmuWjy*p`h@4w(#9%xAJMyx;Nn*QIP@xXqmhK~nzX2f+go8>gSAed;KI;J(s94#s zk)UW3>x!&imW-3J5iuh16NUlQoKbJry8MRn zMZ!^+j?6oPh(JUaXar{0TIs`p);K2Ai_n7cFavakcgd)HY#qU-BM(Nsx}(^tm5aU!Kt>{SSLrD?AgE-U65)s9wlD(P&P@87kg57^J00oJ~` zzXpp(MPx0ld*b&yt%*{gH>Y=s`a09L`iyE{ zzfc5a5Q$!~8PF*to=nm1i=;b5gzSMdaoDYcc97!hi4>X8!4F?LB=}f}F;K=Hk(TWX zzruL!Jw!R^V7qP=%!iq!o}RS@a?+x1iR&K6;PZQZJ`W&oJ|!LUY;kw|dpA1MiW@Ky zXl^x<`THuYCyu~`DHU=Ps72KGNR`9PZ3T>1sBWkTj89OB^2`PYkc$$bfkecl{0s+H zjk}9J(-1_hQp$ms3KR4rAHCe(p`=<(q#=2_Nc&hxyKnO>A*`tGL>54Fj?fKs2B7a4 zVx{3vP&v%kD73)@FAGs&g)iA{4iq()d8;ow*>*i8bWBU# zQIAo}EB2S+PT1(*WzQMck_Xc!yN^5^k&J40d(+-gDNzX&O%&mXdLnoCaTDxaVsqg+ zRV;wA0K?ifo1EK|dAiK$V^UE`bHIZO427dWsVNZ4t(aU>ispIz#`TSU!Y*wz6qJxT z8fnk=Ryek{*wYwiIU00F4K!YLWtWH8=zMQ^g5vs^&q5AV%|T8D3GZl1#g9dX=S6nn zS>yZ97basqCX7WG;L=2i=6t+({Jc7laU@8cy_VhtA1?HyiRSt``MY>;K=m*=PQ`e! zvzerc5+orQ&PcQ|FqFi(qI=pbW6)gr_L+(7W2M1qSRKb5U~gG!;USjKnZEkJ9i~7y z_#}@wSM;5 zd+YxW>gakN%GmxsAGV(pu4&H?$)7N}iN8^5lW6n{l16DIF6`(+&3(P*Hi)e+x zWh+O@{rB#_ub97Y9QnB$5=KHR;GIzOlvNj+)8EyYg^+X}M2Qjzh?CTNZI4ceZ;-iXc5%lNbB`K0iv`7pNf5bw zg5a?aC&u^xx4+&&!6EA7kcrce7CPNHl1bz|e;6|doGqC2d9JfPXdtiXilhBvj#vST z3JI4$NxXx7i&_ohfbI=Hal%*>0bDZZUtDHH`Ib!efT{^DJG@Er$ z2!K*RA^}MNhy^(COEB9UiYk|HX6e+WUAf6hyiPeXwQH_&ty+64oLX8*G^c+Y^ImR6 zRaVrkS6&{DtxDH}o0h655-AjwT>yzBmF=Q~42BY_fJCu2s_x^JRSe!X!RZ#;A7gpn z-f&}Y0aREZsgyDJP$Ulks`Pr_KdX1ofrQnH^X01Qyq!=F5ItdCcgh4=hrj?E1ov}N zuSLh(d9dmKT^@@MNloC7VOP5Me)FB}eqZ8w%lSS(?|R&fy`K`?Hh#0?wwYy?s-~XJ zHgIt+ODwX>EV9civC%Pm7prT}^Q~IWJ)-}shPvyny6djG>#n>PE@Q!cR`tAB@7=|V z2mStD=MCqo*LvArO$-;8v|q*-bHZ zjT=J_4E&>Trn8cM-FL}f@K|(v>Y9w*Wn}ejH8n2+#?sdcnqW+Qc7J4wp5;}IzB<1l zXe_NVHkR@jks?K{y}i6*K+Zh)qJ5q3SH8nsU7&IAfYYa475j5cjoWg`1QE*|u}Lj~ z%(F1~tMMTHZ`orRU7wj%n^eb6xj9KmNob;|aHQnKLNgPapP#q&AX4kBy9_*YpHHE8 zE@@hH(@ynUZnph?td1uqV!uA)xOye?ANJlya`aCtvGZt}7;H-v5=Sb?s@{j0KPH}k zE5-b;9?SOUcl*2N-=2BqyY40Qu3Yb^^>1RwUJQ6~EJ^2E^Rvd zYO8FjuIh0YL{*>_AGZO2E(aKpPypv@ni$453ZmNx1XNsIw`aFaB5l*WO~dJ(!_&G4 zK5z=f?ghli)sJjJ-q~wKNOZzv$>)=@={6qg$jT7weDQ%qc&3mt;W7zl`{F?SO`DIN zT(71~e>Xp9`z85so-J5M0!eNG6eMj3q#<^A9kxs4>&P7XH`9NW>0jVsSoG}bUny>le%Wz z^ttx*lX*Q8@6Skwp_h2}N97k~o{{tnHi-|#Ur@KEB61$6<%%o9Pyn7J`usv6k;Jon zGUMy-3=yXe8cYdiLjd2E*-j{+9TDx$i2DWHrd3fN%LUfJLD|(HWd{vt6m+)Dvpfjc zE@tBl$2p?hRSg*wQPk{kG2R*6Iu_fPOnxfdcJq$q+;g8X+?)0k|H_`2egJxB$~QX% zho<)4dU7X0Je2gq@B@qt_#Io*IW*r%j_q7BilJk|xFt;5aw(QfB-EKhlRFdm%%S{np$+-3E5Au=_C?{{m|$N4Yovw-cvl6> z#BE5h@5j!#kqW?!p%w!;7YHDLJ#YtxW{(64AU6Uid+bqhd8_e*G#ZCY^{KB-%b9QB z>36#1om1Am`_~vH%h1F`8KkX-<&CuEIBeXuY^-gHFf~Y#C;@&FHc+S%!DtjJg`*Bk z*i&XxW}M(>EZbJt)q_ykX@<47E*Cb+*iLY5xeTD(!HsOqa#}$Ost`g%2S$RB7NUtF zqavvyC`c&3Zo;$x^sBlJgl{s!!83Q;W~L>YhOEkFS~hw2b`8wkGTXNtkyKK}o1EBK z=59ESGExfL!sBtJib$5UES0LQZ6r5X;lgJ*wpinuX)NM#mR(Gob1P+IjyS0)*EbGx zb6Kq>&U2Kro0+MV#hXPCDoUV|s-r8LMT-(8q8AYs!En*TjvQ-Rjm^$7(W?}!wY3>k zD`jj&rIJfwAtu{q|J#nb#xs$f_?gr?R-0l&7|Ay{<6zrzW1E)+WQ2z!C4~hWRyI`% zW-6;QQL%%Tc#9oaY*9?g+cQ>UW+cj0YHdcTB-w^_uxT_@9ZW>(qA$mQsJCFGfy&0- zstkoEYdW*TH_O|@Hzsqxm%{LP5UUmnmrJ720Rg%uI09xAMNA@tjNThfm9bkjt+QB$ zVHt~IO`9#I)=Z|uNtLxSlS4`kjk9w$+S5!ACA6B@nrpV>irJdlN~>1YlVz=CQ8s5Z zX#>`eDY<~*Hr&>2ovhfb*sZJn6Ukj$HslK1Q<5~LD%vYnt5p)KYiiO}wNkRCrL?OS zsuOLqTD63drK==ONV2u9ERwQ}WU7+UOC*zK%1KogqP1+>y*BV$Zdtc1VvUuJl+DgK z$^{ymR&{ORs>;bLDc{a!p{lKGSz5O3&9t*pwx)!OOJbQy)GJY`l*+Xvm8)8nYP1xl zQAP_@X;e}bsFhh&P?JclTT^XCX&RA6GrJbsHH!y)qiu-{Qw>{YXJF7xwr$`!v5qpa zY*?(bR%m85v=YH-lq4*+p*3pT9tdm>?J%{B;8KaVtj+2duXvnnrHP8wL0d6Fw}E4F z%rWHl(T=vTlVd2@+0~3F+ijj0Fh!lqJP_FF8zqgNCR=#A&CHt_j{1W(Jf@jhs0f)u z&J!UdJzvjl=Ci3-tV~2tk7`2D@n!l6T5fwoVvbHKgosFj0f`Y+keNzV8a8PVXo?H* zq9AM}Lg9c?O@p-`Ej1xFw%c0Pq|#Q2lSHJ8Y0bgJ(#a($OG<-nO3e|n zA|fm@Ac?n`+inr)5zR%9AFg4k(o%}dc@x0g%H-LYB1=#1Ig|2qmIeAh;Mb z(G+lkTuqW}b(U_4w^FmZwNgt|fZElmEhuJHR;B@Dz*iL2vFF?MWnDPTNGG=g$0dmwr<^x#@jnP zCkGHU#m4AH+ekK}Y^>S0ZUc-tHq_G1WMyrd%QKj*gl&W5?yMBD%PbKL2#z8e6gdLx zzahqrh9R#E0j)O1F!axQ4ugt1ZH?PAI(3InP;`6hM~aC_Mx4UNGZD)+5!+$46hNp{ zrXo;9Mg$kEYG{Erkh=x~;Q<*6gHTnZAYY!YVhJP3*1;@@d#mXH3lLE%3Qlm*0CilKi7_6*I3+3CU zsKKRailqAE!TEb1z%##!CQ&%d*@C2rK@mAw9b(Ken!eB@Bnmj-N3*pCSXs(DL?|J; z#&XPaHsH7tyd>N4L1ID&32o3KO3Pys2tqJ|4lw{BVDaP5^tqQc>KjAh{?F)meE(Cz zUyGhQE>H+TBd0wOXd_Q}n(qWQc2k>eNQ>ral1+mlb@N^zZGn%cV&NbeQwp{8335ZT z9{Zn4c(sUmnajf(Wv2V|fOeOLi?Ds%E?uBYy0O*`moCs>OV_Vj0C&f8F}ITNQy^oZ z0)i(f2!oa@;|>*gLAwf3!GjoD)Lal$xCkI1(P&U6^~Lb9n9Yw7guGJ&fUA&i^pId! zRgeKE%(eOQVT6p^`&l_Xeds17F$D+`Dh9TI8jL_<5d}XSATQvwwy+&unLvl=Ji6Y3 z2HD;gvM0}fwfWADmEwlyZ;WgrX&?vHg0v=M0TfOvfi4nk#SFf%>J|~aU_DI3ab+$T zn-1x>U>lhnp~&>ZZwwVf0pC08V~U2mf0J@tiSQwCMOT!DO9rZjZ)-!OF#*iz+>EPy9p6TkW;^8viLbRnu?gAt)Pd@BU| zmX!-{dka-Y;N30@KnN|N7*wJ{t8}xw#_j}l9m%*?UR+Q%Dgdxz%pF3v^j`grbW5)q z$XkhMM#@TuI`L(?$xxis#EQb=CXn8fwghoU017#Q%<#nV3u{G@Gw}WPGd|HgpJR4C z!EZcGyH@PfEKnDr5W7AyizVg@0sTRfY{~S=2l+Sw_0$aRp|1ys$#ADoOf4|SK&XT^ z0U|9h?$S=X!u93ELUM};fhX`ir!!ej8Ga_oLrXdWM8tzhKGd{0`Ta8ZN zOm{AT(a}gzt*5PHUz1tMBp6(1Bi}mt1Z(0BmIQH85R$$F$#k7pj}4mzzB5dA0mw~p z5eVSC`~B%ICP3Cu6gAf;-BKDLFtN8SBwIsisKQICqY>p0+qf}R%<$8FDi&fn5HL@s z>wx*Odzf-NWTX^LD7x7#s2ez}>wk38tI6ndd&WqH#c75ba!=FSw#eLS>iVcu2@v#c-jhh}OecLI!YM&Vd=S znjc>v%aCnRfjLr5kuuvb8Idr>N~${qL=&Frwx?SCT``NROC=sAI{O)+&8l~jU=URi zQ5vk#wI#Tg-lAT()~eC4X%~NB#cSk{geN%YW6C5S5NI(e5@XA`Mropp2H?XFS~k2< zOd*dOH}T%3_=AU3at5N{HXx#QSlT!qugowiQ^gd#~H7)~mz ztzghl9<J4Zxh6@YgA-*QEWgHI0?-wxMy%kKkObBDqWiC)0 zEz+zJ0+$jS;B3+#j$<|vLx9x@Ych+hRj=8o_$K`((vB_C;>fCDd#GIVa~A=wCW;g@ zf;1z-AvO%J&L*u55jMk>VtEs10ALkF#;B^8qr@wRFKX`#V77#{2^~^MIFBYVmk@n8 zeZcC4Kw2PR;36qu{Dk2r>*vw31O6~zRK-gunl zgPSg6O^i7K)Wc}o9l%6iJ|YR~G^xJ?XIqvHA}M*x9r4}6 zXNMp(hEsf1Mu7y=kb#k;MI%tw2oxouMIQ9s6OT3eRhXKML|B7^p(!hf zk+fv!s7f_laz{{w1A!%6R{gLW4P8z|=R;RcaB-aunCb~~fpVeK#^sEOAVwZ`BdfyP zY8eYrp-!evT{@Ly9YOrcaSNhkJ!C*F2u8&$M;NRcZ3WIqacyRqC1cWPL4sOCCtMAv zEDCEFkT#S+RS<2*eK#@C7sSh!N7NC?iWL%f5EM-wwONpi2!JFy2gGAy5&<#9`O3~Q{5#;YtbdAnoXuzZnsy8e zQ|CU6r~n@)+4w)zbS|KXCBmobN$RSqrIJ~tzw|5I-yA}HyEP!2J8tOjZfAILyt11F zST+*YEG(TmxJ#C%G!Oju97OEvw&?7#S?8;)v zy5`4O9GxgwP*VuXA!Mr>q8>?kTSCPG(hUR)+f{>|a+XkS1tbcJ1R$>TL~ekoP*L21 ztwB=&fS{OF0lK1;;UKLM+tW&L_sbXqU?Wj&C@Bnv|AZQ{qFM|F)OX77tU9-KO`1hyhkcE^CWf>q1*2?g1v6~V8_+q1 z>Hs%VAm3o9QbL%G_-=B&H%r`pEuQoE)BB#sxzhGqAJ52Fmg!TX4y-IXZWI+GhT zb=|pe{i$VczBw#EleIZ6RauH_9 zt9Q+t@*7V-=YG6^3sF90$4Pu7aGlzSw4>rTD;irKlWH&h@j>(JG(B zSXZ<6`&<}5_aWS|bI7d7Y^0*=Vs`)F!5=Qbg5cTtKA5r{_@gvo+QD2eM&>`E?`ArK znI=5Z#83EC4`PLYI|GKB$;})bZ_AQx_cqM7qy~y0zt;K^Hyx*A*-~-n`T>O!8V((X z!sibWp4L5{OF`BtLdUiclF48Mmta2Y;&+Kl(V9G$L80?@uxpMr~pLaPmO4_oQ4aZ9hfvb$7n(tOW|_c9Uq za=hT%)H9;WWp%z6>A=U}Iy6l5ZTK!Z+&`iIMD95E{?Oz~BWU;f79ZT(1t>a{pNhFd< zB$7!mu1dA7>v^qCjCg^}cKt*A_3}_gB5%<>HU}@62j+&Td&aRCVT=WqS!I@4Y}dQK zu1!$CVIB4&ZdUvCw$J&x9?qOO)%rRvYyihszrpK0+itibcBbEE<99svk^g1SLv6*a z8|>b(zW8q(xv1Q;&h?hp^1haCym)o)WjIn; zX2dcrUvIFEF6G4Z}ttF!_HA i%KZP6{7;-Buc8LEDle4;N4Ld?{}*yaI8czsSoe%GtddIr literal 13133 zcmV-TGqTJ=T4*^jL0KkKS?0UNbpYU7fB*mg|NsC0|NsC0|NsC0|NsC0|NsC0|Nrm* z|NH;{|Nr1UK7Fn4ohP4Yc>C+mwv@a(y@92B|16HJXX#AIL*f?)=XL4?rL zCX9_V84WQs8Vv>k0LiAFp@`E7kT6X!Mwu{%O&Vknq)KXNDi2d?Xw6SivY7^&G-x!_ zN2t>wnvYTH05*_%jDe<1fb^OMo}kg7$j|}l41hEm8Udz&0MGydrkIaV0i$XNltBO{ zfuIS238nU$fav*A3 z#^4}QNQy{E$RYMI0K{QOe0x2%iUWqbW)?rbaihR;W(H-2rFg|-{?RZ=0kQq86#@!s zmAUwcJ3JE#ZZJ1}%`%9eV?j6NAeTl&DHY}MM4r6^(^sGo!14$}`6~!dDV|40fAcUr z7cF}Sfko=NOdO+BhLDd8%Svlfo<=%Kl%njMTm^W(ILI9CHv+;-MG?<)Ic`ZXMb(@& z@=kqnYC_SEkr6vlsg9HgGZ$KwFySg0TQWw!SVD-jh7@yL<;LY#8gqEnaK5cq2_ne} z6jvN^(U%B~&f|@x8GW8==a$PX$_NYmf*>3hUWn{JV%p=U^R^l{=XZdex* zl*CiN1+oQ@3V?VY*?#Bf{m(b*E&ZO`*~r@FN(4;}FiZyy7*0xh+ATh#S*C>%8IlmN z5+mzmK0_VIFh&%Ts--;Gh(&3lrml2tPN`0u!YihcL@U?ED|Ycoh5$NH5PTZ5S$!HxeP0V4X!x(<6G=k42rl`}E+r z@aW7nnW(+C-GXTMcjdYopGfED@TiDGVjI2#IK5tbq(rs=CO-`?0GqDrW zjE-?e!dWgc&|!SHEo}URuxAfh!nE07GWK+lkXwe;MME5QpMG&}R_i-I8Zw%1X))uq z{Ozzu4e4xj&}mSOMm^eeNib6d)3PF9j>=O6L0f3aDiU*et}bU0TD8I<1qdlIClDAT z;ZwY!*B}rCb%!Qy*LZXo-#nYNoB!GL*KdUtJYq-77KlM-eo9e=d0 z8&ZV9LnLZQNb68=Lac<@r3%7`s{*#2pVMJY<+a-@*; zuUN@fu~zXNhyWqtBuYRc1jKD)MYfWw8)&tPpa^dJUn9=>82tYv`2Qo0dmruj|5s=R zD$53B!!p{BBp@Bx4gjMlk2pmUxm00BQ4~bgG@5Ndm`gxYvH^S(ZA1uq^^VhSB;hTo zOa*$`hi3O+S*5y*z*uJYaGsb-vpJoh7{b4bKViU97E(It>&Rnx$9^%ZL{%6s267Zo z)Ut{k6>zvB;DNt3V7oBnfnAn(3Elt*qskH5astxBc8+2Vc#dj3RBusTo#Zb0ZBGRr zS~n#g`tD{C=T{Z>P`5+nxHxk<{QEt)byI6u8&FR@CI zlNFVz^{_o~JFtCl1@*mD&YzEyf8_VIe5~hnaix~WWWH;vX&6!fwvtrg^l1=}Tjlkw z=CT*(>d9V{0>K5gxHztQZYK2O-)7dBElG0#AY-Nq&?bFx^8b z9VMSM=jLCba+ij^khHrBb&QXEilE61kmC%DwaYpQcn4VeDIzh?5<#GIg7K`kN@ zWrSSt>C)Zgz_f>SI!hW>5r#tJTId+aec_QUjn*R_k2;p5+?z2mE6pplSupcXDzMBsOq7uf8#p<<$a&m7%Hg2TIbFSK#HlXLg(R(@ z>zWoLtpxW<`fDVb4J#0D!NiegkH2zhy|Y#)9c7p8H`HtjSLE-M+w49OFS zrgE=oVQ~75rGsEn>-d*5Mg<9W1|Aw3I`&Z>8ezg!s3A*6&$hh8%+B51T9#*8u&&TU z^yS~TX}j*+xmMPqN-UzpH_eIYyxHDzL#txLW7EL&V$Vk(!Q#6oQkeONAiiWn?H4@wsn)Ytt9tZ`Ivet>e9mp@5 zLPZDZL`2KEBSledGFCjvw1)aE!nchMmT!rDH$2+tG{vH+EJm@lu7Oq4$l)sCTCDbz z_Io=|BB2CII48L}1v1F+fEAtaHxp#~d(+T<@D9&9|H6wC(0irB$b*iQiE}~#a`B&12BSM)XN16% z^nSXhtA>|hDF*nRQ}VvpGBE^uVxk3zI2hKenzUg9N^vV3@=AqOTy7&lguHP-nxWb-nfprFi@P+nnyT53C9 zm1fatUFiZ@_^NF8Xka`e7h`u!Hu}T`gfze)8^Z)v;;15ILz7`J2x=w+*zS?J8@*B9 ziYMzukkW@JphpOLy!T}`hP-*g|4y0NzGfGDUreh6c!#F36 zFwe+N#FNU`Jn_;lC$YLlf@dSXeR=W<8BYM7=sLy`&SHdN(Uz6;>y{9Zwjm-0KEPq9 zhfK4V$@exs!yM5kh@;atzb(68SGJVF9>C{R|4L#*E@KdJeD1UPC{{2;7sg`!zm3`entTI-0(*q6yyb;HDszhp z(S#bKc(w&;iD>K_gh;cJYSbl~S3~mc3Y|qty@Feo6CvTLS!qTlV#&m}a;ULz7YtkV z&c|u2Zls&nw5@k}o4%8K1dVp?(`?Zy=@itd1SE>mJOOE-{_Zcrt?#`5clfwELgZTu zf@Q508;hnx#xrQy;S`Jp15;Mi$g-*>2EzP+;3uSJyxrqg$46@v|azg#87R2G&pqLdh=1Xb6fTd}5S z#)h{{FMSC>XzZC=1yL1UpO0X2JI9`@O=4&X$x4QQtDU_OjN-3dL0l1u#@tRqX`l-S z%w_gwDo>P=iY`x32+i2%D&T2OFskN$F9XR6Tj1w`Inx&B|hZVayPGsUDc#pt(e zDh!vzVPCNeY@zJAuUXVau8^Z?V2RR9Y3oqsbJu8;MPrGJi;U4m$YdDw?h&kE(w3&A z-9eKE_T9?ew@k4l&yA{8C1d7{*Qw=lt3(C$rdtRsa9jmbBy@T+;==*T(&V-VLK;REY`w&`obW}BucgB6(MjwN6PpX2DqN2m&`=w%P;e9YAm`la-_?Z zg3o8)nL0+mw)nc+`~Q5}yRz2N%MpmTT7*bojzJnqlo|p=rZ7m9g||wDdJdpzhGxG+ zLt9tC!ip{CU7WSf;LhpSf~J06Mi2d5hdd_%MPUKeiXrXF(98nRv>>qHV)d(5Cy;{J z+Y`QE%xhN&&*1JM>xWqm0-OsG*Kt8!*r2TK1P&r&k5t^s0v$L`oj9fi4DMo3WHD2V zl@G3~P&gnP=LCUgm#b7NFDb-Y<{4+sP8m>O6&nNrA%HeNYUWe5-EBF<*m@Bwa;A{LIW6;^@cOalchu%k@@R9?~qIS&yr zsn$DT7nb)0k!GJWkH+LQ9y`q2ugy1tlw>#OEDB#iX)E?vDm~8sI;O2wX-N(O<9JzK!pO$XQY96f5d~9Qv9x8ZZ3`gVF?~q{PH!cR2^z=Sa*jYyrIiSCA2K9W$ z<>ifYzS1t35Nxr1rn0>Ci%4%UJduvmBWtp)zHVgabeSgU3c%UxP^3>@%0DJ2vmtyX zY4+ixizkbo4<4!y>USs+$DL{vGV%E*wnVYPj8d+ZZ{^@~p<&$6d>#lV{MfE#e+*lUnE$usH+Xhp6wsCR5p8Wxl5U z_orG3d^d;&8d}NVLtf&og{WaImJCJOZ;D#V{?(Y@Q8*E-WJ4p%M*J>k%ar=UlfU5E zIs{5!g^FotuUNt>E+$*laDY|JfzcBR9}GA!=3c}TJWL&06|#?*M_SOQ$~=CJ?RjZ& z>-_}!bluf{E(FgH%l0Yq^3c-?>PkfKiS8U-e~aT^v(E(h3WLK)sb>ReCQiYzQYbXs z#|@@8+Z?}G+QuRme1q+s=(K#F@jKyj#QuI-J{Mdy?LxSouAH2^Zw*AP7 zZ8DnVmdLTVMYp8S+|5J%g+7}nj(8dWpz(9Eee0;iCE(P9T*+}9nb|7*eS|S4@Uz=} z7F|x+pD0qen)=b0O}TmW=LQ((JY2)=5i^9ap>dmci(zp*wEKG{{`DDO@+ zdpoSNpn?@0l_*qtB~m`Ddw9+%w`a$NmKT*@7R7YMu=HTum}nG!4W<76(L9X}qEycx zb@f!u0~vGItCm&sy$YS%%S`qdAk`v7_kVX=`;;5~t?5*W+tSy@gJML9?mL)0-@d`^ zVr$!n>EL!?Y&%p6Ia(x0ks>B$V{mEl-xAludw)!rDLN;9INM!5`#P%p0f;EUf8ZszNMnlN& znfDhW?~F&PMkvU+CBZ~URZ6kkK+i>5&Bjxdz|yzv~X2Z5zc4KY-2a}%?sviLJgux++nMaZrFNc8=Y=FCo#6esUowmbo*M? zO>|-r!(7%oNm@#_c-}N~D!A2(s|#GLv+d0B&z`wx%&F^NGsnAh?!&|8FXXQu(&!)o z(uCZlC>bKrlD@|)l6k2HlfOYt0BWInta|7UrQs#!gY9|fgMu%@(Tc-2aTbX}07#5L zqX0>*HB)O$vuqJ*r9h^eCe({mPTo)v{p(} z2#FR-Lk5Ex84*JyVkBhJS#4{fu8LU8GZQQ$Oe!iyeqU1_Nvr`4j`fcU0}2gG0T56K zMA1PMQNK_pO7#yMx5>IVhcf7!X0lLLMQJFdAh6M(>%6pYme)UO=5s=e2BIjSp{xNO zoTvpNhap3*-av)NVb#ca-oJuEM0%2?1tMl*Bg7dBbR#8uv=2r@ML;~!!G46BkZS^v zzHL}gER2B&Dg2qOR482wYnWU0iXyq-vH%hTaeRlDN<#(%(W~?3xAT4VZ=dOJl++B+ zL77tG7}lsL5K(gumJ?KqQ39zx@6?t5Cf&L9|M9VTX(VJKy+effK!7Vkg%be4FC8un zvcU(l@b-dof{K#->T5eY0aK!_4pw%bJ{n+Dj` z0!g=qXhbVFYe0w;kRk;n2!Ty>CD_{<1c4-kgp*@Ayj(Em^SM_bY~|ZIT#<+fDstz! zS0*u+%v@X(CP_1&<8a1rqug@QN!6Dqst{@FWh|N`A+)+xt5|^#bXey+L`Ww6>OKpk;)(9qUYTMDkdG zk|pzeubn~?M!(XFu$fc8uRkaMcfXhEPu1pkS}n!HdL3s~)&1^2Wg`*vS;>LN9EvBO ziWsBtq;=O_b=O^W*IR}0R+~~(sc0$p=ILmyJoC>y^Upl<&oriTMMSd4TD0GrgUNF* zg__vwZbjNUrXAfhf6h#-FUi<*ByQKco`U+n4z&75K-o$^cs#9 zr_to`^0BaTF7E5EGx_}Y@ZjO%WLwr1nV5V`Yt41-ef|aphYZnS=J8`gQlmph z_oH^~qs;RuN;>JhIp(c3F3;PE1mdWWcSSK+DhjbsD$`z)t}EEGMZ@OI%*_LaWI$15+M@ODja}$uZV=4$6M9xy$Rd6P=YH6Log$WwU5&5+BUIq6;fx3z0}KNJP(jROp3ZfPCWnIo z?6C_OkorF--9q!ufDn@EO~5KGK~R5>6n@S;ibWLzmqDG3VtazXkdlIzmFZt2>mnZw z9fsj*=gSj}w~zIT2Ccqfm$8=W7@1%sugAu6*L!5+0KE)`@96(|E48%*p_PXEnc4no{pJQ7F?Hh9L}Q zq@;>-*6}|JV-=<;pgQ658T6mh$qKEKLdXfOxt4;hNTMj80t=! z@dJDi#cvSLod?Y6xyZangltUViJ*FbW-7JD3sB;jgwIinktDhvgWQ0r4vP8fn2&gl ziqRZxkrI?h%mfU93Pnl;LWELeQJ4n6j8vpTGX+;50+0+r5MdOuTtPE1015?41t}G2 z6{J)<$B-4lPkGN44wc~FLwq{*ih zDiX*t2CXA;Tw@lCTuRE~ZU+iDW7XiC-@st7=`Vg{ZhnnUN5wj4)6(RAh$1u(38nQw2?}TQQoX zkeJd^#HNB~4Kg%ptpPJJG?HywM35|_8D1c1NQFuhniW!j8G&h5HHxho**1#YHzuxl zEhK2U+;OZ`fti(D-Xm5^QE0NHrcJ1{k&29JEL3d>WQ=Ao!zzfhQEOtWOc^Bzv5G|z zH3_7n8yi_EGFl}iYij2r*4Cmy3P1vdQ8XaR$fCTARRa{HFt8Frk`{%46fN>xQSAdw;^03s(b4I-d)mxBR_A(RwDbqb1yePC%ypitwSJ|qk( zebn-yuGPJhIcnrjg^!Tl31mMpF6_#Cf41eQE{Gr+@Y{eDh)61~)F`f*1a}I6cV(7X z#f2S`kVuB*QStB@AVf?dO#!r4h_zJNtW`x(Nv1?u0Jyn!q-qEVT@XOF2pQsS;;dxD z1zsRf3K3u-2Z=yTgkl_(5Ouo5r4`(O!zchG$g?VAB*KZ4wIFBAs?<~>)_{M~7QPA~ zVgvbHy?X&LkrD|;BuJikQ>hwaVMGalJnSG2;Q=8OU}hK*m4|Jtl%+(eY};9omPpEN zEYgW7Qb?O*L_|ch*iAf2-N}M$$m`!b4F^MuYt3Wg$r4-NqgoNJrp+K=Bzb+y3t3}yEz=yps zyo1*x@y7+G(dC8YhU|YZ=|bTVV&?3%W;VDSQFGvzF#HAviKsG_KH#Z4tgQ)9+Rr`XHEQ7IwEz) zOQgeL3WrafIYcG$#(@RZ*j?u71w6c&DblUd$|%ngvq z8gWsgz=qI~KN+ngnA53>9h-0|(U)!5ZMhK*56gh#hmt$a2^z(jYluh_6Bb@MFeo-VYLu zAQ&q_5e7sG!lnz0Zna%!bft(k{u&L{Y{pP|l9I>-#H52gKODM!9%Gtjc)s%=C%N%l z36zLoM+-ngMOQc^j8tL~5cx0f*0mF^ea#dJ{lVO%^>Nuq5AVXc%fS&j>Y!xz2W-_~iQC9F6-0L3@|mJHkr+dz?1t@2MhbA9 z$^tsiA5x&3Mgz~DB5)!`*}|FS*XNhD9?eL16IaDuKd4O&&j? zr|(P)p724zFtQa$4x^sj;X$en&}s=h^<->0WEPt9U(;)eJ|Z1zt6^W)bIRu!@0R>G zc7U_q>0t`lRjpARI(L{GUa@#tXQnsIBX_971hA`?oCTbzUkWCPqtiF$^d)Rc$4GCc z7_ufzKMwOA$vppgX0wNETaRtES{X_d0>L60vf(&vORte&KLEvy#r|SJ`>TNa&;uiw z*NW0=l(Nztu&UuIwi(dKwD_tSQ zcvm78WzJIId|@N4xFzLGdA>OASplKdNKoqt8T>Ul_6jF=THz~dnarYnkSO(dr!CMD z*}C}DQ;JMyiuqvtV%SL%V!&b3vp!5AtcoaU96s)dW`M?~q;}GUgj*rmt3$e+ktp31 z)JZscFzj!;M9^&ILq4}-a=^Utt#OVQQ7E8^ofn@Ca)#aYb+bC>H;1XJNjm*c8%>N+ zR=U2|4W4m}SgtuVWwd}bRI<8@ZY)jdo-qs&$Xbg`$q6A+$gg{Xj)k(Ny|t-K)J{A+ zWkkwPys%BI&4>7M81qb#sVONW(xxio0~#gQ&=eO*1O(@E*Dr7_(TLB@rkP8O=4;m3 zw{&h?vIrT1G9;QXusJJ6C9+f1N=(or3O)#^jIC{wLJ*vXJ`74h{C@FL>U}p9URJQQo&^XUh;A)_W{q0^&{>VX3Pl%6P>%E=uP+uU0I?hS9 z2JBa&6F=MN*y<+@Lx{UCGTTG}IxR6-<{J3JW_N}otq4d2jnJou;eLB)DyBl+>{R1m zA-j}D1O_S_a&yt=iJ;M7+UriQR{?69II%#B-GK4|-RTkr5uxS`O=L1bxf42Ag@KEM zeZJjwH?#*@G9Ph7qyj4ifTSdj#5h!xLXN5=$+=)*&)>Zv^nj?I+40J=WX2f8;xuF= z2?-ehfGT9&cL3?{8@hAo9@naDFNDZaLv0)Pc+M!U?Ft&XONJdhbn9p5kZ6<1f5q8#i(JL@YNaUK*Dq_T3 zLk4Q8U;$xYM9vZlC)GHu{_$96*f2{_ky2Ng;)_291BHB4ITOKfnu`jEsdcVW$B!Xm z8p2G_5akA-ZjvDdYm2}N^eF^c0Bl}?!oga*NC>614I^cOTzpX(2Q-d6eO!f&#{o^G zyzcr1urQsb!yhm0euZ+6Hm&SNX!4NCY}&OXzQTzjww_-q z!N-dko~7J?<-2Ix9ZCYe_6P~%X9s!-u8GVF3*U$WZDcr0u254!)V0<&h^5j36{P{m z3XC(`S_anH#PD3L#8Kt@;)-p=umiNNJT`NA9QfqguC>?Jh7T^r#tRLGF-2K)2V*PD zqRLT8BYq7c4h2z*9hH_-Vlrr~tj%j10x={=HN7&x z#$%+0(rstcUhjw4^Sv&b-OYU=*pf)D1BDE^xVWjWqjJ1{9*^v)bE!G{D(T_|f(lx3 z4pH5^XNL+j6A`|m3q%1lVIzg0f{~;&geWCpLWbYOHx$Qic}^O&@~Nh{{EBY5Z-1YI z!|rk!EKGGNp^28V%Q9M+(BVl-3LF)0-z5|%iohk^4NQkCuIAwS^waS|n|D42D7SOo5 z(yCk&M>;UH!-eBYQ6j<4WolFcg&G7PL+lMURI!N&loCZq08p(86)5d&>Q3(1BrB4| zZY!YO@G-~oc+27a&L7FJkSW?=^=4a+oRaBI+6ClMGL73F?|KNaz*`jxGkg{tmZMgJpq^ zKS|4DvSV*F=qfY{EBc@Bh{CGIYb zsRMDMSW6)1yaa5cguuYV7@PwPJCmJEL^MYykw!6FR1>5JW6!gVvpgY&1VNZ#Mqz|#(~M&V++kc4bh^MGo_LMmpeqy?dc?59 zER2EIdck!AK|pg)SqBh|K=Tldl2XYiwxLpKteO}o2UEpLfl8dx3$hsym{yclWEG_W z3IHht0i;lPNEBq1<$50v>Ad&tfA4kafA1da$$ejuJ|8=dr=UBJ{MUfmsi0Z-fg{6g zGL#x}$DU2KIIis%g}n+>A8#Lj^f50}6D@?;??r_Jgj3KXLAM;FJUi(O@fI&1V@U@J zpNIOQbUE{EWJm6E)m4ylVN-n7A*kS#EeE?F+uSw)V*@LHcH41Q*K@T)f7KaG;|~Q- z03MdGsXXweE-|Cy+=Y$t6C%*SQZ<9v>g7Yx1zKuQJSgmcp}nuLc-|}t2HjIDC~Yh{Hsf~YQSAp6AeVH zXt5tP(6S9xw;t*B*J3x9^x8s($}Kt4_c6+l8$^b9ju^UoI2P^Qk|52Ts~%KD3)lw(9fiPgBUUl>}?Z%3t&AyZ8HFU%YR}@2~wm-LB>u^4$HO6^Gf%vKr{|`d7P*7`&dFAMI}b z?^*PEmA>Z>xcfUiEp{9ouS@Cwm7S$$S}SlBe|g3AVkYiuTs*@t=4W*wc!l`@Ndeka zy39A)`k!5r`)s%9J%;-)FKh241?ZH6_sNeK_b+PC)KbqA#W4aPsi7)SDN-hysv~3& z)J~hJQG-PZuxhAIlZmLCO;}@O5tQb%Ml}|bRu&~LP}NPv2^@}?qNbx08W6_Yq>>sM zKP0CaWI!Akkx3P+Ry>9_!t^Anu3+eji&4ax n0LR4i9G}MM`QKJ5FP1~9FjvvYsPC)n&;J*4ML1B9=DWpp5zsP{ diff --git a/inst/create-list-available-forecasts.R b/inst/create-list-available-forecasts.R index fcac2950c..fc4926797 100644 --- a/inst/create-list-available-forecasts.R +++ b/inst/create-list-available-forecasts.R @@ -28,8 +28,8 @@ metrics_quantile <- list( "underprediction" = underprediction, "dispersion" = dispersion, "bias" = bias_quantile, - "coverage_50" = \(...) {run_safely(..., range = 50, fun = interval_coverage_quantile)}, - "coverage_90" = \(...) {run_safely(..., range = 90, fun = interval_coverage_quantile)}, + "coverage_50" = \(...) {do.call(interval_coverage_quantile, c(list(...), range = 50))}, + "coverage_90" = \(...) {do.call(interval_coverage_quantile, c(list(...), range = 90))}, "coverage_deviation" = interval_coverage_deviation_quantile, "ae_median" = ae_median_quantile ) From 6c213fc71da27056276dd77e7eeb04e024384d38 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Tue, 14 Nov 2023 16:14:54 +0100 Subject: [PATCH 6/6] Add global variable to fix failing test --- R/z_globalVariables.R | 1 + 1 file changed, 1 insertion(+) diff --git a/R/z_globalVariables.R b/R/z_globalVariables.R index 28bcfb95b..89cfc2eab 100644 --- a/R/z_globalVariables.R +++ b/R/z_globalVariables.R @@ -62,6 +62,7 @@ globalVariables(c( "rel_to_baseline", "relative_skill", "rn", + "sample_id", "scoringutils_InternalDuplicateCheck", "scoringutils_InternalNumCheck", "se_mean",