From 4ea595204b187a2f374d9303537f67bf2263ed22 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 9 Nov 2023 14:27:30 +0100 Subject: [PATCH 1/5] add three functions for overprediction, underprediction and dispersion --- NAMESPACE | 3 +++ R/metrics-quantile.R | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/NAMESPACE b/NAMESPACE index f29d00f38..fab2e17d3 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -29,6 +29,7 @@ export(bias_sample) export(brier_score) export(correlation) export(crps_sample) +export(dispersion) export(dss_sample) export(get_duplicate_forecasts) export(interval_coverage_quantile) @@ -42,6 +43,7 @@ export(make_NA) export(make_na) export(merge_pred_and_obs) export(new_scoringutils) +export(overprediction) export(pairwise_comparison) export(pit) export(pit_sample) @@ -67,6 +69,7 @@ export(summarise_scores) export(summarize_scores) export(theme_scoringutils) export(transform_forecasts) +export(underprediction) export(validate) export(validate_general) export(wis) diff --git a/R/metrics-quantile.R b/R/metrics-quantile.R index 4f3774a96..e7e6fbfed 100644 --- a/R/metrics-quantile.R +++ b/R/metrics-quantile.R @@ -62,6 +62,27 @@ wis <- function(observed, } } +#' @export +#' @rdname wis +dispersion <- function(observed, predicted, quantile) { + assert_input_quantile(observed, predicted, quantile) + wis(observed, predicted, quantile, separate_results = TRUE)$dispersion +} + +#' @export +#' @rdname wis +overprediction <- function(observed, predicted, quantile) { + assert_input_quantile(observed, predicted, quantile) + wis(observed, predicted, quantile, separate_results = TRUE)$overprediction +} + +#' @export +#' @rdname wis +underprediction <- function(observed, predicted, quantile) { + assert_input_quantile(observed, predicted, quantile) + wis(observed, predicted, quantile, separate_results = TRUE)$underprediction +} + #' @title Interval Coverage (For Quantile-Based Forecasts) #' @description Check whether the observed value is within a given central From 4c075674c25bb3d632b880069b3dbf0c4b519123 Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 9 Nov 2023 15:41:34 +0100 Subject: [PATCH 2/5] Update `metrics_quantile` --- data/metrics_quantile.rda | Bin 11747 -> 6730 bytes inst/create-list-available-forecasts.R | 3 +++ 2 files changed, 3 insertions(+) diff --git a/data/metrics_quantile.rda b/data/metrics_quantile.rda index f1e231d773161dc0df58ab5efd8fb8ca1a0741fc..2f2607933179842f7fcdb9c19e8ef140fe59117d 100644 GIT binary patch literal 6730 zcmV-Q8nxv@T4*^jL0KkKStp_Q;{aPpfB*mg|NsC0|NsC0|NQ^||Nr@i|Nr~{|Nj5~ zG2|Oqxfj>VBmCQ1sGz6GKKI(@g=RKmnsoG-5OWGyuqHq3Hm@ z8Z-ug(@jJwJyX$6(z(#=6O$<#m zG#NB#XwYQSOc8|B6GluSN>eF>n5U+YVj43@10ZPAL690502%-^0D6D`$N&HafDHkl z000000000D0x2qFn^by7rYY((O+k}Jm?lO~LNw4aF*MT?Mg%fo1kr?fModjK4Ff>J zWW-=hO)v?B(GZGZ55a_r0}$;w($Wz)v`B&}|8e3=hA6 zl$OenA3A^YYL}h z6AMfnf}$vhs-d>J=H8riX{Q(de_UpWQN74)Q6)yDLFOCYXU}yX*UbRV9d>4_6fCEr zav){|jNnJlk`ZVbJo2`ZIR@374u3P6s#fq+Udpgw)K>T&T=3lHbHOzvBp!+Zd7S4z z8bGpU6Rd0v8P`;hQ^mqii0QU91!E{~$V;&h8-CbMKGz~x$$%`YTxGt^u;sRiL9$lo zKK9_;w0Lc>)p{)K@@&p@)bl zq?mQ+V^&UO31JCP4p3?DEYY5im*Ux@%b=>19Ugm*({XDXDBR!&1#lmWV z>5yyFA0-K|oXyA%b_CWyY{|QWTyt}(qL-TOSSCj^F5Sy)=DUu!`&oC~ciYU|AT<#L z?p(&0@T_Ts^N?U5v5f4D8VpS{hhh(vA_z_{NUykZO=47}LPwB@!SV&jmFcDGXTmZx z^y%9Am^~@Ckc_VL-RP^%5?~penUqL_REU5ifUyxnJ|>rovCG%~JU{R4Gx)MO&JZCK z*QId#^aHSo%lHu$d0ARTpr(M5Sbd3IAwm#+`Vi-g?#K)QM8c6LosD_3z3p)G*>#63 z>0V;^P`TBeW3Lad5+OVW145pjlDp-{UX4GFxa!>j44~?8)c97S_XHLd0AT?52Hwh= zGlw|40*SnC86Xl4tT_*$H&YEF?Pb@r)UUjKDd88u`emePKh8&-y_z~FaP@~dFBtR^ z#e6c0D1%q-@h>RN;hxan`kiA!sA0y$-ZsIPRWcq#!L(LQm#WL$jF_6;OHVm|BM47G zJ<_g5zGe*-Go2SF6vh2bY#a8(AiX=;d(o(uldsc%HxDj;+`fER+e;|pqtnPjJ?zbs zccH*u+_p>`4$qd?X+cay%E^HS>IJ+-jwWk>rOnf=rZ+1FF4F<<~)qY)!=68dgiBY$MW0#8;Qsfjlhih~#LJG$w z0Io)+@dHA*Fes2n1b{juPOnE7XJ2O)=@t&mO+F)&*4zwpN?n+uU4T1VJqE}I!ECA< zNEZW1;x?XRf)XhpDb^SiX}j9jxbe=@(DdJhR=1_gP4Jw}W0%kKAwhlSDE0!^zfMV2 z(7XiW;x@3eTwig^|4WT2B6?L}!6o{;ww= zSBmp$sbvcB*&%#D7mz{8?({Tp68#@$A8po>2lpiOulCZ)@$-&hJ(0*%CIK5mYbJ$* zRgmHgFObrpg2T8cR5`73|y{>Gy z#>+~|vKbv5!yd2O)1@zxG8p*NDX=?KOPNW_gELz8y&|QDG%U2xNr~L6zBwe@;z-pP zHO8$4p}CC=}sMF?lmVXo{u+Rom(1!Db&yK#?a@YKUQQ@SFnB;D_`frrgET zVcUn-X6Uvb z8YFs#En$E#_}{uXFhwxIUq><5%@p(TB;RZHjE1H;g8zlY$LaJb%os9ISjI#tKnSvc zXh^Np)ERqmlX)GcxVLx`t?=Ll#H)a+*Cmc58JO2sodlk&?l;v#=|~Cp;jcUV_mp@u z8X)G?moIL<%GgO^ExiEpd{f3`Y*D&KSmrx8s=`IGRp{X3qL|XI?Q4y(hVXc;d63tl zq-AvSx3hcrE4tf}Vd&_JpTG|aHV8B{z(5Q=xn+-2f55M&)pkQO|*IjHpL>Lb}B*&<@M^{+Z)3pz4a z;&WSZ4!<%b?Bv_cwe4HX7`DjZKM-d}3j_-Y6TzsHMLSE11W zinCv?v~9~3U8m9jV?s(TP5eyj^50Kc7Mmx)1uCEl)$@2E*jrk3(t*IkKv|_oC_<mMZSWE|K4MG;kRDsS#B)73Uj84BmVu@%t zXDlHVpE5+mXQn~fs7A9HP^#P_FpT^$Z60`20Xd#(r;N6A9wLsB6i zB|zs+0{i@XxX4C8yfnGA>g-ohjT||=NeFj`xS`Zd3hb}YkUT`}Iy|l&2u%2S&d{LY z#u3Gkvcz~=au&tPTwizPc}o z%F@Pq&AP!1(NNd{DuE+dx@8)$HDfjCsi*-2z{m5fTT-g!EqLv{ zZ>`{LV&q8z0nh;ie9v{rvEvC*9?Eu63|;n02(022POh8VrI-j0vjC~=P>M+q0MkKC zfa5?fp;cD#cm<$IFI81gWG>)4xN^V}UEST|R)9nTk^vA3NCZODeV}PoTC*lF#KNEx zWI*Df6TyIp;GiTyLnj#nl>@>RSnqWH1^gJDA(0hhsC<3u1BTc%7pTyZJv8;yds@Z~Xo3QBVFZd*1iUMD#uHd*1iHd5?Qd(bUsTG}BEq z(Ak@9xy<0J$@F}6;kEO8ZhM$m9Ug+0Kk5{&bRYif=ASqI?WAhD>~B%~_r5G2wKt(V zqd#Kag%0{p*cQ^MTu+}qeJWI_(iE)EA-utDN^{iGB$Q3INQJ^FEy0JZ8dl}sXu(|46uB+;eKGs$$(Q=O4+IvlI40m%%i ze1a}<7GK+JRw3e$V#9UIkW8v<0v^aAVj@QHb)kI6*#qhZRX;%)&{3R1Q4O&R1Qc64 z!cR~^av4N71Qc41**yfwJd$?_e0tR9C_hVnysoxf#L-J1ly@XMpEtsJQ@bIVtUIK9 zVVLeEg(#6A-J|Er(B5zeyV3^$kROUgPKFMVAT%d3j`eX@LP1bAi9k3Kftm}5A~}rC z2=c_BnrVmvW>|=fO4))2LV+1Z2$=vBs6C;6`w2WpTxd2k&Dg=O8E(m5|AwV>OAxhANAX1cxgacEA<$;-zl4TSYxWjK+ zYBbfX(@kK(n5G1z1e8R|$dNe}N|~8!Lt?-Y_XMhmszFr>5d>xm0EuWQh@b>lRP7Q1 z8cMVwMGDdaD-sb^F)ay0Bnkj%DscUe4=)4SbiIhZ;?V)$Kl2G0ypg7)%>S-_{ zL4W~yc(gLr_6fgcOn@6%qtg)DtsAMNtwfMMRM*gh4`qL8Vd@v_uU8(=xhX zr4rO74GjuZR0_=nB^5vvutbW6?gR#q6wp#qP!a@Tq=6(MXjwo)Ss8?&VU!_5VpUa; zlAq)NzL~o_O>+gPxtp_Li?zXDjKt z4G?#+zkv<(g(#?LA|#9eIufHm6jBgE6palOg(F%L3XVa;LI{}x2$mQYNJ8jgR#1D; z2C#EoW_n_aKLChuVNl3LAw+_R1_=eEP@;(-BuR=2NMT z_aikuQ*M2zr~EdNp^*it3ITN^hsy97(^aGzs)}PvGSD;N(LlpPIz$o(WYi@{mlKX( zy%=beF#dFINLwsGBQ!BX3k3YQ;T9Q4#kOqLF&LiKVSFr|=jxA-8)crH*=Dkwqp!xo zxItWhoO5Auz7!0LP!$4ICPr9zYk`Unz(KfK5GGxk_0HFcEQ6WkHz>k4_~8nJ^)IMJ_hCnYgIx>6>3hil`J9wnDmZvyu#cW ztF~(-7795Llq7&A8LgAmC=`}OPiL&5P$|Wlj>{xo*7B5GuyO&JAPI=M&}E)UB$5)= zxJcTP`$Z-*_AXKJxta8HmYPC*vY|%8LJg6SC6#J!1yy5dNe*pz?qkSx=F<{Ei)mbr zn_SlLyL@9wb!}TxNJ0}K#D8RK02!k-En^U&>535uB53fypv3d32r5tjb?y30z0wGF zd&GFY-+ToM<+>AKcB1e?Y3b_jnj7l?OE%E^k|Kl^2ZAUsMC2vEUQE0B^T0wa+#9|UHwr2Nu?w$p~*S4d)>m>Z{EbvC`*@`KM zm}HU+3eY!;YNS;M7_e7y7p~#7ShY1>xt}J%sUV&wmj)?huR%Vsi^-G%c4kG!Mdm>) z$kNa!3(LjEF>K_`WBE0fReF1oGcu8YT5lZCUA#o1(n+Ypeix$67>Xe!(%*p!<7i^a z+3ZlzNwpTBgNh^E-#EZzWuxy1&b8Xuqr2IL;?Ud7!n8}XJ)O#zB5@4Cm6`$AV{h} zuP~!0Ct1+s;;V$Q3sD1e(UxowW3kEVw^2?}*dNi)?%&q*S)C5Y@Nl~NZHf`h%VTE~N8#(?1x z@wnp&5FR29CJi0Da7jo=FXGW+!edorZ4v(n1Lx&LjVzK<>xaOkC0*&G=z|n5{iXHk96Eq@B0)#>l3W6~JqI&;d z>-&E4Z~6cC)|r_u#hkW8oAB82`YsOdEPBwlVHw&k@~HFTy8(_iiW>JkM+hP`E@I26 z-k+Xg{2ud#WW9dxccmYt%C;9_R_)?!pb_C6Kmb$2!ME8_tGr6K_$}s zFk_NNz_GXwhS1d2Vj|cNK`_vy2{dm)5J@2!D`enHESEsd&lmuS&ko$>1NWrt_LpUi zFLy^iu2~v(u=M+F_tF2q8kZjCCzH+HRR0{>_4ZQl#tx5Hr2(8^#_q(6FU~aV!@V)16=qu*>{^z^cPl-@2ae-doWEggT| zdQR5Jrd4}E*sD+rz!vXF7o=eOUQv;8%)Rsgg8oj5y`bhi37NL9j% gBU&zft(vd8!C=r(^+S&&C%^G`BvXY60(u{QFkCG$1ONa4 literal 11747 zcmV<9EgaH9T4*^jL0KkKSuAMxy8y#1fB*mg|NsC0|NsC0|NsC0|NsC0|Np=L|Nrf8 z|NsC0|Nr1NTzrS6-R^?%&vz(B-+9(<+vlCbfabk7GcT?-H`~s|wdZA}JE$6orfRCT z-OhHI?%#YipaP7P*@EZAPW#iFpK<2S01rR_000UF0049*ytLwa7k4}Cc-R7>6dLqs zb-mr^qLKjjkop+_^V0B1BoQ#083RBXWB>q$CMoJ>qY1Q)H9Rz_{-)BWrl#1Y8dLR8 z(^UN)ik?qY@CiJV^)hLM^))>-o~G1zliC#gsgu%po~infdQVKJsM<}G^*p0#sM=vo zU6Ovw^s6C+8cCPqdL2Br~+!2=Ub8)Bw-{rh_M_Y@h~*r~n3l&;V!;D9``_pa-ZuKr#RTM5G`z5M%&NF)>W1vYx5r z#Z8p-+NYq?B6>AFBSzF|w2x7t>SQ!+BTWDR003wJWDNiS0iXZ?007VfNPOUc%1VHL z=wk?>??3&3sYrp!K}?m@3S=*vjD$){%;ExMp%5WdFU87pJb?7E?0Ce*K<3fj$Ch`W z6W!EFT#=A0pnGJrKo3R#a5I2o&8X8q`+{q__ysDdZPFb%6zXGjcBCY+-TWd>uh&ya`)hBq}k zMiaiBIvadI$qHzspqS}qRRr&eofzbrhZCc^>FbTXw%#B8U+0>RaMTL84DY4Ht)N|| zjdWCjg$@z*mI|7t1@dRoFcqgJ9k<1(={njZB1SrnoSedPqdFxQCsn9~B1nLpHNobv z)`yllv`L8A(rcXH9$eJtzu$J;if9=HnkI5Cq49H1WG_h@{I0ZsQVL>}EzB_D#HI}n zvz-8;0vX+S6txREBqLvxQ)LueV{cI5%V+PF#aOl*3cZ1xy2ye6<6UDJDJU@$KY?oF zZy`*m^87&l) z1t1|oF+0Fuhhzt8BLHdm9DP-^za%6*(i9XNxBR{hL#M{kr6q+j3yjAdD_yW}niT^&@L|m6SWLC;5tXM$RhYmy zEwL+_;ISDR26;YK#g`rvOUAqW=DL%H&o-@R=P8TMXfO{8%iI1tW#_!r#i3z)`7wa#?L79nHyop-nH2VToMS;% z8Dc@t!!=bz7lX$ph@P%$-q_m5TA6OTjB^vIgr=R?VzsJ8<6?^ltU6d(-!1N#Y3RER+5lNApnR%AXXqj zT!!a`t&`*K{G$`ZA0_UK^tD4{Zn7|tJsDD>IpB%w z4@r6&^(xYyYm6)P0uVuA>~PX;#agmLGVNoFrfk>vQ)%ZnP}Rg zTlON-Yfc(V3UuAPsNvTOo`$`Hr)j%xPQUZ_Ejd` z{bzO^*@c@385nn-ws@W8jN{L%!{uu3QEl|J$FFcqd{Z&qiIRAK43KL0-FYIzFnBOa z6nZ${pmtmmJ5GftJ7ty-6Ig6t+<0pPO`1ZRLYhqM9gLtnLV|3%1_e3#H5E2z4nS&ayz zV%sIl*rN3v;xUcjXp$Ff7ow%@372r6V?eRYfy*U?VURf1x?31eij9t!7*!();4;*t7O#H% zb2CpW5}mP80o4*w9fTPrNkw_Dc#|rkX40N*7OvlX(}o#_5Vz1xEVuPxV=eIyxe7oK z(27Urwb6mDr(YCNM81Dn&)tTYIdV10`Ef$Q)+OXDsZXR?@J;3@ynyH!pV*Snhy%4!jnw8 zk_a^}#fv*lm98Co+SCWg%T@YVCqVKEWrFPv2~tUOJW{o`Kro3JYEdgzD6bv% z2c@ec1}7BsOk3%BNnLN_#Rbi70BF%{%P%A8N2k}zRTrVdHW1f>l|IeBS^hQ5dN(iG zYtz3bl0TB}QA2u~b}NK)wIi@Qp?fchwdkV6xc8-7dpD0uVHimXCP_TOflx;ha6Qm; z4_%ks9ReYXEUgGa@8e>?Ox#09pmmu~CSE%q@j2 z70S5@Gl@r1fCem3DT)=Il_=9>Zb!O8r%Vt9?br4Up=q;$BWgpeV8^2XSwqn{W_a_&G!u~V8qiPfL@H+T1{QgI`8jOvbQl_2EY-maKeD%eh zatms*Arz8?Ah19_8Cp44`9y7=i~UO|jQ*|%py?b#6mtog6y{JyOp-y|k?Ell$d3#M zG^n|)rN~6iL<})1gSFFMGtxDzTg==unkv*(YsB+hOan-Acxny_?cJ0z;wGFoB-33y)EH)#3p$T%8wb%ru2qaNxGxSXh+`MJ|d5 zQ*dLu1_3+ zI@6ABPenTVrT_HLn=;g@qsb z==$}XZ%gZY`qFr?wRWU9RbiY%xd4nnAeeaB)zH zW`0zA^*xK%;iC0n#_v?M6ZkG-GdYb)-sD4Ao(scEvpB>x;dRifQ&RhaAW{-asE<+_ zbx;c67m128CiuMBski67@V$32lc0(jqthiI5RJ!q#GQ&Z&5wS}U*UIxQGJv43+2tyKAA!O8bx4JJjHMMa9(BVPJp2)%Cqi0{RYHTw48Y8l zI{~E|khu_~a`y!f7C4D5?a$ta@=TXK{-N)t>P|S+kuQp1y4Exlz~GtEIG~Hc&)bQ? zofXJlBT_I)h~#efe3x#~xa6N4&@0*<_+qgdT?69SjwzPHZgi~a1x(9W(i+|^?zSM$ zdgdcJfzaSiQd1ccsx@ zPb_1S#}F4NV7zF|qm@ooO-TF{rMqeX45`&(R)$3#L@|JejUcIDXt}t8COU?uEn(EI z1(ykw<2jerYi9S(8ruN!bgOo>O?ool$eWl%xa0!*3iHKvGH8t$X9jEw4#P1Va5xto z0pRpXK!ok7L?Qs4uj%?`05eeXI~GTuYzKq_Y)~1nKtbDG$Su%Hg`S0L0^wt^BQH3f z(gj^P1_7(tL2R0)C1It;H?ynSp<)F74?ExVV>Q=6tP~n!OiRlT-T`A->AH#TxUvD{ zAPi4g`q@?s5RNzfMdAjmbv3hyLk!(r04S?E38rev9sKxNy`G1W2a0nC$56p4mTK)} zuRuCz7#0PI6IdPH7%A$MR0Az_`5BB%i8N`|Hbvn!@kgOPJyxD=fWw-~AV`=gmX#n9 z(ay)+RkGag6n-OlJAQfN5Z0>p<;mw$76y_$}RR0I-(V*;f+z`&cV~aN0{Kx`nP2lBOK_3LD>VhuLm@lZWfp?`*;1>XhCd2Y~BFa;5dOi)OmOlVF0 zmNA0yL41pf0sRQLGH?XbZO)-ItOe92=L&Pfa6;1^25WVqX~EKlK9v{TV3%Uy##)~| z=5gK4znu5IPE@%H5+V>FI0%KzwM3aXB-}zx%9nhL3GYXAc)~d}_;;mJ?defgxQUZZ zc3(ChSkOy(Qx(Rm=R?${#OTAch9uHL2R#Q(Gb|!9T*&8M1lkiETwp^)Aa4Pk`k$Dt zGdaN2z8MWY3fn@roYZZ(gyDy_@2^DZ(5kYx0VZYL)uXU2LV5@bP~z+bt!@t(B|<_z zpn11@v!psa)oxT_jpAr>P<#lwqpqJ|n?cH!opHBL_Q4_%+dbV}#5ayN}qVF^)#f?xF!$@S@2iUdSV3 zYW91?~M0yj{9GqId>tTTK(^pRBvSNpk&r9

aw~@pW z3k94P+g4;BknfIGJbJd%(DgDsX!stBrQ%-v>QgVEgNY#sLLAZ2DkwRK1w-Byg+w(t zulP7J_F`z+f6Tz%fr;OsQN+<9Az@&G775E=xE`ApNm429wr;+MilRHjA-K7 zIcc}6{Y{Od<6oU`z(S5P3Sw*;~QBCQ}SjUf)+K*{;&QiHKHQV&cxr){k#2d@Jp_$aracK4_* z1;0hUzFwo51}@Hxh(VXJ_NJS+PDzOAXGX=MNCrbIZZaZ;qd`*r(5YSLa6cr6sLr_xS`1t4P zhe<ZQQPUWJI%?9G^0UiS58bSym_aU7NTTSYSXv`ua7%=5Zpf_o& zm~X*@4X|)?6VI~xsB2`jh9LwUSD7@=bacZ^!3PX5_MHmxf$|;)rs)7hd>H_TCg-x~ z)+#Hf1wh9E(m@~sfK-S`y#c1uR>vf)w*@e5+DRi5yGE5aXhse~Fl=oDZb>Az{)%I6 zvd~j*t@9=@z{0lf^lNN0FvuusiYm-9y4Ymf_j)x^Rd1Oz6g1mmk#pujRZDAq8j7Nd zYN!fnjz(cIF%dJ4V@%9X2Qh|eqA98cPK86n$O-x~2(IFSCXgW*g8j<|!){5_@%wLy zZh3|8N0ERV>rdaYa7ki~zoJu;G0HI|AqD>$qO+?nZnAyuxVPv%u3u2lv_EOkLkpbe zj_M?V(8CNcvqKDDZ_|Slzxr_`#`CwIUQb~;-rTB>=A-x{(SCZGk8(R|^>IZnBoIdwFvO5GNUeTMyoviC3ANl#?JVVZ_g#`m zcBwq2STNqN4R~Dj8{)&3JR?FH}3e{Vat}bn^mBP6$ zZm!ZVvRRp##rq@}J!7|w8`oxNIX=6t3zNtMVn8Aa1p|nu2=*C%=g9qsqV!*Xz(b4- zLL@@O3R2^#U3fHA0*Tf2*Zz|lUU=T$O^&ASu_StZU@db;4k@kX#sE@d2DL@HOF?rt zaVucc+F1Lbp$1NeOW8#UB$8oOF_&|SjG!o-1S03+$w*N^HOk_+hzWfdQ4!OwEv^%b zON7rQ&UUoy3Iu*Gn{r+hP&CA}63^qvG?CgOE`*L6!cV!0R(K=wsyGN4M^ z#2BI|eB^=$>_EN}P#RF~SE#s%=#>;hn1KZS`B5`M;wY<>U`B|6+8_vXb*%NT+P+hj z=QhtPr;zlAuq(qnps5H^@*@o{Qd}Uk4ickWCHNOme>p0z7sY!?O(YBuF5M7h1*EOImJTQJ(85 zoN-Q&Sw3|;Y0kgixkW=zhy!)?PDfF@p;6R-Ew1d|xrF+QIZO|}5XElDo_f!el$|m! zksREck%>Tc;9sjX(1n;|oDx@A_4yBYAt*c*^_PN=ypIWBDAZZ}2W}MAxLkF=IgT8o zw-j$7ilLAI5tL%8WB?S12}p^C3Q3Bi@WSg65VWo$ree5)87hEKB2pt6h^{~-C=DV3 zN})0W%mPLPV67?fXvG^!K{N~=O80I#Kc_^u(`XPI3gx-b%~ z@P(ivN(Lr@nGyC~#8WhdwipBuQb7bc0zr$05r9PrM0^g)3TE30MF_!$IRgY_6DDR8 z4b5UxXvQ-PAc?Dgc?(E@^MZgTr3Dc<%qUW%%8Vp{t0IIbK_Vd`C_y7Y(14WC zQqv5K5KStgpg4+=Bmg~9A%TWUf~imQVc@SB*YO>149u?M6{F? z6g3qs6G2i`%27i>N(}`F1l2J~6cp6e5Y12?#=hFoMcZkR*gj3qr~wB(lVWO0y+I#LP7^6fg zjKCrR!lV@d@o5Oyn7~U9HJBXms2bm`(yhpD9xA}!QTkd6XFbEXv&nuJ;^-H$fF9O5 z&<3O_6+;*lJ40*{3?Tv3D)x+~%Q(QPsYGZ3B4A2NKuQs!l>mup5`mzI8bzSo(~1a& znKuFyL#q$az#{DmuF?Y%0EY0hA`UQ$fJ~s^!UBZSIjblD4LWGBb*K;+E#m3-TWbKE zi3E$4#gR0mA}q?`5h4kxRMP;a3JC)wFask9g&-(QGYt|D1gOzN5lj*=QACkrVJj3- zMHEptwcsL{lC-JUfWU(w!UbTJ1HDcBE0ZzqY!op zQ3!xwgdA0mw&ul6p;J%>GBlGYJRqRlN(3YWNeYCJrKHIxv=1B|+1PwT#!8&zHt17S zzAdb6i4rI*NU14AnE=3%SRm*mi9Au_&X{Y10PvJw+2Ugw!)xfsH3FI&snx?|fS3e^ zZUAG~>~T#Z`=lelBti&8G)TA*FhUJ5%7ifbr+|3VTf(irTAH3ojH}vV*{Eoc7bp-g zZqT6t3fQrTR>n02EP~b$QU=w5Q9=y^J^lSznq#5S| zjE}TZ(JjCV7_T^tL@Qd*(C!1c?)$OvGr0m_B9%PrpS_);U~i*Cpc)pDJWV zm?$2AsCbGC*{VUK07d}P#nvHV9~LYg0cioG7ha79_j^^r+rnG`yfxpwT92>5CGotq zfrb~aUrLm2gMY>!7;C+LgJstJ`s!e<@FT`{IcPTtLI@5Lrvef<;=YV{VC#GraviP? z*kdcdh`>P;L9MjtA$&}8ocy@(+*c5F{0KIQ*v274ATact5GwwwHnHEiaz=M}H@elJ z7XY~7sa8Nz)CmX*LXaq+PiFfzl_6e zxdq)VG2WPT*AcVg4Q6aPNTxJ=bF)LjZRPi$dej#wqlifmwJE8g+n>@P=coE?C&S4fyrJ7Ns@rq+T#iljrA5n-6}suLmp$!YKI(c3?r9ZN7g7>S zmz0a_hK+zkmoUckDMMgu^TSwz7D2g$n%cGlX2u7>tQ=?v9C`Z}eL?~q_R$^_h2z&z z0i8Dt)B{>8(S*nE*)+D^ok&@3^1uK$`sNG1V8N#zoE)oUI^C@>qLyo3rqcxsTAHAc zB0x;=ofhfQ$5Z2P2qK}ijbG>jqP*qw zq=iI_*bXCoRb&^zL7LTRY-Z5TCVY_qEijpa2&7Ri>DiBn+3~Juz>ts#Kn!GDGN_tx zTp4W>S5y@cGhy2p277O>Ei@Q-7X>r28?li^!1^K=P=Es9DhNX9M*~Bqx2z>3!Fiby z5Ne-F=$JSQb@u>tHvrr84}fmME-z69)zqR3jG;1y1VT1(s5&U?ma_Cy!t$-y-ruad z3x|K%QuYw8A&CJ3mcqE*&zh`+ZX_;5Lpqnxu;;DfGuDEI>&1jF+fWsP?P8Y&43(G7 zN0X(l_a{U!>Ko>V2Q*y;OAVrkC|6s6*r1q^m|&r{4MqJ`E!4Ma_f&dY6%)QD5;X7M zz$|GIn0=Mfy9<-n!~ODBj@4(9GWU^M0Ali&_H>h02%Ez zE=z(4XpZ%cft-NdP;IPeWV2AwptcYffBT97TuRl~%YOBn_VVy=NqVc6U4>VJ_vxf>8whlwgost94uBcp zj-HOBUbX%nZTGlR#~S8_is89i7DTKBpc-rzJ{CZR%fyV(5 ztPxGQO|FiWMx9^jDO@Zpu}K#OT{MOl7_^YslM%2)hIUd=Frm&mmDUb@;2%#6GODT7 zP(}RF0xt>yx&J-RQANl0cd^NjkbNt4ilZR>$ovli;1r5U1P&;AdFP$2Y;g8|zM%5} zJM4yC1TYL=)uh- z>pA#YB=hbBX(tAAXvaf4y6{kxULlsDC|nZpiVQmwr+9`89lGrh(F{Uy z958$Rj@ou3Vj~=4F%L)bc!t~91?4bIc@cw-1_?q$1f2C<5{#x;Nz;H;Dg$?5D7BRW zqAr1?WU?$O5e0OBHUrl*q6+XLqL5ZP4FQ-24J5F~2`~)6DB5&7Qw#$fXafQkEGJMN zep9O&lfZ+Lk^#9U&L$=)jXVt^I*=$LRHYF-B%(VzVgiK@4rT^rV3RIj>{w!sAcmFz zL2#kGHV#FGISbd20w8eKDh?3@F%VE81W7E?6bab0L`s23B`6vsg-MYLKpBJvfK{Xd zkx@!P5b!;l!*`xr^?E-yt>=B8d)(^3vQF%Gi7WgM6T5remFRjgF3-9|UL1lmI}pR^ zr1X6Z665IDFTw96D{yzCB*NgHi=puEXcQq>SM-Dn5JQ`{waEV_mTCR>|7+7+Yj<>i z3w`@Tg*dZ(l4AA5%;lLo%^qpBp#aQ=_Wm)c1`?-zbO9XzO^ zj7RmK#(T)`xvwe8DtG%cI`vmdB_HzG!4!x!WF)c`$mbA|4`%!JN^|}u3~P%6<1ozW zdU&jqh>Hfjl**ZM_4cYzv%^U)&UWL*spAh-{lwx>cjfP}@+1U$rr6%?5lhX;yNH6~ zBTuAr1-k9`^3thw zyBKycIZdBo+T;vd&aG?@B3FmJa=0LOU&jJ}5bgmwT`Vcp1ocyCQ*Qe2eAxM2s@~;x z!L_yG@Kc`CRr|jWdB*%6Z{_(t-&^x*Eb?&=CYHl*()k^=hDV#Z(6-^)($Yow%`3Os z=5d-&m3dDCtlv9BU&{5byVNI-`O^z%F?;20BQdc1G~ zcW=aWni*=Sb6&p#{4d1&@4oj;&%XbKnOUM}Z_UNGv1zQPJ+|%{t31d~Jh_cCv-dCd zT<31g$M!Cy^jr9ux43!!kIx~`BkkGmrQP?1PNRjX;+3#=9e46ocGfv`ll!TBnO~Ct z?4`@%!CQZO4ta zl}}?$8cd&JMxWY-pN(bOztYJ0J*$g Date: Thu, 9 Nov 2023 15:42:26 +0100 Subject: [PATCH 3/5] remove hack in the new `score.scoringutils_quantile_new` proposal that deals with the wis components --- R/score.R | 5 ----- 1 file changed, 5 deletions(-) diff --git a/R/score.R b/R/score.R index e4290287f..3d1dc7f07 100644 --- a/R/score.R +++ b/R/score.R @@ -295,11 +295,6 @@ score.scoringutils_quantile_new <- function(data, metrics = metrics_quantile, .. fun <- metrics[[i]] matching_args <- filter_function_args(fun, args) - if ("separate_results" %in% names(matching_args) && - matching_args$separate_results) { - metric_name <- c(metric_name, "dispersion", "underprediction", "overprediction") - } - data[, eval(metric_name) := do.call( fun, c(list(observed), list(predicted), list(quantile), matching_args) )] From 32239bf8220db7edad5584afcc4a55255dc0cd9d Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Thu, 9 Nov 2023 15:42:59 +0100 Subject: [PATCH 4/5] Update documentation for `wis()` and its components --- R/metrics-quantile.R | 92 +++++++++++++++++++++++++++++++++++++- man/metrics_quantile.Rd | 2 +- man/wis.Rd | 98 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 188 insertions(+), 4 deletions(-) diff --git a/R/metrics-quantile.R b/R/metrics-quantile.R index e7e6fbfed..f3598690e 100644 --- a/R/metrics-quantile.R +++ b/R/metrics-quantile.R @@ -2,13 +2,92 @@ # Metrics with a many-to-one relationship between input and score ################################################################################ -#' Weighted Interval Score +#' Weighted Interval Score (WIS) +#' @description +#' The WIS is a proper scoring rule used to evaluate forecasts in an interval- / +#' quantile-based format. See Bracher et al. (2021). Smaller values are better. +#' +#' As the name suggest the score assumes that a forecast comes in the form of +#' one or multiple central prediction intervals. A prediction interval is +#' characterised by a lower and an upper bound formed by a pair of predictive +#' quantiles. For example, a 50% central prediction interval is formed by the +#' 0.25 and 0.75 quantiles of the predictive distribution. +#' +#' **Interval score** +#' +#' The interval score (IS) is the sum of three components: +#' overprediction, underprediction and dispersion. For a single prediction +#' interval only one of the components is non-zero. If for a single prediction +#' interval the observed value is below the lower bound, then the interval +#' score is equal to the absolute difference between the lower bound and the +#' observed value ("underprediction"). "Overprediction" is defined analogously. +#' If the observed value falls within the bounds of the prediction interval, +#' then the interval score is equal to the width of the prediction interval, +#' i.e. the difference between the upper and lower bound. For a single interval, +#' we therefore have: +#' +#' \deqn{ +#' \textrm{IS} = (\textrm{upper} - \textrm{lower}) + \frac{2}{\alpha}(\textrm{lower} +#' - \textrm{observed}) * +#' \mathbf{1}(\textrm{observed} < \textrm{lower}) + +#' \frac{2}{\alpha}(\textrm{observed} - \textrm{upper}) * +#' \mathbf{1}(\textrm{observed} > \textrm{upper}) +#' }{ +#' score = (upper - lower) + 2/alpha * (lower - observed) * +#' 1(observed < lower) + 2/alpha * (observed - upper) * +#' 1(observed > upper) +#' } +#' where \eqn{\mathbf{1}()}{1()} is the indicator function and +#' indicates how much is outside the prediction interval. +#' \eqn{\alpha}{alpha} is the decimal value that indicates how much is outside +#' the prediction interval. For a 90% prediction interval, for example, +#' \eqn{\alpha}{alpha} is equal to 0.1. No specific distribution is assumed, +#' but the range has to be symmetric (i.e you can't use the 0.1 quantile +#' as the lower bound and the 0.7 quantile as the upper). +#' Non-symmetric quantiles can be scored using the function [quantile_score()]. +#' +#' Usually the interval score is weighted by a factor that makes sure that the +#' average score across an increasing number of equally spaced +#' quantiles, converges to the continuous ranked probability score (CRPS). This +#' weighted score is called the weihted interval score (WIS). +#' The weight commonly used is \eqn{\alpha / 2}{alpha / 2}. +#' +#' **Quantile score** +#' +#' In addition to the interval score, there also exists a quantile score (QS) +#' (see [quantile_score()]), which is equal to the so-called pinball loss. +#' The quantile score can be computed for a single quantile (whereas the +#' interval score requires two quantiles that form an interval). However, +#' the intuitive decomposition into overprediction, underprediction and +#' dispersion does not exist for the quantile score. +#' +#' **Two versions of the weighted interval score** +#' +#' There are two ways to conceptualise the weighted interval score across +#' several quantiles / prediction intervals and the median. +#' +#' In one view, you would treat the WIS as the average of quantile scores (and +#' the median as 0.5-quantile) (this is the default for `wis()`). In another +#' view, you would treat the WIS as the average of several interval scores + +#' the difference between observed value and median forecast. The effect of +#' that is that in contrast to the first view, the median has twice as much +#' weight (because it is weighted like a prediction interval, rather than like +#' a single quantile). Both are valid ways to conceptualise the WIS and you +#' can control the behvaviour with the `count_median_twice`-argument. +#' +#' **WIS components**: +#' WIS components can be computed individually using the functions +#' `overprediction`, `underprediction`, and `dispersion.` +#' #' @inheritParams interval_score #' @param predicted vector of size n with the predicted values #' @param quantile vector with quantile levels of size N #' @param count_median_twice if TRUE, count the median twice in the score #' @param na.rm if TRUE, ignore NA values when computing the score #' @importFrom stats weighted.mean +#' @return +#' `wis()`: a numeric vector with WIS values (one per observation), or a list +#' with separate entries if `separate_results` is `TRUE`. #' @export wis <- function(observed, predicted, @@ -62,6 +141,9 @@ wis <- function(observed, } } + +#' @return +#' `dispersion()`: a numeric vector with dispersion values (one per observation) #' @export #' @rdname wis dispersion <- function(observed, predicted, quantile) { @@ -69,6 +151,10 @@ dispersion <- function(observed, predicted, quantile) { wis(observed, predicted, quantile, separate_results = TRUE)$dispersion } + +#' @return +#' `overprediction()`: a numeric vector with overprediction values (one per +#' observation) #' @export #' @rdname wis overprediction <- function(observed, predicted, quantile) { @@ -76,6 +162,10 @@ overprediction <- function(observed, predicted, quantile) { wis(observed, predicted, quantile, separate_results = TRUE)$overprediction } + +#' @return +#' `underprediction()`: a numeric vector with underprediction values (one per +#' observation) #' @export #' @rdname wis underprediction <- function(observed, predicted, quantile) { diff --git a/man/metrics_quantile.Rd b/man/metrics_quantile.Rd index e96dcc836..9fda39f03 100644 --- a/man/metrics_quantile.Rd +++ b/man/metrics_quantile.Rd @@ -5,7 +5,7 @@ \alias{metrics_quantile} \title{Default metrics for quantile-based forecasts.} \format{ -An object of class \code{list} of length 4. +An object of class \code{list} of length 7. } \usage{ metrics_quantile diff --git a/man/wis.Rd b/man/wis.Rd index fbf8bc6f7..a76b3c35f 100644 --- a/man/wis.Rd +++ b/man/wis.Rd @@ -2,7 +2,10 @@ % Please edit documentation in R/metrics-quantile.R \name{wis} \alias{wis} -\title{Weighted Interval Score} +\alias{dispersion} +\alias{overprediction} +\alias{underprediction} +\title{Weighted Interval Score (WIS)} \usage{ wis( observed, @@ -13,6 +16,12 @@ wis( count_median_twice = FALSE, na.rm = TRUE ) + +dispersion(observed, predicted, quantile) + +overprediction(observed, predicted, quantile) + +underprediction(observed, predicted, quantile) } \arguments{ \item{observed}{A vector with observed values of size n} @@ -36,6 +45,91 @@ Default: \code{TRUE}.} \item{na.rm}{if TRUE, ignore NA values when computing the score} } +\value{ +\code{wis()}: a numeric vector with WIS values (one per observation), or a list +with separate entries if \code{separate_results} is \code{TRUE}. + +\code{dispersion()}: a numeric vector with dispersion values (one per observation) + +\code{overprediction()}: a numeric vector with overprediction values (one per +observation) + +\code{underprediction()}: a numeric vector with underprediction values (one per +observation) +} \description{ -Weighted Interval Score +The WIS is a proper scoring rule used to evaluate forecasts in an interval- / +quantile-based format. See Bracher et al. (2021). Smaller values are better. + +As the name suggest the score assumes that a forecast comes in the form of +one or multiple central prediction intervals. A prediction interval is +characterised by a lower and an upper bound formed by a pair of predictive +quantiles. For example, a 50\% central prediction interval is formed by the +0.25 and 0.75 quantiles of the predictive distribution. + +\strong{Interval score} + +The interval score (IS) is the sum of three components: +overprediction, underprediction and dispersion. For a single prediction +interval only one of the components is non-zero. If for a single prediction +interval the observed value is below the lower bound, then the interval +score is equal to the absolute difference between the lower bound and the +observed value ("underprediction"). "Overprediction" is defined analogously. +If the observed value falls within the bounds of the prediction interval, +then the interval score is equal to the width of the prediction interval, +i.e. the difference between the upper and lower bound. For a single interval, +we therefore have: + +\deqn{ +\textrm{IS} = (\textrm{upper} - \textrm{lower}) + \frac{2}{\alpha}(\textrm{lower} + - \textrm{observed}) * +\mathbf{1}(\textrm{observed} < \textrm{lower}) + +\frac{2}{\alpha}(\textrm{observed} - \textrm{upper}) * +\mathbf{1}(\textrm{observed} > \textrm{upper}) +}{ +score = (upper - lower) + 2/alpha * (lower - observed) * +1(observed < lower) + 2/alpha * (observed - upper) * +1(observed > upper) +} +where \eqn{\mathbf{1}()}{1()} is the indicator function and +indicates how much is outside the prediction interval. +\eqn{\alpha}{alpha} is the decimal value that indicates how much is outside +the prediction interval. For a 90\% prediction interval, for example, +\eqn{\alpha}{alpha} is equal to 0.1. No specific distribution is assumed, +but the range has to be symmetric (i.e you can't use the 0.1 quantile +as the lower bound and the 0.7 quantile as the upper). +Non-symmetric quantiles can be scored using the function \code{\link[=quantile_score]{quantile_score()}}. + +Usually the interval score is weighted by a factor that makes sure that the +average score across an increasing number of equally spaced +quantiles, converges to the continuous ranked probability score (CRPS). This +weighted score is called the weihted interval score (WIS). +The weight commonly used is \eqn{\alpha / 2}{alpha / 2}. + +\strong{Quantile score} + +In addition to the interval score, there also exists a quantile score (QS) +(see \code{\link[=quantile_score]{quantile_score()}}), which is equal to the so-called pinball loss. +The quantile score can be computed for a single quantile (whereas the +interval score requires two quantiles that form an interval). However, +the intuitive decomposition into overprediction, underprediction and +dispersion does not exist for the quantile score. + +\strong{Two versions of the weighted interval score} + +There are two ways to conceptualise the weighted interval score across +several quantiles / prediction intervals and the median. + +In one view, you would treat the WIS as the average of quantile scores (and +the median as 0.5-quantile) (this is the default for \code{wis()}). In another +view, you would treat the WIS as the average of several interval scores + +the difference between observed value and median forecast. The effect of +that is that in contrast to the first view, the median has twice as much +weight (because it is weighted like a prediction interval, rather than like +a single quantile). Both are valid ways to conceptualise the WIS and you +can control the behvaviour with the \code{count_median_twice}-argument. + +\strong{WIS components}: +WIS components can be computed individually using the functions +\code{overprediction}, \code{underprediction}, and \code{dispersion.} } From a9e89bbb87256aedc6e8b8127721437a8dc6b2cc Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Wed, 15 Nov 2023 15:51:26 +0100 Subject: [PATCH 5/5] Allow passing additional arguments to `wis()` from `overprediction()`, `underprediction()` and `dispersion()` --- R/metrics-quantile.R | 20 ++++++++++++++------ man/wis.Rd | 9 ++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/R/metrics-quantile.R b/R/metrics-quantile.R index f3598690e..1b74a932c 100644 --- a/R/metrics-quantile.R +++ b/R/metrics-quantile.R @@ -144,11 +144,15 @@ wis <- function(observed, #' @return #' `dispersion()`: a numeric vector with dispersion values (one per observation) +#' @param ... Additional arguments passed on to `wis()` from functions +#' `overprediction()`, `underprediction()` and `dispersion()` #' @export #' @rdname wis -dispersion <- function(observed, predicted, quantile) { +dispersion <- function(observed, predicted, quantile, ...) { + args <- list(...) + args$separate_results <- TRUE assert_input_quantile(observed, predicted, quantile) - wis(observed, predicted, quantile, separate_results = TRUE)$dispersion + do.call(wis, c(list(observed), list(predicted), list(quantile), args))$dispersion } @@ -157,9 +161,11 @@ dispersion <- function(observed, predicted, quantile) { #' observation) #' @export #' @rdname wis -overprediction <- function(observed, predicted, quantile) { +overprediction <- function(observed, predicted, quantile, ...) { + args <- list(...) + args$separate_results <- TRUE assert_input_quantile(observed, predicted, quantile) - wis(observed, predicted, quantile, separate_results = TRUE)$overprediction + do.call(wis, c(list(observed), list(predicted), list(quantile), args))$overprediction } @@ -168,9 +174,11 @@ overprediction <- function(observed, predicted, quantile) { #' observation) #' @export #' @rdname wis -underprediction <- function(observed, predicted, quantile) { +underprediction <- function(observed, predicted, quantile, ...) { + args <- list(...) + args$separate_results <- TRUE assert_input_quantile(observed, predicted, quantile) - wis(observed, predicted, quantile, separate_results = TRUE)$underprediction + do.call(wis, c(list(observed), list(predicted), list(quantile), args))$underprediction } diff --git a/man/wis.Rd b/man/wis.Rd index a76b3c35f..0b4853e76 100644 --- a/man/wis.Rd +++ b/man/wis.Rd @@ -17,11 +17,11 @@ wis( na.rm = TRUE ) -dispersion(observed, predicted, quantile) +dispersion(observed, predicted, quantile, ...) -overprediction(observed, predicted, quantile) +overprediction(observed, predicted, quantile, ...) -underprediction(observed, predicted, quantile) +underprediction(observed, predicted, quantile, ...) } \arguments{ \item{observed}{A vector with observed values of size n} @@ -44,6 +44,9 @@ Default: \code{TRUE}.} \item{count_median_twice}{if TRUE, count the median twice in the score} \item{na.rm}{if TRUE, ignore NA values when computing the score} + +\item{...}{Additional arguments passed on to \code{wis()} from functions +\code{overprediction()}, \code{underprediction()} and \code{dispersion()}} } \value{ \code{wis()}: a numeric vector with WIS values (one per observation), or a list