A previous post introduced the crossvalidation package for R. This time, the focus is on probabilistic forecasting — evaluating not just how accurate point forecasts are, but how well-calibrated prediction intervals are, using empirical coverage rates and Winkler scores – and crossvalidation.

install.packages("remotes")

install.packages("forecast")

remotes::install_github("Techtonique/crossvalidation")

library(crossvalidation)

Example 1

require(forecast)
data("AirPassengers")



eval_metric <- function(predicted, observed)
{
  error <- observed - predicted$mean

  me <- mean(error)
  rmse <- sqrt(mean(error^2))
  mae <- mean(abs(error))

  # ----- 80% interval -----

  lower80 <- predicted$lower[, 1]
  upper80 <- predicted$upper[, 1]

  coverage80 <- mean(
    observed >= lower80 & observed <= upper80
  )

  alpha80 <- 0.20

  winkler80 <- ifelse(
    observed < lower80,
    (upper80 - lower80) + (2 / alpha80) * (lower80 - observed),
    ifelse(
      observed > upper80,
      (upper80 - lower80) + (2 / alpha80) * (observed - upper80),
      (upper80 - lower80)
    )
  )

  # ----- 95% interval -----

  lower95 <- predicted$lower[, 2]
  upper95 <- predicted$upper[, 2]

  coverage95 <- mean(
    observed >= lower95 & observed <= upper95
  )

  alpha95 <- 0.05

  winkler95 <- ifelse(
    observed < lower95,
    (upper95 - lower95) + (2 / alpha95) * (lower95 - observed),
    ifelse(
      observed > upper95,
      (upper95 - lower95) + (2 / alpha95) * (observed - upper95),
      (upper95 - lower95)
    )
  )

  c(
    ME = me,
    RMSE = rmse,
    MAE = mae,
    Coverage80 = coverage80,
    Winkler80 = mean(winkler80),
    Coverage95 = coverage95,
    Winkler95 = mean(winkler95)
  )
}

(res <- crossval_ts(y=AirPassengers, initial_window = 10,
horizon = 3, fcast_func = forecast::thetaf, eval_metric = eval_metric))
print(colMeans(res))

Loading required package: forecast

  |======================================================================| 100%

A matrix: 132 × 7 of type dbl
	ME	RMSE	MAE	Coverage80	Winkler80	Coverage95	Winkler95
result.1	-28.794660	29.300287	28.794660	0.0000000	153.10992	0.3333333	207.58384
result.2	16.198526	16.894302	16.198526	1.0000000	45.01795	1.0000000	68.84902
result.3	11.201494	15.993359	12.578276	1.0000000	45.05996	1.0000000	68.91326
result.4	21.430125	22.483895	21.430125	0.6666667	63.01207	1.0000000	68.84778
result.5	10.055765	11.527746	10.055765	1.0000000	45.99967	1.0000000	70.35043
result.6	-2.640822	10.676714	9.999466	1.0000000	46.56907	1.0000000	71.22125
result.7	14.296434	23.709132	20.531135	0.6666667	75.04186	1.0000000	67.58381
result.8	38.247497	39.529998	38.247497	0.0000000	198.74990	0.3333333	212.44029
result.9	23.043159	23.947630	23.043159	0.3333333	93.83463	1.0000000	64.19366
result.10	-21.689067	27.907560	21.689067	0.6666667	90.23377	1.0000000	84.12361
result.11	-41.782157	46.664199	41.782157	0.3333333	222.06310	0.3333333	345.16553
result.12	-34.934831	36.512081	34.934831	0.3333333	162.38092	0.6666667	212.58117
result.13	-4.002700	12.728771	9.999100	1.0000000	59.64475	1.0000000	91.21878
result.14	30.349582	30.588761	30.349582	0.6666667	72.14355	1.0000000	99.76932
result.15	21.192349	25.806712	21.192349	0.6666667	71.39094	1.0000000	101.02401
result.16	23.193143	25.914875	23.193143	0.6666667	91.70660	1.0000000	76.57925
result.17	30.081542	30.679960	30.081542	0.3333333	111.58689	1.0000000	75.78459
result.18	-6.530509	9.111376	6.999059	1.0000000	69.51704	1.0000000	106.31714
result.19	19.907586	23.010762	19.907586	1.0000000	67.03506	1.0000000	102.52128
result.20	17.631089	19.829355	17.631089	1.0000000	67.97573	1.0000000	103.95991
result.21	11.738022	14.718185	12.229846	1.0000000	61.61617	1.0000000	94.23380
result.22	-21.787490	28.489509	21.787490	0.6666667	93.30920	1.0000000	88.70090
result.23	-43.557571	47.527244	43.557571	0.3333333	206.77368	0.6666667	216.50078
result.24	-34.473558	35.514155	34.473558	0.3333333	146.63288	0.6666667	173.17046
result.25	-4.699360	10.498595	7.201224	1.0000000	60.07550	1.0000000	91.87755
result.26	25.974138	26.581272	25.974138	1.0000000	63.01942	1.0000000	96.37989
result.27	16.905109	19.474600	16.905109	1.0000000	58.04472	1.0000000	88.77173
result.28	15.218760	16.352917	15.218760	1.0000000	55.27721	1.0000000	84.53920
result.29	7.625241	8.933828	7.625241	1.0000000	55.27718	1.0000000	84.53916
result.30	2.261970	17.595326	15.666212	1.0000000	57.13292	1.0000000	87.37725
⋮	⋮	⋮	⋮	⋮	⋮	⋮	⋮
result.103	95.047754	111.26440	95.04775	0.3333333	485.7096	0.6666667	594.3549
result.104	121.335201	125.76554	121.33520	0.0000000	646.5750	0.3333333	772.4818
result.105	27.661546	53.66952	52.33567	0.6666667	149.4669	1.0000000	226.7499
result.106	-82.928463	106.53675	87.39838	0.3333333	439.0476	0.6666667	391.0034
result.107	-168.429957	174.86402	168.42996	0.0000000	1125.8534	0.0000000	2680.3671
result.108	-86.047368	89.34969	86.04737	0.6666667	241.5086	1.0000000	281.3325
result.109	-35.392983	38.64620	35.39298	1.0000000	192.3314	1.0000000	294.1455
result.110	32.273683	33.69167	32.27368	1.0000000	199.9978	1.0000000	305.8702
result.111	35.911969	45.52857	35.91197	1.0000000	195.2069	1.0000000	298.5432
result.112	28.584481	41.79144	38.16654	1.0000000	196.5409	1.0000000	300.5833
result.113	78.144295	79.31310	78.14430	1.0000000	196.9343	1.0000000	301.1850
result.114	37.152546	52.61404	39.21044	1.0000000	192.5487	1.0000000	294.4778
result.115	95.078342	110.88602	95.07834	0.6666667	366.3676	1.0000000	274.9151
result.116	109.166178	116.17612	109.16618	0.3333333	406.7397	1.0000000	277.4405
result.117	41.289554	62.02085	57.33490	0.3333333	215.1577	1.0000000	222.4127
result.118	-92.399494	116.61777	92.82407	0.3333333	466.7285	0.6666667	445.3571
result.119	-175.618445	183.27955	175.61845	0.0000000	1143.5479	0.0000000	2574.2409
result.120	-94.580461	97.36039	94.58046	0.6666667	277.7847	1.0000000	293.2590
result.121	-27.751828	32.93559	27.75183	1.0000000	202.1374	1.0000000	309.1425
result.122	36.177008	38.16646	36.17701	1.0000000	208.6352	1.0000000	319.0800
result.123	5.992278	14.16185	13.99743	1.0000000	200.0098	1.0000000	305.8885
result.124	12.637863	33.65269	27.98030	1.0000000	200.1828	1.0000000	306.1532
result.125	71.834372	76.95073	71.83437	1.0000000	200.5753	1.0000000	306.7534
result.126	85.518711	93.75094	85.51871	0.6666667	252.5638	1.0000000	295.0496
result.127	94.429064	115.52397	94.42906	0.6666667	407.3636	0.6666667	417.2566
result.128	173.325805	177.66652	173.32580	0.0000000	1129.6141	0.0000000	2547.8618
result.129	33.890665	63.84191	61.66861	0.6666667	242.6901	1.0000000	230.3885
result.130	-119.059067	137.73685	119.05907	0.3333333	619.4166	0.3333333	668.9786
result.131	-180.821172	190.45241	180.82117	0.0000000	1152.4949	0.0000000	2469.3936
result.132	-103.156396	108.61881	103.15640	0.6666667	330.0400	1.0000000	302.1675

         ME        RMSE         MAE  Coverage80   Winkler80  Coverage95 
  2.6570822  51.4271704  46.5118747   0.6590909 218.4527816   0.8459596 
  Winkler95 
312.1383104 

Example 2

eval_metric <- function(predicted, observed)
{
  error <- observed - predicted$mean

  me <- mean(error)
  rmse <- sqrt(mean(error^2))
  mae <- mean(abs(error))

  # Only one interval returned
  lower <- predicted$lower
  upper <- predicted$upper

  coverage <- mean(
    observed >= lower & observed <= upper
  )

  alpha <- 0.05

  winkler <- ifelse(
    observed < lower,
    (upper - lower) + (2 / alpha) * (lower - observed),
    ifelse(
      observed > upper,
      (upper - lower) + (2 / alpha) * (observed - upper),
      (upper - lower)
    )
  )

  c(
    ME = me,
    RMSE = rmse,
    MAE = mae,
    Coverage95 = coverage,
    Winkler95 = mean(winkler)
  )
}

fcast_func <- function(y, h, ...)
{
  forecast::thetaf(
    y,
    h = h,
    level = 95
  )
}

res <- crossval_ts(
  y = AirPassengers,
  initial_window = 10,
  horizon = 3,
  fcast_func = fcast_func,
  eval_metric = eval_metric
)

print(colMeans(res))

  |======================================================================| 100%
         ME        RMSE         MAE  Coverage95   Winkler95 
  2.6570822  51.4271704  46.5118747   0.8459596 312.1383104 

boxplot(res[, "Coverage95"])

image-title-here

Comments powered by Talkyard.

@misc{ tmoudiki20260516, author = { T. Moudiki }, title = { Probabilistic Time Series Cross-Validation with R package crossvalidation }, url = { https://thierrymoudiki.github.io/blog/2026/05/16/r/crossvalidation }, year = { 2026 } }