# Bayesian model comparison: WAIC, LOO, DIC
# https://tidyecology.com/posts/bayesian-model-comparison-waic-loo/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-11-04

## ---- setup ----
knitr::opts_chunk$set(message = FALSE, warning = FALSE,
                      fig.width = 7.5, fig.height = 5.4, dpi = 150)
library(ggplot2)
te <- list(ink="#16241d", body="#2c3a31", forest="#275139", label="#46604a",
           sage="#93a87f", paper="#f5f4ee", line="#dad9ca", faint="#5d6b61",
           gold="#cda23f", mown="#2f8f63", terra="#b5534e")
theme_te <- function(base_size = 13) {
  theme_minimal(base_size = base_size) +
    theme(plot.background = element_rect(fill = te$paper, colour = NA),
          panel.background = element_rect(fill = te$paper, colour = NA),
          panel.grid.major = element_line(colour = te$line, linewidth = 0.3),
          panel.grid.minor = element_blank(),
          axis.text = element_text(colour = te$faint),
          axis.title = element_text(colour = te$body),
          plot.title = element_text(colour = te$ink, face = "bold"),
          plot.subtitle = element_text(colour = te$label),
          legend.text = element_text(colour = te$body),
          plot.caption = element_text(colour = te$faint, size = base_size * 0.75))
}

## ---- sim ----
set.seed(0)
n      <- 80
b_true <- c(1.0, 0.8, -0.7)                 # intercept, linear, quadratic
xsim   <- sort(runif(n, -2.2, 2.2))
Xfull  <- cbind(1, xsim, xsim^2)
y      <- rpois(n, exp(as.vector(Xfull %*% b_true)))
c(n = n, mean_count = round(mean(y), 2), zeros = sum(y == 0), max = max(y))

## ---- mcmc ----
loglik_vec <- function(beta, X) dpois(y, exp(as.vector(X %*% beta)), log = TRUE)
logpost <- function(beta, X, ps = 5) {
  lam <- exp(as.vector(X %*% beta))
  if (any(!is.finite(lam)) || any(lam > 1e8)) return(-Inf)
  sum(dpois(y, lam, log = TRUE)) + sum(dnorm(beta, 0, ps, log = TRUE))
}
run_mcmc <- function(X, iter = 30000, warm = 5000, seed = 1) {
  set.seed(seed)
  d   <- ncol(X)
  fit <- optim(rep(0, d), function(b) -logpost(b, X), method = "BFGS", hessian = TRUE)
  Cp  <- chol(solve(fit$hessian) * (2.4^2 / d))     # proposal Cholesky
  beta <- fit$par; lp <- logpost(beta, X)
  out <- matrix(NA_real_, iter, d); acc <- 0
  for (t in seq_len(iter)) {
    prop <- beta + as.vector(crossprod(Cp, rnorm(d)))
    lpp  <- logpost(prop, X)
    if (log(runif(1)) < lpp - lp) { beta <- prop; lp <- lpp; acc <- acc + 1 }
    out[t, ] <- beta
  }
  draws <- out[(warm + 1):iter, , drop = FALSE]
  ll <- matrix(NA_real_, nrow(draws), length(y))       # S x n log-likelihood
  for (s in seq_len(nrow(draws))) ll[s, ] <- loglik_vec(draws[s, ], X)
  list(draws = draws, ll = ll, acc = acc / iter)
}

Xlist <- list(M0 = Xfull[, 1, drop = FALSE], M1 = Xfull[, 1:2],
              M2 = Xfull[, 1:3], M3 = cbind(Xfull, xsim^3))
fits  <- Map(function(X, s) run_mcmc(X, seed = s), Xlist, c(101, 202, 303, 404))

## ---- naive ----
naive_dev <- sapply(Xlist, function(X) {
  Xm <- X; -2 * as.numeric(logLik(glm(y ~ Xm - 1, family = poisson)))
})
round(naive_dev, 1)

## ---- dic ----
dic_one <- function(m, X) {
  Dbar <- mean(-2 * rowSums(m$ll))
  Dhat <- -2 * sum(loglik_vec(colMeans(m$draws), X))
  pD <- Dbar - Dhat
  c(DIC = Dbar + pD, pD = pD)
}

## ---- waic ----
waic_one <- function(m) {
  cm  <- apply(m$ll, 2, max)                             # log-sum-exp, stably
  lppd <- cm + log(colMeans(exp(sweep(m$ll, 2, cm))))
  pw   <- apply(m$ll, 2, var)
  list(WAIC = -2 * sum(lppd - pw), pWAIC = sum(pw),
       pt = lppd - pw)                                   # pointwise elpd
}

## ---- gpd ----
gpdfit <- function(x) {                       # x: exceedances, ascending, > 0
  N <- length(x); m <- 30 + floor(sqrt(N))
  bs <- 1 - sqrt(m / (seq_len(m) - 0.5))
  bs <- bs / (3 * x[floor(N / 4 + 0.5)]) + 1 / x[N]
  ks <- vapply(bs, function(b) mean(log1p(-b * x)), numeric(1))
  L  <- N * (log(-bs / ks) - ks - 1)
  w  <- 1 / vapply(seq_len(m), function(j) sum(exp(L - L[j])), numeric(1))
  b  <- sum(bs * w); k <- mean(log1p(-b * x))
  list(k = (k * N + 5) / (N + 10), sigma = -k / b)       # prior-shrunk k, scale
}
qgpd <- function(p, k, sig)
  if (abs(k) < 1e-9) -sig * log1p(-p) else sig / k * ((1 - p)^(-k) - 1)

psis_i <- function(logr) {                    # log importance ratios, one point
  S <- length(logr); r <- exp(logr - max(logr))
  M <- ceiling(min(0.2 * S, 3 * sqrt(S)))
  ord <- order(r); u <- r[ord][S - M]; tail_i <- ord[(S - M + 1):S]
  g <- gpdfit(r[tail_i] - u)
  r[tail_i] <- pmin(u + qgpd((seq_len(M) - 0.5) / M, g$k, g$sigma), max(r))
  list(w = r / sum(r), k = g$k)
}

loo_one <- function(m) {
  nn <- ncol(m$ll); ei <- numeric(nn); kh <- numeric(nn)
  for (i in seq_len(nn)) {
    ps <- psis_i(-m$ll[, i])                             # ratios 1 / p_i
    a  <- m$ll[, i]; mx <- max(a)
    ei[i] <- mx + log(sum(ps$w * exp(a - mx)))           # weighted predictive
    kh[i] <- ps$k
  }
  list(LOO = -2 * sum(ei), elpd = ei, khat = kh)
}

## ---- table ----
ics <- lapply(names(Xlist), function(nm) {
  m <- fits[[nm]]; X <- Xlist[[nm]]
  d <- dic_one(m, X); w <- waic_one(m); l <- loo_one(m)
  list(in_sample = naive_dev[[nm]], DIC = d["DIC"], pD = d["pD"],
       WAIC = w$WAIC, pWAIC = w$pWAIC, pt = w$pt,
       LOO = l$LOO, khat = l$khat, elpd_loo = l$elpd)
})
names(ics) <- names(Xlist)
aic <- sapply(Xlist, function(X) { Xm <- X; AIC(glm(y ~ Xm - 1, family = poisson)) })

tab <- data.frame(
  model = names(Xlist), parameters = sapply(Xlist, ncol),
  in_sample = round(naive_dev, 1),
  DIC = round(sapply(ics, `[[`, "DIC"), 1), pD = round(sapply(ics, `[[`, "pD"), 2),
  WAIC = round(sapply(ics, `[[`, "WAIC"), 1),
  pWAIC = round(sapply(ics, `[[`, "pWAIC"), 2),
  LOO = round(sapply(ics, `[[`, "LOO"), 1), AIC = round(aic, 1),
  max_k = round(sapply(ics, function(z) max(z$khat)), 2), row.names = NULL)
tab

## ---- fig-scores ----
np  <- sapply(Xlist, ncol)
seW <- sapply(names(Xlist), function(nm) 2 * sqrt(n * var(ics[[nm]]$pt)))
mk  <- function(v, lab) data.frame(npar = np, score = v, kind = lab)
df1 <- rbind(mk(naive_dev, "in-sample (-2 log-lik)"), mk(sapply(ics, `[[`, "DIC"), "DIC"),
             mk(sapply(ics, `[[`, "LOO"), "PSIS-LOO"), mk(sapply(ics, `[[`, "WAIC"), "WAIC"))
# WAIC is drawn last and dotted: DIC and PSIS-LOO land on top of it otherwise
df1$kind <- factor(df1$kind, levels = c("in-sample (-2 log-lik)", "DIC", "PSIS-LOO", "WAIC"))
best <- which.min(sapply(ics, `[[`, "WAIC"))
dfW <- data.frame(npar = np, WAIC = sapply(ics, `[[`, "WAIC"), se = seW)
ggplot(df1, aes(npar, score, colour = kind)) +
  geom_errorbar(data = dfW, inherit.aes = FALSE,
                aes(npar, ymin = WAIC - se, ymax = WAIC + se), width = 0.08,
                colour = te$forest, alpha = 0.6) +
  geom_line(aes(linetype = kind), linewidth = 0.9) + geom_point(size = 2.4) +
  annotate("point", x = np[best], y = sapply(ics, `[[`, "WAIC")[best],
           size = 5, shape = 21, stroke = 1.1, colour = te$ink, fill = NA) +
  scale_colour_manual(values = c("in-sample (-2 log-lik)" = te$terra, "DIC" = te$gold,
                                 "WAIC" = te$forest, "PSIS-LOO" = te$sage), name = NULL) +
  scale_linetype_manual(values = c("in-sample (-2 log-lik)" = "solid", "DIC" = "solid",
                                   "PSIS-LOO" = "solid", "WAIC" = "dotted"), name = NULL) +
  scale_x_continuous(breaks = np, labels = paste0(names(Xlist), " (", np, "p)")) +
  labs(x = "candidate model", y = "deviance scale (lower is better)",
       title = "In-sample fit never gets worse; WAIC, LOO and DIC do",
       subtitle = "Poisson counts, true structure is the quadratic model M2") +
  theme_te() + theme(legend.position = "top")

## ---- fig-pareto ----
kk <- ics[[best]]$khat
ggplot(data.frame(y = y, k = kk), aes(y, k)) +
  geom_hline(yintercept = 0.7, linetype = "dashed", colour = te$terra) +
  geom_hline(yintercept = 0.5, linetype = "dotted", colour = te$gold) +
  geom_jitter(width = 0.12, height = 0, size = 2.3, colour = te$forest, alpha = 0.85) +
  annotate("text", x = max(y), y = 0.72, hjust = 1, vjust = 0, size = 3.4,
           colour = te$terra, label = "k = 0.7: importance sampling unreliable") +
  annotate("text", x = max(y), y = 0.52, hjust = 1, vjust = 0, size = 3.4,
           colour = te$gold, label = "k = 0.5") +
  coord_cartesian(ylim = c(min(kk) - 0.05, 0.85)) +
  labs(x = "observed count y", y = "Pareto k-hat",
       title = "PSIS diagnostic for the selected model M2",
       subtitle = "every k below 0.5: the LOO approximation is trustworthy here") +
  theme_te()
