# Checking a multi-model analysis
# https://tidyecology.com/posts/checking-a-multimodel-analysis/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-11-04

## ---- selbias ----
aicc <- function(m) { k <- length(coef(m)) + 1; N <- nobs(m); AIC(m) + 2 * k * (k + 1) / (N - k - 1) }
P <- 6; n <- 70
preds   <- paste0("x", 1:P)
subsets <- unlist(lapply(0:P, function(k) combn(preds, k, simplify = FALSE)), recursive = FALSE)
forms   <- vapply(subsets, function(s)
  if (length(s) == 0) "y ~ 1" else paste("y ~", paste(s, collapse = " + ")), character(1))

# one large fixed hold-out set representing the truth (only x1 has an effect)
set.seed(6061)
Nte <- 3000
Xte <- as.data.frame(matrix(rnorm(Nte * P), Nte, P)); names(Xte) <- preds
Xte$y <- 2 + 1.0 * Xte$x1 + rnorm(Nte, 0, 2.0)
r2 <- function(m, D) { yh <- predict(m, D); 1 - sum((D$y - yh)^2) / sum((D$y - mean(D$y))^2) }

set.seed(606)
reps <- 300
retained <- 0; null_coefs <- c(); r2_in <- c(); r2_out <- c()
for (i in seq_len(reps)) {
  X <- as.data.frame(matrix(rnorm(n * P), n, P)); names(X) <- preds
  X$y <- 2 + 1.0 * X$x1 + rnorm(n, 0, 2.0)
  fits <- lapply(forms, function(f) lm(as.formula(f), data = X))
  best <- fits[[which.min(vapply(fits, aicc, numeric(1)))]]
  chosen <- names(coef(best))[-1]
  nulls  <- setdiff(preds, "x1")                 # x2..x6 have no true effect
  got    <- intersect(chosen, nulls)
  retained <- retained + (length(got) > 0)
  for (v in got) null_coefs <- c(null_coefs, abs(coef(best)[v]))
  r2_in  <- c(r2_in,  r2(best, X))
  r2_out <- c(r2_out, r2(best, Xte))
}
c(p_null_retained = retained / reps,
  mean_null_coef  = mean(null_coefs),
  r2_in  = mean(r2_in),
  r2_out = mean(r2_out))

## ---- selnums ----
p_ret   <- retained / reps
mean_nc <- mean(null_coefs)
r2i     <- mean(r2_in); r2o <- mean(r2_out)

## ---- fig-nullcoef ----
library(ggplot2)
te_canvas <- theme(plot.background  = element_rect(fill = "#f5f4ee", colour = NA),
                   panel.background = element_rect(fill = "#f5f4ee", colour = NA))
ggplot(data.frame(coef = null_coefs), aes(coef)) +
  geom_histogram(bins = 30, fill = "#c44e52", colour = "white") +
  geom_vline(xintercept = 0, linetype = "dashed", colour = "grey40") +
  labs(x = "absolute coefficient of a selected null predictor (true value 0)",
       y = "count across simulations") +
  theme_minimal(base_size = 12) + te_canvas

## ---- setdep ----
set.seed(202)
n2 <- 55
moisture <- rnorm(n2); nitrogen <- rnorm(n2); aspect <- rnorm(n2)
biomass  <- 2 + 1.0 * moisture + 0.22 * nitrogen + 0 * aspect + rnorm(n2, 0, 1.6)
set.seed(2021)
moisture2 <- moisture + rnorm(n2, 0, 0.25)     # a near-duplicate sensor
cor_dup <- cor(moisture, moisture2)

weights_for <- function(vars, data) {
  ss <- unlist(lapply(0:length(vars), function(k) combn(vars, k, simplify = FALSE)), recursive = FALSE)
  ff <- vapply(ss, function(s)
    if (length(s) == 0) "biomass ~ 1" else paste("biomass ~", paste(s, collapse = " + ")), character(1))
  fits <- lapply(ff, function(f) lm(as.formula(f), data = data))
  w <- exp(-0.5 * (vapply(fits, aicc, numeric(1)) - min(vapply(fits, aicc, numeric(1))))); w <- w / sum(w)
  imp <- vapply(vars, function(v) sum(w[vapply(ss, function(s) v %in% s, logical(1))]), numeric(1))
  list(best_weight = max(w), importance = imp)
}

dat <- data.frame(biomass, moisture, nitrogen, aspect, moisture2)
before <- weights_for(c("moisture", "nitrogen", "aspect"), dat)
after  <- weights_for(c("moisture", "nitrogen", "aspect", "moisture2"), dat)

## ---- setdepnums ----
bw_before <- before$best_weight; bw_after <- after$best_weight
imp_moist_before <- before$importance["moisture"]
imp_moist_after  <- after$importance["moisture"]
imp_moist2_after <- after$importance["moisture2"]

## ---- fig-setdep ----
impdf <- rbind(
  data.frame(predictor = names(before$importance), wplus = as.numeric(before$importance), set = "original set"),
  data.frame(predictor = names(after$importance),  wplus = as.numeric(after$importance),  set = "with duplicate"))
impdf$predictor <- factor(impdf$predictor, levels = c("moisture", "moisture2", "nitrogen", "aspect"))
ggplot(impdf, aes(predictor, wplus, fill = set)) +
  geom_col(position = position_dodge(preserve = "single"), width = 0.7) +
  scale_fill_manual(values = c("original set" = "#4c72b0", "with duplicate" = "#dd8452"), name = NULL) +
  labs(x = NULL, y = "summed Akaike weight (w+)") +
  theme_minimal(base_size = 12) + te_canvas + theme(legend.position = "top")

## ---- truthset ----
set.seed(808)
n3 <- 90
moisture <- rnorm(n3); carbon <- rnorm(n3); aspect <- rnorm(n3)
biomass  <- 2 + 1.0 * moisture + 0.8 * carbon + 1.3 * moisture * carbon + 0 * aspect + rnorm(n3, 0, 1.5)
dt <- data.frame(biomass, moisture, carbon, aspect)

main_vars <- c("moisture", "carbon", "aspect")
ss <- unlist(lapply(0:3, function(k) combn(main_vars, k, simplify = FALSE)), recursive = FALSE)
ff <- vapply(ss, function(s)
  if (length(s) == 0) "biomass ~ 1" else paste("biomass ~", paste(s, collapse = " + ")), character(1))
fits <- lapply(ff, function(f) lm(as.formula(f), data = dt))
w <- exp(-0.5 * (vapply(fits, aicc, numeric(1)) - min(vapply(fits, aicc, numeric(1))))); w <- w / sum(w)
best_i <- which.max(w)
best_main <- fits[[best_i]]
best_weight <- w[best_i]

## ---- truthnums ----
set.seed(8081)
Dte <- data.frame(moisture = rnorm(4000), carbon = rnorm(4000), aspect = rnorm(4000))
Dte$biomass <- 2 + 1.0 * Dte$moisture + 0.8 * Dte$carbon + 1.3 * Dte$moisture * Dte$carbon + rnorm(4000, 0, 1.5)
r2 <- function(m, D) { yh <- predict(m, D); 1 - sum((D$biomass - yh)^2) / sum((D$biomass - mean(D$biomass))^2) }
oracle <- lm(biomass ~ moisture * carbon, data = dt)
r2_best   <- r2(best_main, Dte)
r2_oracle <- r2(oracle, Dte)
p_interaction <- summary(lm(resid(best_main) ~ I(moisture * carbon), data = dt))$coefficients["I(moisture * carbon)", "Pr(>|t|)"]

## ---- sci-tex-helper ----
sci_tex <- function(x, digits = 2) {
  s <- formatC(x, format = "e", digits = digits)
  paste0("$", sub("e.*", "", s), " \\times 10^{", as.integer(sub(".*e", "", s)), "}$")
}

## ---- fig-resid ----
rdf <- data.frame(product = dt$moisture * dt$carbon, resid = resid(best_main))
ggplot(rdf, aes(product, resid)) +
  geom_point(alpha = 0.6, colour = "grey35") +
  geom_smooth(method = "lm", se = FALSE, colour = "#c44e52", formula = y ~ x) +
  geom_hline(yintercept = 0, linetype = "dashed", colour = "grey55") +
  labs(x = "moisture x carbon (omitted interaction)", y = "residual from best main-effects model") +
  theme_minimal(base_size = 12) + te_canvas
