# Checking missing-data assumptions
# https://tidyecology.com/posts/checking-missing-data-assumptions/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-15

## ---- setup ----
library(ggplot2); library(dplyr); library(tidyr); library(MASS)
ink <- "#2b2b2b"; grid_col <- "#d9d5cc"; paper <- "#f5f4ee"
theme_te <- function(base = 12) {
  theme_minimal(base_size = base) +
    theme(panel.grid.minor = element_blank(),
          panel.grid.major = element_line(colour = grid_col, linewidth = 0.3),
          plot.background = element_rect(fill = paper, colour = NA),
          panel.background = element_rect(fill = paper, colour = NA),
          strip.text = element_text(face = "bold", colour = ink),
          plot.title = element_text(face = "bold", colour = ink),
          axis.title = element_text(colour = ink),
          legend.position = "top")
}
## proper Bayesian linear imputation and Rubin pooling, as built in the previous post
step <- function(target, mask, preds) {
  obs <- !mask; Xo <- cbind(1, preds[obs, , drop = FALSE]); Vo <- target[obs]
  XtXinv <- chol2inv(chol(crossprod(Xo))); bhat <- XtXinv %*% crossprod(Xo, Vo)
  dfree <- sum(obs) - ncol(Xo); s2 <- sum((Vo - Xo %*% bhat)^2) / dfree
  s2s <- s2 * dfree / rchisq(1, dfree); bstar <- bhat + t(chol(s2s * XtXinv)) %*% rnorm(ncol(Xo))
  out <- target; Xm <- cbind(1, preds[mask, , drop = FALSE])
  out[mask] <- as.numeric(Xm %*% bstar) + rnorm(sum(mask), 0, sqrt(s2s)); out
}
pool <- function(ests, ses, m, nu_com) {
  Qbar <- mean(ests); Ubar <- mean(ses^2); B <- var(ests); Tt <- Ubar + (1 + 1 / m) * B
  lambda <- (1 + 1 / m) * B / Tt; nu_old <- (m - 1) / lambda^2
  nu_obs <- (nu_com + 1) / (nu_com + 3) * nu_com * (1 - lambda)
  df <- nu_old * nu_obs / (nu_old + nu_obs); c(Q = Qbar, se = sqrt(Tt), df = df)
}

## ---- dgp ----
set.seed(3672)
ng <- 250; g <- rep(0:1, each = ng); n <- 2 * ng
x <- rnorm(n); tau <- 0.6; beta <- 0.5
y_true <- 0 + tau * g + beta * x + rnorm(n, 0, 1)
## missing at random on x: higher x makes y more likely to be missing (about 35%)
aM <- uniroot(function(a) mean(plogis(a + x)) - 0.35, c(-6, 6))$root
my <- runif(n) < plogis(aM + x)
y <- y_true; y[my] <- NA

## ---- mar-fit ----
m <- 40; nu_com <- n - 3
imps <- lapply(1:m, function(k) step(y, my, cbind(g, x)))
eff <- function(yc) { f <- lm(yc ~ g + x); c(coef(f)["g"], summary(f)$coef["g", 2]) }
E0 <- sapply(imps, eff)
p0 <- pool(E0[1, ], E0[2, ], m, nu_com); tcrit <- qt(0.975, p0["df"])
c(effect = unname(p0["Q"]),
  lo = unname(p0["Q"] - tcrit * p0["se"]), hi = unname(p0["Q"] + tcrit * p0["se"]))

## ---- mcar-check ----
mcar <- glm(my ~ g + x, family = binomial)
round(summary(mcar)$coefficients, 3)

## ---- fig-obs-imp ----
obs_y <- y[!my]; imp_y <- unlist(lapply(imps, function(v) v[my]))
df1 <- rbind(data.frame(y = obs_y, src = "Observed"),
             data.frame(y = imp_y, src = "Imputed (pooled)"))
mu1 <- df1 |> group_by(src) |> summarise(m = mean(y), .groups = "drop")
ggplot(df1, aes(y, colour = src, fill = src)) +
  geom_density(alpha = 0.16, linewidth = 0.8) +
  geom_vline(data = mu1, aes(xintercept = m, colour = src),
             linetype = "dashed", linewidth = 0.5, show.legend = FALSE) +
  scale_colour_manual(values = c("Observed" = "#356170", "Imputed (pooled)" = "#c08a3e")) +
  scale_fill_manual(values = c("Observed" = "#356170", "Imputed (pooled)" = "#c08a3e")) +
  labs(title = "Observed against imputed: a model check, not a mechanism test",
       subtitle = "A gap between the two is expected under missing at random",
       x = "Outcome y", y = "Density", colour = NULL, fill = NULL) + theme_te()

## ---- delta-sweep ----
deltas <- seq(0, 1.5, 0.025)
trtmiss <- my & (g == 1)
sweep <- t(sapply(deltas, function(dl) {
  ests <- ses <- numeric(m)
  for (k in 1:m) { yc <- imps[[k]]; yc[trtmiss] <- yc[trtmiss] - dl
    e <- eff(yc); ests[k] <- e[1]; ses[k] <- e[2] }
  p <- pool(ests, ses, m, nu_com); tc <- qt(0.975, p["df"])
  c(delta = dl, est = p["Q"], lo = p["Q"] - tc * p["se"], hi = p["Q"] + tc * p["se"])
}))
sweep <- as.data.frame(sweep); colnames(sweep) <- c("delta", "est", "lo", "hi")
tip <- sweep$delta[which(sweep$lo <= 0)[1]]
c(mar_effect = round(sweep$est[1], 3), tipping_delta = round(tip, 3),
  outcome_sd = round(sd(y_true), 3), tipping_in_outcome_sd = round(tip / sd(y_true), 3))

## ---- fig-sensitivity ----
p_sens <- ggplot(sweep, aes(delta, est)) +
  geom_ribbon(aes(ymin = lo, ymax = hi), fill = "#4a7a8c", alpha = 0.18) +
  geom_line(colour = "#356170", linewidth = 0.9) +
  geom_hline(yintercept = 0, colour = "#a24b4b", linetype = "dashed", linewidth = 0.6) +
  geom_vline(xintercept = tip, colour = ink, linetype = "dotted", linewidth = 0.6) +
  annotate("point", x = 0, y = sweep$est[1], colour = ink, size = 2.6) +
  annotate("text", x = 0.03, y = sweep$est[1] + 0.06, label = "MAR (delta = 0)",
           hjust = 0, size = 3.3, colour = ink) +
  annotate("text", x = tip, y = 0.7, label = sprintf("tipping point\ndelta = %.2f", tip),
           hjust = 1.08, size = 3.3, colour = ink, fontface = "bold") +
  labs(title = "How far from MAR before the interval reaches zero",
       subtitle = "Red line: no effect. The conclusion holds until the departure exceeds the tipping point",
       x = "Delta: assumed shortfall of treatment dropouts (outcome units)",
       y = "Estimated treatment effect") + theme_te()
p_sens

## ---- thumb ----
ggsave("thumbnail.png",
       p_sens + labs(title = NULL, subtitle = NULL) +
         theme(plot.margin = margin(8, 14, 8, 8)),
       width = 7.5, height = 5.4, dpi = 100, bg = paper)
