# Single imputation: bias and variance
# https://tidyecology.com/posts/single-imputation-bias-variance/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-11-15

## ---- setup ----
library(ggplot2); library(dplyr); library(tidyr)
ink <- "#2b2b2b"; grid_col <- "#d9d5cc"; paper <- "#f5f4ee"
imp_col <- c("Mean imputation" = "#c08a3e",
             "Regression imputation" = "#a24b4b",
             "Stochastic imputation" = "#4a7a8c")
theme_te <- function(base = 12) {
  theme_minimal(base_size = base) +
    theme(panel.grid.minor = element_blank(),
          panel.grid.major = element_line(colour = grid_col, linewidth = 0.3),
          plot.background = element_rect(fill = paper, colour = NA),
          panel.background = element_rect(fill = paper, colour = NA),
          strip.text = element_text(face = "bold", colour = ink),
          plot.title = element_text(face = "bold", colour = ink),
          axis.title = element_text(colour = ink),
          legend.position = "top")
}

## ---- dgp ----
set.seed(2947)
n <- 500; c0 <- 1; c1 <- 1.5; sdx <- 1.2; sde <- 1.6
x <- rnorm(n, 0, sdx)
y <- c0 + c1 * x + rnorm(n, 0, sde)
full_slope <- unname(coef(lm(y ~ x))[2]); full_r <- cor(x, y); full_sdY <- sd(y)

## missing at random: higher x makes y more likely to be missing (about 40%)
a1 <- 1.1
a0 <- uniroot(function(a) mean(plogis(a + a1 * x)) - 0.40, c(-5, 5))$root
miss <- runif(n) < plogis(a0 + a1 * x); obs <- !miss

impute <- function(kind) {
  yo <- y; yo[miss] <- NA
  fit <- lm(yo[obs] ~ x[obs]); sig <- summary(fit)$sigma
  yhat <- coef(fit)[1] + coef(fit)[2] * x[miss]
  if (kind == "mean")   yo[miss] <- mean(y[obs])                        # constant fill
  if (kind == "detreg") yo[miss] <- yhat                               # on the fitted line
  if (kind == "stoch")  yo[miss] <- yhat + rnorm(sum(miss), 0, sig)     # line plus residual noise
  yo
}

## ---- fig-fills ----
lev <- c("Mean imputation", "Regression imputation", "Stochastic imputation")
mkp <- function(yo, nm) data.frame(x = x, y = yo, nm = nm,
                                   kind = ifelse(miss, "Imputed", "Observed"))
df1 <- rbind(mkp(impute("mean"), lev[1]), mkp(impute("detreg"), lev[2]),
             mkp(impute("stoch"), lev[3]))
df1$nm <- factor(df1$nm, levels = lev)
fl <- coef(lm(y ~ x))
ggplot(df1, aes(x, y)) +
  geom_point(data = subset(df1, kind == "Observed"), colour = "#8a8578", size = 0.9, alpha = 0.6) +
  geom_point(data = subset(df1, kind == "Imputed"), aes(colour = nm), size = 1.1, alpha = 0.85) +
  geom_abline(intercept = fl[1], slope = fl[2], colour = ink, linewidth = 0.5, linetype = "dashed") +
  facet_wrap(~nm) + scale_colour_manual(values = imp_col, guide = "none") +
  labs(title = "How each single-imputation method fills the gaps",
       subtitle = "Grey: observed. Coloured: imputed",
       x = "Predictor x", y = "Response y") + theme_te()

## ---- point-estimates ----
estim <- function(yo) { ok <- !is.na(yo); f <- lm(yo[ok] ~ x[ok])
  c(slope = unname(coef(f)[2]), r = cor(x[ok], yo[ok]), sdY = sd(yo[ok])) }
set.seed(29471)
tab <- rbind(
  Full                    = estim(y),
  `Complete-case`         = { z <- y; z[miss] <- NA; estim(z) },
  `Mean imputation`       = estim(impute("mean")),
  `Regression imputation` = estim(impute("detreg")),
  `Stochastic imputation` = estim(impute("stoch")))
round(tab, 3)

## ---- monte-carlo ----
M <- 2000; keep <- c("cc", "mean", "detreg", "stoch")
sl <- se <- cv <- matrix(NA, M, 4, dimnames = list(NULL, keep))
set.seed(11072026)
for (i in 1:M) {
  xx <- rnorm(n, 0, sdx); yy <- c0 + c1 * xx + rnorm(n, 0, sde)
  a0i <- uniroot(function(a) mean(plogis(a + a1 * xx)) - 0.40, c(-6, 6))$root
  mi <- runif(n) < plogis(a0i + a1 * xx); ob <- !mi
  fit <- lm(yy[ob] ~ xx[ob]); sig <- summary(fit)$sigma
  yh <- coef(fit)[1] + coef(fit)[2] * xx[mi]
  comp <- list(cc = { z <- yy; z[mi] <- NA; z },
               mean = { z <- yy; z[mi] <- mean(yy[ob]); z },
               detreg = { z <- yy; z[mi] <- yh; z },
               stoch = { z <- yy; z[mi] <- yh + rnorm(sum(mi), 0, sig); z })
  for (k in keep) { z <- comp[[k]]; ok <- !is.na(z); f <- lm(z[ok] ~ xx[ok])
    b <- unname(coef(f)[2]); s <- summary(f)$coef[2, 2]
    sl[i, k] <- b; se[i, k] <- s
    cv[i, k] <- (b - 1.96 * s <= c1) & (c1 <= b + 1.96 * s) }
}
summ <- rbind(bias = colMeans(sl) - c1, emp_SD = apply(sl, 2, sd),
              mean_SE = colMeans(se), coverage = colMeans(cv))
round(summ, 3)

## ---- fig-se ----
labs_m <- c(cc = "Complete-case", mean = "Mean imputation",
            detreg = "Regression imputation", stoch = "Stochastic imputation")
df2 <- data.frame(method = factor(labs_m, levels = rev(labs_m)),
                  emp_SD = summ["emp_SD", ], mean_SE = summ["mean_SE", ],
                  coverage = summ["coverage", ])
dfl <- pivot_longer(df2, c(emp_SD, mean_SE), names_to = "q", values_to = "v")
dfl$q <- factor(dfl$q, levels = c("emp_SD", "mean_SE"),
                labels = c("Empirical SD of slope", "Mean reported SE"))
ggplot(df2, aes(y = method)) +
  geom_segment(aes(x = mean_SE, xend = emp_SD, yend = method), colour = "#b9b4a8", linewidth = 1.1) +
  geom_point(data = dfl, aes(x = v, colour = q), size = 3) +
  geom_text(aes(x = pmax(emp_SD, mean_SE) + 0.006,
                label = sprintf("%.0f%% cover", 100 * coverage)),
            hjust = 0, size = 3.3, colour = ink) +
  scale_colour_manual(values = c("Empirical SD of slope" = "#2b2b2b", "Mean reported SE" = "#a24b4b")) +
  coord_cartesian(xlim = c(0.03, 0.135)) +
  labs(title = "A right point estimate can still report the wrong uncertainty",
       subtitle = "Reported error below the true spread means intervals undercover",
       x = "Standard error of the slope", y = NULL, colour = NULL) + theme_te()
