# Penalised regression splines from scratch
# https://tidyecology.com/posts/penalised-regression-splines/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-15

## ---- setup ----
library(splines)
library(mgcv)
library(ggplot2)

theme_te <- function() {
  theme_minimal(base_size = 12) +
    theme(
      panel.grid.minor = element_blank(),
      panel.grid.major = element_line(colour = "#e5e4dc", linewidth = 0.3),
      plot.background  = element_rect(fill = "#f5f4ee", colour = NA),
      panel.background = element_rect(fill = "#f5f4ee", colour = NA),
      text      = element_text(colour = "#16241d"),
      axis.text = element_text(colour = "#16241d"),
      plot.title = element_text(face = "bold")
    )
}
col_truth <- "#16241d"; col_naive <- "#c0622d"
col_pen   <- "#275139"; col_sage  <- "#93a87f"

## ---- simulate-and-basis ----
set.seed(4206)
n <- 200
x <- sort(runif(n))
f <- function(x) sin(2 * pi * x) * (0.8 + 0.9 * x)
sigma <- 0.45
y <- f(x) + rnorm(n, 0, sigma)

## cubic B-spline basis with K functions, equally spaced knots
K <- 20; deg <- 3
rng <- range(x)
nik <- K - deg - 1
dx  <- diff(rng) / (nik + 1)
knots <- c(rng[1] - dx * (deg:1),
           seq(rng[1], rng[2], length = nik + 2),
           rng[2] + dx * (1:deg))
B <- splineDesign(knots, x, ord = deg + 1, outer.ok = TRUE)

## second-order difference penalty on adjacent coefficients
D <- diff(diag(K), differences = 2)
S <- crossprod(D)

## ---- penalised-fit ----
BtB <- crossprod(B); Bty <- crossprod(B, y)
fit_ps <- function(lam) {
  A  <- BtB + lam * S
  Ai <- solve(A)
  beta <- Ai %*% Bty
  fit  <- as.vector(B %*% beta)
  H    <- B %*% Ai %*% t(B)          # influence matrix
  edf  <- sum(diag(H))               # effective degrees of freedom
  rss  <- sum((y - fit)^2)
  gcv  <- n * rss / (n - edf)^2      # generalised cross-validation
  list(beta = beta, fit = fit, edf = edf, gcv = gcv)
}

## select lambda by GCV over a log-spaced grid
lgrid <- 10^seq(-4, 6, length = 200)
gcvv  <- sapply(lgrid, function(l) fit_ps(l)$gcv)
p6_lam    <- lgrid[which.min(gcvv)]
ps_best   <- fit_ps(p6_lam)
ps_unpen  <- fit_ps(1e-8)
p6_edf    <- ps_best$edf
p6_edfun  <- ps_unpen$edf

## ---- fig-fit ----
poly12 <- lm(y ~ poly(x, 12))
grid <- data.frame(x = seq(min(x), max(x), length = 400))
Bg <- splineDesign(knots, grid$x, ord = deg + 1, outer.ok = TRUE)
grid$pen  <- as.vector(Bg %*% ps_best$beta)
grid$pl12 <- predict(poly12, grid)
grid$tru  <- f(grid$x)

ggplot() +
  geom_point(data = data.frame(x, y), aes(x, y),
             colour = col_sage, alpha = 0.6, size = 1.4) +
  geom_line(data = grid, aes(x, tru),  colour = col_truth,
            linetype = "dotted", linewidth = 0.8) +
  geom_line(data = grid, aes(x, pl12), colour = col_naive, linewidth = 0.9) +
  geom_line(data = grid, aes(x, pen),  colour = col_pen,   linewidth = 1) +
  coord_cartesian(ylim = c(-2.4, 2.4)) +
  labs(x = "gradient x", y = "response y",
       title = "Global polynomial versus penalised spline") +
  theme_te()

## ---- mgcv-check ----
d <- data.frame(x = x, y = y)
m_gcv  <- gam(y ~ s(x, bs = "ps", k = 20, m = c(2, 2)), data = d, method = "GCV.Cp")
m_reml <- gam(y ~ s(x, bs = "ps", k = 20, m = c(2, 2)), data = d, method = "REML")

p6_edfmg   <- sum(m_gcv$edf)                 # total, including intercept
p6_edfsm   <- summary(m_gcv)$edf             # smooth term only, GCV
p6_edfreml <- summary(m_reml)$edf            # smooth term only, REML
p6_maxdiff <- max(abs(ps_best$fit - as.vector(predict(m_gcv))))

## ---- criteria-sim ----
stopifnot(identical(formals(gam)$method, "GCV.Cp"),   # the default measured here,
          gam(round(3 * exp(f(x))) ~ s(x), family = poisson)$method == "UBRE")  # UBRE for counts
crit_reps <- 400; crit_slope <- 1.5
crit_ps <- function(v) sprintf("s(%s, bs = 'ps', k = 20, m = c(2, 2))", v)
crit_pair <- function(fm, dat) list(g = gam(fm, data = dat, method = "GCV.Cp"),
                                    r = gam(fm, data = dat, method = "REML"))
crit_wiggly <- function() {                 # this post's curve, n and sigma
  dw <- data.frame(xw = sort(runif(n))); dw$yw <- f(dw$xw) + rnorm(n, 0, sigma)
  fp <- crit_pair(as.formula(paste("yw ~", crit_ps("xw"))), dw)
  c(edf_g = summary(fp$g)$edf, edf_r = summary(fp$r)$edf,
    rmse_g = sqrt(mean((fitted(fp$g) - f(dw$xw))^2)),
    rmse_r = sqrt(mean((fitted(fp$r) - f(dw$xw))^2)))
}
crit_straight <- function() {               # x straight, z no effect at all
  ds <- data.frame(xs = runif(n), zs = runif(n))
  ds$ys <- crit_slope * ds$xs + rnorm(n, 0, sigma)
  fp <- crit_pair(as.formula(paste("ys ~", crit_ps("xs"), "+", crit_ps("zs"))), ds)
  sg <- summary(fp$g)$s.table; sr <- summary(fp$r)$s.table
  c(lin_g = sg[1, "edf"], lin_r = sr[1, "edf"], nul_g = sg[2, "edf"],
    nul_r = sr[2, "edf"], p_g = sg[2, "p-value"], p_r = sr[2, "p-value"])
}
set.seed(14302)
crit_w <- as.data.frame(t(replicate(crit_reps, crit_wiggly())))
crit_s <- as.data.frame(t(replicate(crit_reps, crit_straight())))
## share of samples, with its Monte Carlo standard error
crit_share <- function(v) c(p = mean(v), se = sqrt(mean(v) * (1 - mean(v)) / length(v)))
crit_lin <- sapply(crit_s[c("lin_g", "lin_r")], function(e) crit_share(e > 2))
crit_nul <- sapply(crit_s[c("nul_g", "nul_r")], function(e) crit_share(e > 2))
crit_fp  <- sapply(crit_s[c("p_g", "p_r")], function(p) crit_share(p < 0.05))
crit_q95 <- sapply(crit_s[c("nul_g", "nul_r")], quantile, probs = 0.95)
crit_more <- crit_share(crit_w$edf_r > crit_w$edf_g)
crit_qw <- sapply(crit_w[c("edf_g", "edf_r")], quantile, probs = c(0.05, 0.5, 0.95))
crit_ratio <- crit_w$rmse_g / crit_w$rmse_r
crit_bad_g <- crit_share(crit_ratio > 1.25); crit_bad_r <- crit_share(crit_ratio < 0.8)
crit_bad_up <- (crit_w$edf_g > crit_w$edf_r)[crit_ratio > 1.25]

## ---- fig-criteria ----
crit_arms <- c("straight term s(x)", "null term s(z)", "wiggly curve of this post")
crit_pts <- data.frame(arm = factor(rep(crit_arms, each = crit_reps), levels = crit_arms),
                       reml = c(crit_s$lin_r, crit_s$nul_r, crit_w$edf_r),
                       gcv  = c(crit_s$lin_g, crit_s$nul_g, crit_w$edf_g),
                       bad  = c(rep(FALSE, 2 * crit_reps), crit_ratio > 1.25))
ggplot(crit_pts[order(crit_pts$bad), ], aes(reml, gcv)) +
  geom_abline(intercept = 0, slope = 1, colour = col_truth, linetype = "dotted", linewidth = 0.6) +
  geom_point(aes(colour = bad), alpha = 0.55, size = 1.3) +
  scale_colour_manual(values = c(`FALSE` = col_pen, `TRUE` = col_naive), name = NULL,
                      labels = c("other samples", "GCV error > 1.25 x REML error")) +
  facet_wrap(~ arm, scales = "free") +
  labs(x = "edf of the smooth term under REML", y = "edf under GCV", title = "The same samples, two criteria") +
  theme_te() + theme(legend.position = "bottom")

## ---- compare ----
poly4 <- lm(y ~ poly(x, 4)); poly12 <- lm(y ~ poly(x, 12))
truef <- f(x)
rmse  <- function(fit) sqrt(mean((fit - truef)^2))

p6_r_p4  <- rmse(fitted(poly4))
p6_r_p12 <- rmse(fitted(poly12))
p6_r_un  <- rmse(ps_unpen$fit)
p6_r_pen <- rmse(ps_best$fit)

xe <- 1.15
Be <- splineDesign(knots, xe, ord = deg + 1, outer.ok = TRUE)
p6_ext_pen <- as.vector(Be %*% ps_best$beta)
p6_ext_p12 <- as.vector(predict(poly12, data.frame(x = xe)))

## how much basis is left at and beyond the edge of the data
p6_row_in  <- sum(splineDesign(knots, max(x), ord = deg + 1, outer.ok = TRUE))
p6_row_mid <- sum(splineDesign(knots, 1.05,   ord = deg + 1, outer.ok = TRUE))
p6_row_out <- sum(Be)

## ---- fig-gcv ----
gdf <- data.frame(loglam = log10(lgrid), gcv = gcvv)
ggplot(gdf, aes(loglam, gcv)) +
  geom_line(colour = col_pen, linewidth = 1) +
  geom_vline(xintercept = log10(p6_lam), linetype = "dashed",
             colour = col_naive, linewidth = 0.7) +
  annotate("text", x = log10(p6_lam), y = max(gcvv) * 0.96,
           label = sprintf("edf = %.1f", p6_edf), hjust = -0.1,
           colour = col_truth, size = 3.6) +
  labs(x = "log10 penalty", y = "GCV score",
       title = "The penalty is selected, not chosen") +
  theme_te()
