# Surplus production models in R
# https://tidyecology.com/posts/surplus-production-models/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-15

## ---- setup ----
library(ggplot2)

te_pal <- list(forest = "#275139", green = "#2f8f63", sage = "#93a87f",
               clay = "#b5534e", gold = "#cda23f", line = "#dad9ca",
               ink = "#16241d", paper = "#f5f4ee")

theme_te <- function() {
  theme_minimal(base_size = 12) +
    theme(panel.grid.minor = element_blank(),
          panel.grid.major = element_line(colour = "#e7e6dc"),
          plot.background = element_rect(fill = "#f5f4ee", colour = NA),
          panel.background = element_rect(fill = "#f5f4ee", colour = NA),
          plot.title = element_text(face = "bold", colour = te_pal$ink),
          axis.title = element_text(colour = "#2c3a31"))
}

## ---- simulate ----
set.seed(20260818)

production <- function(bio, rr, kk, shape) (rr / shape) * bio * (1 - (bio / kk)^shape)

sim_stock <- function(rr, kk, qq, fpath, sig_obs, sig_proc = 0) {
  nyr <- length(fpath)
  bio <- numeric(nyr + 1)
  catch <- numeric(nyr)
  bio[1] <- kk
  for (i in seq_len(nyr)) {
    catch[i] <- fpath[i] * bio[i]
    nxt <- bio[i] + production(bio[i], rr, kk, 1) - catch[i]
    if (sig_proc > 0) nxt <- nxt * exp(rnorm(1, -sig_proc^2 / 2, sig_proc))
    bio[i + 1] <- max(nxt, 1e-6)
  }
  cpue <- qq * bio[seq_len(nyr)] * exp(rnorm(nyr, -sig_obs^2 / 2, sig_obs))
  data.frame(year = seq_len(nyr), biomass = bio[seq_len(nyr)],
             catch = catch, cpue = cpue)
}

r_true <- 0.32; k_true <- 1200; q_true <- 3.5e-4; sig_obs <- 0.12; nyr <- 40

f_con <- c(seq(0.02, 0.30, length.out = 14), rep(0.30, 6),
           seq(0.30, 0.05, length.out = 4), rep(0.05, 6), rep(0.15, 10))
f_one <- seq(0.015, 0.34, length.out = nyr)

d_con <- sim_stock(r_true, k_true, q_true, f_con, sig_obs)
d_one <- sim_stock(r_true, k_true, q_true, f_one, sig_obs)

c(years = nyr, index_cv = sig_obs)
round(c(true_r = r_true, true_K = k_true, true_q = q_true,
        true_MSY = r_true * k_true / 4, true_Bmsy = k_true / 2,
        true_Fmsy = r_true / 2), 5)
round(c(contrast_lowest_BK = min(d_con$biomass) / k_true,
        contrast_final_BK  = d_con$biomass[nyr] / k_true,
        oneway_final_BK    = d_one$biomass[nyr] / k_true,
        oneway_declines_every_year = as.numeric(all(diff(d_one$biomass) < 0))), 3)

## ---- fig-series ----
long <- rbind(
  data.frame(d_con, stock = "Contrast: fished down, then rebuilt"),
  data.frame(d_one, stock = "One-way trip: effort only grows"))
long$biomass <- long$biomass / k_true
panels <- rbind(
  data.frame(long[, c("year", "stock")], value = long$biomass,
             panel = "Biomass / K"),
  data.frame(long[, c("year", "stock")], value = long$catch,
             panel = "Catch"),
  data.frame(long[, c("year", "stock")], value = long$cpue,
             panel = "Index"))
panels$panel <- factor(panels$panel, levels = c("Biomass / K", "Catch", "Index"))

ggplot(panels, aes(year, value)) +
  geom_line(colour = te_pal$forest, linewidth = 0.7) +
  geom_point(colour = te_pal$green, size = 0.9) +
  facet_grid(panel ~ stock, scales = "free_y") +
  labs(x = "Year", y = NULL, title = "Same biology, two management histories") +
  theme_te() +
  theme(strip.text = element_text(colour = te_pal$ink, face = "bold"))

## ---- estimator ----
project_biomass <- function(rv, kv, catch, shape = 1) {
  nyr <- length(catch)
  bio <- matrix(0, nrow = length(rv), ncol = nyr)
  bio[, 1] <- kv
  ok <- rep(TRUE, length(rv))
  for (i in seq_len(nyr - 1)) {
    nxt <- bio[, i] + production(bio[, i], rv, kv, shape) - catch[i]
    ok <- ok & is.finite(nxt) & nxt > 0.01 * kv
    bio[, i + 1] <- pmax(nxt, 0.01 * kv)
  }
  list(bio = bio, ok = ok)
}

spm_nll <- function(rv, kv, catch, cpue, shape = 1) {
  path <- project_biomass(rv, kv, catch, shape)
  lres <- matrix(log(cpue), nrow = length(rv), ncol = length(cpue),
                 byrow = TRUE) - log(path$bio)
  s2 <- rowMeans(lres^2) - rowMeans(lres)^2
  s2[!is.finite(s2) | s2 <= 0] <- 1e-12
  out <- 0.5 * length(cpue) * (log(2 * pi * s2) + 1)
  out[!path$ok] <- 1e6
  out
}

fit_spm <- function(dat, shape = 1, init = c(log(0.3), log(1000))) {
  obj <- function(par) spm_nll(exp(par[1]), exp(par[2]), dat$catch, dat$cpue, shape)
  opt <- optim(init, obj, control = list(reltol = 1e-13, maxit = 2000))
  opt <- optim(opt$par, obj, control = list(reltol = 1e-13, maxit = 2000))
  rr <- exp(opt$par[1]); kk <- exp(opt$par[2])
  bio <- as.vector(project_biomass(rr, kk, dat$catch, shape)$bio)
  list(r = rr, K = kk, q = exp(mean(log(dat$cpue) - log(bio))),
       shape = shape, nll = opt$value, bio = bio,
       bmsy = kk * (1 + shape)^(-1 / shape),
       msy = rr * kk * (1 + shape)^(-(1 + shape) / shape),
       fmsy = rr / (1 + shape))
}

fit_con <- fit_spm(d_con)
round(c(r = fit_con$r, K = fit_con$K, q = fit_con$q,
        MSY = fit_con$msy, Bmsy = fit_con$bmsy, Fmsy = fit_con$fmsy), 5)
round(c(r_pct_error = 100 * (fit_con$r / r_true - 1),
        K_pct_error = 100 * (fit_con$K / k_true - 1),
        q_pct_error = 100 * (fit_con$q / q_true - 1),
        MSY_pct_error = 100 * (fit_con$msy / (r_true * k_true / 4) - 1)), 2)

## ---- refpoints ----
bgrid <- seq(1, fit_con$K, length.out = 200001)
pgrid <- production(bgrid, fit_con$r, fit_con$K, 1)
round(c(numeric_Bmsy = bgrid[which.max(pgrid)],
        closed_form_K_over_2 = fit_con$K / 2,
        numeric_MSY = max(pgrid),
        closed_form_rK_over_4 = fit_con$r * fit_con$K / 4,
        numeric_Fmsy = max(pgrid) / bgrid[which.max(pgrid)],
        closed_form_r_over_2 = fit_con$r / 2), 4)

## ---- oneway ----
fit_one <- fit_spm(d_one)
round(c(r = fit_one$r, K = fit_one$K, q = fit_one$q,
        MSY = fit_one$msy, Bmsy = fit_one$bmsy, Fmsy = fit_one$fmsy), 5)

## ---- surface ----
r_cap <- 1
rgrid <- exp(seq(log(0.02), log(r_cap), length.out = 200))
kgrid <- exp(seq(log(250), log(30000), length.out = 200))
c(search_box_r = range(rgrid), search_box_K = range(kgrid))
surf <- expand.grid(r = rgrid, K = kgrid)

surf$dev_con <- spm_nll(surf$r, surf$K, d_con$catch, d_con$cpue)
surf$dev_con <- 2 * (surf$dev_con - min(surf$dev_con))
surf$dev_one <- spm_nll(surf$r, surf$K, d_one$catch, d_one$cpue)
surf$dev_one <- 2 * (surf$dev_one - min(surf$dev_one))

in_con <- surf[surf$dev_con <= 4, ]
in_one <- surf[surf$dev_one <= 4, ]
round(c(grid_points = nrow(surf),
        inside_contrast = nrow(in_con), inside_oneway = nrow(in_one),
        cor_logr_logK_contrast = cor(log(in_con$r), log(in_con$K)),
        cor_logr_logK_oneway = cor(log(in_one$r), log(in_one$K))), 4)

## ---- fig-ridge ----
ridge <- rbind(
  data.frame(surf[, c("r", "K")], dev = surf$dev_con,
             stock = "Contrast: fished down, then rebuilt"),
  data.frame(surf[, c("r", "K")], dev = surf$dev_one,
             stock = "One-way trip: effort only grows"))
best_k <- function(dv) {
  mat <- matrix(dv, nrow = length(rgrid), ncol = length(kgrid))
  kgrid[apply(mat, 1, which.min)]
}
trace_ridge <- rbind(
  data.frame(r = rgrid, K = best_k(surf$dev_con),
             stock = "Contrast: fished down, then rebuilt"),
  data.frame(r = rgrid, K = best_k(surf$dev_one),
             stock = "One-way trip: effort only grows"))
marks <- data.frame(
  r = c(r_true, fit_con$r, r_true, fit_one$r),
  K = c(k_true, fit_con$K, k_true, fit_one$K),
  kind = rep(c("truth", "fitted"), 2),
  stock = rep(c("Contrast: fished down, then rebuilt",
                "One-way trip: effort only grows"), each = 2))

ggplot(ridge, aes(r, K, z = dev)) +
  geom_contour_filled(breaks = c(0, 4, 10, 25, 60, Inf)) +
  geom_line(data = trace_ridge, aes(r, K), inherit.aes = FALSE,
            colour = te_pal$gold, linewidth = 0.5, linetype = "22") +
  geom_point(data = marks, aes(r, K, shape = kind), inherit.aes = FALSE,
             colour = te_pal$clay, size = 3, stroke = 1.2) +
  scale_fill_manual(values = c("#275139", "#3f9970", "#93a87f",
                               "#c4cfb6", "#e6e4d6"), name = "Deviance") +
  scale_shape_manual(values = c(truth = 1, fitted = 4), name = NULL) +
  scale_x_log10() + scale_y_log10() +
  coord_cartesian(xlim = c(0.02, 1), ylim = c(350, 20000)) +
  facet_wrap(~stock) +
  labs(x = "Growth rate r", y = "Carrying capacity K", title = "One likelihood ridge, two lengths") +
  theme_te() +
  theme(strip.text = element_text(colour = te_pal$ink, face = "bold"))

## ---- profiles ----
d_trunc <- d_con[seq_len(22), ]
c(truncated_years = nrow(d_trunc))

# profile: minimise over the nuisance parameter on a grid, then refine locally
prof_min <- function(dat, xv, ygrid, build, shape = 1) {
  cost <- function(xx, yv) {
    pars <- build(xx, yv)
    spm_nll(pars$r, pars$K, dat$catch, dat$cpue, shape)
  }
  dv <- sapply(xv, function(xx) {
    vals <- cost(xx, ygrid)
    j <- which.min(vals)
    lo <- ygrid[max(j - 1, 1)]
    hi <- ygrid[min(j + 1, length(ygrid))]
    if (lo >= hi) return(vals[j])
    min(vals[j],
        optimize(function(ly) cost(xx, exp(ly)), c(log(lo), log(hi)), tol = 1e-9)$objective)
  })
  data.frame(value = xv, dev = 2 * (dv - min(dv)))
}

build_r <- function(xx, yv) list(r = rep(xx, length(yv)), K = yv)
build_k <- function(xx, yv) list(r = yv, K = rep(xx, length(yv)))
build_msy <- function(xx, yv) {
  rr <- xx / (0.25 * yv)
  rr[rr > r_cap] <- 1e3        # outside the search box
  list(r = rr, K = yv)
}

mgrid <- exp(seq(log(5), log(300), length.out = 200))
series <- list(contrast = d_con, truncated = d_trunc, oneway = d_one)
mp <- lapply(series, function(dat) prof_min(dat, rgrid, kgrid, build_r))
kp <- lapply(series, function(dat) prof_min(dat, kgrid, rgrid, build_k))
mm <- lapply(series, function(dat) prof_min(dat, mgrid, kgrid, build_msy))

# where the profile crosses two log-likelihood units, by interpolation
span <- function(pf) {
  ok <- which(pf$dev <= 4)
  cross <- function(i, j) exp(approx(pf$dev[c(i, j)], log(pf$value[c(i, j)]), 4)$y)
  lo <- if (min(ok) == 1) pf$value[1] else cross(min(ok) - 1, min(ok))
  hi <- if (max(ok) == nrow(pf)) pf$value[nrow(pf)] else cross(max(ok) + 1, max(ok))
  c(lo = lo, hi = hi, fold = hi / lo,
    at_edge = as.numeric(min(ok) == 1 || max(ok) == nrow(pf)))
}
prof_tab <- data.frame(r = t(sapply(mp, span)), K = t(sapply(kp, span)),
                       MSY = t(sapply(mm, span)))
print(round(prof_tab, 3))
fit_tr <- fit_spm(d_trunc)
round(c(truncated_MSY_point = fit_tr$msy, truncated_r_point = fit_tr$r,
        truncated_K_point = fit_tr$K,
        truncated_MSY_pct_error = 100 * (fit_tr$msy / (r_true * k_true / 4) - 1)), 3)

## ---- fig-profiles ----
grab_prof <- function(nm) rbind(
  data.frame(mp[[nm]], parameter = "Growth rate r", series = nm),
  data.frame(kp[[nm]], parameter = "Carrying capacity K", series = nm),
  data.frame(mm[[nm]], parameter = "MSY", series = nm))
prof_long <- do.call(rbind, lapply(names(series), grab_prof))
prof_long$parameter <- factor(prof_long$parameter, levels = c(
  "Growth rate r", "Carrying capacity K", "MSY"))
prof_long$series <- factor(prof_long$series, levels = c(
  "contrast", "truncated", "oneway"), labels = c(
  "contrast (40 y)", "truncated (22 y)", "one-way trip (40 y)"))

ggplot(prof_long, aes(value, dev, colour = series)) +
  geom_hline(yintercept = 4, colour = te_pal$line, linewidth = 0.8) +
  geom_line(linewidth = 0.8) +
  facet_wrap(~parameter, scales = "free_x") +
  scale_x_log10() +
  scale_colour_manual(values = c(te_pal$forest, te_pal$gold, te_pal$clay),
                      name = NULL) +
  coord_cartesian(ylim = c(0, 14)) +
  labs(x = "Parameter value", y = "Deviance from best fit", title = "What a rebuild adds to the likelihood") +
  theme_te() +
  theme(legend.position = "top",
        strip.text = element_text(colour = te_pal$ink, face = "bold"))

## ---- shape ----
fit_shape <- function(dat, shape) {
  ff <- fit_spm(dat, shape, init = c(log(0.3), log(1200)))
  data.frame(shape = shape, r = ff$r, K = ff$K, nll = ff$nll, bmsy = ff$bmsy,
             bratio = ff$bmsy / ff$K, msy = ff$msy, fmsy = ff$fmsy)
}

shape_grid <- exp(seq(log(0.1), log(8), length.out = 45))
shape_tab <- do.call(rbind, lapply(shape_grid, function(s) fit_shape(d_con, s)))
shape_tab$dev <- 2 * (shape_tab$nll - min(shape_tab$nll))
shape_span <- span(data.frame(value = shape_tab$shape, dev = shape_tab$dev))
dev_schaefer <- 2 * (fit_shape(d_con, 1)$nll - min(shape_tab$nll))

round(c(best_shape = shape_tab$shape[which.min(shape_tab$dev)],
        shape_lo = unname(shape_span["lo"]), shape_hi = unname(shape_span["hi"]),
        shape_fold = unname(shape_span["fold"]),
        deviance_at_shape_one = dev_schaefer), 2)

## ---- fig-shape ----
shape_long <- rbind(
  data.frame(shape = shape_tab$shape, value = shape_tab$dev,
             panel = "Profile deviance"),
  data.frame(shape = shape_tab$shape, value = shape_tab$bratio,
             panel = "Implied Bmsy / K"))
panel_order <- c("Profile deviance", "Implied Bmsy / K")
shape_long$panel <- factor(shape_long$panel, levels = panel_order)
inside <- data.frame(lo = shape_span["lo"], hi = shape_span["hi"])
cutline <- data.frame(yv = 4,
                      panel = factor("Profile deviance", levels = panel_order))

ggplot(shape_long, aes(shape, value)) +
  geom_rect(data = inside, aes(xmin = lo, xmax = hi, ymin = -Inf, ymax = Inf),
            inherit.aes = FALSE, fill = te_pal$sage, alpha = 0.25) +
  geom_hline(data = cutline, aes(yintercept = yv),
             colour = te_pal$clay, linewidth = 0.5, linetype = "22") +
  geom_line(colour = te_pal$forest, linewidth = 0.8) +
  facet_wrap(~panel, ncol = 1, scales = "free_y") +
  scale_x_log10() +
  labs(x = "Shape parameter p", y = NULL, title = "A shape the data cannot choose") +
  theme_te() +
  theme(strip.text = element_text(colour = te_pal$ink, face = "bold"))

## ---- shape-refpoints ----
s1 <- fit_shape(d_con, 1)
s3 <- fit_shape(d_con, 3)
print(round(rbind(schaefer = unlist(s1), skewed = unlist(s3)), 4))
bmsy_pct_shift <- 100 * (s3$bmsy / s1$bmsy - 1)
round(c(MSY_pct_shift = 100 * (s3$msy / s1$msy - 1),
        Bmsy_pct_shift = bmsy_pct_shift,
        Fmsy_pct_shift = 100 * (s3$fmsy / s1$fmsy - 1),
        deviance_gap = 2 * (s3$nll - s1$nll)), 2)

## ---- error-fits ----
set.seed(4021)
sig_proc <- 0.15
d_proc <- sim_stock(r_true, k_true, q_true, f_con, 0.03, sig_proc)

nll_proc <- function(par, dat) {
  rr <- exp(par[1]); kk <- exp(par[2]); qq <- exp(par[3])
  bio <- dat$cpue / qq
  nn <- length(bio)
  pred <- bio[-nn] + production(bio[-nn], rr, kk, 1) - dat$catch[-nn]
  if (any(!is.finite(pred)) || any(pred <= 0)) return(1e8)
  wres <- log(bio[-1]) - log(pred)
  s2 <- mean(wres^2)
  if (!is.finite(s2) || s2 <= 0) return(1e8)
  0.5 * length(wres) * (log(2 * pi * s2) + 1)
}

fit_proc <- function(dat) {
  init <- c(log(0.3), log(1200), log(median(dat$cpue) / 600))
  opt <- optim(init, nll_proc, dat = dat, control = list(reltol = 1e-13, maxit = 4000))
  opt <- optim(opt$par, nll_proc, dat = dat, control = list(reltol = 1e-13, maxit = 4000))
  rr <- exp(opt$par[1]); kk <- exp(opt$par[2])
  list(r = rr, K = kk, q = exp(opt$par[3]), msy = rr * kk / 4, bmsy = kk / 2)
}

grab <- function(f) c(r = f$r, K = f$K, q = f$q, MSY = f$msy)
print(round(rbind(
  truth              = c(r = r_true, K = k_true, q = q_true, MSY = r_true * k_true / 4),
  index_error_obsfit = grab(fit_spm(d_con)),
  index_error_procfit = grab(fit_proc(d_con)),
  process_error_obsfit = grab(fit_spm(d_proc)),
  process_error_procfit = grab(fit_proc(d_proc))), 5))

## ---- error-mc ----
set.seed(88)
nrep <- 60
mc <- do.call(rbind, lapply(seq_len(nrep), function(i) {
  da <- sim_stock(r_true, k_true, q_true, f_con, sig_obs, 0)
  db <- sim_stock(r_true, k_true, q_true, f_con, 0.03, sig_proc)
  data.frame(rep = i,
             truth = rep(c("index error", "process error"), each = 2),
             assumption = rep(c("observation", "process"), 2),
             msy = c(fit_spm(da)$msy, fit_proc(da)$msy,
                     fit_spm(db)$msy, fit_proc(db)$msy))
}))
mc$ratio <- mc$msy / (r_true * k_true / 4)

c(replicates = nrep)
print(round(tapply(mc$ratio, list(mc$truth, mc$assumption), median), 3))
print(round(tapply(mc$ratio, list(mc$truth, mc$assumption), mad), 3))
