# Integrated population models from scratch
# https://tidyecology.com/posts/integrated-population-model-basics/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-05-21

## ---- setup ----
library(ggplot2)
paper <- "#f5f4ee"; ink <- "#16241d"; body <- "#2c3a31"; forest <- "#275139"
labelc <- "#46604a"; sage <- "#93a87f"; linec <- "#dad9ca"; faint <- "#5d6b61"
gold <- "#c9b458"; rust <- "#b5534e"
theme_te <- function(base_size = 12) {
  theme_minimal(base_size = base_size) +
    theme(
      plot.background  = element_rect(fill = paper, colour = NA),
      panel.background = element_rect(fill = paper, colour = NA),
      panel.grid.minor = element_blank(),
      panel.grid.major = element_line(colour = linec, linewidth = 0.3),
      axis.title = element_text(colour = body),
      axis.text  = element_text(colour = faint),
      plot.title = element_text(colour = ink, face = "bold"),
      plot.subtitle = element_text(colour = labelc),
      legend.title = element_text(colour = body),
      legend.text  = element_text(colour = body),
      strip.text   = element_text(colour = ink)
    )
}

## ---- simulate ----
set.seed(131)
Tyr <- 25; phi_true <- 0.52; f_true <- 0.45; sy_true <- 12; N1_true <- 200

# true demographic process: survivors Binomial, recruits Poisson
N <- numeric(Tyr); N[1] <- N1_true
for (t in 1:(Tyr - 1)) {
  S <- rbinom(1, N[t], phi_true)
  R <- rpois(1, f_true * N[t])
  N[t + 1] <- S + R
}
C <- round(rnorm(Tyr, N, sy_true))                 # annual counts (with error)

# capture-recapture: individual CJS histories collapsed to an m-array
p_true <- 0.5; Kcr <- 9; Rrel <- rep(30, Kcr - 1)
recaps <- vector("list", Kcr - 1); never <- integer(Kcr - 1)
for (i in 1:(Kcr - 1)) {
  alive <- rep(TRUE, Rrel[i]); fr <- integer(Rrel[i]); rc <- numeric(Kcr)
  for (t in (i + 1):Kcr) {
    alive <- alive & (runif(length(alive)) < phi_true)
    got   <- alive & (runif(length(alive)) < p_true) & (fr == 0)
    fr[got] <- t
  }
  for (t in (i + 1):Kcr) rc[t] <- sum(fr == t)
  recaps[[i]] <- rc; never[i] <- Rrel[i] - sum(fr > 0)
}

# productivity survey: recruits from a sample of adults, J years
Jyr <- 10; prodB <- rep(40, Jyr); prodJ <- rpois(Jyr, prodB * f_true)

## ---- likelihood ----
# count component: Kalman filter with a state-dependent process variance
kf_ll <- function(phi, f, sy, N1, C) {
  lam <- phi + f; dv <- phi * (1 - phi) + f
  a <- N1; P <- (2 * sy)^2; ll <- 0
  for (t in seq_along(C)) {
    if (t > 1) { ap <- a; a <- lam * ap; P <- lam^2 * P + dv * max(ap, 1) }
    Fv <- P + sy^2
    ll <- ll + dnorm(C[t], a, sqrt(Fv), log = TRUE)
    K <- P / Fv; a <- a + K * (C[t] - a); P <- (1 - K) * P
  }
  ll
}
# capture-recapture component: m-array multinomial
marray_ll <- function(phi, p) {
  ll <- 0
  for (i in 1:(Kcr - 1)) {
    ts <- (i + 1):Kcr
    q  <- phi^(ts - i) * (1 - p)^(ts - i - 1) * p
    ll <- ll + sum(recaps[[i]][ts] * log(q)) + never[i] * log(1 - sum(q))
  }
  ll
}
# joint negative log-likelihood (drop components via `use` to see their effect)
nll <- function(par, use = c("count", "cr", "prod")) {
  phi <- plogis(par[1]); f <- exp(par[2]); sy <- exp(par[3])
  p   <- plogis(par[4]); N1 <- exp(par[5]); ll <- 0
  if ("count" %in% use) ll <- ll + kf_ll(phi, f, sy, N1, C)
  if ("cr"    %in% use) ll <- ll + marray_ll(phi, p)
  if ("prod"  %in% use) ll <- ll + sum(dpois(prodJ, prodB * f, log = TRUE))
  -ll
}

st  <- c(qlogis(0.5), log(0.5), log(15), qlogis(0.5), log(C[1]))
fit <- optim(st, nll, use = c("count", "cr", "prod"),
             method = "BFGS", hessian = TRUE,
             control = list(maxit = 1000, reltol = 1e-11))
stopifnot(fit$convergence == 0)   # check before inverting the Hessian
V     <- solve(fit$hessian); pr <- fit$par
phi_h <- plogis(pr[1]); f_h <- exp(pr[2]); sy_h <- exp(pr[3])
p_h   <- plogis(pr[4]); N1_h <- exp(pr[5])
se_phi <- sqrt(V[1, 1]) * phi_h * (1 - phi_h)
se_f   <- sqrt(V[2, 2]) * f_h

## ---- ridge-profile ----
phi_grid <- seq(0.32, 0.74, by = 0.015)
prof <- function(phi_fix, use) {
  o <- optim(c(log(0.5), log(15), qlogis(0.5), log(C[1])), function(pp) {
    f <- exp(pp[1]); sy <- exp(pp[2]); p <- plogis(pp[3]); N1 <- exp(pp[4]); ll <- 0
    if ("count" %in% use) ll <- ll + kf_ll(phi_fix, f, sy, N1, C)
    if ("cr"    %in% use) ll <- ll + marray_ll(phi_fix, p)
    if ("prod"  %in% use) ll <- ll + sum(dpois(prodJ, prodB * f, log = TRUE))
    -ll
  }, method = "Nelder-Mead", control = list(maxit = 2000))
  -o$value
}
ll_count <- sapply(phi_grid, prof, use = "count")
ll_joint <- sapply(phi_grid, prof, use = c("count", "cr", "prod"))
w_count <- range(phi_grid[ll_count >= max(ll_count) - 2])
w_joint <- range(phi_grid[ll_joint >= max(ll_joint) - 2])

## ---- fig-ridge ----
pg <- seq(0.35, 0.72, length.out = 60)
fg <- seq(0.28, 0.62, length.out = 60)
gr <- expand.grid(phi = pg, f = fg)
gr$ll <- mapply(function(ph, ff) kf_ll(ph, ff, sy_h, N1_h, C), gr$phi, gr$f)
gr$ll <- gr$ll - max(gr$ll)
gr$ll[gr$ll < -25] <- -25
ggplot(gr, aes(phi, f)) +
  geom_raster(aes(fill = ll), interpolate = TRUE) +
  geom_contour(aes(z = ll), colour = linec, linewidth = 0.25, bins = 10) +
  annotate("point", x = phi_h, y = f_h, colour = gold, size = 3) +
  annotate("point", x = phi_true, y = f_true, colour = rust, size = 3, shape = 17) +
  scale_fill_gradient(low = paper, high = forest, name = "log-lik") +
  coord_equal() +
  labs(x = "adult survival (phi)", y = "recruitment (f)",
       title = "Counts fix the growth rate, not the rates",
       subtitle = "count-only likelihood ridges along constant phi + f") +
  theme_te()

## ---- fig-trajectory ----
lam <- phi_h + f_h; dv <- phi_h * (1 - phi_h) + f_h
ap <- Pp <- af <- Pf <- numeric(Tyr)
a <- N1_h; P <- (2 * sy_h)^2
for (t in 1:Tyr) {
  if (t > 1) { a <- lam * af[t - 1]; P <- lam^2 * Pf[t - 1] + dv * max(af[t - 1], 1) }
  ap[t] <- a; Pp[t] <- P
  Fk <- P + sy_h^2; K <- P / Fk
  af[t] <- a + K * (C[t] - a); Pf[t] <- (1 - K) * P
}
as_ <- af; Ps <- Pf                                   # RTS smoother (backward)
for (t in (Tyr - 1):1) {
  J <- Pf[t] * lam / Pp[t + 1]
  as_[t] <- af[t] + J * (as_[t + 1] - ap[t + 1])
  Ps[t]  <- Pf[t] + J^2 * (Ps[t + 1] - Pp[t + 1])
}
td <- data.frame(year = 1:Tyr, count = C, Nsm = as_,
                 lo = as_ - 1.96 * sqrt(Ps), hi = as_ + 1.96 * sqrt(Ps))
ggplot(td, aes(year)) +
  geom_ribbon(aes(ymin = lo, ymax = hi), fill = sage, alpha = 0.3) +
  geom_line(aes(y = Nsm), colour = forest, linewidth = 0.8) +
  geom_point(aes(y = count), colour = ink, size = 1.8) +
  labs(x = "year", y = "adult abundance",
       title = "Reconstructed population trajectory",
       subtitle = "smoothed abundance separates process from observation error") +
  theme_te()

## ---- thumb ----
th <- ggplot(td, aes(year)) +
  geom_ribbon(aes(ymin = lo, ymax = hi), fill = sage, alpha = 0.3) +
  geom_line(aes(y = Nsm), colour = forest, linewidth = 0.9) +
  geom_point(aes(y = count), colour = ink, size = 1.6) +
  labs(x = NULL, y = NULL) + theme_te() +
  theme(legend.position = "none")
ggsave("thumbnail.png", th, width = 7.5, height = 5.4, dpi = 100, bg = paper)
