# Checking an integrated population model
# https://tidyecology.com/posts/checking-an-integrated-population-model/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-05-21

## ---- setup ----
library(ggplot2)
paper <- "#f5f4ee"; ink <- "#16241d"; body <- "#2c3a31"; forest <- "#275139"
labelc <- "#46604a"; sage <- "#93a87f"; linec <- "#dad9ca"; faint <- "#5d6b61"
gold <- "#c9b458"; rust <- "#b5534e"
theme_te <- function(base_size = 12) {
  theme_minimal(base_size = base_size) +
    theme(
      plot.background  = element_rect(fill = paper, colour = NA),
      panel.background = element_rect(fill = paper, colour = NA),
      panel.grid.minor = element_blank(),
      panel.grid.major = element_line(colour = linec, linewidth = 0.3),
      axis.title = element_text(colour = body),
      axis.text  = element_text(colour = faint),
      plot.title = element_text(colour = ink, face = "bold"),
      plot.subtitle = element_text(colour = labelc),
      legend.title = element_text(colour = body),
      legend.text  = element_text(colour = body),
      strip.text   = element_text(colour = ink)
    )
}

## ---- simulate ----
set.seed(134)
Tyr <- 26; phi_true <- 0.54; f_true <- 0.46; sy_true <- 10; N1_true <- 170
N <- numeric(Tyr); N[1] <- N1_true
for (t in 1:(Tyr - 1)) N[t + 1] <- rbinom(1, N[t], phi_true) + rpois(1, f_true * N[t])
C <- round(rnorm(Tyr, N, sy_true))

p_true <- 0.5; Kcr <- 9; Rrel <- rep(32, Kcr - 1)
recaps <- vector("list", Kcr - 1); never <- integer(Kcr - 1)
for (i in 1:(Kcr - 1)) {
  alive <- rep(TRUE, Rrel[i]); fr <- integer(Rrel[i]); rc <- numeric(Kcr)
  for (t in (i + 1):Kcr) {
    alive <- alive & (runif(length(alive)) < phi_true)
    got   <- alive & (runif(length(alive)) < p_true) & (fr == 0); fr[got] <- t
  }
  for (t in (i + 1):Kcr) rc[t] <- sum(fr == t)
  recaps[[i]] <- rc; never[i] <- Rrel[i] - sum(fr > 0)
}
Jyr <- 11; prodB <- rep(42, Jyr)
prodJ_ok   <- rpois(Jyr, prodB * f_true)             # unbiased survey
prodJ_bias <- rpois(Jyr, prodB * f_true * 1.35)      # productivity over-recorded by 35%

## ---- estimates ----
marray_ll <- function(phi, p) {
  ll <- 0
  for (i in 1:(Kcr - 1)) {
    ts <- (i + 1):Kcr; q <- phi^(ts - i) * (1 - p)^(ts - i - 1) * p
    ll <- ll + sum(recaps[[i]][ts] * log(q)) + never[i] * log(1 - sum(q))
  }
  ll
}
kf_ll <- function(phi, f, sy, N1) {
  lam <- phi + f; dv <- phi * (1 - phi) + f; a <- N1; P <- (2 * sy)^2; ll <- 0
  for (t in seq_along(C)) {
    if (t > 1) { ap <- a; a <- lam * ap; P <- lam^2 * P + dv * max(ap, 1) }
    Fv <- P + sy^2; ll <- ll + dnorm(C[t], a, sqrt(Fv), log = TRUE)
    K <- P / Fv; a <- a + K * (C[t] - a); P <- (1 - K) * P
  }
  ll
}
# survival from capture-recapture alone
ocr <- optim(c(qlogis(.5), qlogis(.5)), function(pp) -marray_ll(plogis(pp[1]), plogis(pp[2])),
             method = "BFGS", hessian = TRUE)
stopifnot(ocr$convergence == 0)   # check before inverting the Hessian
Vcr <- solve(ocr$hessian); phi_cr <- plogis(ocr$par[1])
se_cr <- sqrt(Vcr[1, 1]) * phi_cr * (1 - phi_cr)
# survival implied by counts + productivity (no capture-recapture)
implicit <- function(prodJ) {
  o <- optim(c(qlogis(.5), log(.5), log(11), log(C[1])), function(pp)
    -(kf_ll(plogis(pp[1]), exp(pp[2]), exp(pp[3]), exp(pp[4])) +
      sum(dpois(prodJ, prodB * exp(pp[2]), log = TRUE))),
    method = "BFGS", hessian = TRUE, control = list(reltol = 1e-11))
  V <- solve(o$hessian); ph <- plogis(o$par[1])
  c(phi = ph, se = sqrt(V[1, 1]) * ph * (1 - ph))
}
im_ok <- implicit(prodJ_ok); im_bi <- implicit(prodJ_bias)
zscore <- function(a, sea, b, seb) (a - b) / sqrt(sea^2 + seb^2)
z_ok <- zscore(phi_cr, se_cr, im_ok["phi"], im_ok["se"])
z_bi <- zscore(phi_cr, se_cr, im_bi["phi"], im_bi["se"])

## ---- fig-conflict ----
cf <- data.frame(
  survey = factor(rep(c("unbiased survey", "inflated survey"), each = 2),
                  levels = c("unbiased survey", "inflated survey")),
  source = rep(c("capture-recapture", "counts + productivity"), 2),
  est = c(phi_cr, im_ok["phi"], phi_cr, im_bi["phi"]),
  se  = c(se_cr, im_ok["se"], se_cr, im_bi["se"])
)
ggplot(cf, aes(est, source, colour = source)) +
  geom_vline(xintercept = phi_true, linetype = "dotted", colour = faint) +
  geom_errorbar(aes(xmin = est - 1.96 * se, xmax = est + 1.96 * se),
                orientation = "y", width = 0.18, linewidth = 0.7) +
  geom_point(size = 3.2) +
  facet_wrap(~survey, ncol = 1) +
  scale_colour_manual(values = c("capture-recapture" = forest,
                                 "counts + productivity" = gold), guide = "none") +
  labs(x = "adult survival (phi)", y = NULL,
       title = "Do the data sets agree on survival?",
       subtitle = "direct and implicit estimates should coincide; the dotted line is the truth") +
  theme_te()

## ---- fig-residuals ----
of <- optim(c(qlogis(.5), log(.5), log(11), qlogis(.5), log(C[1])), function(pp)
  -(kf_ll(plogis(pp[1]), exp(pp[2]), exp(pp[3]), exp(pp[5])) +
    marray_ll(plogis(pp[1]), plogis(pp[4])) +
    sum(dpois(prodJ_bias, prodB * exp(pp[2]), log = TRUE))),
  method = "BFGS", control = list(reltol = 1e-11))
phi_f <- plogis(of$par[1]); f_f <- exp(of$par[2]); sy_f <- exp(of$par[3]); N1_f <- exp(of$par[5])
lam <- phi_f + f_f; dv <- phi_f * (1 - phi_f) + f_f; a <- N1_f; P <- (2 * sy_f)^2
innov <- numeric(Tyr)
for (t in 1:Tyr) {
  if (t > 1) { ap <- a; a <- lam * ap; P <- lam^2 * P + dv * max(ap, 1) }
  Fv <- P + sy_f^2; innov[t] <- (C[t] - a) / sqrt(Fv)
  K <- P / Fv; a <- a + K * (C[t] - a); P <- (1 - K) * P
}
rd <- data.frame(year = 1:Tyr, z = innov)
ggplot(rd, aes(year, z)) +
  geom_hline(yintercept = 0, colour = linec) +
  geom_hline(yintercept = c(-2, 2), linetype = "dashed", colour = faint) +
  geom_point(colour = forest, size = 2) +
  labs(x = "year", y = "standardised prediction error",
       title = "The count component looks fine",
       subtitle = "healthy residuals do not rule out a conflict between data sets") +
  theme_te()

## ---- thumb ----
th <- ggplot(cf, aes(est, source, colour = source)) +
  geom_vline(xintercept = phi_true, linetype = "dotted", colour = faint) +
  geom_errorbar(aes(xmin = est - 1.96 * se, xmax = est + 1.96 * se),
                orientation = "y", width = 0.16, linewidth = 0.9) +
  geom_point(size = 3.4) +
  facet_wrap(~survey, ncol = 1) +
  scale_colour_manual(values = c("capture-recapture" = forest,
                                 "counts + productivity" = gold), guide = "none") +
  labs(x = NULL, y = NULL) +
  theme_te() + theme(axis.text.y = element_blank())
ggsave("thumbnail.png", th, width = 7.5, height = 5.4, dpi = 100, bg = paper)
