# Implicit information in integrated models
# https://tidyecology.com/posts/ipm-implicit-information/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-07

## ---- setup ----
library(ggplot2)
paper <- "#f5f4ee"; ink <- "#16241d"; body <- "#2c3a31"; forest <- "#275139"
labelc <- "#46604a"; sage <- "#93a87f"; linec <- "#dad9ca"; faint <- "#5d6b61"
gold <- "#c9b458"; rust <- "#b5534e"
theme_te <- function(base_size = 12) {
  theme_minimal(base_size = base_size) +
    theme(
      plot.background  = element_rect(fill = paper, colour = NA),
      panel.background = element_rect(fill = paper, colour = NA),
      panel.grid.minor = element_blank(),
      panel.grid.major = element_line(colour = linec, linewidth = 0.3),
      axis.title = element_text(colour = body),
      axis.text  = element_text(colour = faint),
      plot.title = element_text(colour = ink, face = "bold"),
      plot.subtitle = element_text(colour = labelc),
      legend.title = element_text(colour = body),
      legend.text  = element_text(colour = body),
      strip.text   = element_text(colour = ink)
    )
}

## ---- simulate ----
set.seed(132)
Tyr <- 25; phi_true <- 0.55; f_true <- 0.48; sy_true <- 11; N1_true <- 180
N <- numeric(Tyr); N[1] <- N1_true
for (t in 1:(Tyr - 1)) {
  N[t + 1] <- rbinom(1, N[t], phi_true) + rpois(1, f_true * N[t])
}
C <- round(rnorm(Tyr, N, sy_true))

p_true <- 0.5; Kcr <- 9; Rrel <- rep(30, Kcr - 1)
recaps <- vector("list", Kcr - 1); never <- integer(Kcr - 1)
for (i in 1:(Kcr - 1)) {
  alive <- rep(TRUE, Rrel[i]); fr <- integer(Rrel[i]); rc <- numeric(Kcr)
  for (t in (i + 1):Kcr) {
    alive <- alive & (runif(length(alive)) < phi_true)
    got   <- alive & (runif(length(alive)) < p_true) & (fr == 0); fr[got] <- t
  }
  for (t in (i + 1):Kcr) rc[t] <- sum(fr == t)
  recaps[[i]] <- rc; never[i] <- Rrel[i] - sum(fr > 0)
}
Jyr <- 10; prodB <- rep(40, Jyr); prodJ <- rpois(Jyr, prodB * f_true)

kf_ll <- function(phi, f, sy, N1, C) {
  lam <- phi + f; dv <- phi * (1 - phi) + f; a <- N1; P <- (2 * sy)^2; ll <- 0
  for (t in seq_along(C)) {
    if (t > 1) { ap <- a; a <- lam * ap; P <- lam^2 * P + dv * max(ap, 1) }
    Fv <- P + sy^2; ll <- ll + dnorm(C[t], a, sqrt(Fv), log = TRUE)
    K <- P / Fv; a <- a + K * (C[t] - a); P <- (1 - K) * P
  }
  ll
}
marray_ll <- function(phi, p) {
  ll <- 0
  for (i in 1:(Kcr - 1)) {
    ts <- (i + 1):Kcr; q <- phi^(ts - i) * (1 - p)^(ts - i - 1) * p
    ll <- ll + sum(recaps[[i]][ts] * log(q)) + never[i] * log(1 - sum(q))
  }
  ll
}

## ---- fits ----
fitv <- function(use) {
  has_p <- "cr" %in% use
  init <- if (has_p) c(qlogis(.5), log(.5), log(12), qlogis(.5), log(C[1]))
          else       c(qlogis(.5), log(.5), log(12), log(C[1]))
  fn <- function(par) {
    phi <- plogis(par[1]); ff <- exp(par[2]); sy <- exp(par[3])
    if (has_p) { p <- plogis(par[4]); N1 <- exp(par[5]) } else { N1 <- exp(par[4]) }
    ll <- 0
    if ("count" %in% use) ll <- ll + kf_ll(phi, ff, sy, N1, C)
    if ("cr"    %in% use) ll <- ll + marray_ll(phi, p)
    if ("prod"  %in% use) ll <- ll + sum(dpois(prodJ, prodB * ff, log = TRUE))
    -ll
  }
  o <- optim(init, fn, method = "BFGS", hessian = TRUE,
             control = list(maxit = 1500, reltol = 1e-11))
  V <- solve(o$hessian); phi <- plogis(o$par[1]); ff <- exp(o$par[2])
  c(phi = phi, f = ff, se_phi = sqrt(V[1, 1]) * phi * (1 - phi), se_f = sqrt(V[2, 2]) * ff)
}
cr   <- fitv(c("count", "cr"))       # counts + capture-recapture (no productivity)
prod <- fitv(c("count", "prod"))     # counts + productivity (no capture-recapture)
full <- fitv(c("count", "cr", "prod"))

## ---- fig-precision ----
sedf <- data.frame(
  model = rep(c("count + CR", "count + productivity", "full"), each = 2),
  rate  = rep(c("survival", "recruitment"), 3),
  se    = c(cr["se_phi"], cr["se_f"], prod["se_phi"], prod["se_f"], full["se_phi"], full["se_f"])
)
sedf$model <- factor(sedf$model, levels = c("count + CR", "count + productivity", "full"))
ggplot(sedf, aes(rate, se, colour = model, group = model)) +
  geom_line(position = position_dodge(0.35), linewidth = 0.5, alpha = 0.6) +
  geom_point(position = position_dodge(0.35), size = 3.4) +
  scale_colour_manual(values = c("count + CR" = sage,
                                 "count + productivity" = gold, "full" = forest)) +
  labs(x = NULL, y = "standard error", colour = NULL,
       title = "What each data set buys in precision",
       subtitle = "the full model is tightest for both rates") +
  theme_te() + theme(legend.position = "top")

## ---- fig-regions ----
# refit the full model to get the nuisance parameters and p for the surfaces
oful <- optim(c(qlogis(.5), log(.5), log(12), qlogis(.5), log(C[1])), function(par) {
  -(kf_ll(plogis(par[1]), exp(par[2]), exp(par[3]), exp(par[5]), C) +
    marray_ll(plogis(par[1]), plogis(par[4])) +
    sum(dpois(prodJ, prodB * exp(par[2]), log = TRUE)))
}, method = "BFGS", control = list(maxit = 1500, reltol = 1e-11))
syf <- exp(oful$par[3]); N1f <- exp(oful$par[5]); pf <- plogis(oful$par[4])
pg <- seq(0.38, 0.72, length.out = 70); fg <- seq(0.30, 0.64, length.out = 70)
gr <- expand.grid(phi = pg, f = fg)
gr$count <- mapply(function(a, b) kf_ll(a, b, syf, N1f, C), gr$phi, gr$f)
gr$cr    <- gr$count + sapply(gr$phi, function(a) marray_ll(a, pf))
gr$prod  <- gr$count + sapply(gr$f, function(b) sum(dpois(prodJ, prodB * b, log = TRUE)))
gr$fulls <- gr$cr + gr$prod - gr$count       # counts once, plus both auxiliary terms
gr$rz <- gr$count - max(gr$count); gr$rz[gr$rz < -25] <- -25
lev <- function(z) max(z) - 2
ggplot(gr, aes(phi, f)) +
  geom_raster(aes(fill = rz), interpolate = TRUE) +
  scale_fill_gradient(low = paper, high = sage, name = "count log-lik") +
  geom_contour(aes(z = cr),    breaks = lev(gr$cr),    colour = forest, linewidth = 0.7) +
  geom_contour(aes(z = prod),  breaks = lev(gr$prod),  colour = gold,   linewidth = 0.7) +
  geom_contour(aes(z = fulls), breaks = lev(gr$fulls), colour = rust,   linewidth = 0.7) +
  annotate("point", x = phi_true, y = f_true, colour = rust, size = 3, shape = 17) +
  annotate("text", x = 0.44, y = 0.62, label = "+ CR", colour = forest, size = 3.4) +
  annotate("text", x = 0.68, y = 0.40, label = "+ productivity", colour = gold, size = 3.4) +
  coord_equal() +
  labs(x = "adult survival (phi)", y = "recruitment (f)",
       title = "Each data set cuts the ridge",
       subtitle = "capture-recapture pins survival, productivity pins recruitment") +
  theme_te()

## ---- bias ----
prodJ_bias <- rpois(Jyr, prodB * f_true * 1.3)          # productivity over-recorded by 30%
ob <- optim(c(qlogis(.5), log(.5), log(12), log(C[1])), function(par) {
  -(kf_ll(plogis(par[1]), exp(par[2]), exp(par[3]), exp(par[4]), C) +
    sum(dpois(prodJ_bias, prodB * exp(par[2]), log = TRUE)))
}, method = "BFGS", control = list(maxit = 1500, reltol = 1e-11))
phi_bias <- plogis(ob$par[1]); f_bias <- exp(ob$par[2])

## ---- thumb ----
th <- ggplot(gr, aes(phi, f)) +
  geom_raster(aes(fill = rz), interpolate = TRUE) +
  scale_fill_gradient(low = paper, high = sage, guide = "none") +
  geom_contour(aes(z = cr),    breaks = lev(gr$cr),    colour = forest, linewidth = 0.8) +
  geom_contour(aes(z = prod),  breaks = lev(gr$prod),  colour = gold,   linewidth = 0.8) +
  geom_contour(aes(z = fulls), breaks = lev(gr$fulls), colour = rust,   linewidth = 0.8) +
  coord_equal() + labs(x = NULL, y = NULL) + theme_te()
ggsave("thumbnail.png", th, width = 7.5, height = 5.4, dpi = 100, bg = paper)
