# Demographic and environmental variance in an IPM
# https://tidyecology.com/posts/demographic-and-environmental-variance/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-11-15

## ---- setup ----
library(ggplot2)
paper <- "#f5f4ee"; ink <- "#16241d"; body <- "#2c3a31"; forest <- "#275139"
labelc <- "#46604a"; sage <- "#93a87f"; linec <- "#dad9ca"; faint <- "#5d6b61"
gold <- "#c9b458"; rust <- "#b5534e"
theme_te <- function(base_size = 12) {
  theme_minimal(base_size = base_size) +
    theme(
      plot.background  = element_rect(fill = paper, colour = NA),
      panel.background = element_rect(fill = paper, colour = NA),
      panel.grid.minor = element_blank(),
      panel.grid.major = element_line(colour = linec, linewidth = 0.3),
      axis.title = element_text(colour = body),
      axis.text  = element_text(colour = faint),
      plot.title = element_text(colour = ink, face = "bold"),
      plot.subtitle = element_text(colour = labelc),
      legend.title = element_text(colour = body),
      legend.text  = element_text(colour = body),
      strip.text   = element_text(colour = ink)
    )
}

## ---- simulate ----
set.seed(133)
Tyr <- 30; phi_bar <- 0.52; f_bar <- 0.48; sig_e <- 0.06; sy_true <- 8; N1_true <- 110

# environmental shocks perturb the rates each year; demographic draws on top
N <- numeric(Tyr); N[1] <- N1_true
for (t in 1:(Tyr - 1)) {
  e    <- rnorm(1, 0, sig_e)
  phit <- plogis(qlogis(phi_bar) + e / (phi_bar + f_bar) * 2)
  ft   <- f_bar * exp(e / (phi_bar + f_bar))
  N[t + 1] <- rbinom(1, N[t], phit) + rpois(1, ft * N[t])
}
C <- round(rnorm(Tyr, N, sy_true))

# capture-recapture and productivity: pin the mean rates (hence demographic size)
p_true <- 0.55; Kcr <- 10; Rrel <- rep(35, Kcr - 1)
recaps <- vector("list", Kcr - 1); never <- integer(Kcr - 1)
for (i in 1:(Kcr - 1)) {
  alive <- rep(TRUE, Rrel[i]); fr <- integer(Rrel[i]); rc <- numeric(Kcr)
  for (t in (i + 1):Kcr) {
    alive <- alive & (runif(length(alive)) < phi_bar)
    got   <- alive & (runif(length(alive)) < p_true) & (fr == 0); fr[got] <- t
  }
  for (t in (i + 1):Kcr) rc[t] <- sum(fr == t)
  recaps[[i]] <- rc; never[i] <- Rrel[i] - sum(fr > 0)
}
Jyr <- 12; prodB <- rep(45, Jyr); prodJ <- rpois(Jyr, prodB * f_bar)

## ---- fit ----
marray_ll <- function(phi, p) {
  ll <- 0
  for (i in 1:(Kcr - 1)) {
    ts <- (i + 1):Kcr; q <- phi^(ts - i) * (1 - p)^(ts - i - 1) * p
    ll <- ll + sum(recaps[[i]][ts] * log(q)) + never[i] * log(1 - sum(q))
  }
  ll
}
P1 <- 20^2   # initial state variance, fixed: tied to sy, the likelihood had no maximum as sy -> 0
kf <- function(phi, f, se2, sy, N1, demog = TRUE) {
  lam <- phi + f; dv <- phi * (1 - phi) + f; a <- N1; P <- P1; ll <- 0
  for (t in seq_along(C)) {
    if (t > 1) {
      ap <- a; a <- lam * ap
      Q  <- (if (demog) dv * max(ap, 1) else 0) + se2 * max(ap, 1)^2
      P  <- lam^2 * P + Q
    }
    Fv <- P + sy^2; ll <- ll + dnorm(C[t], a, sqrt(Fv), log = TRUE)
    K <- P / Fv; a <- a + K * (C[t] - a); P <- (1 - K) * P
  }
  ll
}
nll <- function(par, demog = TRUE) {
  -(kf(plogis(par[1]), exp(par[2]), exp(par[3]), exp(par[4]), exp(par[6]), demog) +
    marray_ll(plogis(par[1]), plogis(par[5])) +
    sum(dpois(prodJ, prodB * exp(par[2]), log = TRUE)))
}
st <- c(qlogis(.5), log(.5), log(0.003), log(9), qlogis(.5), log(C[1]))
of <- optim(st, nll, demog = TRUE, method = "BFGS", hessian = TRUE,
            control = list(maxit = 2000, reltol = 1e-11))
stopifnot(of$convergence == 0)   # check before inverting the Hessian
Vf    <- solve(of$hessian)
phi_h <- plogis(of$par[1]); f_h <- exp(of$par[2]); se2_h <- exp(of$par[3])
sig_e_h <- sqrt(se2_h)
ci_lo <- sqrt(exp(of$par[3] - 1.96 * sqrt(Vf[3, 3])))
ci_hi <- sqrt(exp(of$par[3] + 1.96 * sqrt(Vf[3, 3])))
dv_h  <- phi_h * (1 - phi_h) + f_h
Nstar <- dv_h / se2_h

## ---- fig-components ----
Ng <- seq(min(N) * 0.8, max(N) * 1.15, length.out = 200)
comp <- rbind(
  data.frame(N = Ng, v = dv_h / Ng, part = "demographic"),
  data.frame(N = Ng, v = se2_h,     part = "environmental")
)
ggplot(comp, aes(N, v, colour = part)) +
  annotate("rect", xmin = min(N), xmax = max(N), ymin = -Inf, ymax = Inf,
           fill = sage, alpha = 0.13) +
  geom_vline(xintercept = Nstar, linetype = "dashed", colour = faint) +
  geom_line(linewidth = 0.9) +
  annotate("text", x = Nstar, y = max(comp$v) * 0.95,
           label = paste0("N* = ", round(Nstar)), colour = faint, hjust = -0.1, size = 3.4) +
  scale_colour_manual(values = c(demographic = rust, environmental = forest), name = NULL) +
  labs(x = "population size (N)", y = "variance of per-capita growth rate",
       title = "Which noise dominates depends on abundance",
       subtitle = "demographic variance falls with N, environmental variance does not") +
  theme_te() + theme(legend.position = "top")

## ---- env-only ----
oe <- optim(st, nll, demog = FALSE, method = "BFGS",
            control = list(maxit = 2000, reltol = 1e-11))
sig_e_env <- sqrt(exp(oe$par[3]))

## ---- fig-decomp ----
lam <- phi_h + f_h
ap <- Pp <- af <- Pf <- numeric(Tyr); a <- exp(of$par[6]); P <- P1
for (t in 1:Tyr) {
  if (t > 1) { a <- lam * af[t - 1]; P <- lam^2 * Pf[t - 1] + dv_h * max(af[t - 1], 1) + se2_h * max(af[t - 1], 1)^2 }
  ap[t] <- a; Pp[t] <- P
  Fk <- P + exp(of$par[4])^2; K <- P / Fk
  af[t] <- a + K * (C[t] - a); Pf[t] <- (1 - K) * P
}
as_ <- af
for (t in (Tyr - 1):1) { J <- Pf[t] * lam / Pp[t + 1]; as_[t] <- af[t] + J * (as_[t + 1] - ap[t + 1]) }
dd <- data.frame(N = as_,
                 demographic = dv_h * as_,
                 environmental = se2_h * as_^2)
dd <- dd[order(dd$N), ]
dl <- rbind(data.frame(N = dd$N, v = dd$demographic, part = "demographic"),
            data.frame(N = dd$N, v = dd$environmental, part = "environmental"))
ggplot(dl, aes(N, v, fill = part)) +
  geom_area(alpha = 0.85, colour = paper, linewidth = 0.2) +
  scale_fill_manual(values = c(demographic = rust, environmental = forest), name = NULL) +
  labs(x = "population size (N)", y = "one-step process variance",
       title = "The noise budget shifts as the population grows",
       subtitle = "environmental term grows with N squared, demographic share fades") +
  theme_te() + theme(legend.position = "top")

## ---- thumb ----
th <- ggplot(comp, aes(N, v, colour = part)) +
  annotate("rect", xmin = min(N), xmax = max(N), ymin = -Inf, ymax = Inf,
           fill = sage, alpha = 0.13) +
  geom_vline(xintercept = Nstar, linetype = "dashed", colour = faint) +
  geom_line(linewidth = 1.1) +
  scale_colour_manual(values = c(demographic = rust, environmental = forest), guide = "none") +
  labs(x = NULL, y = NULL) + theme_te()
ggsave("thumbnail.png", th, width = 7.5, height = 5.4, dpi = 100, bg = paper)
