# N-mixture models for abundance in R
# https://tidyecology.com/posts/n-mixture-models-abundance/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-11-14

## ---- setup ----
library(ggplot2)

ink <- "#16241d"; body <- "#2c3a31"; forest <- "#275139"; gold <- "#c9b458"
paper <- "#f5f4ee"; line_c <- "#dad9ca"; faint <- "#5d6b61"; sage <- "#93a87f"

theme_te <- function(base = 12) {
  theme_minimal(base_size = base) +
    theme(panel.grid.minor = element_blank(),
          panel.grid.major = element_line(colour = line_c, linewidth = 0.3),
          plot.background = element_rect(fill = paper, colour = NA),
          panel.background = element_rect(fill = paper, colour = NA),
          axis.title = element_text(colour = body),
          axis.text = element_text(colour = faint),
          plot.title = element_text(colour = ink, face = "bold"),
          plot.subtitle = element_text(colour = faint),
          legend.title = element_text(colour = body),
          legend.text = element_text(colour = faint))
}

## ---- simulate ----
set.seed(2027)
R <- 150            # sites
K <- 4              # repeat visits
lambda_true <- 2.6  # mean local abundance
p_true <- 0.35      # per-visit detection probability

N <- rpois(R, lambda_true)                       # latent abundance, never observed
y <- matrix(NA_integer_, R, K)
for (k in 1:K) y[, k] <- rbinom(R, N, p_true)    # binomial counts given N

head(y)

## ---- naive ----
maxcount <- apply(y, 1, max)
meancount <- rowMeans(y)

c(true_mean_N = mean(N),
  naive_mean_of_max = mean(maxcount),
  naive_mean_of_mean = mean(meancount))

c(true_total_N = sum(N), naive_total_from_max = sum(maxcount))

## ---- fit-nmix ----
Nmax <- 60   # upper bound for the sum; well above any plausible abundance here

negll <- function(par) {
  lam <- exp(par[1])
  p   <- plogis(par[2])
  ll  <- 0
  for (i in 1:R) {
    lo <- max(y[i, ])          # abundance cannot be below the largest count seen
    Ns <- lo:Nmax
    pmfN <- dpois(Ns, lam)
    lik_i <- 0
    for (j in seq_along(Ns)) {
      lik_i <- lik_i + pmfN[j] * prod(dbinom(y[i, ], Ns[j], p))
    }
    ll <- ll + log(lik_i)
  }
  -ll
}

fit <- optim(c(log(2), qlogis(0.3)), negll, method = "BFGS", hessian = TRUE)
lam_hat <- exp(fit$par[1])
p_hat   <- plogis(fit$par[2])
c(lambda_hat = lam_hat, p_hat = p_hat)

## ---- nmix-se ----
se_link <- sqrt(diag(solve(fit$hessian)))
se_lam <- se_link[1] * lam_hat               # derivative of exp is exp
se_p   <- se_link[2] * p_hat * (1 - p_hat)   # derivative of plogis
lam_ci <- exp(fit$par[1] + c(-1.96, 1.96) * se_link[1])
p_ci   <- plogis(fit$par[2] + c(-1.96, 1.96) * se_link[2])

round(c(lambda_hat = lam_hat, se = se_lam, lo = lam_ci[1], hi = lam_ci[2]), 3)
round(c(p_hat = p_hat, se = se_p, lo = p_ci[1], hi = p_ci[2]), 3)

## ---- fig-naive-vs-model ----
d1 <- data.frame(
  est = factor(c("Naive: mean\nmax count", "N-mixture\nlambda-hat", "Truth\n(lambda)"),
               levels = c("Naive: mean\nmax count", "N-mixture\nlambda-hat", "Truth\n(lambda)")),
  val = c(mean(maxcount), lam_hat, lambda_true),
  lo  = c(NA, lam_ci[1], NA),
  hi  = c(NA, lam_ci[2], NA),
  fill = c("naive", "model", "truth"))

ggplot(d1, aes(est, val, fill = fill)) +
  geom_col(width = 0.62) +
  geom_errorbar(aes(ymin = lo, ymax = hi), width = 0.16, colour = ink, na.rm = TRUE) +
  geom_hline(yintercept = lambda_true, linetype = "dashed", colour = faint, linewidth = 0.4) +
  scale_fill_manual(values = c(naive = gold, model = forest, truth = sage), guide = "none") +
  labs(title = "Ignoring detection undercounts abundance",
       subtitle = "Mean local abundance: naive max count versus the N-mixture estimate",
       x = NULL, y = "Individuals per site") +
  theme_te()

## ---- fig-marginal-fit ----
cmax <- max(y)
cs <- 0:cmax
obs <- as.numeric(table(factor(as.vector(y), levels = cs))) / (R * K)
expf <- sapply(cs, function(c) {
  Ns <- c:Nmax
  sum(dpois(Ns, lam_hat) * dbinom(c, Ns, p_hat))
})

d2 <- data.frame(count = rep(cs, 2),
                 freq = c(obs, expf),
                 src = rep(c("Observed", "Fitted N-mixture"), each = length(cs)))
d2$src <- factor(d2$src, levels = c("Observed", "Fitted N-mixture"))

ggplot(d2, aes(factor(count), freq, fill = src)) +
  geom_col(position = position_dodge(width = 0.7), width = 0.62) +
  scale_fill_manual(values = c("Observed" = sage, "Fitted N-mixture" = forest), name = NULL) +
  labs(title = "The fitted model reproduces the count distribution",
       subtitle = "Relative frequency of per-visit counts, pooled over sites and visits",
       x = "Count on a visit", y = "Relative frequency") +
  theme_te() +
  theme(legend.position = "inside", legend.position.inside = c(0.8, 0.8))

## ---- fig-site-abundance ----
postN <- numeric(R)
for (i in 1:R) {
  lo <- max(y[i, ]); Ns <- lo:Nmax
  w <- dpois(Ns, lam_hat) * sapply(Ns, function(Nj) prod(dbinom(y[i, ], Nj, p_hat)))
  postN[i] <- sum(Ns * w) / sum(w)
}
c(estimated_total = sum(postN), naive_max_total = sum(maxcount), true_total = sum(N))

d3 <- data.frame(maxc = maxcount, post = postN)
ggplot(d3, aes(maxc, post)) +
  geom_abline(slope = 1, intercept = 0, linetype = "dashed", colour = faint) +
  geom_point(position = position_jitter(width = 0.12, height = 0, seed = 7),
             colour = forest, alpha = 0.5, size = 1.9) +
  labs(title = "Site abundance is corrected upward from the raw maximum",
       subtitle = "Posterior-mean local abundance versus the largest count seen at a site",
       x = "Observed maximum count", y = "Estimated abundance (posterior mean)") +
  theme_te()
