# Mark resight abundance estimation in R
# https://tidyecology.com/posts/mark-resight-abundance/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-07

## ---- data ----
set.seed(3140)

sim_mr <- function(N = 400, n = 80, J = 12, lam = 0.35, sigma = 0.8, seed = 1) {
  set.seed(seed)
  # individual sighting rate, log-normal, with mean fixed at lam
  alpha <- rlnorm(N, log(lam) - sigma^2 / 2, sigma)
  mk <- sample(N, n)
  y <- matrix(rpois(N * J, rep(alpha, J)), N, J)
  list(N = N, n = n, J = J,
       ym = y[mk, , drop = FALSE],                  # marked: identified individually
       Tu = sum(y[-mk, , drop = FALSE]))            # unmarked: a count, nothing more
}
d <- sim_mr(seed = 7)
ytot <- rowSums(d$ym)
p314_Tm <- sum(ytot); p314_Tu <- d$Tu
c(true_N = d$N, marked = d$n, occasions = d$J,
  marked_sightings = p314_Tm, unmarked_sightings = p314_Tu)

## ---- estimator ----
lam_hat <- sum(ytot) / (d$n * d$J)
U_hat <- d$Tu / (d$J * lam_hat)
N_hat <- d$n + U_hat
p314_lam <- lam_hat; p314_N <- N_hat
c(rate = lam_hat, unmarked_est = U_hat, N_hat = N_hat, true = d$N)

## ---- overdispersion ----
p314_mean <- mean(ytot); p314_var <- var(ytot)
p314_ratio <- p314_var / p314_mean
c(mean_of_individual_totals = p314_mean,
  variance_of_individual_totals = p314_var,
  ratio = p314_ratio)

## ---- gh ----
gh_nodes <- function(K) {                # Golub-Welsch for the weight exp(-x^2)
  i <- 1:(K - 1)
  Jm <- matrix(0, K, K)
  Jm[cbind(i, i + 1)] <- sqrt(i / 2)
  Jm[cbind(i + 1, i)] <- sqrt(i / 2)
  e <- eigen(Jm, symmetric = TRUE)
  o <- order(e$values)
  list(x = e$values[o], w = sqrt(pi) * e$vectors[1, o]^2)
}
GH <- gh_nodes(300)
# validate on integrals we know: E[Z^2] = 1 and E[exp(Z)] = exp(1/2)
p314_ez2 <- sum(GH$w / sqrt(pi) * (sqrt(2) * GH$x)^2)
p314_eez <- sum(GH$w / sqrt(pi) * exp(sqrt(2) * GH$x))
c(E_Z2 = p314_ez2, E_expZ = p314_eez, target = exp(0.5))

## ---- pln ----
mk_tab <- function(y) { tb <- table(y); list(v = as.integer(names(tb)), k = as.integer(tb)) }
nll_pln <- function(par, tab, J) {
  lam_i <- exp(par[1] + sqrt(2) * exp(par[2]) * GH$x)     # rate at each node
  M <- outer(tab$v, J * lam_i, dpois)                     # unique totals x nodes
  -sum(tab$k * log(as.numeric(M %*% (GH$w / sqrt(pi)))))
}
tab <- mk_tab(ytot)
f <- optim(c(log(0.3), log(0.8)), nll_pln, tab = tab, J = d$J, method = "BFGS")
mu_h <- f$par[1]; sg_h <- exp(f$par[2])
lam_pln <- exp(mu_h + sg_h^2 / 2)
p314_sg <- sg_h; p314_lam_pln <- lam_pln
c(sigma = sg_h, mean_rate = lam_pln, moment_rate = lam_hat)

## ---- variance ----
J <- d$J
cv2 <- exp(sg_h^2) - 1                          # squared CV of the individual rate
var_yi <- J * lam_hat + J^2 * lam_hat^2 * cv2   # variance of one animal's total
v_lam <- var_yi / (d$n * J^2)
v_Tu_naive <- d$Tu                              # Poisson: variance equals the count
v_Tu_hon <- U_hat * (J * lam_hat + J^2 * lam_hat^2 * cv2)

se_naive <- sqrt(v_Tu_naive / (J * lam_hat)^2 + (U_hat / lam_hat)^2 * (lam_hat / (d$n * J)))
se_hon <- sqrt(v_Tu_hon / (J * lam_hat)^2 + (U_hat / lam_hat)^2 * v_lam)
p314_se_n <- se_naive; p314_se_h <- se_hon
c(se_naive = se_naive, se_honest = se_hon, ratio = se_hon / se_naive)

## ---- mc ----
fit_one <- function(d) {
  yt <- rowSums(d$ym); J <- d$J; n <- d$n
  lam <- sum(yt) / (n * J)
  U <- d$Tu / (J * lam); N <- n + U
  se_n <- sqrt(d$Tu / (J * lam)^2 + (U / lam)^2 * (lam / (n * J)))
  tb <- mk_tab(yt)
  ff <- optim(c(log(lam), log(0.8)), nll_pln, tab = tb, J = J, method = "BFGS")
  sg <- exp(ff$par[2]); cv2 <- exp(sg^2) - 1
  v_lam <- (J * lam + J^2 * lam^2 * cv2) / (n * J^2)
  v_Tu <- U * (J * lam + J^2 * lam^2 * cv2)
  se_h <- sqrt(v_Tu / (J * lam)^2 + (U / lam)^2 * v_lam)
  c(N = N, se_naive = se_n, se_hon = se_h, sigma = sg)
}
M <- 300
res <- t(sapply(1:M, function(i) fit_one(sim_mr(seed = 8000 + i))))
p314_bias <- 100 * (mean(res[, "N"]) / 400 - 1)
p314_truecv <- sd(res[, "N"]) / mean(res[, "N"])
p314_cv_n <- mean(res[, "se_naive"] / res[, "N"])
p314_cv_h <- mean(res[, "se_hon"] / res[, "N"])
p314_sg_mc <- mean(res[, "sigma"])
p314_cov_n <- mean(abs(res[, "N"] - 400) <= 1.96 * res[, "se_naive"])
p314_cov_h <- mean(abs(res[, "N"] - 400) <= 1.96 * res[, "se_hon"])
data.frame(replicates = M, rel_bias_pct = p314_bias, mean_sigma = p314_sg_mc,
           cv_actual = p314_truecv, cv_naive = p314_cv_n, cv_honest = p314_cv_h,
           cover_naive = p314_cov_n, cover_honest = p314_cov_h)

## ---- sweep ----
sweep_sigma <- function(sg, R = 150) {
  r <- t(sapply(1:R, function(i) fit_one(sim_mr(sigma = sg, seed = 9000 + i))))
  data.frame(sigma = sg,
             bias = 100 * (mean(r[, "N"]) / 400 - 1),
             cv_true = sd(r[, "N"]) / mean(r[, "N"]),
             cv_naive = mean(r[, "se_naive"] / r[, "N"]),
             cv_honest = mean(r[, "se_hon"] / r[, "N"]),
             cover_naive = mean(abs(r[, "N"] - 400) <= 1.96 * r[, "se_naive"]),
             cover_honest = mean(abs(r[, "N"] - 400) <= 1.96 * r[, "se_hon"]))
}
sw <- do.call(rbind, lapply(c(0, 0.4, 0.8, 1.2), sweep_sigma))
p314_sw <- sw
print(round(sw, 4), row.names = FALSE)

## ---- fig-sweep ----
library(ggplot2)
theme_te <- function() {
  theme_minimal(base_size = 11) +
    theme(
          plot.background  = element_rect(fill = "#f5f4ee", colour = NA),
          panel.background = element_rect(fill = "#f5f4ee", colour = NA),
          panel.grid.minor = element_blank(),
          panel.grid.major = element_line(colour = "#dad9ca", linewidth = 0.3),
          plot.title = element_text(colour = "#16241d", face = "bold", size = 11),
          axis.title = element_text(colour = "#46604a", size = 9.5),
          axis.text = element_text(colour = "#5d6b61", size = 8.5),
          legend.position = "top", legend.title = element_blank(),
          legend.text = element_text(colour = "#2c3a31", size = 8.5))
}
d1 <- rbind(
  data.frame(s = sw$sigma, v = sw$cv_naive, k = "naive Poisson"),
  data.frame(s = sw$sigma, v = sw$cv_honest, k = "overdispersion corrected"),
  data.frame(s = sw$sigma, v = sw$cv_true, k = "actual variability"))
pA <- ggplot(d1, aes(s, v, colour = k, linetype = k)) +
  geom_line(linewidth = 0.9) + geom_point(size = 2) +
  scale_colour_manual(values = c("naive Poisson" = "#c9b458",
                                 "overdispersion corrected" = "#275139",
                                 "actual variability" = "#b5534e")) +
  scale_linetype_manual(values = c("naive Poisson" = "solid",
                                   "overdispersion corrected" = "solid",
                                   "actual variability" = "dashed")) +
  labs(title = "The naive interval misses heterogeneity",
       x = "sigma (individual heterogeneity)", y = "coefficient of variation of N") +
  theme_te()
d2 <- rbind(data.frame(s = sw$sigma, v = sw$cover_naive, k = "naive Poisson"),
            data.frame(s = sw$sigma, v = sw$cover_honest, k = "overdispersion corrected"))
pB <- ggplot(d2, aes(s, v, colour = k)) +
  geom_hline(yintercept = 0.95, linetype = "dashed", colour = "#5d6b61", linewidth = 0.4) +
  geom_line(linewidth = 0.9) + geom_point(size = 2) +
  scale_colour_manual(values = c("naive Poisson" = "#c9b458",
                                 "overdispersion corrected" = "#275139")) +
  scale_y_continuous(limits = c(0.3, 1)) +
  labs(title = "Coverage of a nominal 95% interval",
       x = "sigma (individual heterogeneity)", y = "measured coverage") +
  theme_te()
grid::grid.newpage()
grid::pushViewport(grid::viewport(layout = grid::grid.layout(1, 2)))
print(pA, vp = grid::viewport(layout.pos.row = 1, layout.pos.col = 1))
print(pB, vp = grid::viewport(layout.pos.row = 1, layout.pos.col = 2))

## ---- collar ----
sim_eta <- function(N = 400, n = 80, J = 12, lam = 0.35, sigma = 0.8, eta = 1, seed = 1) {
  set.seed(seed)
  alpha <- rlnorm(N, log(lam) - sigma^2 / 2, sigma)
  mk <- sample(N, n)
  a2 <- alpha; a2[mk] <- a2[mk] * eta           # the collar effect, marked only
  y <- matrix(rpois(N * J, rep(a2, J)), N, J)
  list(N = N, n = n, J = J, ym = y[mk, , drop = FALSE], Tu = sum(y[-mk, , drop = FALSE]))
}
collar_row <- function(eta, R = 120) {
  r <- t(sapply(1:R, function(i) {
    dd <- sim_eta(eta = eta, seed = 7000 + i)
    out <- fit_one(dd)
    c(out, ratio = var(rowSums(dd$ym)) / mean(rowSums(dd$ym)))
  }))
  data.frame(eta = eta, mean_N = mean(r[, "N"]), bias = 100 * (mean(r[, "N"]) / 400 - 1),
             var_mean_ratio = mean(r[, "ratio"]), sigma_hat = mean(r[, "sigma"]))
}
cs <- do.call(rbind, lapply(c(1.0, 0.9, 0.8, 0.7), collar_row))
p314_cs <- cs
print(round(cs, 3), row.names = FALSE)
