# Checking a dynamic occupancy model
# https://tidyecology.com/posts/checking-a-dynamic-occupancy-model/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-09-30

## ---- setup ----
Tn <- 5; K <- 4; R <- 400; psi1 <- 0.55; gamma <- 0.20; eps <- 0.25; M <- Tn * K
Hgrid <- as.matrix(expand.grid(rep(list(0:K), Tn)))   # every possible count history
tot_grid <- rowSums(Hgrid)

fwd_ll <- function(H, pr, K) {                         # log P(count history), vectorised over rows
  ps <- pr[1]; ga <- pr[2]; ep <- pr[3]; pp <- pr[4]
  Phi <- matrix(c(1 - ga, ga, ep, 1 - ep), 2, 2, byrow = TRUE); n <- nrow(H)
  em <- function(t) { dc <- H[, t]; cbind(as.numeric(dc == 0), dbinom(dc, K, pp)) }
  a <- matrix(c(1 - ps, ps), n, 2, byrow = TRUE) * em(1); s <- rowSums(a); ll <- log(s); a <- a / s
  for (t in 2:ncol(H)) { a <- (a %*% Phi) * em(t); s <- rowSums(a); ll <- ll + log(s); a <- a / s }
  ll
}
nll    <- function(par, d, K) -sum(fwd_ll(d, plogis(par), K))
fitfun <- function(d) optim(c(0, qlogis(0.2), qlogis(0.2), qlogis(0.3)), nll,
                            d = d, K = K, method = "BFGS")
gen <- function(pvec, ps = psi1, ga = gamma, ep = eps) {   # dynamics default to the truth
  z <- matrix(0L, R, Tn); z[, 1] <- rbinom(R, 1, ps)
  for (t in 2:Tn) z[, t] <- rbinom(R, 1, ifelse(z[, t - 1] == 1, 1 - ep, ga))
  d <- matrix(0L, R, Tn); for (t in 1:Tn) d[, t] <- rbinom(R, K, z[, t] * pvec); d
}
set.seed(7);   dA <- gen(rep(0.35, R))                         # homogeneous detection
set.seed(505); pv <- ifelse(runif(R) < 0.5, 0.23, 0.49); dB <- gen(pv)  # two detection groups
round(rbind(homogeneous = plogis(fitfun(dA)$par),
            heterogeneous = plogis(fitfun(dB)$par)), 3)

## ---- statistic ----
exact_dist <- function(pr) {                 # exact P(total detections = j), j = 0..M
  ph <- exp(fwd_ll(Hgrid, pr, K)); agg <- tapply(ph, tot_grid, sum)
  out <- numeric(M + 1); out[as.integer(names(agg)) + 1] <- agg; out
}
X2 <- function(tot, E) {                      # Pearson statistic on the total-detection table
  O <- tabulate(tot + 1, nbins = M + 1); k <- E > 0.5   # tiny cells pooled into one
  sum((O[k] - E[k])^2 / E[k]) + (sum(O[!k]) - sum(E[!k]))^2 / sum(E[!k])
}
round(c(exact_dist_sums_to = sum(exact_dist(c(0.5, 0.2, 0.25, 0.35)))), 3)

## ---- gof ----
gof <- function(d, seed, Bb = 200) {
  pr <- plogis(fitfun(d)$par); Eo <- exact_dist(pr) * R
  x2o <- X2(rowSums(d), Eo)
  set.seed(seed); x2b <- numeric(Bb)
  for (b in 1:Bb) {
    db <- gen(rep(pr[4], R), ps = pr[1], ga = pr[2], ep = pr[3])   # simulate from the fit
    fb <- plogis(fitfun(db)$par)
    x2b[b] <- X2(rowSums(db), exact_dist(fb) * R)
  }
  list(pr = pr, Eo = Eo, Oobs = tabulate(rowSums(d) + 1, nbins = M + 1),
       x2o = x2o, x2b = x2b, chat = x2o / mean(x2b), p = (1 + sum(x2b >= x2o)) / (Bb + 1))
}
gA <- gof(dA, seed = 107); gB <- gof(dB, seed = 506)
round(c(chat_homogeneous = gA$chat, p_homogeneous = gA$p,
        chat_heterogeneous = gB$chat, p_heterogeneous = gB$p), 3)

## ---- fig-total-dist ----
suppressMessages({library(ggplot2); library(dplyr); library(tidyr)})
theme_te <- function(bs = 12) theme_minimal(base_size = bs) + theme(
  plot.background = element_rect(fill = "#f5f4ee", colour = NA),
  panel.background = element_rect(fill = "#f5f4ee", colour = NA),
  panel.grid.major = element_line(colour = "#dad9ca", linewidth = 0.3),
  panel.grid.minor = element_blank(),
  axis.text = element_text(colour = "#2c3a31"), axis.title = element_text(colour = "#16241d"),
  plot.title = element_text(colour = "#16241d", face = "bold"),
  plot.subtitle = element_text(colour = "#46604a"),
  legend.position = "top", strip.text = element_text(colour = "#16241d", face = "bold"))

pA <- sprintf("Correctly specified (c-hat %.2f)", gA$chat)
pB <- sprintf("Detection heterogeneity (c-hat %.2f)", gB$chat)
mk <- function(g, lab) bind_rows(
  data.frame(tot = 0:M, count = g$Oobs, kind = "Observed", panel = lab),
  data.frame(tot = 0:M, count = g$Eo,   kind = "Expected", panel = lab))
dd1 <- bind_rows(mk(gA, pA), mk(gB, pB)); dd1$panel <- factor(dd1$panel, levels = c(pA, pB))
ggplot(dd1, aes(tot, count)) +
  geom_col(data = subset(dd1, kind == "Observed"), fill = "#93a87f", width = 0.85) +
  geom_line(data = subset(dd1, kind == "Expected"), aes(group = 1), colour = "#b5534e", linewidth = 0.9) +
  geom_point(data = subset(dd1, kind == "Expected"), colour = "#b5534e", size = 1.3) +
  facet_wrap(~panel) + coord_cartesian(xlim = c(0, M)) +
  labs(x = "Total detections per site (5 seasons x 4 visits)", y = "Number of sites") +
  theme_te()

## ---- fig-bootstrap ----
dd2 <- bind_rows(data.frame(x2 = gA$x2b, panel = pA), data.frame(x2 = gB$x2b, panel = pB))
dd2$panel <- factor(dd2$panel, levels = c(pA, pB))
vl <- data.frame(panel = factor(c(pA, pB), levels = c(pA, pB)), x2o = c(gA$x2o, gB$x2o))
ggplot(dd2, aes(x2)) +
  geom_histogram(bins = 28, fill = "#93a87f", colour = "#f5f4ee", linewidth = 0.2) +
  geom_vline(data = vl, aes(xintercept = x2o), colour = "#b5534e", linewidth = 1) +
  geom_text(data = vl, aes(x = x2o, y = Inf, label = sprintf("obs %.0f", x2o)),
            colour = "#b5534e", hjust = 1.1, vjust = 1.6, size = 3, family = "mono") +
  facet_wrap(~panel, scales = "free_x") +
  labs(x = "Chi-square statistic", y = "Bootstrap replicates") +
  theme_te()
