# Estimating transition probabilities
# https://tidyecology.com/posts/estimating-transition-probabilities/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-14

## ---- setup ----
library(ggplot2)

theme_te <- function(bs = 12) theme_minimal(base_size = bs) + theme(
  text = element_text(colour = "#2c3a31"),
  plot.title = element_text(colour = "#16241d", face = "bold", size = rel(1.05)),
  plot.subtitle = element_text(colour = "#5d6b61", size = rel(0.9)),
  axis.title = element_text(colour = "#46604a"), axis.text = element_text(colour = "#46604a"),
  panel.grid.major = element_line(colour = "#dad9ca", linewidth = 0.3),
  panel.grid.minor = element_blank(),
  plot.background = element_rect(fill = "#f5f4ee", colour = NA),
  panel.background = element_rect(fill = "#f5f4ee", colour = NA),
  strip.text = element_text(colour = "#16241d", face = "bold"),
  legend.position = "bottom", legend.title = element_blank())
forest <- "#275139"; brick <- "#b5534e"; ink <- "#16241d"

# two sites; latent 1 = alive-A, 2 = alive-B, 3 = dead
Gam <- function(phiA, phiB, psiAB, psiBA) matrix(c(
  phiA * (1 - psiAB), phiA * psiAB,     1 - phiA,
  phiB * psiBA,       phiB * (1 - psiBA), 1 - phiB,
  0,                  0,                1), nrow = 3, byrow = TRUE)

## ---- truth ----
phiA_t <- 0.72; phiB_t <- 0.62      # survival
psiAB_t <- 0.30; psiBA_t <- 0.20    # movement (the target)
pA_t <- 0.60                         # detection at A (fixed)
k <- 8; nA0 <- 320; nB0 <- 280; N <- nA0 + nB0
G_true <- Gam(phiA_t, phiB_t, psiAB_t, psiBA_t)
init_state <- c(rep(1L, nA0), rep(2L, nB0))
start <- c(qlogis(0.5), qlogis(0.5), qlogis(0.25), qlogis(0.25), qlogis(0.5), qlogis(0.4))

sim_data <- function(pB, seed) {
  set.seed(seed)
  y <- matrix(1L, N, k)
  for (i in 1:N) {
    s <- init_state[i]; y[i, 1] <- if (s == 1L) 2L else 3L
    for (t in 2:k) {
      s <- sample.int(3, 1, prob = G_true[s, ])
      if (s == 1L)      y[i, t] <- sample.int(3, 1, prob = c(1 - pA_t, pA_t, 0))
      else if (s == 2L) y[i, t] <- sample.int(3, 1, prob = c(1 - pB, 0, pB))
    }
  }
  y
}

## ---- fitters ----
mle_fit <- function(y) {
  nll <- function(par) {
    phiA <- plogis(par[1]); phiB <- plogis(par[2]); psiAB <- plogis(par[3])
    psiBA <- plogis(par[4]); pA <- plogis(par[5]); pB <- plogis(par[6])
    G <- Gam(phiA, phiB, psiAB, psiBA)
    Pm <- list(c(1 - pA, 1 - pB, 1), c(pA, 0, 0), c(0, pB, 0)); ll <- 0
    for (i in 1:N) {
      a <- rep(0, 3); a[init_state[i]] <- 1; lg <- 0
      for (t in 2:k) { a <- (a %*% G) * Pm[[y[i, t]]]; s <- sum(a)
        if (s <= 0) return(1e10); a <- a / s; lg <- lg + log(s) }
      ll <- ll + lg
    }
    -ll
  }
  plogis(optim(start, nll, method = "BFGS", control = list(maxit = 500))$par)
}

# naive: count only the transitions you actually saw on consecutive occasions
naive_psi <- function(y) {
  nAA <- sum(y[, -k] == 2L & y[, -1] == 2L); nAB <- sum(y[, -k] == 2L & y[, -1] == 3L)
  nBA <- sum(y[, -k] == 3L & y[, -1] == 2L); nBB <- sum(y[, -k] == 3L & y[, -1] == 3L)
  c(psiAB = nAB / (nAA + nAB), psiBA = nBA / (nBA + nBB))
}

## ---- gap-experiment ----
pB_grid <- c(0.60, 0.50, 0.40, 0.30, 0.20)
R_rep <- 12
rows <- list()
for (pB in pB_grid) {
  mm <- matrix(NA, R_rep, 2); nn <- matrix(NA, R_rep, 2)
  for (r in 1:R_rep) {
    y <- sim_data(pB, seed = 136 * 10 + r)
    mm[r, ] <- mle_fit(y)[3:4]; nn[r, ] <- naive_psi(y)
  }
  rows[[length(rows) + 1]] <- data.frame(
    gap = pA_t - pB, pB = pB,
    psiAB_naive = mean(nn[, 1]), psiBA_naive = mean(nn[, 2]),
    psiAB_mle = mean(mm[, 1]),   psiBA_mle = mean(mm[, 2]))
}
tab <- do.call(rbind, rows)
round(tab, 3)

## ---- fig-gap ----
long <- rbind(
  data.frame(gap = tab$gap, param = "psi[AB]", method = "Naive", value = tab$psiAB_naive),
  data.frame(gap = tab$gap, param = "psi[AB]", method = "Multi-state", value = tab$psiAB_mle),
  data.frame(gap = tab$gap, param = "psi[BA]", method = "Naive", value = tab$psiBA_naive),
  data.frame(gap = tab$gap, param = "psi[BA]", method = "Multi-state", value = tab$psiBA_mle))
tru <- data.frame(param = c("psi[AB]", "psi[BA]"), truth = c(psiAB_t, psiBA_t))
ggplot(long, aes(gap, value, colour = method)) +
  geom_hline(data = tru, aes(yintercept = truth), linetype = "dashed", colour = ink, linewidth = 0.5) +
  geom_line(linewidth = 1) + geom_point(size = 2.6) +
  facet_wrap(~param, labeller = label_parsed) +
  scale_colour_manual(values = c("Multi-state" = forest, "Naive" = brick)) +
  labs(title = "State-specific detection makes naive transitions drift",
       subtitle = "As detection at B drops below A, the naive estimate leaves the truth (dashed)",
       x = "Detection gap (p_A minus p_B)", y = "Estimated transition probability") + theme_te()

## ---- mc-gap ----
S <- 30
mcM <- matrix(NA, S, 2); mcN <- matrix(NA, S, 2)
for (s in 1:S) {
  ys <- sim_data(0.30, seed = 2000 + s)
  mcM[s, ] <- mle_fit(ys)[3:4]; mcN[s, ] <- naive_psi(ys)
}
rbind(multistate = round(colMeans(mcM), 3), naive = round(colMeans(mcN), 3))

## ---- fig-mc ----
mc <- rbind(
  data.frame(param = "psi[AB]", method = "Naive", value = mcN[, 1]),
  data.frame(param = "psi[AB]", method = "Multi-state", value = mcM[, 1]),
  data.frame(param = "psi[BA]", method = "Naive", value = mcN[, 2]),
  data.frame(param = "psi[BA]", method = "Multi-state", value = mcM[, 2]))
ggplot(mc, aes(x = method, y = value, fill = method)) +
  geom_boxplot(width = 0.5, colour = "#3a3a3a", outlier.size = 0.6) +
  geom_hline(data = tru, aes(yintercept = truth), linetype = "dashed", colour = ink, linewidth = 0.5) +
  facet_wrap(~param, labeller = label_parsed, scales = "free_y") +
  scale_fill_manual(values = c("Multi-state" = forest, "Naive" = brick)) +
  labs(title = "At a detection gap of 0.3, the naive estimate is systematically off",
       subtitle = "30 simulations; multi-state centres on the truth (dashed), naive does not",
       x = NULL, y = "Estimate") + theme_te() + theme(legend.position = "none")
