# Spatial autocorrelation in occupancy models
# https://tidyecology.com/posts/spatial-autocorrelation-in-occupancy/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-11-15

## ---- setup ----
knitr::opts_chunk$set(echo = TRUE, message = FALSE, warning = FALSE,
                      fig.width = 7.5, fig.height = 4.6, dpi = 150, dev = "png")
library(ggplot2)
te <- list(ink="#16241d", body="#2c3a31", forest="#275139", label="#46604a",
           sage="#93a87f", paper="#f5f4ee", line="#dad9ca", card="#ffffff",
           faint="#5d6b61", grazed="#cda23f", mown="#2f8f63", abandoned="#b5534e",
           grad_low="#c9b458", grad_high="#1d5b4e")
theme_te <- function(base_size = 12) {
  theme_minimal(base_size = base_size) +
    theme(text=element_text(colour=te$body),
          plot.title=element_text(colour=te$ink, face="bold", size=base_size+1),
          plot.subtitle=element_text(colour=te$faint, size=base_size-1),
          axis.title=element_text(colour=te$label),
          axis.text=element_text(colour=te$faint, size=base_size-2),
          panel.grid.major=element_line(colour=te$line, linewidth=0.3),
          panel.grid.minor=element_blank(),
          panel.background=element_rect(fill=te$paper, colour=NA),
          plot.background=element_rect(fill=te$paper, colour=NA),
          legend.title=element_text(colour=te$label, size=base_size-2),
          legend.text=element_text(colour=te$faint, size=base_size-2),
          strip.text=element_text(colour=te$ink, face="bold", size=base_size-1),
          strip.background=element_rect(fill=te$paper, colour=NA),
          plot.margin=margin(10,12,10,10))
}

## ---- lattice ----
# a smooth field on a g x g grid, standardised
smooth_field <- function(g, l, seed) {
  set.seed(seed)
  u <- (seq_len(g) - 0.5) / g
  D <- outer(u, u, function(a, b) exp(-(a - b)^2 / (2 * l^2)))
  Sm <- D / rowSums(D)
  as.numeric(scale(as.numeric(Sm %*% matrix(rnorm(g * g), g, g) %*% t(Sm))))
}

g <- 16; R <- g * g; K <- 4; p_true <- 0.5
cx <- rep((seq_len(g) - 0.5) / g, times = g)
cy <- rep((seq_len(g) - 0.5) / g, each = g)

# rook adjacency: neighbours share an edge
idx <- matrix(1:R, g, g); W <- matrix(0, R, R)
for (r in 1:g) for (c in 1:g) {
  h <- idx[r, c]
  if (r > 1) W[h, idx[r - 1, c]] <- 1; if (r < g) W[h, idx[r + 1, c]] <- 1
  if (c > 1) W[h, idx[r, c - 1]] <- 1; if (c < g) W[h, idx[r, c + 1]] <- 1
}

# Moran's I for a vector on this lattice
moran <- function(z, W) {
  z <- z - mean(z)
  (length(z) / sum(W)) * sum(W * outer(z, z)) / sum(z^2)
}

## ---- eigenvectors ----
q <- 30                                   # number of spatial eigenvectors to offer
one <- rep(1, R)
Cmat <- diag(R) - outer(one, one) / R     # double-centring operator
ev <- eigen(Cmat %*% W %*% Cmat, symmetric = TRUE)
Esel <- ev$vectors[, 1:q, drop = FALSE]   # the broadest spatial patterns

## ---- simulate ----
sig_s <- 1.6; sf_len <- 0.20; x0_len <- 0.08
field <- sig_s * smooth_field(g, sf_len, 101)   # latent driver of occupancy (hidden)
x0    <- smooth_field(g, x0_len, 101 + 5000)     # spatially patterned null covariate
set.seed(101)
z <- rbinom(R, 1, plogis(field))                 # true occupancy from the field alone
y <- rbinom(R, K, z * p_true)                    # detections over K visits
c(true_occupancy = round(mean(plogis(field)), 3), ever_detected = sum(y > 0), sites = R)

## ---- fits ----
nll <- function(par, Xd, y, K) {
  np <- ncol(Xd); psi <- plogis(Xd %*% par[1:np]); p <- plogis(par[np + 1])
  -sum(log(psi * dbinom(y, K, p) + (1 - psi) * (y == 0)))
}
fit <- function(Xd, y, K) {
  np <- ncol(Xd)
  f <- optim(c(rep(0, np), 0), nll, Xd = Xd, y = y, K = K, method = "BFGS", hessian = TRUE)
  se <- tryCatch(sqrt(diag(solve(f$hessian))), error = function(e) rep(NA, np + 1))
  list(f = f, se = se, np = np)
}
Xn <- cbind(1, x0);        mn <- fit(Xn, y, K)          # naive
Xr <- cbind(1, x0, Esel);  mr <- fit(Xr, y, K)          # MEM filtered
bN <- mn$f$par[2]; sN <- mn$se[2]
bR <- mr$f$par[2]; sR <- mr$se[2]

# Pearson residuals on the detection scale and their Moran's I
resid_p <- function(m, Xd) {
  np <- m$np; psi <- plogis(Xd %*% m$f$par[1:np]); p <- plogis(m$f$par[np + 1])
  pi <- psi * (1 - (1 - p)^K); o <- as.numeric(y > 0); (o - pi) / sqrt(pi * (1 - pi))
}
rN <- as.numeric(resid_p(mn, Xn)); rR <- as.numeric(resid_p(mr, Xr))
miN <- moran(rN, W); miR <- moran(rR, W)
round(c(b_naive = bN, se_naive = sN, moran_naive = miN,
        b_mem = bR, se_mem = sR, moran_mem = miR), 3)

## ---- fig-correlogram ----
dxy <- as.matrix(dist(cbind(cx, cy))); brks <- seq(0, 0.5, length.out = 8)
corl <- function(rv) sapply(1:(length(brks) - 1), function(b) {
  Wb <- (dxy > brks[b] & dxy <= brks[b + 1]) * 1
  if (sum(Wb) == 0) return(NA); moran(rv, Wb)
})
lag <- (brks[-1] + brks[-length(brks)]) / 2
cg <- data.frame(lag = rep(lag, 2), I = c(corl(rN), corl(rR)),
                 model = rep(c("naive (no spatial term)", "MEM-filtered"), each = length(lag)))
ggplot(cg, aes(lag, I, colour = model, shape = model)) +
  geom_hline(yintercept = 0, colour = te$faint, linewidth = 0.3) +
  geom_line(linewidth = 0.8) + geom_point(size = 2.6) +
  scale_colour_manual(values = c("naive (no spatial term)" = te$abandoned, "MEM-filtered" = te$forest)) +
  scale_shape_manual(values = c("naive (no spatial term)" = 16, "MEM-filtered" = 17)) +
  labs(x = "distance lag", y = "Moran's I of residuals",
       title = "Filtering removes the residual spatial signal",
       subtitle = "Naive residuals stay clustered at short range; filtered ones dip slightly below zero",
       colour = NULL, shape = NULL) +
  theme_te() + theme(legend.position = "inside", legend.position.inside = c(0.72, 0.85))

## ---- type-one ----
fit_b <- function(Xd, y, K) {
  np <- ncol(Xd)
  # BFGS alone stops at its default iteration limit on many filtered fits:
  # start it from an nlminb fit and give it room to converge
  f0 <- nlminb(c(rep(0, np), 0), nll, Xd = Xd, y = y, K = K,
               control = list(iter.max = 3000, eval.max = 6000))
  f <- optim(f0$par, nll, Xd = Xd, y = y, K = K, method = "BFGS", hessian = TRUE,
             control = list(maxit = 2000))
  se <- tryCatch(sqrt(diag(solve(f$hessian)))[2], error = function(e) NA)
  c(f$par[2], se, f$convergence)
}
M <- 200; out <- matrix(NA, M, 2); nconv <- 0
for (m in 1:M) {
  s  <- sig_s * smooth_field(g, sf_len, 7000 + m)
  x0m <- smooth_field(g, x0_len, 13000 + m)
  zz <- rbinom(R, 1, plogis(s)); yy <- rbinom(R, K, zz * p_true)
  a <- fit_b(cbind(1, x0m), yy, K)
  b <- fit_b(cbind(1, x0m, Esel), yy, K)
  out[m, ] <- c(a[1] / a[2], b[1] / b[2])
  nconv <- nconv + (a[3] != 0) + (b[3] != 0)
}
tiN <- 100 * mean(abs(out[, 1]) > 1.96, na.rm = TRUE)
tiR <- 100 * mean(abs(out[, 2]) > 1.96, na.rm = TRUE)
c(naive_type_I = round(tiN, 1), mem_type_I = round(tiR, 1), nominal = 5,
  unconverged_fits = nconv, missing_se = sum(is.na(out)))

## ---- fig-typeone ----
d2 <- data.frame(model = factor(c("naive\n(no spatial term)", "MEM-filtered"),
                                levels = c("naive\n(no spatial term)", "MEM-filtered")),
                 rate = c(tiN, tiR))
ggplot(d2, aes(model, rate, fill = model)) +
  geom_col(width = 0.55) +
  geom_hline(yintercept = 5, colour = te$ink, linetype = "dashed") +
  annotate("text", x = 1.68, y = 6.5, label = "nominal 5%", hjust = 1, size = 3.2, colour = te$ink) +
  geom_text(aes(label = sprintf("%.1f%%", rate)), vjust = -0.4, size = 4, colour = te$ink) +
  scale_fill_manual(values = c("naive\n(no spatial term)" = te$abandoned, "MEM-filtered" = te$mown), guide = "none") +
  scale_y_continuous(limits = c(0, 42), expand = expansion(mult = c(0, 0.02))) +
  labs(x = NULL, y = "false-positive rate for a null covariate",
       title = "Unmodelled autocorrelation inflates false positives",
       subtitle = "200 simulated data sets; the covariate has no true effect") +
  theme_te()
