# Checking a community occupancy model
# https://tidyecology.com/posts/checking-a-community-occupancy-model/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-01

## ---- setup ----
library(ggplot2); library(dplyr); library(tidyr)
logit <- qlogis; expit <- plogis

te_ink<-"#16241d"; te_body<-"#2c3a31"; te_forest<-"#275139"; te_faint<-"#5d6b61"
te_sage<-"#93a87f"; te_paper<-"#f5f4ee"; te_line<-"#dad9ca"; te_brick<-"#b5534e"
theme_te <- function(base_size=12){
  theme_minimal(base_size=base_size) +
    theme(plot.background=element_rect(fill=te_paper,colour=NA),
          panel.background=element_rect(fill="#f5f4ee",colour=NA),
          panel.grid.minor=element_blank(),
          panel.grid.major=element_line(colour=te_line,linewidth=0.3),
          axis.title=element_text(colour=te_body), axis.text=element_text(colour=te_faint),
          plot.title=element_text(colour=te_ink,face="bold"),
          plot.subtitle=element_text(colour=te_faint,size=rel(0.9)),
          legend.text=element_text(colour=te_faint,size=rel(0.8)),
          strip.text=element_text(colour=te_ink,face="bold"))
}

## ---- simdat ----
simdat <- function(seed, hetero=FALSE, sigma_site=1.2){
  set.seed(seed)
  S <- 35L; R <- 90L; K <- 4L
  lpsi <- rnorm(S, logit(0.40), 1.0); lp <- rnorm(S, logit(0.35), 0.7)
  psi <- expit(lpsi); z <- matrix(rbinom(S*R,1,psi), S, R)
  if (hetero){ u <- rnorm(R, 0, sigma_site)                  # site survey-condition effect
    P <- expit(outer(lp, rep(1,R)) + matrix(u, S, R, byrow=TRUE))
  } else { P <- matrix(expit(lp), S, R) }                    # detection constant across sites
  y <- matrix(rbinom(S*R, K, ifelse(z==1, P, 0)), S, R)
  list(y=y, S=S, R=R, K=K)
}
dat_h <- simdat(606,  hetero=FALSE)   # detection homogeneous across sites
dat_g <- simdat(4602, hetero=TRUE)    # detection varies by site (unmodelled)

## ---- fit-ppc ----
fit_ppc <- function(dat, ni=14000L, nb=5000L, nt=2L){
  y<-dat$y; S<-dat$S; R<-dat$R; K<-dat$K
  ndet<-rowSums(y>0); nzero<-R-ndet; Sy<-rowSums(y*(y>0)); SKy<-rowSums((K-y)*(y>0))
  LL <- function(lpsi,lp){ psi<-expit(lpsi); p<-expit(lp)                    # occupancy state summed out
    ndet*log(psi) + Sy*log(p) + SKy*log(1-p) + nzero*log(psi*(1-p)^K + (1-psi)) }
  lpsi<-logit(pmin(pmax((ndet+.5)/(R+1),.02),.98)); lp<-rep(logit(.35),S)
  mps<-0; sps<-1; mp<-logit(.35); sp<-1
  t_ls<-.3; t_lp<-.26; t_mu<-.14; t_lsd<-.14
  keep<-seq(nb+1,ni,by=nt); nk<-length(keep); kk<-0L
  Dobs<-colSums(y)                                                           # observed detections per site
  Tobs<-Trep<-numeric(nk); ge<-logical(nk); DREP<-matrix(NA,nk,R)
  for (it in 1:ni){
    ll<-LL(lpsi,lp)
    pr<-lpsi+rnorm(S,0,t_ls); a<-LL(pr,lp)+dnorm(pr,mps,sps,log=TRUE)-(ll+dnorm(lpsi,mps,sps,log=TRUE)); au<-log(runif(S))<a; lpsi[au]<-pr[au]; ll<-LL(lpsi,lp)
    pr<-lp+rnorm(S,0,t_lp);   a<-LL(lpsi,pr)+dnorm(pr,mp,sp,log=TRUE)-(ll+dnorm(lp,mp,sp,log=TRUE));   au<-log(runif(S))<a; lp[au]<-pr[au]
    pr<-mps+rnorm(1,0,t_mu); if(log(runif(1))<sum(dnorm(lpsi,pr,sps,log=TRUE))-sum(dnorm(lpsi,mps,sps,log=TRUE)))mps<-pr
    pr<-exp(log(sps)+rnorm(1,0,t_lsd)); if(log(runif(1))<sum(dnorm(lpsi,mps,pr,log=TRUE))+dnorm(pr,0,2.5,log=TRUE)+log(pr)-sum(dnorm(lpsi,mps,sps,log=TRUE))-dnorm(sps,0,2.5,log=TRUE)-log(sps))sps<-pr
    pr<-mp+rnorm(1,0,t_mu);  if(log(runif(1))<sum(dnorm(lp,pr,sp,log=TRUE))-sum(dnorm(lp,mp,sp,log=TRUE)))mp<-pr
    pr<-exp(log(sp)+rnorm(1,0,t_lsd));  if(log(runif(1))<sum(dnorm(lp,mp,pr,log=TRUE))+dnorm(pr,0,2.5,log=TRUE)+log(pr)-sum(dnorm(lp,mp,sp,log=TRUE))-dnorm(sp,0,2.5,log=TRUE)-log(sp))sp<-pr
    if (it>nb && ((it-nb)%%nt==0)){ kk<-kk+1L
      psi<-expit(lpsi); p<-expit(lp); ED<-sum(psi*K*p)                       # expected detections per site
      Tobs[kk]<-sum((sqrt(Dobs)-sqrt(ED))^2)                                 # Freeman-Tukey, observed
      zr<-matrix(rbinom(S*R,1,psi),S,R); yr<-matrix(rbinom(S*R,K,ifelse(zr==1,p,0)),S,R)
      Drep<-colSums(yr); Trep[kk]<-sum((sqrt(Drep)-sqrt(ED))^2); ge[kk]<-Trep[kk]>=Tobs[kk]; DREP[kk,]<-Drep
    }
  }
  list(bp=mean(ge), chat=mean(Tobs)/mean(Trep), Tobs=Tobs, Trep=Trep, Dobs=Dobs, DREP=DREP)
}
rh <- fit_ppc(dat_h)   # homogeneous data
rg <- fit_ppc(dat_g)   # heterogeneous data
round(c(bp_homogeneous=rh$bp, chat_homogeneous=rh$chat,
        bp_heterogeneous=rg$bp, chat_heterogeneous=rg$chat), 3)

## ---- fig-ppc ----
mk <- function(r,lab) data.frame(To=r$Tobs, Tr=r$Trep, scen=lab)
d1 <- rbind(mk(rh, sprintf("homogeneous detection  (p = %.2f)", rh$bp)),
            mk(rg, sprintf("unmodelled site heterogeneity  (p = %.2f)", rg$bp)))
ggplot(d1, aes(To, Tr)) +
  geom_abline(slope=1, intercept=0, colour=te_faint, linewidth=0.5) +
  geom_point(colour=te_forest, alpha=0.25, size=0.8) +
  geom_blank(aes(x=Tr, y=To)) +  # each panel spans both ranges, so the identity line is drawn in both
  facet_wrap(~scen, scales="free") +
  labs(x="discrepancy for observed data  T(y)", y="discrepancy for replicated data  T(y_rep)",
       title="A posterior predictive check separates fit from misfit",
       subtitle="points below the identity line: the model under-predicts the observed discrepancy") +
  theme_te()

## ---- fig-dispersion ----
den <- function(r,lab) rbind(
  data.frame(v=r$Dobs,             src="observed",              scen=lab),
  data.frame(v=as.numeric(r$DREP), src="posterior-predictive", scen=lab))
d2 <- rbind(den(rh,"homogeneous detection"), den(rg,"unmodelled site heterogeneity"))
ggplot(d2, aes(v)) +
  geom_density(data=subset(d2,src=="posterior-predictive"), aes(colour="posterior-predictive"),
               fill=te_sage, alpha=0.25, linewidth=0.5) +
  geom_density(data=subset(d2,src=="observed"), aes(colour="observed"), fill=NA, linewidth=0.9) +
  scale_colour_manual(NULL, values=c("observed"=te_brick,"posterior-predictive"=te_forest)) +
  facet_wrap(~scen, scales="free") +
  labs(x="detections per site (summed over species and visits)", y="density",
       title="The misfit is over-dispersion across sites",
       subtitle="observed spread far exceeds what the constant-detection model predicts") +
  theme_te() + theme(legend.position="top")
