# Species richness by data augmentation
# https://tidyecology.com/posts/estimating-species-richness-augmentation/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-01

## ---- setup ----
library(ggplot2); library(dplyr)
logit <- qlogis; expit <- plogis

te_ink<-"#16241d"; te_body<-"#2c3a31"; te_forest<-"#275139"; te_faint<-"#5d6b61"
te_sage<-"#93a87f"; te_paper<-"#f5f4ee"; te_line<-"#dad9ca"; te_brick<-"#b5534e"; te_gold<-"#cda23f"
theme_te <- function(base_size=12){
  theme_minimal(base_size=base_size) +
    theme(plot.background=element_rect(fill=te_paper,colour=NA),
          panel.background=element_rect(fill="#f5f4ee",colour=NA),
          panel.grid.minor=element_blank(),
          panel.grid.major=element_line(colour=te_line,linewidth=0.3),
          axis.title=element_text(colour=te_body), axis.text=element_text(colour=te_faint),
          plot.title=element_text(colour=te_ink,face="bold"),
          plot.subtitle=element_text(colour=te_faint,size=rel(0.9)),
          legend.text=element_text(colour=te_faint,size=rel(0.8)))
}

## ---- simulate ----
set.seed(221)
Strue <- 60L; R <- 70L; K <- 3L
lpsi_t <- rnorm(Strue, logit(0.22), 1.15)         # community occupancy
lp_t   <- rnorm(Strue, logit(0.16), 1.10)         # community detection (a low tail)
psi_t  <- expit(lpsi_t); p_t <- expit(lp_t)
z_t    <- matrix(rbinom(Strue*R, 1, psi_t), Strue, R)
Yfull  <- matrix(rbinom(Strue*R, K, ifelse(z_t==1, p_t, 0)), Strue, R)

detected <- rowSums(Yfull > 0) > 0
Yobs <- Yfull[detected, , drop=FALSE]             # only species seen at least once
Sobs <- nrow(Yobs)
c(true_richness = Strue, observed = Sobs, missed = Strue - Sobs)

## ---- chao ----
inc <- rowSums(Yobs > 0)                           # sites each observed species was seen at
Q1 <- sum(inc==1); Q2 <- sum(inc==2)
chao2   <- if (Q2>0) Sobs + Q1^2/(2*Q2) else Sobs + Q1*(Q1-1)/2
chao2bc <- Sobs + ((R-1)/R) * Q1*(Q1-1) / (2*(Q2+1))   # bias-corrected form
round(c(Q1=Q1, Q2=Q2, chao2=chao2, chao2_bc=chao2bc), 1)

## ---- augment ----
Maug <- 120L; M <- Sobs + Maug
Y <- rbind(Yobs, matrix(0L, Maug, R))              # padded species are all zero
ndet <- rowSums(Y>0); nzero <- R - ndet
Sy <- rowSums(Y*(Y>0)); SKy <- rowSums((K-Y)*(Y>0))
is_obs <- c(rep(TRUE, Sobs), rep(FALSE, Maug))
# marginal occupancy log-likelihood for one species (occupancy state summed out)
logLocc <- function(lpsi, lp){
  psi <- expit(lpsi); p <- expit(lp)
  ndet*log(psi) + Sy*log(p) + SKy*log(1-p) + nzero*log(psi*(1-p)^K + (1-psi))
}

## ---- mcmc ----
ni <- 206000L; nb <- 6000L; nt <- 10L
lpsi <- rep(logit(0.2), M); lp <- rep(logit(0.2), M); w <- rep(1L, M)
mps <- logit(0.2); sps <- 1; mp <- logit(0.2); sp <- 1; Omega <- 0.5
t_ls<-0.4; t_lp<-0.4; t_mu<-0.15; t_lsd<-0.15
keep <- seq(nb+1, ni, by=nt); nk <- length(keep); kk <- 0L
Nsup <- numeric(nk); HYP <- matrix(NA, nk, 4)

for (it in 1:ni){
  ll <- logLocc(lpsi, lp)
  prop <- lpsi + rnorm(M,0,t_ls)               # occupancy logits (Metropolis)
  a <- w*logLocc(prop,lp)+dnorm(prop,mps,sps,log=TRUE) - (w*ll+dnorm(lpsi,mps,sps,log=TRUE))
  au <- log(runif(M)) < a; lpsi[au] <- prop[au]; ll <- logLocc(lpsi, lp)
  prop <- lp + rnorm(M,0,t_lp)                 # detection logits (Metropolis)
  a <- w*logLocc(lpsi,prop)+dnorm(prop,mp,sp,log=TRUE) - (w*ll+dnorm(lp,mp,sp,log=TRUE))
  au <- log(runif(M)) < a; lp[au] <- prop[au]
  # inclusion indicators for padded all-zero species (Gibbs)
  ll0 <- logLocc(lpsi, lp)                     # probability of all zeros if present
  pw <- Omega*exp(ll0) / (Omega*exp(ll0) + (1-Omega))
  w <- ifelse(is_obs, 1L, rbinom(M,1,pw))
  Omega <- rbeta(1, 1+sum(w), 1+M-sum(w))      # inclusion rate (Gibbs)
  # community means and spreads (Metropolis)
  pr <- mps+rnorm(1,0,t_mu)
  if (log(runif(1)) < sum(dnorm(lpsi,pr,sps,log=TRUE))-sum(dnorm(lpsi,mps,sps,log=TRUE))) mps <- pr
  pr <- exp(log(sps)+rnorm(1,0,t_lsd))
  if (log(runif(1)) < sum(dnorm(lpsi,mps,pr,log=TRUE))+dnorm(pr,0,2.5,log=TRUE)+log(pr)-
                      sum(dnorm(lpsi,mps,sps,log=TRUE))-dnorm(sps,0,2.5,log=TRUE)-log(sps)) sps <- pr
  pr <- mp+rnorm(1,0,t_mu)
  if (log(runif(1)) < sum(dnorm(lp,pr,sp,log=TRUE))-sum(dnorm(lp,mp,sp,log=TRUE))) mp <- pr
  pr <- exp(log(sp)+rnorm(1,0,t_lsd))
  if (log(runif(1)) < sum(dnorm(lp,mp,pr,log=TRUE))+dnorm(pr,0,2.5,log=TRUE)+log(pr)-
                      sum(dnorm(lp,mp,sp,log=TRUE))-dnorm(sp,0,2.5,log=TRUE)-log(sp)) sp <- pr
  if (it>nb && ((it-nb) %% nt == 0)){ kk<-kk+1L; Nsup[kk]<-sum(w); HYP[kk,]<-c(mps,sps,mp,sp) }
}
qN <- quantile(Nsup, c(0.025,0.5,0.975))
round(c(post_mean=mean(Nsup), median=qN[2], lower=qN[1], upper=qN[3]), 1)

## ---- fig-richness ----
dfN <- data.frame(N=Nsup)
ggplot(dfN, aes(N)) +
  geom_histogram(binwidth=1, fill=te_sage, colour="#ffffff", linewidth=0.15) +
  geom_vline(xintercept=Sobs, colour=te_faint, linetype="dotted", linewidth=0.7) +
  geom_vline(xintercept=chao2, colour=te_gold, linetype="dashed", linewidth=0.7) +
  geom_vline(xintercept=Strue, colour=te_brick, linewidth=0.8) +
  annotate("text", x=Sobs, y=Inf, label="observed", colour=te_faint, angle=90, vjust=-0.4, hjust=1.1, size=3) +
  annotate("text", x=chao2, y=Inf, label="Chao2", colour=te_gold, angle=90, vjust=1.2, hjust=1.1, size=3) +
  annotate("label", x=Strue, y=Inf, label="true richness", colour=te_brick, angle=90, vjust=-0.4, hjust=1.1,
           size=3, fill="#f5f4ee", linewidth = 0) +
  labs(x="community richness N (species present, detected or not)", y="posterior draws",
       title="The community holds more species than were seen",
       subtitle="observed count and Chao2 sit below the truth; the posterior reaches it") +
  theme_te()

## ---- fig-incidence ----
tab <- as.data.frame(table(factor(inc, levels=1:max(inc)))); names(tab) <- c("sites","species")
tab$sites <- as.integer(as.character(tab$sites))
und  <- mean(Nsup) - Sobs
zero <- data.frame(sites=0, species=und)
ggplot() +
  geom_col(data=tab, aes(sites, species), fill=te_forest, width=0.8) +
  geom_col(data=zero, aes(sites, species), fill=te_brick, width=0.8, alpha=0.55) +
  annotate("text", x=0, y=und+0.6, label="inferred\nundetected", colour=te_brick, size=3, lineheight=0.9) +
  scale_x_continuous(breaks=seq(0, max(inc), 5)) +
  labs(x="number of sites where a species was detected (incidence)", y="number of species",
       title="Rare species dominate, and some are never seen",
       subtitle="green: observed incidence counts; red: species inferred at zero incidence") +
  theme_te()
