# Estimating species richness beyond your sample
# https://tidyecology.com/posts/asymptotic-richness-estimators/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-14

## ---- setup ----
library(ggplot2); library(dplyr); library(tidyr)
te <- c(forest = "#2f5d50", moss = "#7a9b76", rust = "#b5651d", gold = "#c9a227",
        slate = "#3d4b53", sand = "#d9cbb2", sky = "#5b8aa6", brick = "#8c3b2e")
theme_te <- function(base_size = 12) {
  theme_minimal(base_size = base_size) +
    theme(panel.grid.minor = element_blank(),
          panel.grid.major = element_line(colour = "#e7e1d5", linewidth = 0.3),
          axis.title = element_text(colour = "#3d4b53"),
          axis.text  = element_text(colour = "#5c6670"),
          plot.title = element_text(colour = "#2f3a40", face = "bold", size = base_size + 1),
          plot.subtitle = element_text(colour = "#5c6670", size = base_size - 1),
          legend.position = "bottom",
          legend.title = element_text(colour = "#3d4b53"),
          plot.background  = element_rect(fill = "#f5f4ee", colour = NA),
          panel.background = element_rect(fill = "#f5f4ee", colour = NA))
}

## ---- simulate ----
set.seed(4190)
S_true <- 220
rel <- exp(rnorm(S_true, mean = 0, sd = 1.6)); rel <- rel / sum(rel)
N <- 400
counts <- as.vector(rmultinom(1, N, rel))     # abundances in the sample
obs <- counts[counts > 0]
S_obs <- length(obs)
f <- tabulate(obs)                            # f[k] = number of species with abundance k
f1 <- f[1]; f2 <- f[2]                        # singletons and doubletons
c(S_true = S_true, N = N, S_obs = S_obs, f1 = f1, f2 = f2)

## ---- chao1 ----
chao1_classic <- S_obs + f1^2 / (2 * f2)
f0_hat        <- (N - 1) / N * f1 * (f1 - 1) / (2 * (f2 + 1))   # estimated unseen
chao1_bc      <- S_obs + f0_hat
c(chao1_classic = round(chao1_classic, 1),
  chao1_bc = round(chao1_bc, 1), f0_hat = round(f0_hat, 1))

## ---- ace ----
k_cut  <- 10
rare   <- obs[obs <= k_cut]; S_rare <- length(rare); S_abund <- sum(obs > k_cut)
N_rare <- sum(rare)
C_ace  <- 1 - f1 / N_rare                                   # coverage of the rare group
i  <- 1:k_cut; fi <- sapply(i, function(k) sum(obs == k))
g2 <- max(S_rare / C_ace * sum(i * (i - 1) * fi) / (N_rare * (N_rare - 1)) - 1, 0)
S_ace <- S_abund + S_rare / C_ace + f1 / C_ace * g2
round(c(C_ace = C_ace, gamma2 = g2, S_ACE = S_ace), 3)

## ---- incidence ----
set.seed(5190)
Tq <- 15
det_p <- 1 - exp(-rel * 90)                                 # per-unit detection probability
inc <- matrix(rbinom(S_true * Tq, 1, rep(det_p, times = Tq)), nrow = S_true)
detected <- rowSums(inc); obsI <- detected[detected > 0]
S_obsI <- length(obsI)
Q  <- tabulate(obsI, nbins = Tq); Q1 <- Q[1]; Q2 <- Q[2]
chao2 <- S_obsI + (Tq - 1) / Tq * Q1 * (Q1 - 1) / (2 * (Q2 + 1))
jack1 <- S_obsI + Q1 * (Tq - 1) / Tq
jack2 <- S_obsI + (Q1 * (2 * Tq - 3) / Tq - Q2 * (Tq - 2)^2 / (Tq * (Tq - 1)))
round(c(S_obsI = S_obsI, Q1 = Q1, Q2 = Q2,
        Chao2 = chao2, jack1 = jack1, jack2 = jack2), 1)

## ---- fig-estimators ----
est <- data.frame(
  estimator = factor(c("S_obs","Chao1","ACE","S_obs (inc.)","Chao2","Jack1","Jack2"),
                     levels = c("S_obs","Chao1","ACE","S_obs (inc.)","Chao2","Jack1","Jack2")),
  value = c(S_obs, chao1_bc, S_ace, S_obsI, chao2, jack1, jack2),
  kind  = rep(c("Abundance sample","Incidence (15 units)"), c(3, 4)))
ggplot(est, aes(estimator, value, colour = kind)) +
  geom_hline(yintercept = S_true, linetype = "dashed", colour = te["brick"], linewidth = 0.6) +
  geom_point(size = 3.6) +
  geom_text(aes(label = round(value)), vjust = -1.0, size = 3.1, show.legend = FALSE) +
  scale_colour_manual(values = c("Abundance sample" = unname(te["forest"]),
                                 "Incidence (15 units)" = unname(te["rust"])), name = NULL) +
  scale_y_continuous(limits = c(0, 235)) +
  labs(title = "Observed richness undersamples; estimators add back the unseen",
       subtitle = "One community, true richness 220 (dashed); each estimator lifts S_obs toward it",
       x = NULL, y = "estimated species richness") +
  theme_te() + theme(axis.text.x = element_text(angle = 20, hjust = 1))

## ---- mc-bias ----
set.seed(6190)
Ns <- c(150, 300, 600, 1200); R <- 400
mc <- lapply(Ns, function(nn) {
  so <- ch <- numeric(R)
  for (r in 1:R) {
    cc <- as.vector(rmultinom(1, nn, rel)); o <- cc[cc > 0]
    so[r] <- length(o); ff <- tabulate(o)
    ff1 <- ff[1]; ff2 <- if (length(ff) >= 2) ff[2] else 0
    ch[r] <- length(o) + (nn - 1) / nn * ff1 * (ff1 - 1) / (2 * (ff2 + 1))
  }
  data.frame(N = nn, Sobs = mean(so), Chao1 = mean(ch))
})
mcdf <- do.call(rbind, mc); round(mcdf, 1)

## ---- fig-bias ----
mcl <- pivot_longer(mcdf, c(Sobs, Chao1), names_to = "quantity", values_to = "richness")
mcl$quantity <- factor(mcl$quantity, levels = c("Chao1","Sobs"),
                       labels = c("Chao1 (bias-corrected)", "observed S_obs"))
ggplot(mcl, aes(N, richness, colour = quantity)) +
  geom_hline(yintercept = S_true, linetype = "dashed", colour = te["brick"], linewidth = 0.6) +
  annotate("text", x = 160, y = S_true + 7, label = "true richness = 220",
           colour = te["brick"], size = 3.3, hjust = 0) +
  geom_line(linewidth = 0.9) + geom_point(size = 2.6) +
  scale_x_log10(breaks = Ns) +
  scale_colour_manual(values = c("Chao1 (bias-corrected)" = unname(te["forest"]),
                                 "observed S_obs" = unname(te["sky"])), name = NULL) +
  scale_y_continuous(limits = c(0, 235)) +
  labs(title = "More sampling shrinks the gap, but Chao1 stays a lower bound",
       subtitle = "Mean over 400 samples per sample size; Chao1 tracks below 220 throughout",
       x = "individuals sampled (log scale)", y = "mean richness") +
  theme_te()
