# Coverage-based rarefaction and extrapolation
# https://tidyecology.com/posts/coverage-based-rarefaction/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-08

## ---- setup ----
library(ggplot2); library(dplyr); library(tidyr)
te <- c(forest = "#2f5d50", moss = "#7a9b76", rust = "#b5651d", gold = "#c9a227",
        slate = "#3d4b53", sand = "#d9cbb2", sky = "#5b8aa6", brick = "#8c3b2e")
theme_te <- function(base_size = 12) {
  theme_minimal(base_size = base_size) +
    theme(panel.grid.minor = element_blank(),
          panel.grid.major = element_line(colour = "#e7e1d5", linewidth = 0.3),
          axis.title = element_text(colour = "#3d4b53"),
          axis.text  = element_text(colour = "#5c6670"),
          plot.title = element_text(colour = "#2f3a40", face = "bold", size = base_size + 1),
          plot.subtitle = element_text(colour = "#5c6670", size = base_size - 1),
          legend.position = "bottom",
          legend.title = element_text(colour = "#3d4b53"),
          plot.background  = element_rect(fill = "#f5f4ee", colour = NA),
          panel.background = element_rect(fill = "#f5f4ee", colour = NA))
}

## ---- helpers ----
lchoose_safe <- function(n, k) ifelse(k > n | k < 0, -Inf, lchoose(n, k))
rare_S <- function(x, m)                                   # Hurlbert rarefied richness
  sapply(m, function(mm) { n <- sum(x)
    sum(1 - exp(lchoose_safe(n - x, mm) - lchoose_safe(n, mm))) })
rare_C <- function(x, m) {                                 # rarefied sample coverage
  n <- sum(x)
  sapply(m, function(mm)                                   # the formula holds for m < n only;
    if (mm >= n) covhat(x) else                            # at m = n it is the Good-Turing value
      1 - sum((x / n) * exp(lchoose_safe(n - x, mm) - lchoose_safe(n - 1, mm))))
}
covhat <- function(x) { n <- sum(x); f1 <- sum(x == 1); f2 <- sum(x == 2)
  A <- if (f2 > 0) (n - 1) * f1 / ((n - 1) * f1 + 2 * f2) else   # f2 = 0: the iNEXT rule
    (n - 1) * (f1 - 1) / ((n - 1) * (f1 - 1) + 2)
  1 - (f1 / n) * A }
chao1  <- function(x) { n <- sum(x); f1 <- sum(x == 1); f2 <- sum(x == 2)
  length(x) + (n - 1) / n * f1 * (f1 - 1) / (2 * (f2 + 1)) }
extr_S <- function(x, m) { n <- sum(x); f1 <- sum(x == 1); f2 <- sum(x == 2)
  f0 <- (n - 1) / n * f1 * (f1 - 1) / (2 * (f2 + 1)); So <- length(x)
  sapply(m, function(mm) if (mm <= n)
    sum(1 - exp(lchoose_safe(n - x, mm) - lchoose_safe(n, mm)))
    else So + f0 * (1 - (1 - f1 / (n * f0 + f1))^(mm - n))) }

## ---- goods-coverage ----
set.seed(4193)                                       # same community as the checking post
S_known   <- 180
rel_known <- exp(rnorm(S_known, 0, 1.6)); rel_known <- rel_known / sum(rel_known)
good_cov  <- function(x) 1 - sum(x == 1) / sum(x)   # Good's (Turing's) estimator

sizes_cov <- c(50, 100, 250, 500, 1500, 5000); reps_cov <- 2000
cov_draws <- lapply(sizes_cov, function(nn) t(replicate(reps_cov, {
  x_s  <- as.vector(rmultinom(1, nn, rel_known)); seen <- x_s > 0
  c(good = good_cov(x_s[seen]), chao_jost = covhat(x_s[seen]),
    true_coverage = sum(rel_known[seen]), species_seen = mean(seen)) })))
cov_table <- data.frame(n = sizes_cov, t(sapply(cov_draws, colMeans)))
cov_err   <- sapply(cov_draws, function(d) d[, "good"] - d[, "true_coverage"])
cov_bias  <- colMeans(cov_err)                       # mean error of Good's estimate
cov_bias_se <- apply(cov_err, 2, sd) / sqrt(reps_cov)
cov_mae   <- colMeans(abs(cov_err))                  # typical error of one sample
cj_shift  <- abs(cov_table$chao_jost - cov_table$good)
rare_2pc  <- sum(cumsum(sort(rel_known)) <= 0.02)    # rarest species holding 2 per cent
good_shortfall <- sapply(sizes_cov, function(nn)    # exact E(true coverage) - E(Good)
  sum(rel_known^2 * (1 - rel_known)^(nn - 1)))
cov_z     <- (cov_bias + good_shortfall) / cov_bias_se  # simulation against the formula
round(cov_table, 3)
data.frame(n = sizes_cov, exact_gap = round(good_shortfall, 4),
           simulated_gap = round(-cov_bias, 4), mc_se = round(cov_bias_se, 4))

## ---- simulate ----
set.seed(13)
SA <- 90;  relA <- exp(rnorm(SA, 0, 0.55)); relA <- relA / sum(relA)
SB <- 200; relB <- exp(rnorm(SB, 0, 1.70)); relB <- relB / sum(relB)
n <- 250
xA <- as.vector(rmultinom(1, n, relA)); xA <- xA[xA > 0]
xB <- as.vector(rmultinom(1, n, relB)); xB <- xB[xB > 0]
SobsA <- length(xA); SobsB <- length(xB); CA <- covhat(xA); CB <- covhat(xB)
round(c(SobsA = SobsA, SobsB = SobsB, coverageA = CA, coverageB = CB), 4)

## ---- extrapolate ----
SA_2 <- extr_S(xA, 2 * n); SB_2 <- extr_S(xB, 2 * n)
c(A_at_500 = round(SA_2, 1), B_at_500 = round(SB_2, 1),
  Chao1_A = round(chao1(xA), 1), Chao1_B = round(chao1(xB), 1))

## ---- fig-curves ----
grid <- seq(1, 2 * n, by = 5)
mk <- function(x, lab) data.frame(m = grid, S = extr_S(x, grid), assemblage = lab,
                                  part = ifelse(grid <= n, "interpolation", "extrapolation"))
allc <- rbind(mk(xA, "A (even, true S = 90)"), mk(xB, "B (uneven, true S = 200)"))
cols <- c("A (even, true S = 90)" = unname(te["forest"]),
          "B (uneven, true S = 200)" = unname(te["rust"]))
p_curves <- ggplot(allc, aes(m, S, colour = assemblage)) +
  geom_vline(xintercept = n, linetype = "dotted", colour = te["slate"], linewidth = 0.4) +
  geom_line(aes(linetype = part), linewidth = 0.9) +
  geom_point(data = data.frame(m = n, S = c(SobsA, SobsB),
             assemblage = names(cols)), size = 3, show.legend = FALSE) +
  annotate("text", x = n, y = 20, label = "reference sample n = 250",
           angle = 90, vjust = -0.4, hjust = 0, size = 3, colour = te["slate"]) +
  scale_colour_manual(values = cols, name = NULL) +
  scale_linetype_manual(values = c("interpolation" = "solid", "extrapolation" = "22"), name = NULL) +
  labs(title = "Size-based rarefaction and extrapolation for two assemblages",
       subtitle = "Solid = interpolation, dashed = extrapolation",
       x = "number of individuals", y = "expected species richness") +
  theme_te()
p_curves

## ---- thumb ----
ggsave("thumbnail.png", p_curves, width = 7.5, height = 5.4, dpi = 100)

## ---- equal-coverage ----
Clow <- min(CA, CB)
m_for_C <- function(x, Ct) { ms <- 1:sum(x); ms[which.min(abs(rare_C(x, ms) - Ct))] }
mA_low <- m_for_C(xA, Clow)
SA_low <- rare_S(xA, mA_low)              # A rarefied to B's coverage
SB_low <- SobsB                           # B is already at this coverage
c(target_coverage = round(Clow, 4), A_richness = round(SA_low, 1),
  A_at_m = mA_low, B_richness = round(SB_low, 1))

## ---- fig-coverage ----
ms <- 1:n
cc <- rbind(
  data.frame(C = rare_C(xA, ms), S = rare_S(xA, ms), assemblage = "A (even, true S = 90)"),
  data.frame(C = rare_C(xB, ms), S = rare_S(xB, ms), assemblage = "B (uneven, true S = 200)"))
ggplot(cc, aes(C, S, colour = assemblage)) +
  geom_vline(xintercept = Clow, linetype = "dotted", colour = te["slate"], linewidth = 0.4) +
  geom_line(linewidth = 0.9) +
  geom_point(data = data.frame(C = c(CA, CB), S = c(SobsA, SobsB),
             assemblage = names(cols)), size = 3, show.legend = FALSE) +
  annotate("text", x = Clow, y = 15, label = "equal coverage 0.852",
           angle = 90, vjust = 1.2, hjust = 0, size = 3, colour = te["slate"]) +
  scale_colour_manual(values = cols, name = NULL) +
  # zoom the axis, do not subset the data: scale limits would censor the points outside
  # the window and stop each line short of the panel edge
  coord_cartesian(xlim = c(0.3, 0.95)) +
  labs(title = "Expected richness against sample coverage",
       subtitle = "Dotted line = the lower of the two sample coverages",
       x = "sample coverage", y = "expected species richness") +
  theme_te()

## ---- equal-coverage-repeat ----
set.seed(99)
rep_cov <- replicate(1000, {                    # B minus A at equal coverage, fresh samples
  a <- as.vector(rmultinom(1, n, relA)); a <- a[a > 0]
  b <- as.vector(rmultinom(1, n, relB)); b <- b[b > 0]
  Cl <- min(covhat(a), covhat(b))
  sa <- if (covhat(a) > Cl) rare_S(a, m_for_C(a, Cl)) else length(a)
  sb <- if (covhat(b) > Cl) rare_S(b, m_for_C(b, Cl)) else length(b)
  sb - sa })
c(B_ahead = mean(rep_cov > 0), median_B_minus_A = round(median(rep_cov), 1))
