# Checking a diversity estimate
# https://tidyecology.com/posts/checking-a-diversity-estimate/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-01

## ---- setup ----
library(ggplot2)
te <- c(forest = "#2f5d50", moss = "#7a9b76", rust = "#b5651d", gold = "#c9a227",
        slate = "#3d4b53", sand = "#d9cbb2", sky = "#5b8aa6", brick = "#8c3b2e")
theme_te <- function(base_size = 12) {
  theme_minimal(base_size = base_size) +
    theme(panel.grid.minor = element_blank(),
          panel.grid.major = element_line(colour = "#e7e1d5", linewidth = 0.3),
          axis.title = element_text(colour = "#3d4b53"),
          axis.text  = element_text(colour = "#5c6670"),
          plot.title = element_text(colour = "#2f3a40", face = "bold", size = base_size + 1),
          plot.subtitle = element_text(colour = "#5c6670", size = base_size - 1),
          legend.position = "bottom",
          legend.title = element_text(colour = "#3d4b53"),
          plot.background  = element_rect(fill = "#f5f4ee", colour = NA),
          panel.background = element_rect(fill = "#f5f4ee", colour = NA))
}
covCJ <- function(x) { n <- sum(x); f1 <- sum(x == 1); f2 <- sum(x == 2)
  if (f2 == 0) f2 <- f1 * (f1 - 1) / 2
  1 - (f1 / n) * ((n - 1) * f1 / ((n - 1) * f1 + 2 * f2)) }
chao1 <- function(x) { n <- sum(x); f1 <- sum(x == 1); f2 <- sum(x == 2)
  length(x) + (n - 1) / n * f1 * (f1 - 1) / (2 * (f2 + 1)) }

## ---- community ----
set.seed(4193)
S_true <- 180
rel <- exp(rnorm(S_true, 0, 1.6)); rel <- rel / sum(rel)

## ---- coverage-deficit ----
n_low <- 250; n_high <- 1500
x_lo <- as.vector(rmultinom(1, n_low,  rel)); x_lo <- x_lo[x_lo > 0]
x_hi <- as.vector(rmultinom(1, n_high, rel)); x_hi <- x_hi[x_hi > 0]
data.frame(effort = c("low (n = 250)", "high (n = 1500)"),
           S_obs   = c(length(x_lo), length(x_hi)),
           coverage = round(c(covCJ(x_lo), covCJ(x_hi)), 4),
           Chao1    = round(c(chao1(x_lo), chao1(x_hi)), 1),
           unseen   = round(c(chao1(x_lo) - length(x_lo), chao1(x_hi) - length(x_hi)), 1))

## ---- singleton-sensitivity ----
x <- x_lo; So <- length(x); f1 <- sum(x == 1); f2 <- sum(x == 2); nn <- sum(x)
chao1_f1 <- function(f1v) { d <- f1v - f1   # singleton species removed (d < 0) or added (d > 0)
  (So + d) + (nn + d - 1) / (nn + d) * f1v * (f1v - 1) / (2 * (f2 + 1)) }
c(f1_observed = f1, f2 = f2,
  Chao1_observed = round(chao1_f1(f1), 1),
  Chao1_if_20pct_spurious = round(chao1_f1(round(f1 * 0.8)), 1),
  Chao1_if_20pct_more     = round(chao1_f1(round(f1 * 1.2)), 1),
  slope_per_singleton = round(chao1_f1(f1 + 1) - chao1_f1(f1), 2))

## ---- fig-singleton ----
f1grid <- seq(max(2, round(f1 * 0.6)), round(f1 * 1.4), by = 1)
fb <- data.frame(f1 = f1grid, chao1 = sapply(f1grid, chao1_f1))
marks <- data.frame(f1 = c(round(f1 * 0.8), f1, round(f1 * 1.2)),
                    chao1 = chao1_f1(c(round(f1 * 0.8), f1, round(f1 * 1.2))))
p_singleton <- ggplot(fb, aes(f1, chao1)) +
  geom_line(colour = te["forest"], linewidth = 0.9) +
  geom_point(data = marks, colour = te["rust"], size = 3) +
  geom_text(data = marks, aes(label = round(chao1)), vjust = -1.1, size = 3, colour = te["slate"]) +
  labs(title = "Chao1 rides on the singleton count",
       subtitle = paste0("A 20% change in singletons (miscounts, contamination) moves the estimate by ",
                         round(min(abs(diff(marks$chao1)))), " to ", round(max(abs(diff(marks$chao1)))), " species"),
       x = "number of singletons (species seen once)", y = "Chao1 richness estimate") +
  theme_te()
p_singleton

## ---- thumb ----
ggsave("thumbnail.png",
       p_singleton +
         labs(title = paste(strwrap("Chao1 as a function of the singleton count", 46),
                            collapse = "\n"),
              subtitle = NULL) +
         theme(plot.title = element_text(colour = "#2f3a40", face = "bold", size = 13),
               plot.subtitle = element_text(colour = "#5c6670", size = 9.5),
               plot.margin = margin(8, 14, 8, 8)),
       width = 7.5, height = 5.4, dpi = 100, bg = "#f5f4ee")

## ---- ci-helpers ----
chao1_ci <- function(x, level = 0.95) { n <- sum(x); So <- length(x)
  f1 <- sum(x == 1); f2 <- sum(x == 2); Sc <- chao1(x); D <- Sc - So
  r <- f1 / f2; v <- f2 * (0.5 * r^2 + r^3 + 0.25 * r^4)     # Chao 1987 variance
  z <- qnorm(1 - (1 - level) / 2)
  wald <- c(Sc - z * sqrt(v), Sc + z * sqrt(v))
  K <- exp(z * sqrt(log(1 + v / D^2)))                        # log-transformed
  list(Sc = Sc, wald = wald, logci = c(So + D / K, So + D * K)) }
ci <- chao1_ci(x_lo)
c(Chao1 = round(ci$Sc, 1), Chao1_uncorrected = round(So + f1^2 / (2 * f2), 1),
  Wald_lo = round(ci$wald[1], 1), Wald_hi = round(ci$wald[2], 1),
  log_lo = round(ci$logci[1], 1), log_hi = round(ci$logci[2], 1))

## ---- ci-coverage ----
set.seed(5193); R <- 2000
Sc <- wl <- wh <- ll <- lh <- numeric(R)
for (r in 1:R) { xx <- as.vector(rmultinom(1, n_low, rel)); xx <- xx[xx > 0]
  ci <- chao1_ci(xx); Sc[r] <- ci$Sc
  wl[r] <- ci$wald[1]; wh[r] <- ci$wald[2]; ll[r] <- ci$logci[1]; lh[r] <- ci$logci[2] }
E_chao1 <- mean(Sc)
data.frame(target = c("E[Chao1] (estimator)", "true richness"),
  Wald = round(c(mean(wl <= E_chao1 & wh >= E_chao1), mean(wl <= S_true & wh >= S_true)), 3),
  log  = round(c(mean(ll <= E_chao1 & lh >= E_chao1), mean(ll <= S_true & lh >= S_true)), 3))

## ---- fig-coverage ----
fc <- data.frame(
  method = factor(rep(c("symmetric Wald", "log-transformed"), each = 2),
                  levels = c("symmetric Wald", "log-transformed")),
  target = rep(c("E[Chao1] (estimator target)", "true richness"), 2),
  coverage = c(mean(wl <= E_chao1 & wh >= E_chao1), mean(wl <= S_true & wh >= S_true),
               mean(ll <= E_chao1 & lh >= E_chao1), mean(ll <= S_true & lh >= S_true)))
ggplot(fc, aes(method, coverage, fill = target)) +
  geom_hline(yintercept = 0.95, linetype = "dashed", colour = te["brick"], linewidth = 0.5) +
  geom_col(position = position_dodge(width = 0.7), width = 0.6) +
  geom_text(aes(label = sprintf("%.2f", coverage)), position = position_dodge(width = 0.7),
            vjust = -0.4, size = 3, colour = te["slate"]) +
  scale_fill_manual(values = c("E[Chao1] (estimator target)" = unname(te["forest"]),
                               "true richness" = unname(te["sky"])), name = NULL) +
  scale_y_continuous(limits = c(0, 1.05)) +
  labs(title = "The interval covers the lower bound, not the truth",
       subtitle = paste0("n = 250, 2000 samples; dashed line 0.95:\n",
                         "log CI reaches the estimator target, neither reaches true richness 180"),
       x = NULL, y = "interval coverage") +
  theme_te()

## ---- frame-fraction ----
data("BCI", package = "vegan")          # the data set only; vegan is not attached
stopifnot(identical(dim(BCI), c(50L, 225L)), sum(BCI) == 21457, all(colSums(BCI) > 0))
bci_inc <- (BCI > 0) * 1; bci_K <- nrow(bci_inc); bci_S <- ncol(bci_inc)   # incidence
bci_est <- function(y, K) { k <- nrow(y); qq <- colSums(y); qq <- qq[qq > 0]
  S <- length(qq); Q1 <- sum(qq == 1); Q2 <- sum(qq == 2); q <- k / K
  c(S_obs = S, Q2 = Q2, chao2 = S + (k - 1) / k * Q1^2 / (2 * Q2),
    wor = if (q < 1) S + Q1^2 / (2 * k / (k - 1) * Q2 + q / (1 - q) * Q1) else S) }
set.seed(27092601); bci_q <- c(0.2, 0.3, 0.4, 0.5, 0.7, 0.9); bci_reps <- 300
bci_draws <- lapply(bci_q, function(q) t(replicate(bci_reps,
  bci_est(bci_inc[sample.int(bci_K, round(q * bci_K)), ], bci_K))))
stopifnot(all(sapply(bci_draws, function(m) min(m[, "Q2"])) > 0))
bci_col <- function(v, f) sapply(bci_draws, function(m) f(m[, v]))
bci_tab <- data.frame(q = bci_q, k = round(bci_q * bci_K),
  err_S_obs = bci_col("S_obs", mean) - bci_S, err_chao2 = bci_col("chao2", mean) - bci_S,
  err_wor = bci_col("wor", mean) - bci_S, chao2_over = bci_col("chao2", function(v) mean(v > bci_S)),
  wor_over = bci_col("wor", function(v) mean(v > bci_S)))
bci_mcse <- max(bci_col("chao2", sd), bci_col("wor", sd)) / sqrt(bci_reps)
bci_at <- function(q) bci_tab[abs(bci_tab$q - q) < 1e-9, ]
bci_whole <- bci_est(bci_inc, Inf)       # the frame read as a sample from an unlimited supply
bci_Q1 <- sum(colSums(bci_inc) == 1); bci_Q2 <- sum(colSums(bci_inc) == 2)
bci_over <- (bci_K - 1) / bci_K * bci_Q1^2 / (2 * bci_Q2)
bci_single <- sum(colSums(BCI) == 1)     # species with one tree in the whole census
stopifnot(abs(bci_whole[["chao2"]] - bci_S - bci_over) < 1e-9,
          abs(vegan::specpool(BCI)$chao - bci_whole[["chao2"]]) < 1e-9)
round(bci_tab, 2)

## ---- fig-frame-fraction ----
bci_long <- do.call(rbind, lapply(seq_along(bci_q), function(i)
  do.call(rbind, lapply(c("S_obs", "chao2", "wor"), function(v) { vv <- bci_draws[[i]][, v]
    data.frame(q = bci_q[i], what = v, mean = mean(vv), lo = quantile(vv, 0.1, names = FALSE),
               hi = quantile(vv, 0.9, names = FALSE)) }))))
bci_long$what <- factor(bci_long$what, levels = c("S_obs", "chao2", "wor"),
  labels = c("species observed", "Chao2 (assumes replacement)", "bound without replacement"))
bci_cols <- setNames(unname(te[c("slate", "rust", "forest")]), levels(bci_long$what))
ggplot(bci_long, aes(q, mean, colour = what, fill = what)) +
  geom_hline(yintercept = bci_S, linetype = "dashed", colour = te["brick"], linewidth = 0.5) +
  geom_ribbon(data = bci_long[bci_long$what != "species observed", ],
              aes(ymin = lo, ymax = hi), alpha = 0.15, colour = NA) +
  geom_line(linewidth = 0.9) + geom_point(size = 2) +
  annotate("point", x = 1, y = bci_whole[["chao2"]], shape = 21, size = 3, stroke = 1,
           colour = te["rust"], fill = "#f5f4ee") +
  annotate("text", x = 0.2, y = bci_S + 3, label = paste("census richness", bci_S), hjust = 0,
           size = 3, colour = te["brick"]) +
  scale_colour_manual(values = bci_cols, name = NULL, aesthetics = c("colour", "fill")) +
  scale_x_continuous(breaks = c(bci_q, 1)) +
  labs(title = "On a finite frame Chao2 overshoots as the sampled share grows",
       subtitle = "BCI, 50 one-hectare quadrats drawn without replacement, 300 draws per share",
       x = "share of the quadrats sampled (q = k / 50)", y = "richness") +
  guides(fill = "none") + theme_te()
