# Hill numbers: estimating diversity in R
# https://tidyecology.com/posts/estimating-hill-number-diversity/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-05-30

## ---- setup ----
library(ggplot2); library(dplyr); library(tidyr)
te <- c(forest = "#2f5d50", moss = "#7a9b76", rust = "#b5651d", gold = "#c9a227",
        slate = "#3d4b53", sand = "#d9cbb2", sky = "#5b8aa6", brick = "#8c3b2e")
theme_te <- function(base_size = 12) {
  theme_minimal(base_size = base_size) +
    theme(panel.grid.minor = element_blank(),
          panel.grid.major = element_line(colour = "#e7e1d5", linewidth = 0.3),
          axis.title = element_text(colour = "#3d4b53"),
          axis.text  = element_text(colour = "#5c6670"),
          plot.title = element_text(colour = "#2f3a40", face = "bold", size = base_size + 1),
          plot.subtitle = element_text(colour = "#5c6670", size = base_size - 1),
          legend.position = "bottom",
          legend.title = element_text(colour = "#3d4b53"),
          plot.background  = element_rect(fill = "#f5f4ee", colour = NA),
          panel.background = element_rect(fill = "#f5f4ee", colour = NA))
}

## ---- helpers ----
hill <- function(p, q) { p <- p[p > 0]
  if (abs(q - 1) < 1e-9) exp(-sum(p * log(p))) else (sum(p^q))^(1 / (1 - q)) }
chao1 <- function(x) { n <- sum(x); f1 <- sum(x == 1); f2 <- sum(x == 2)
  length(x) + (n - 1) / n * f1 * (f1 - 1) / (2 * (f2 + 1)) }
covGT <- function(x) 1 - sum(x == 1) / sum(x)                    # Good-Turing coverage
shannon_CS <- function(x) { n <- sum(x); C <- covGT(x); pa <- C * (x / n)   # Chao-Shen
  -sum(pa * log(pa) / (1 - (1 - pa)^n)) }
simpson_hat <- function(x) { n <- sum(x); 1 / sum(x * (x - 1) / (n * (n - 1))) }  # nearly unbiased

## ---- simulate ----
set.seed(4192)
S_true <- 160
rel <- exp(rnorm(S_true, 0, 1.5)); rel <- rel / sum(rel)
n <- 300
x <- as.vector(rmultinom(1, n, rel)); x <- x[x > 0]
Sobs <- length(x); phat <- x / n
c(S_true = S_true, S_obs = Sobs, coverage = round(covGT(x), 4))

## ---- three-orders ----
truD <- c(hill(rel, 0),  hill(rel, 1),  hill(rel, 2))
empD <- c(hill(phat, 0), hill(phat, 1), hill(phat, 2))
estD <- c(chao1(x),      exp(shannon_CS(x)), simpson_hat(x))
data.frame(q = c(0, 1, 2), true = round(truD, 1),
           empirical = round(empD, 1), estimated = round(estD, 1),
           rel_bias_empirical = round((empD - truD) / truD, 3))

## ---- fig-profile ----
qgrid <- seq(0, 3, by = 0.1)
prof <- data.frame(q = rep(qgrid, 2),
  D = c(sapply(qgrid, function(q) hill(rel, q)),
        sapply(qgrid, function(q) hill(phat, q))),
  series = rep(c("true", "empirical (plug-in)"), each = length(qgrid)))
estpts <- data.frame(q = c(0, 1, 2), D = estD)
ggplot(prof, aes(q, D, colour = series, linetype = series)) +
  geom_line(linewidth = 0.9) +
  geom_point(data = estpts, aes(q, D), inherit.aes = FALSE,
             colour = te["rust"], size = 3.4, shape = 17) +
  annotate("text", x = 0.08, y = truD[1] + 8, label = "richness (q = 0): biggest gap",
           hjust = 0, size = 3.1, colour = te["slate"]) +
  scale_colour_manual(values = c("true" = unname(te["forest"]),
                                 "empirical (plug-in)" = unname(te["sky"]),
                                 "estimated (bias-corrected)" = unname(te["rust"])), name = NULL) +
  scale_linetype_manual(values = c("true" = "solid", "empirical (plug-in)" = "22",
                                   "estimated (bias-corrected)" = "solid"), name = NULL) +
  labs(title = "The plug-in diversity profile is biased low, worst at q = 0",
       subtitle = "One community, true S = 160, n = 300; triangles are bias-corrected estimates at q = 0, 1, 2",
       x = "diversity order q", y = "Hill number (effective species)") +
  theme_te()

## ---- mc ----
set.seed(5192); R <- 500
mat <- replicate(R, { xx <- as.vector(rmultinom(1, n, rel)); xx <- xx[xx > 0]; pp <- xx / sum(xx)
  c(hill(pp, 0), hill(pp, 1), hill(pp, 2),
    chao1(xx), exp(shannon_CS(xx)), simpson_hat(xx)) })
mm <- rowMeans(mat)
mcbias <- data.frame(q = c(0, 1, 2),
  rb_emp = (mm[1:3] - truD) / truD,
  rb_est = (mm[4:6] - truD) / truD)
round(mcbias, 3)

## ---- fig-bias ----
mbl <- data.frame(q = factor(rep(c(0, 1, 2), 2)),
  rbias = c(mcbias$rb_emp, mcbias$rb_est),
  method = rep(c("empirical (plug-in)", "estimated (bias-corrected)"), each = 3))
ggplot(mbl, aes(q, rbias, fill = method)) +
  geom_hline(yintercept = 0, colour = te["slate"], linewidth = 0.4) +
  geom_col(position = position_dodge(width = 0.7), width = 0.6) +
  geom_text(aes(label = sprintf("%+.0f%%", 100 * rbias)),
            position = position_dodge(width = 0.7),
            vjust = ifelse(mbl$rbias < 0, 1.3, -0.5), size = 2.9, colour = te["slate"]) +
  scale_fill_manual(values = c("empirical (plug-in)" = unname(te["sky"]),
                               "estimated (bias-corrected)" = unname(te["rust"])), name = NULL) +
  scale_y_continuous(labels = scales::percent, limits = c(-0.65, 0.12)) +
  labs(title = "Estimability of diversity depends on the order q",
       subtitle = "Mean relative bias over 500 samples; the plug-in gap collapses as q rises, q = 0 stays hardest",
       x = "diversity order q", y = "relative bias") +
  theme_te()
