# Combining species trends into one index
# https://tidyecology.com/posts/a-multi-species-index-and-its-arithmetic/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-08-09

## ---- setup ----
library(ggplot2)
library(patchwork)

te_paper  <- "#f5f4ee"
te_ink    <- "#16241d"
te_body   <- "#2c3a31"
te_forest <- "#275139"
te_rust   <- "#b5534e"
te_gold   <- "#c9b458"
te_line   <- "#dad9ca"

theme_datasheet <- function() {
  theme_minimal(base_size = 12) +
    theme(plot.background  = element_rect(fill = te_paper, colour = NA),
          panel.background = element_rect(fill = te_paper, colour = NA),
          panel.grid.major = element_line(colour = te_line, linewidth = 0.3),
          panel.grid.minor = element_blank(),
          text             = element_text(colour = te_body),
          plot.title       = element_text(colour = te_ink, face = "bold"),
          plot.subtitle    = element_text(colour = te_body),
          axis.text        = element_text(colour = te_body))
}

n_sp   <- 30
n_yr   <- 20
sd_log <- 0.04

set.seed(2026)
trend_log <- rnorm(n_sp, 0, sd_log)
idx_sp    <- exp(outer(trend_log, 0:n_yr))
arith_one <- colMeans(idx_sp)
geom_one  <- exp(colMeans(log(idx_sp)))
med_one   <- apply(idx_sp, 2, median)

kind_three <- c("arithmetic mean", "geometric mean", "median species")
mean_trend_one <- mean(trend_log)
n_up      <- sum(trend_log > 0)
best_one  <- max(idx_sp[, n_yr + 1])
worst_one <- min(idx_sp[, n_yr + 1])

## ---- sci-tex-helper ----
sci_tex <- function(x, digits = 2) {
  s <- formatC(x, format = "e", digits = digits)
  paste0("$", sub("e.*", "", s), " \\times 10^{", as.integer(sub(".*e", "", s)), "}$")
}

## ---- fig-onecommunity ----
sp_long <- data.frame(
  year = rep(0:n_yr, each = n_sp), index = as.vector(idx_sp),
  species = rep(seq_len(n_sp), times = n_yr + 1))
avg_long <- data.frame(
  year = rep(0:n_yr, 3), index = c(arith_one, geom_one, med_one),
  kind = rep(kind_three, each = n_yr + 1))
ggplot(sp_long, aes(year, index)) +
  geom_line(aes(group = species), colour = te_body, alpha = 0.22, linewidth = 0.35) +
  geom_line(data = avg_long, aes(colour = kind), linewidth = 1.1) +
  scale_colour_manual(values = c(te_rust, te_forest, te_gold), name = NULL) +
  scale_y_log10(breaks = c(0.2, 0.5, 1, 2, 5)) +
  labs(x = "year", y = "index (first year = 1, log scale)",
       title = "One community, three summaries",
       subtitle = "on a log axis the geometric mean sits in the middle of the bundle") +
  theme_datasheet() +
  theme(legend.position = "bottom")

## ---- manycommunities ----
n_rep <- 2000
set.seed(1867)
sim_acc <- replicate(n_rep, {
  trend_i <- rnorm(n_sp, 0, sd_log)
  path_i  <- exp(outer(trend_i, 0:n_yr))
  rbind(colMeans(path_i), exp(colMeans(log(path_i))), apply(path_i, 2, median))
})
traj_out <- apply(sim_acc, c(1, 2), mean)
sim_out  <- sim_acc[, n_yr + 1, ]
arith_bar <- mean(sim_out[1, ])
geom_bar  <- mean(sim_out[2, ])
med_bar   <- mean(sim_out[3, ])
arith_pct <- 100 * (arith_bar - 1)
share_up  <- mean(sim_out[1, ] > 1)
mc_se     <- sd(sim_out[1, ]) / sqrt(n_rep)
arith_theory <- exp((sd_log * n_yr)^2 / 2)
arith_cond   <- exp((sd_log * n_yr)^2 * (1 - 1 / n_sp) / 2)
geom_theory  <- exp((sd_log * n_yr)^2 / (2 * n_sp))

## ---- sweep ----
sd_grid  <- seq(0, 0.08, by = 0.01)
n_sweep  <- 20000
set.seed(4004)
sweep_rep <- lapply(sd_grid, function(sd_i) replicate(n_sweep, {
  final_i <- exp(rnorm(n_sp, 0, sd_i) * n_yr)
  c(mean(final_i), exp(mean(log(final_i))))
}))
sweep_out <- vapply(sweep_rep, rowMeans, numeric(2))
sd_fine  <- seq(0, max(sd_grid), length.out = 200)
sd_hi    <- max(sd_grid)
hi_both  <- sweep_out[, length(sd_grid)]
hi_theory <- exp((sd_hi * n_yr)^2 / 2)
hi_se    <- sd(sweep_rep[[length(sd_grid)]][1, ]) / sqrt(n_sweep)
sp_hi    <- exp(sd_hi * n_yr)

## ---- fig-replicated ----
traj_long <- data.frame(
  year = rep(0:n_yr, 3), index = as.vector(t(traj_out)),
  kind = rep(kind_three, each = n_yr + 1))
p_traj <- ggplot(traj_long, aes(year, index, colour = kind)) +
  geom_line(linewidth = 1.1) +
  scale_colour_manual(values = c(te_rust, te_forest, te_gold), name = NULL) +
  labs(x = "year", y = "mean indicator value",
       title = "Trends centred on zero",
       subtitle = "averaged over 2000 communities") +
  theme_datasheet() +
  theme(legend.position = "bottom", legend.direction = "horizontal")
sweep_long <- data.frame(
  sd_trend = rep(sd_grid, 2), index = as.vector(t(sweep_out)),
  kind     = rep(kind_three[1:2], each = length(sd_grid)))
theory_long <- data.frame(
  sd_trend = rep(sd_fine, 2), kind = rep(kind_three[1:2], each = length(sd_fine)),
  index = exp((sd_fine * n_yr)^2 / rep(c(2, 2 * n_sp), each = length(sd_fine))))
p_sweep <- ggplot(sweep_long, aes(sd_trend, index, colour = kind)) +
  geom_line(data = theory_long, linewidth = 0.7, alpha = 0.7) +
  geom_point(size = 2.2) +
  scale_colour_manual(values = c(te_rust, te_forest), name = NULL) +
  labs(x = "standard deviation of species log trend",
       y = "indicator at year 20",
       title = "How fast the gap opens",
       subtitle = "lines: lognormal expectation") +
  theme_datasheet() +
  theme(legend.position = "none")
p_traj + p_sweep + plot_layout(guides = "collect") +
  plot_annotation(theme = theme_datasheet() + theme(legend.position = "bottom"))

## ---- poisson ----
lam_set    <- c(40, 5, 2)
zero_floor <- 0.5
n_pois     <- 4000
index_pair <- function(cnt) {
  y_use <- pmax(cnt, zero_floor)
  ratio <- y_use / y_use[, 1]
  rbind(colMeans(ratio), exp(colMeans(log(ratio))))
}
set.seed(5150)
pois_rep <- lapply(lam_set, function(lam) {
  replicate(n_pois, index_pair(matrix(rpois(n_sp * (n_yr + 1), lam), n_sp)))
})
pois_traj <- lapply(pois_rep, function(a_i) apply(a_i, c(1, 2), mean))
pois20 <- vapply(pois_traj, function(m_i) m_i[, n_yr + 1], numeric(2))

k_grid  <- 0:400
pois_ex <- function(lam, f_i) sum(f_i(k_grid) * dpois(k_grid, lam))
inv_y   <- function(k_i) 1 / pmax(k_i, zero_floor)
flat_ex <- function(lam) pois_ex(lam, function(k_i) pmax(k_i, zero_floor)) *
                         pois_ex(lam, inv_y)
recip_lo    <- pois_ex(lam_set[3], inv_y)
recip_infl  <- recip_lo * lam_set[3]
flat_lo     <- flat_ex(lam_set[3])
wobble_lo   <- range(pois_traj[[3]][1, -1])
v_log_lo    <- pois_ex(lam_set[3], function(k_i) log(pmax(k_i, zero_floor))^2) -
               pois_ex(lam_set[3], function(k_i) log(pmax(k_i, zero_floor)))^2
geo_pred_lo <- exp(2 * v_log_lo / (2 * n_sp))
geo_se_lo   <- sd(pois_rep[[3]][2, n_yr + 1, ]) / sqrt(n_pois)
excess_ratio <- min((pois20[1, ] - 1) / (pois20[2, ] - 1))
pois_se <- vapply(pois_rep, function(a_i) sd(a_i[1, n_yr + 1, ]), 0) / sqrt(n_pois)

## ---- poissondrop ----
set.seed(7150)
drop_out <- vapply(lam_set, function(lam) {
  rowMeans(replicate(n_pois, {
    cnt  <- matrix(rpois(n_sp * (n_yr + 1), lam), n_sp)
    keep <- cnt[, 1] > 0 & cnt[, n_yr + 1] > 0
    c(mean(cnt[keep, n_yr + 1] / cnt[keep, 1]), mean(keep))
  }))
}, numeric(2))
drop_theory <- function(lam) {
  p_pos <- dpois(k_grid[-1], lam) / (1 - dpois(0, lam))
  sum(k_grid[-1] * p_pos) * sum(p_pos / k_grid[-1])
}
floor_share <- vapply(lam_set, function(lam)
  100 * (flat_ex(lam) - drop_theory(lam)) / (flat_ex(lam) - 1), 0)
p0_hi     <- dpois(0, lam_set[1])
drop_lo   <- drop_theory(lam_set[3])
discard_lo <- 100 * (1 - (1 - dpois(0, lam_set[3]))^2)

## ---- fig-poisson ----
lam_lab <- sprintf("mean count %d", lam_set)
pois_long <- do.call(rbind, lapply(seq_along(lam_set), function(k) data.frame(
  year = rep(0:n_yr, 2), index = as.vector(t(pois_traj[[k]])),
  kind = rep(kind_three[1:2], each = n_yr + 1),
  lam  = factor(lam_lab[k], levels = lam_lab))))
ggplot(pois_long, aes(year, index, colour = lam, linetype = kind)) +
  geom_line(linewidth = 0.9) +
  scale_colour_manual(values = c(te_forest, te_gold, te_rust), name = NULL) +
  scale_linetype_manual(values = c("solid", "22"), name = NULL) +
  labs(x = "year", y = "indicator value",
       title = "Stable populations, moving indicator",
       subtitle = "every population has a constant expected count") +
  theme_datasheet() +
  theme(legend.position = "bottom", legend.box = "vertical",
        legend.spacing.y = unit(0, "pt"))

## ---- identity ----
set.seed(6120)
trend_id <- rnorm(n_sp, 0, sd_log)
count_id <- exp(outer(trend_id, 0:n_yr)) *
  matrix(rlnorm(n_sp * (n_yr + 1), 0, 0.15), n_sp)
log_id     <- log(count_id)
base_form  <- exp(colMeans(log_id - log_id[, 1]))
step_mean  <- colMeans(log_id[, -1] - log_id[, -(n_yr + 1)])
chain_form <- exp(c(0, cumsum(step_mean)))
gap_id   <- max(abs(base_form - chain_form))
equal_id <- isTRUE(all.equal(base_form, chain_form))

## ---- fp-guard-1 ----
stopifnot(max(abs(base_form - chain_form)) < 1e-10)

## ---- stagger ----
n_late    <- 10
yr_enter  <- 10
rate_late <- -0.05
sd_stable <- 0.01
n_stag    <- 2000
stagger_once <- function() {
  trend_i <- c(rnorm(n_sp - n_late, 0, sd_stable), rep(rate_late, n_late))
  log_i   <- log(exp(outer(trend_i, 0:n_yr)) *
                 matrix(rlnorm(n_sp * (n_yr + 1), 0, 0.10), n_sp))
  log_i[(n_sp - n_late + 1):n_sp, 1:yr_enter] <- NA
  keep0  <- !is.na(log_i[, 1])
  base_v <- exp(colMeans(log_i[keep0, , drop = FALSE] - log_i[keep0, 1]))
  step_v <- vapply(2:(n_yr + 1), function(j) {
    ok <- !is.na(log_i[, j]) & !is.na(log_i[, j - 1])
    mean(log_i[ok, j] - log_i[ok, j - 1])
  }, 0)
  rbind(base_v, exp(c(0, cumsum(step_v))), exp(mean(trend_i) * (0:n_yr)))
}
set.seed(303)
stag <- apply(replicate(n_stag, stagger_once()), c(1, 2), mean)
stag_pct <- 100 * (1 - exp(rate_late * (n_yr - yr_enter)))

## ---- zerobreak ----
break_once <- function(lam) {
  cnt <- matrix(rpois(n_sp * (n_yr + 1), lam), n_sp)
  log_i <- log(cnt)
  log_i[!is.finite(log_i)] <- NA
  keep0  <- !is.na(log_i[, 1]) & !is.na(log_i[, n_yr + 1])
  step_v <- vapply(2:(n_yr + 1), function(j) {
    ok <- !is.na(log_i[, j]) & !is.na(log_i[, j - 1])
    mean(log_i[ok, j] - log_i[ok, j - 1])
  }, 0)
  c(exp(mean(log_i[keep0, n_yr + 1] - log_i[keep0, 1])), exp(sum(step_v)))
}
gap_cut <- 0.1
set.seed(909)
brk       <- replicate(n_pois, break_once(lam_set[3]))
gap_break <- abs(brk[1, ] - brk[2, ])
gap_share <- mean(gap_break > gap_cut)

## ---- patchprice ----
n_dec     <- 6
lam_start <- 20
rate_dec  <- -0.25
trend_dec <- c(rep(0, n_sp - n_dec), rep(rate_dec, n_dec))
lam_grid  <- lam_start * exp(outer(trend_dec, 0:n_yr))
truth_dec <- exp(mean(trend_dec) * n_yr)
loss_dec  <- 100 * (1 - exp(rate_dec * n_yr))
gm_last   <- function(m_i) exp(mean(log(m_i[, n_yr + 1]) - log(m_i[, 1])))
floor_one <- 1
set.seed(515)
patch_out <- replicate(n_pois, {
  cnt  <- matrix(rpois(length(lam_grid), lam_grid), n_sp)
  keep <- cnt[, 1] > 0 & cnt[, n_yr + 1] > 0
  floored <- pmax(cnt, zero_floor)
  c(gm_last(floored),
    gm_last(pmax(cnt, floor_one)),
    gm_last(cnt + floor_one),
    exp(mean(log(cnt[keep, n_yr + 1] / cnt[keep, 1]))),
    mean(keep),
    mean(floored[, n_yr + 1] / floored[, 1]))
})
patch_mean <- rowMeans(patch_out)
names(patch_mean) <- c("floor 0.5", "floor 1", "add 1", "drop",
                       "kept", "arithmetic mean")

## ---- fig-chain ----
stag_long <- data.frame(
  year = rep(0:n_yr, 3), index = as.vector(t(stag)),
  kind = rep(c("base year form", "chained form", "all species, if monitored"),
             each = n_yr + 1))
p_stag <- ggplot(stag_long, aes(year, index, colour = kind, linetype = kind)) +
  geom_line(linewidth = 1) +
  scale_colour_manual(values = c(te_body, te_forest, te_rust), name = NULL) +
  scale_linetype_manual(values = c("22", "solid", "solid"), name = NULL) +
  labs(x = "year", y = "indicator value",
       title = "Ten species join in year 10",
       subtitle = "and they are the ones declining") +
  theme_datasheet() +
  theme(legend.position = "bottom", legend.direction = "horizontal")
patch_df <- data.frame(index = unname(patch_mean[1:4]),
  patch = factor(names(patch_mean)[1:4], levels = names(patch_mean)[1:4]))
p_patch <- ggplot(patch_df, aes(patch, index)) +
  geom_hline(yintercept = truth_dec, colour = te_body,
             linetype = "22", linewidth = 0.6) +
  geom_point(size = 3.4, colour = te_forest) +
  scale_y_continuous(limits = c(0, 1)) +
  labs(x = "zero handling", y = "indicator at year 20",
       title = "Four patches, four answers",
       subtitle = "dashed: the true geometric indicator") +
  theme_datasheet()
p_stag + p_patch + plot_layout(guides = "collect") +
  plot_annotation(theme = theme_datasheet() + theme(legend.position = "bottom"))
