# Compositional analysis and the zero problem
# https://tidyecology.com/posts/compositional-analysis-and-zeros/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-14

## ---- setup ----
library(ggplot2)
clr <- function(x) { lx <- log(x); lx - mean(lx) }
te_ink <- "#16241d"; te_forest <- "#275139"; te_faint <- "#5d6b61"
te_paper <- "#f5f4ee"; te_line <- "#dad9ca"; te_red <- "#b5534e"
theme_te <- function(base = 12) theme_minimal(base_size = base) + theme(
  plot.background = element_rect(fill = te_paper, colour = NA),
  panel.background = element_rect(fill = te_paper, colour = NA),
  panel.grid.major = element_line(colour = te_line, linewidth = 0.3),
  panel.grid.minor = element_blank(),
  axis.text = element_text(colour = "#2c3a31"), axis.title = element_text(colour = te_ink),
  plot.title = element_text(colour = te_ink, face = "bold"),
  plot.subtitle = element_text(colour = te_faint, size = rel(0.9)),
  legend.position = "top")
theme_set(theme_te())

## ---- sci-tex-helper ----
sci_tex <- function(x, digits = 2) {
  s <- formatC(x, format = "e", digits = digits)
  paste0("$", sub("e.*", "", s), " \\times 10^{", as.integer(sub(".*e", "", s)), "}$")
}

## ---- simulate ----
set.seed(4472)
D <- 6L; ng <- 60L
base_mu <- c(3.4, 2.6, 2.4, 2.2, 2.0, 2.3); sd_log <- rep(0.45, D)
delta1  <- log(2.6)                              # taxon 1 rises; taxa 2..6 unchanged
sim_abs <- function(mu) sapply(1:D, function(j) rlnorm(ng, mu[j], sd_log[j]))
Xc <- sim_abs(base_mu)
mu_t <- base_mu; mu_t[1] <- mu_t[1] + delta1
Xt <- sim_abs(mu_t)
Pc <- Xc / rowSums(Xc); Pt <- Xt / rowSums(Xt)

p_raw <- sapply(1:D, function(j) t.test(Pc[,j], Pt[,j])$p.value)
setNames(round(p_raw, 6), paste0("t", 1:D))

## ---- fig-falsepos ----
dfp <- data.frame(taxon = factor(1:D), nl = -log10(p_raw),
                  truth = ifelse(1:D == 1, "truly changed (absolute)", "unchanged (absolute)"))
p_falsepos <- ggplot(dfp, aes(taxon, nl, fill = truth)) +
  geom_col(width = 0.65) +
  geom_hline(yintercept = -log10(0.05), linetype = "dashed", colour = te_faint) +
  scale_fill_manual(values = c("truly changed (absolute)" = te_forest,
                               "unchanged (absolute)" = te_red), name = NULL) +
  labs(title = "Closure turns one real change into five spurious ones",
       subtitle = "Only taxon 1 changed; the naive proportion test flags every taxon",
       x = "taxon", y = "naive proportion test  (-log10 p)")
p_falsepos

## ---- thumb ----
ggsave("thumbnail.png",
       p_falsepos + labs(title = NULL, subtitle = NULL) +
         theme(plot.margin = margin(8, 14, 8, 8)),
       width = 7.5, height = 5.4, dpi = 100, bg = te_paper)

## ---- coda-localise ----
pairs <- t(combn(1:D, 2))
p_lr <- apply(pairs, 1, function(pr) {
  t.test(log(Xc[,pr[1]] / Xc[,pr[2]]), log(Xt[,pr[1]] / Xt[,pr[2]]))$p.value })
has1 <- apply(pairs, 1, function(pr) 1 %in% pr)
c(among_unchanged_sig = sum(p_lr[!has1] < 0.05), of = sum(!has1),
  involving_t1_sig     = sum(p_lr[has1] < 0.05),  of_ = sum(has1))

## ---- monte-carlo ----
set.seed(55)
B <- 1500L
mc <- replicate(B, {
  a <- sim_abs(base_mu); mt <- base_mu; mt[1] <- mt[1] + delta1; b <- sim_abs(mt)
  pa <- a / rowSums(a); pb <- b / rowSums(b)
  pr_raw <- sapply(2:D, function(j) t.test(pa[,j], pb[,j])$p.value)
  prs <- t(combn(2:D, 2))
  pr_lr <- apply(prs, 1, function(pr) t.test(log(a[,pr[1]]/a[,pr[2]]), log(b[,pr[1]]/b[,pr[2]]))$p.value)
  c(raw = mean(pr_raw < 0.05), lr = mean(pr_lr < 0.05))
})
round(c(naive_proportion = mean(mc["raw",]), coda_logratio = mean(mc["lr",])), 3)

## ---- pca ----
P <- rbind(Pc, Pt); CLR <- t(apply(P, 1, clr))
vr_raw <- summary(prcomp(P,   center = TRUE))$importance[2, 1:2]
vr_clr <- summary(prcomp(CLR, center = TRUE))$importance[2, 1:2]
round(rbind(raw_proportions = vr_raw, clr = vr_clr), 3)

## ---- zeros ----
set.seed(6190)
Dz <- 6L; nz <- 80L
rel_mu <- c(0.30, 0.24, 0.20, 0.14, 0.10, 0.02)     # taxon 6 genuinely rare
Comp <- t(apply(matrix(rgamma(nz * Dz, rep(rel_mu * 40, each = nz)), nz, Dz), 1,
                function(z) z / sum(z)))
Nlib <- sample(120:180, nz, replace = TRUE)
Counts <- t(sapply(1:nz, function(i) rmultinom(1, Nlib[i], Comp[i,])[,1]))

nz_cells <- Counts == 0
c(zero_cells = sum(nz_cells), of = nz * Dz,
  rare_taxon_zero_in = sum(nz_cells[,6]), samples = nz)

## ---- clr-breaks ----
any(!is.finite(clr((Counts / rowSums(Counts))[which(nz_cells[,6])[1], ])))

## ---- mult-repl ----
mult_repl <- function(cnt, delta) {
  p <- cnt / sum(cnt); z <- p == 0; k <- sum(z)
  if (k == 0) return(p)
  p[z] <- delta; p[!z] <- p[!z] * (1 - k * delta); p
}
i0 <- which(nz_cells[,6])[1]
c(observed_ratio_small_delta = log(mult_repl(Counts[i0,], 0.2 / Nlib[i0])[1] /
                                    mult_repl(Counts[i0,], 0.2 / Nlib[i0])[2]),
  observed_ratio_large_delta = log(mult_repl(Counts[i0,], 1.0 / Nlib[i0])[1] /
                                    mult_repl(Counts[i0,], 1.0 / Nlib[i0])[2]),
  rare_ratio_small_delta     = log(mult_repl(Counts[i0,], 0.2 / Nlib[i0])[6] /
                                    mult_repl(Counts[i0,], 0.2 / Nlib[i0])[1]),
  rare_ratio_large_delta     = log(mult_repl(Counts[i0,], 1.0 / Nlib[i0])[6] /
                                    mult_repl(Counts[i0,], 1.0 / Nlib[i0])[1]))

## ---- fig-zeros ----
deltas <- c(0.1, 0.2, 0.35, 0.5, 0.65, 0.8, 1.0)
mclr6 <- sapply(deltas, function(dd) {
  Pr <- t(sapply(1:nz, function(i) mult_repl(Counts[i,], dd / Nlib[i]))); mean(t(apply(Pr,1,clr))[,6]) })
mlr12 <- sapply(deltas, function(dd) {
  Pr <- t(sapply(1:nz, function(i) mult_repl(Counts[i,], dd / Nlib[i]))); mean(log(Pr[,1] / Pr[,2])) })
dz <- rbind(data.frame(delta = deltas, val = mclr6, series = "mean clr of rare taxon, all samples"),
            data.frame(delta = deltas, val = mlr12, series = "log-ratio of two common taxa"))
ggplot(dz, aes(delta, val, colour = series)) +
  geom_line(linewidth = 0.9) + geom_point(size = 2) +
  scale_colour_manual(values = c("mean clr of rare taxon, all samples" = te_red,
                                 "log-ratio of two common taxa" = te_forest), name = NULL) +
  labs(title = "The zero replacement is an untestable choice",
       subtitle = "Observed ratios are invariant; the rare-taxon value tracks the imputed delta",
       x = "imputed value delta  (units of 1/N)", y = "mean log-ratio quantity")

## ---- bayes-repl ----
bayes_repl <- function(cnt, a) (cnt + a) / (sum(cnt) + length(cnt) * a)
c(dirichlet_half = mean(sapply(1:nz, function(i) clr(bayes_repl(Counts[i,], 0.5))[6])),
  dirichlet_one  = mean(sapply(1:nz, function(i) clr(bayes_repl(Counts[i,], 1.0))[6])))
