# The false discovery rate and Benjamini-Hochberg
# https://tidyecology.com/posts/false-discovery-rate-benjamini-hochberg/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-01

## ---- setup ----
library(ggplot2)
te_ink<-"#16241d"; te_body<-"#2c3a31"; te_forest<-"#275139"; te_faint<-"#5d6b61"
te_paper<-"#f5f4ee"; te_line<-"#dad9ca"
c_bonf<-"#cda23f"; c_bh<-"#275139"
theme_te <- function(base=13) theme_minimal(base_size=base) +
  theme(plot.background=element_rect(fill=te_paper,colour=NA),
        panel.background=element_rect(fill=te_paper,colour=NA),
        panel.grid.minor=element_blank(),
        panel.grid.major=element_line(colour=te_line,linewidth=.3),
        axis.text=element_text(colour=te_body), axis.title=element_text(colour=te_ink),
        plot.title=element_text(colour=te_ink,face="bold"),
        plot.subtitle=element_text(colour=te_faint),
        strip.text=element_text(colour=te_ink,face="bold"))

## ---- bh-fun ----
BH <- function(p, q = 0.05){
  m <- length(p); o <- order(p); s <- p[o]
  below <- s <= (seq_len(m) / m) * q
  k <- if(any(below)) max(which(below)) else 0
  rej <- rep(FALSE, m); if(k > 0) rej[o[seq_len(k)]] <- TRUE
  rej
}
set.seed(99); p_chk <- runif(50)^2
identical(BH(p_chk, 0.1), p.adjust(p_chk, "BH") <= 0.1)

## ---- single-run ----
m <- 200; m1 <- 40; m0 <- m - m1; mu <- 3; q <- 0.05
nonnull <- c(rep(TRUE, m1), rep(FALSE, m0))
set.seed(77)
z <- rnorm(m, mean = ifelse(nonnull, mu, 0))
p <- 2 * pnorm(-abs(z))                     # two-sided p-values
bonf_rej <- p.adjust(p, "bonferroni") < q
bh_rej   <- BH(p, q)
bh_thr   <- { s <- sort(p); below <- s <= (seq_len(m)/m)*q
              if(any(below)) s[max(which(below))] else 0 }
fdp <- function(rej) if(sum(rej) == 0) 0 else sum(rej & !nonnull) / sum(rej)
c(bonferroni_rejections = sum(bonf_rej), bonferroni_true = sum(bonf_rej & nonnull),
  bh_rejections = sum(bh_rej), bh_true = sum(bh_rej & nonnull), bh_fdp = round(fdp(bh_rej), 3))

## ---- fig-bh ----
so <- sort(p); rank <- seq_len(m)
ggplot(data.frame(rank = rank, p = so), aes(rank, p)) +
  geom_line(data = data.frame(rank = rank, y = (rank/m)*q), aes(rank, y),
            colour = c_bh, linewidth = .9) +
  geom_hline(yintercept = q/m, colour = c_bonf, linewidth = .9) +
  geom_point(size = 1.1, colour = te_body) +
  coord_cartesian(xlim = c(0, 60), ylim = c(0, 0.03)) +
  annotate("text", x = 58, y = q/m + .0012, label = "Bonferroni alpha/m",
           colour = c_bonf, hjust = 1, size = 3.3) +
  annotate("text", x = 52, y = (52/m)*q + .0016, label = "BH (k/m)q",
           colour = c_bh, hjust = 1, size = 3.3) +
  labs(x = "rank of p-value", y = "p-value (sorted)",
       title = "Benjamini-Hochberg rejects far more than Bonferroni") +
  theme_te()

## ---- mc ----
set.seed(2024)
R <- 3000
fdp_bh <- fdp_bf <- pow_bh <- pow_bf <- numeric(R)
for(r in 1:R){
  z <- rnorm(m, mean = ifelse(nonnull, mu, 0)); p <- 2 * pnorm(-abs(z))
  rb <- BH(p, q); rf <- p.adjust(p, "bonferroni") < q
  fdp_bh[r] <- if(sum(rb) == 0) 0 else sum(rb & !nonnull) / sum(rb)
  fdp_bf[r] <- if(sum(rf) == 0) 0 else sum(rf & !nonnull) / sum(rf)
  pow_bh[r] <- mean(rb[nonnull]); pow_bf[r] <- mean(rf[nonnull])
}
fdr_bh <- mean(fdp_bh); fdr_bf <- mean(fdp_bf)
power_bh <- mean(pow_bh); power_bf <- mean(pow_bf)
round(c(bh_FDR = fdr_bh, bh_power = power_bh, bonf_FDR = fdr_bf, bonf_power = power_bf), 4)

## ---- fig-fdr-power ----
d <- data.frame(
  method = factor(rep(c("Bonferroni","BH"), 2), levels = c("Bonferroni","BH")),
  metric = rep(c("power","FDR"), each = 2),
  value  = c(power_bf, power_bh, fdr_bf, fdr_bh))
ggplot(d, aes(method, value, fill = method)) +
  geom_col(width = .6) + facet_wrap(~metric, scales = "free_y") +
  geom_text(aes(label = sprintf("%.3f", value)), vjust = -0.35, size = 3.4, colour = te_ink) +
  geom_hline(data = data.frame(metric = "FDR", yint = q), aes(yintercept = yint),
             linetype = "dashed", colour = te_faint) +
  scale_fill_manual(values = c(c_bonf, c_bh), guide = "none") +
  scale_y_continuous(expand = expansion(mult = c(0, .15))) +
  labs(x = NULL, y = NULL, title = "FDR control buys more power at a small, bounded cost") +
  theme_te()
