# Family-wise error and the Bonferroni correction
# https://tidyecology.com/posts/family-wise-error-and-bonferroni/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-08

## ---- setup ----
library(ggplot2)
te_ink<-"#16241d"; te_body<-"#2c3a31"; te_forest<-"#275139"; te_faint<-"#5d6b61"
te_paper<-"#f5f4ee"; te_line<-"#dad9ca"
c_raw<-"#b5534e"; c_bonf<-"#cda23f"; c_holm<-"#2f8f63"
theme_te <- function(base=13) theme_minimal(base_size=base) +
  theme(plot.background=element_rect(fill=te_paper,colour=NA),
        panel.background=element_rect(fill=te_paper,colour=NA),
        panel.grid.minor=element_blank(),
        panel.grid.major=element_line(colour=te_line,linewidth=.3),
        axis.text=element_text(colour=te_body), axis.title=element_text(colour=te_ink),
        plot.title=element_text(colour=te_ink,face="bold"),
        plot.subtitle=element_text(colour=te_faint),
        legend.text=element_text(colour=te_body), legend.title=element_text(colour=te_ink),
        strip.text=element_text(colour=te_ink,face="bold"))

## ---- fwer-grid ----
alpha <- 0.05
m_grid <- c(1, 5, 10, 20, 50, 100)
fwer <- 1 - (1 - alpha)^m_grid
setNames(round(fwer, 3), m_grid)

## ---- fig-fwer ----
mg <- seq(1, 120)
ggplot(data.frame(m=mg, fwer=1-(1-alpha)^mg), aes(m, fwer)) +
  geom_hline(yintercept=alpha, linetype="dashed", colour=te_faint) +
  geom_line(colour=te_forest, linewidth=1.1) +
  geom_point(data=data.frame(m=20, fwer=1-(1-alpha)^20), size=3, colour=c_raw) +
  annotate("text", x=22, y=1-(1-alpha)^20-.04, label="m=20: 0.64",
           colour=c_raw, hjust=0, size=3.6) +
  scale_y_continuous(limits=c(0,1)) +
  labs(x="number of tests (m)", y="family-wise error rate",
       title="Running m tests at 0.05 inflates the family-wise error") +
  theme_te()

## ---- single-run ----
set.seed(1)
mtax <- 20; n <- 30
grpA <- matrix(rnorm(mtax*n), n, mtax)   # habitat A
grpB <- matrix(rnorm(mtax*n), n, mtax)   # habitat B, identical distribution
pvals <- sapply(1:mtax, function(j) t.test(grpA[,j], grpB[,j])$p.value)
raw_hits <- which(pvals < alpha)
raw_hits
round(sort(pvals)[1:3], 4)

## ---- corrections ----
bonf <- function(p) pmin(p * length(p), 1)
holm <- function(p){
  m <- length(p); o <- order(p)
  adj <- cummax((m - seq_len(m) + 1) * p[o])
  out <- numeric(m); out[o] <- pmin(adj, 1); out
}
sidak_thr <- 1 - (1 - alpha)^(1/mtax)
c(bonferroni = max(abs(bonf(pvals) - p.adjust(pvals, "bonferroni"))),
  holm       = max(abs(holm(pvals) - p.adjust(pvals, "holm"))))

## ---- mc-fwer ----
welch_p <- function(a, b){
  n1<-nrow(a); n2<-nrow(b); m1<-colMeans(a); m2<-colMeans(b)
  v1<-apply(a,2,var); v2<-apply(b,2,var); se2<-v1/n1+v2/n2
  tt<-(m1-m2)/sqrt(se2)
  df<-se2^2/((v1/n1)^2/(n1-1)+(v2/n2)^2/(n2-1))
  2*pt(-abs(tt), df)
}
sidak_p <- function(p) pmin(1 - (1-p)^length(p), 1)
set.seed(4242)
R <- 3000
fw <- c(raw=0, bonferroni=0, holm=0, sidak=0)
for(r in 1:R){
  A<-matrix(rnorm(mtax*n),n,mtax); B<-matrix(rnorm(mtax*n),n,mtax)
  p<-welch_p(A,B)
  fw["raw"]        <- fw["raw"]        + (min(p) < alpha)
  fw["bonferroni"] <- fw["bonferroni"] + (min(p.adjust(p,"bonferroni")) < alpha)
  fw["holm"]       <- fw["holm"]       + (min(p.adjust(p,"holm")) < alpha)
  fw["sidak"]      <- fw["sidak"]      + (min(sidak_p(p)) < alpha)
}
round(fw / R, 3)

## ---- power-demo ----
set.seed(31337)
m1 <- 5; delta <- 0.8
true_eff <- c(rep(TRUE, m1), rep(FALSE, mtax - m1))
mu_vec <- rep(c(delta, 0), c(m1, mtax - m1))
Rp <- 3000
pw <- matrix(0, Rp, 3); fwv <- matrix(0, Rp, 3)
holm_never_worse <- TRUE
for(r in 1:Rp){
  A <- matrix(rnorm(mtax*n), n, mtax)
  B <- sweep(matrix(rnorm(mtax*n), n, mtax), 2, mu_vec, "+")
  p <- welch_p(A, B)
  rj <- cbind(p < alpha, p.adjust(p,"bonferroni") < alpha, p.adjust(p,"holm") < alpha)
  if(sum(rj[,3]) < sum(rj[,2])) holm_never_worse <- FALSE
  pw[r,]  <- colMeans(rj[true_eff, , drop=FALSE])
  fwv[r,] <- apply(rj[!true_eff, , drop=FALSE], 2, any)
}
power <- setNames(round(colMeans(pw), 3), c("raw","bonferroni","holm"))
fwerp <- setNames(round(colMeans(fwv), 3), c("raw","bonferroni","holm"))
rbind(power = power, fwer = fwerp)

## ---- fig-power ----
d <- data.frame(
  method = factor(rep(c("raw (p<0.05)","Bonferroni","Holm"), 2),
                  levels=c("raw (p<0.05)","Bonferroni","Holm")),
  metric = rep(c("power (true effects)","FWER (>=1 false +)"), each=3),
  value  = c(power, fwerp))
ggplot(d, aes(method, value, fill=method)) +
  geom_col(width=.68) + facet_wrap(~metric) +
  geom_text(aes(label=sprintf("%.2f", value)), vjust=-0.35, size=3.4, colour=te_ink) +
  scale_fill_manual(values=c(c_raw,c_bonf,c_holm), guide="none") +
  scale_y_continuous(limits=c(0,1), expand=expansion(mult=c(0,.12))) +
  labs(x=NULL, y=NULL, title="Correction trades power for controlled error") +
  theme_te() + theme(axis.text.x=element_text(angle=12, hjust=1))
