# Checking a multiple-testing analysis
# https://tidyecology.com/posts/checking-a-multiple-testing-analysis/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-15

## ---- setup ----
library(ggplot2)
te_ink<-"#16241d"; te_body<-"#2c3a31"; te_forest<-"#275139"; te_faint<-"#5d6b61"
te_paper<-"#f5f4ee"; te_line<-"#dad9ca"
c_bh<-"#275139"; c_bonf<-"#cda23f"
theme_te <- function(base=13) theme_minimal(base_size=base) +
  theme(plot.background=element_rect(fill=te_paper,colour=NA),
        panel.background=element_rect(fill=te_paper,colour=NA),
        panel.grid.minor=element_blank(),
        panel.grid.major=element_line(colour=te_line,linewidth=.3),
        axis.text=element_text(colour=te_body), axis.title=element_text(colour=te_ink),
        plot.title=element_text(colour=te_ink,face="bold"),
        plot.subtitle=element_text(colour=te_faint),
        strip.text=element_text(colour=te_ink,face="bold"))
BH <- function(p, q = 0.05){
  m <- length(p); o <- order(p); s <- p[o]
  below <- s <= (seq_len(m) / m) * q
  k <- if(any(below)) max(which(below)) else 0
  rej <- rep(FALSE, m); if(k > 0) rej[o[seq_len(k)]] <- TRUE
  rej
}

## ---- histograms ----
m <- 200; m1 <- 40; m0 <- m - m1; mu <- 3
nonnull <- c(rep(TRUE, m1), rep(FALSE, m0))
gof_unif <- function(p, B = 10){
  ct <- table(cut(p, breaks = seq(0, 1, length.out = B + 1), include.lowest = TRUE))
  chisq.test(as.numeric(ct))$p.value
}
set.seed(303)
p_null <- 2 * pnorm(-abs(rnorm(m)))                            # valid, all null
p_mis  <- 2 * pnorm(-abs(rnorm(m, sd = 1.4)))                  # overdispersed, all null
p_mix  <- 2 * pnorm(-abs(rnorm(m, mean = ifelse(nonnull, mu, 0))))  # 40 true effects
c(valid_GOF = round(gof_unif(p_null), 3),
  misspecified_GOF = signif(gof_unif(p_mis), 3),
  mixture_GOF = signif(gof_unif(p_mix), 3))
c(frac_below_05_valid = mean(p_null < 0.05),
  frac_below_05_misspecified = mean(p_mis < 0.05),
  frac_below_05_mixture = mean(p_mix < 0.05))

## ---- fig-histograms ----
d <- data.frame(p = c(p_null, p_mis, p_mix),
  panel = factor(rep(c("valid (all null)","misspecified (all null)","real signal (40 true)"),
                     each = m),
                 levels = c("valid (all null)","misspecified (all null)","real signal (40 true)")))
p_hist <- ggplot(d, aes(p)) +
  geom_histogram(breaks = seq(0, 1, .05), fill = c_bh, colour = te_paper, linewidth = .2) +
  facet_wrap(~panel) +
  geom_hline(yintercept = m/20, linetype = "dashed", colour = te_faint) +
  scale_x_continuous(breaks = c(0, 0.5, 1)) +
  labs(x = "p-value", y = "count", title = "The p-value histogram is the first diagnostic") +
  theme_te() + theme(panel.spacing.x = unit(1, "lines"))
p_hist

## ---- thumb ----
ggsave("thumbnail.png",
       p_hist +
         labs(title = paste(strwrap("P-value histograms in three scenarios", 46),
                            collapse = "\n")) +
         theme(plot.title = element_text(size = 13, face = "bold"),
               plot.subtitle = element_text(size = 9.5),
               plot.margin = margin(8, 14, 8, 8)),
       width = 7.5, height = 5.4, dpi = 100, bg = te_paper)

## ---- pi0 ----
pi0_hat <- function(p, lam) min(1, mean(p > lam) / (1 - lam))
sweep <- sapply(c(0.3, 0.5, 0.8), function(l)
  c(mixture = pi0_hat(p_mix, l), valid_null = pi0_hat(p_null, l),
    misspecified = pi0_hat(p_mis, l)))
colnames(sweep) <- c("lambda=0.3","lambda=0.5","lambda=0.8")
round(sweep, 3)

## ---- sensitivity ----
set.seed(2718)
z <- rnorm(m, mean = ifelse(nonnull, mu, 0)); p <- 2 * pnorm(-abs(z))
padj <- p.adjust(p, "BH")
q_grid <- c(0.01, 0.02, 0.05, 0.10, 0.15, 0.20)
Rq <- sapply(q_grid, function(qq) sum(padj <= qq))
setNames(Rq, q_grid)
c(bonferroni_at_05 = sum(p.adjust(p, "bonferroni") < 0.05), bh_at_05 = sum(padj <= 0.05))

## ---- tipping ----
ord <- order(padj)
borderline <- ord[which(padj[ord] > 0.05)[1]]
round(padj[borderline], 4)

## ---- fig-sensitivity ----
qg <- seq(0.005, 0.20, length.out = 60)
Rqc <- sapply(qg, function(qq) sum(padj <= qq))
ggplot(data.frame(q = qg, R = Rqc), aes(q, R)) +
  geom_step(colour = c_bh, linewidth = 1) +
  geom_hline(yintercept = sum(p.adjust(p, "bonferroni") < 0.05),
             linetype = "dashed", colour = c_bonf) +
  geom_vline(xintercept = 0.05, linetype = "dotted", colour = te_faint) +
  annotate("text", x = 0.02, y = sum(p.adjust(p, "bonferroni") < 0.05) + 1.5,
           label = "Bonferroni (q=0.05): 10", colour = c_bonf, hjust = 0, size = 3.3) +
  annotate("text", x = 0.052, y = 6, label = "q=0.05", colour = te_faint, hjust = 0, size = 3.2) +
  labs(x = "target FDR level q", y = "number of discoveries",
       title = "The discovery set is a function of q, not a fact") +
  theme_te()
