# Heterozygosity-fitness correlations and power
# https://tidyecology.com/posts/heterozygosity-fitness-correlations/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-09-06

## ---- setup ----
library(ggplot2)
library(patchwork)

te_paper  <- "#f5f4ee"
te_ink    <- "#16241d"
te_body   <- "#2c3a31"
te_forest <- "#275139"
te_rust   <- "#b5534e"
te_gold   <- "#c9b458"
te_line   <- "#dad9ca"
te_sage   <- "#7fa98a"

theme_datasheet <- function() {
  theme_minimal(base_size = 12) +
    theme(plot.background  = element_rect(fill = te_paper, colour = NA),
          panel.background = element_rect(fill = te_paper, colour = NA),
          panel.grid.major = element_line(colour = te_line, linewidth = 0.3),
          panel.grid.minor = element_blank(),
          text             = element_text(colour = te_body),
          plot.title       = element_text(colour = te_ink, face = "bold"),
          plot.subtitle    = element_text(colour = te_body),
          axis.text        = element_text(colour = te_body))
}

## ---- simfuns ----
make_panel <- function(n_ind, het, fprob, fval, miss = 0.03) {
  n_loc <- length(het)
  f_ind <- fval[sample.int(length(fval), n_ind, replace = TRUE, prob = fprob)]
  ibd   <- runif(n_ind * n_loc) < f_ind
  is_het <- (runif(n_ind * n_loc) < rep(het, each = n_ind)) & !ibd
  typed <- runif(n_ind * n_loc) >= miss
  h0 <- matrix(is_het & typed, n_ind, n_loc) * 1
  tt <- matrix(typed, n_ind, n_loc) * 1
  poly <- colSums(h0) > 0                  # drop loci with no heterozygote
  h0 <- h0[, poly, drop = FALSE]; tt <- tt[, poly, drop = FALSE]
  loc_mean <- colSums(h0) / colSums(tt)
  # Coltman et al. (1999) sMLH, as in inbreedR::sMLH
  smlh <- (rowSums(h0) / rowSums(tt)) / as.vector(tt %*% loc_mean / rowSums(tt))
  list(f = f_ind, h0 = h0, tt = tt, smlh = smlh)
}

surv_int   <- qlogis(0.6)
surv_slope <- -6
het_ms     <- 0.7
set.seed(4412)
maf_snp <- runif(100, 0.05, 0.5)
panels <- list("12 microsatellites" = rep(het_ms, 12),
               "30 microsatellites" = rep(het_ms, 30),
               "100 SNPs"           = 2 * maf_snp * (1 - maf_snp))
scenarios <- list(
  "no variance in f"     = list(p = 1, v = 0),
  "pedigree mixture"     = list(p = c(0.7, 0.2, 0.1), v = c(0, 0.125, 0.25)),
  "rare close inbreeding" = list(p = c(0.95, 0.05), v = c(0, 0.25)))

f_mean  <- sum(scenarios[[2]]$p * scenarios[[2]]$v)
f_var   <- sum(scenarios[[2]]$p * scenarios[[2]]$v^2) - f_mean^2
g2_true <- f_var / (1 - f_mean)^2
surv_fs <- plogis(surv_int + surv_slope * 0.25)

set.seed(1907)
demo_ms  <- make_panel(200, panels[[1]], scenarios[[2]]$p, scenarios[[2]]$v)
demo_snp <- make_panel(200, panels[[3]], scenarios[[2]]$p, scenarios[[2]]$v)
r_demo_ms  <- cor(demo_ms$smlh, demo_ms$f)
r_demo_snp <- cor(demo_snp$smlh, demo_snp$f)
q_ms <- tapply(demo_ms$smlh, demo_ms$f, median)
overlap_ms <- mean(demo_ms$smlh[demo_ms$f == 0.25] >
                   median(demo_ms$smlh[demo_ms$f == 0]))

## ---- fig-keyhole ----
keyhole_df <- rbind(
  data.frame(f = demo_ms$f, smlh = demo_ms$smlh, panel = "12 microsatellites"),
  data.frame(f = demo_snp$f, smlh = demo_snp$smlh, panel = "100 SNPs"))
keyhole_df$panel <- factor(keyhole_df$panel, levels = c("12 microsatellites", "100 SNPs"))
keyhole_df$f_lab <- factor(sprintf("%.3f", keyhole_df$f))

ggplot(keyhole_df, aes(f_lab, smlh)) +
  geom_jitter(width = 0.18, height = 0, colour = te_forest, alpha = 0.45, size = 1.4) +
  geom_boxplot(fill = NA, colour = te_ink, outlier.shape = NA, width = 0.5,
               linewidth = 0.5) +
  facet_wrap(~ panel) +
  labs(x = "pedigree inbreeding coefficient f", y = "standardised heterozygosity",
       title = "Inbred individuals are less heterozygous, on average",
       subtitle = "the classes overlap widely, more so at twelve loci") +
  theme_datasheet()

## ---- g2check ----
g2_david <- function(h0, tt) {
  p_ll <- crossprod(h0)                    # sum_i h_il h_il'
  c_ll <- crossprod(tt)                    # typed at both
  q_ll <- outer(colSums(h0), colSums(h0)) - p_ll   # sum over i != j
  d_ll <- outer(colSums(tt), colSums(tt)) - c_ll
  num <- sum(p_ll / c_ll) - sum(diag(p_ll) / diag(c_ll))
  den <- sum(q_ll / d_ll) - sum(diag(q_ll) / diag(d_ll))
  num / den - 1
}

g2_brute <- function(h0, tt) {
  n_ind <- nrow(h0); n_loc <- ncol(h0); num <- 0; den <- 0
  for (l1 in seq_len(n_loc)) for (l2 in seq_len(n_loc)) {
    if (l1 == l2) next
    s_same <- 0; n_same <- 0; s_diff <- 0; n_diff <- 0
    for (i in seq_len(n_ind)) {
      if (tt[i, l1] == 1 && tt[i, l2] == 1) {
        s_same <- s_same + h0[i, l1] * h0[i, l2]; n_same <- n_same + 1
      }
      for (j in seq_len(n_ind)) {
        if (j != i && tt[i, l1] == 1 && tt[j, l2] == 1) {
          s_diff <- s_diff + h0[i, l1] * h0[j, l2]; n_diff <- n_diff + 1
        }
      }
    }
    num <- num + s_same / n_same; den <- den + s_diff / n_diff
  }
  num / den - 1
}

sub_rows <- 1:60
g2_fast_sub  <- g2_david(demo_ms$h0[sub_rows, ], demo_ms$tt[sub_rows, ])
g2_brute_sub <- g2_brute(demo_ms$h0[sub_rows, ], demo_ms$tt[sub_rows, ])
n_miss_sub   <- sum(demo_ms$tt[sub_rows, ] == 0)

g2_perm_p <- function(h0, tt, n_perm = 99) {
  n_ind <- nrow(h0); n_loc <- ncol(h0)
  key <- rep(seq_len(n_loc), each = n_ind)
  g2_obs <- g2_david(h0, tt); n_ge <- 0
  for (k in seq_len(n_perm)) {             # shuffle each locus across individuals
    o <- order(key + runif(n_ind * n_loc), method = "radix")
    n_ge <- n_ge + (g2_david(matrix(h0[o], n_ind, n_loc),
                             matrix(tt[o], n_ind, n_loc)) >= g2_obs)
  }
  c(g2 = g2_obs, p = (1 + n_ge) / (n_perm + 1))
}
set.seed(77)
demo_test <- g2_perm_p(demo_ms$h0, demo_ms$tt)
n_boot <- 199
boot_g2 <- replicate(n_boot, {
  rows <- sample.int(nrow(demo_ms$h0), replace = TRUE)
  g2_david(demo_ms$h0[rows, ], demo_ms$tt[rows, ])
})
boot_ci <- quantile(boot_g2, c(0.025, 0.975))

## ---- grid ----
one_study <- function(n_ind, het, sc) {
  pan <- make_panel(n_ind, het, sc$p, sc$v)
  surv <- rbinom(n_ind, 1, plogis(surv_int + surv_slope * pan$f))
  has_f <- var(pan$f) > 0
  p_hfc <- coef(summary(glm(surv ~ pan$smlh, family = binomial)))[2, 4]
  p_f <- if (has_f) coef(summary(glm(surv ~ pan$f, family = binomial)))[2, 4] else NA
  g2t <- g2_perm_p(pan$h0, pan$tt)
  c(p_hfc = p_hfc, p_f = p_f, g2 = unname(g2t["g2"]), p_g2 = unname(g2t["p"]),
    r_hf = if (has_f) cor(pan$smlh, pan$f) else NA,
    r_fw = if (has_f) cor(pan$f, surv) else NA,
    r_hw = cor(pan$smlh, surv), v_h = var(pan$smlh))
}

n_rep <- 60
cells <- rbind(
  expand.grid(panel = names(panels), scen = names(scenarios), n_ind = 200,
              stringsAsFactors = FALSE),
  expand.grid(panel = names(panels), scen = names(scenarios)[2],
              n_ind = c(100, 400), stringsAsFactors = FALSE))

set.seed(2010)
cell_out <- lapply(seq_len(nrow(cells)), function(i) {
  t(replicate(n_rep, one_study(cells$n_ind[i], panels[[cells$panel[i]]],
                               scenarios[[cells$scen[i]]])))
})

summ <- cells
summ$hfc   <- vapply(cell_out, function(r) mean(r[, "p_hfc"] < 0.05), 0)
summ$truef <- vapply(cell_out, function(r) mean(r[, "p_f"] < 0.05), 0)
summ$g2sig <- vapply(cell_out, function(r) mean(r[, "p_g2"] < 0.05), 0)
summ$g2    <- vapply(cell_out, function(r) mean(r[, "g2"]), 0)
summ$r_hf  <- vapply(cell_out, function(r) mean(r[, "r_hf"]), 0)
summ$r_fw  <- vapply(cell_out, function(r) mean(r[, "r_fw"]), 0)
summ$r_hw  <- vapply(cell_out, function(r) mean(r[, "r_hw"]), 0)
summ$v_h   <- vapply(cell_out, function(r) mean(r[, "v_h"]), 0)
summ$r2_hf <- vapply(cell_out, function(r) mean(r[, "r_hf"]^2), 0)

pick <- function(pan, sc = "pedigree mixture", n_ind = 200) {
  summ[summ$panel == pan & summ$scen == sc & summ$n_ind == n_ind, ]
}
m12 <- pick("12 microsatellites"); m30 <- pick("30 microsatellites")
s100 <- pick("100 SNPs")
z12 <- pick("12 microsatellites", "no variance in f")
z30 <- pick("30 microsatellites", "no variance in f")
zsn <- pick("100 SNPs", "no variance in f")
rare12 <- pick("12 microsatellites", "rare close inbreeding")
rare_sn <- pick("100 SNPs", "rare close inbreeding")
perm_size <- mean((1 + 0:99) / 100 < 0.05)   # exact size of p < 0.05 with 99 permutations

## ---- fig-power ----
long_pow <- do.call(rbind, lapply(c("truef", "hfc", "g2sig"), function(v) {
  data.frame(panel = summ$panel, scen = summ$scen, n_ind = summ$n_ind,
             test = v, rate = summ[[v]])
}))
long_pow <- long_pow[long_pow$n_ind == 200 & !is.na(long_pow$rate), ]
long_pow$se <- sqrt(long_pow$rate * (1 - long_pow$rate) / n_rep)
long_pow$test <- factor(long_pow$test, levels = c("truef", "hfc", "g2sig"),
                        labels = c("survival on true f", "survival on sMLH (HFC)",
                                   "g2 permutation test"))
long_pow$panel <- factor(long_pow$panel, levels = names(panels))
long_pow$scen <- factor(long_pow$scen, levels = names(scenarios))

ggplot(long_pow, aes(panel, rate, colour = test)) +
  geom_hline(yintercept = 0.05, linetype = "dashed", colour = te_rust, linewidth = 0.5) +
  geom_errorbar(aes(ymin = pmax(rate - se, 0), ymax = pmin(rate + se, 1)),
                width = 0, linewidth = 0.6, position = position_dodge(width = 0.6)) +
  geom_point(size = 2.4, position = position_dodge(width = 0.6)) +
  facet_wrap(~ scen) +
  scale_colour_manual(values = c(te_ink, te_sage, te_gold), name = NULL) +
  scale_y_continuous(limits = c(0, 1)) +
  labs(x = NULL, y = "share of studies significant",
       title = "Real inbreeding depression, weak marker tests",
       subtitle = "dashed red: the nominal five per cent level") +
  theme_datasheet() +
  theme(legend.position = "bottom",
        axis.text.x = element_text(angle = 25, hjust = 1))

## ---- closedform ----
cf <- summ[summ$scen != "no variance in f", ]
cf$r_hw_pred  <- cf$r_hf * cf$r_fw
cf$r2_hf_pred <- cf$g2 / cf$v_h
gap_rhw  <- max(abs(cf$r_hw - cf$r_hw_pred))
gap_r2   <- max(abs(cf$r2_hf - cf$r2_hf_pred))
m12_cf   <- cf[cf$panel == "12 microsatellites" & cf$scen == "pedigree mixture" &
               cf$n_ind == 200, ]
rel_12   <- m12_cf$r2_hf

## ---- fig-closedform ----
cf$panel <- factor(cf$panel, levels = names(panels))
p_a <- ggplot(cf, aes(r_hw_pred, r_hw, colour = panel, shape = panel)) +
  geom_abline(slope = 1, intercept = 0, colour = te_body, linetype = "dashed") +
  geom_point(size = 2.4) +
  scale_colour_manual(values = c(te_sage, te_rust, te_ink), name = NULL) +
  scale_shape_manual(values = c(16, 17, 15), name = NULL) +
  labs(x = "r(sMLH, f) times r(f, W)", y = "simulated r(sMLH, W)",
       title = "Correlation with survival") +
  theme_datasheet()
p_b <- ggplot(cf, aes(r2_hf_pred, r2_hf, colour = panel, shape = panel)) +
  geom_abline(slope = 1, intercept = 0, colour = te_body, linetype = "dashed") +
  geom_point(size = 2.4) +
  scale_colour_manual(values = c(te_sage, te_rust, te_ink), name = NULL) +
  scale_shape_manual(values = c(16, 17, 15), name = NULL) +
  labs(x = "g2 / var(sMLH)", y = "simulated r(sMLH, f) squared",
       title = "Reliability of sMLH") +
  theme_datasheet()
(p_a | p_b) + plot_layout(guides = "collect") +
  plot_annotation(theme = theme_datasheet()) &
  theme(legend.position = "bottom")

## ---- joint ----
n_joint <- 400
set.seed(1202)
joint_out <- t(replicate(n_joint, one_study(200, panels[[1]], scenarios[[2]])))
hfc_sig <- joint_out[, "p_hfc"] < 0.05
g2_sig  <- joint_out[, "p_g2"] < 0.05
share_g2_ns   <- mean(!g2_sig)
share_hfc_sig <- mean(hfc_sig)
hfc_given_g2  <- mean(hfc_sig[g2_sig])
hfc_given_ns  <- mean(hfc_sig[!g2_sig])
both_ns       <- mean(!hfc_sig & !g2_sig)
g2ns_among_hfc <- mean(!g2_sig[hfc_sig])
se_among_hfc  <- sqrt(g2ns_among_hfc * (1 - g2ns_among_hfc) / sum(hfc_sig))
se_both       <- sqrt(both_ns * (1 - both_ns) / n_joint)
se_joint <- sqrt(share_g2_ns * (1 - share_g2_ns) / n_joint)
se_hfc   <- sqrt(share_hfc_sig * (1 - share_hfc_sig) / n_joint)
se_given_g2 <- sqrt(hfc_given_g2 * (1 - hfc_given_g2) / sum(g2_sig))
se_given_ns <- sqrt(hfc_given_ns * (1 - hfc_given_ns) / sum(!g2_sig))
hfc_sig_g2_ns <- mean(hfc_sig & !g2_sig)
het_snp_mean <- mean(panels[[3]])

nax <- summ[summ$scen == "pedigree mixture", ]
m12_100 <- pick("12 microsatellites", n_ind = 100)
m12_400 <- pick("12 microsatellites", n_ind = 400)
s_100 <- pick("100 SNPs", n_ind = 100)
s_400 <- pick("100 SNPs", n_ind = 400)

## ---- fig-samplesize ----
n_long <- rbind(
  data.frame(panel = nax$panel, n_ind = nax$n_ind, rate = nax$hfc,
             test = "survival on sMLH (HFC)"),
  data.frame(panel = nax$panel, n_ind = nax$n_ind, rate = nax$g2sig,
             test = "g2 permutation test"),
  data.frame(panel = nax$panel, n_ind = nax$n_ind, rate = nax$truef,
             test = "survival on true f"))
n_long$panel <- factor(n_long$panel, levels = names(panels))
n_long$test <- factor(n_long$test, levels = c("survival on true f",
                      "survival on sMLH (HFC)", "g2 permutation test"))

ggplot(n_long, aes(n_ind, rate, colour = panel, shape = panel)) +
  geom_line(linewidth = 0.8) +
  geom_point(size = 2.2) +
  facet_wrap(~ test) +
  scale_x_continuous(breaks = c(100, 200, 400)) +
  scale_y_continuous(limits = c(0, 1)) +
  scale_colour_manual(values = c(te_sage, te_rust, te_ink), name = NULL) +
  scale_shape_manual(values = c(16, 17, 15), name = NULL) +
  labs(x = "individuals sampled", y = "share of studies significant",
       title = "Power grows with individuals, not only with loci",
       subtitle = "pedigree mixture, 70/20/10 per cent at f = 0, 0.125, 0.25") +
  theme_datasheet() +
  theme(legend.position = "bottom")
