# Extra-pair paternity from parentage mismatches
# https://tidyecology.com/posts/extra-pair-paternity-from-mismatches/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-08

## ---- setup ----
library(ggplot2)
library(patchwork)

te_paper  <- "#f5f4ee"
te_ink    <- "#16241d"
te_body   <- "#2c3a31"
te_forest <- "#275139"
te_rust   <- "#b5534e"
te_gold   <- "#c9b458"
te_line   <- "#dad9ca"

theme_datasheet <- function() {
  theme_minimal(base_size = 12) +
    theme(plot.background  = element_rect(fill = te_paper, colour = NA),
          panel.background = element_rect(fill = te_paper, colour = NA),
          panel.grid.major = element_line(colour = te_line, linewidth = 0.3),
          panel.grid.minor = element_blank(),
          text             = element_text(colour = te_body),
          plot.title       = element_text(colour = te_ink, face = "bold"),
          plot.subtitle    = element_text(colour = te_body),
          axis.text        = element_text(colour = te_body),
          strip.text       = element_text(colour = te_ink, face = "bold"))
}

## ---- tools ----
n_allele  <- 8
dir_shape <- 0.8

draw_freqs <- function(n_loc, n_al, shape) {
  lapply(seq_len(n_loc), function(i) { x <- rgamma(n_al, shape); x / sum(x) })
}
draw_geno <- function(n, fr) {
  geno <- array(0L, c(n, 2, length(fr)))
  for (l in seq_along(fr)) {
    geno[, , l] <- sample.int(length(fr[[l]]), 2 * n, TRUE, fr[[l]])
  }
  geno
}
add_error <- function(geno, e_rate, fr) {
  if (e_rate == 0) return(geno)
  for (l in seq_along(fr)) {
    hit <- runif(2 * dim(geno)[1]) < e_rate
    if (any(hit)) {
      calls <- geno[, , l]
      calls[hit] <- sample.int(length(fr[[l]]), sum(hit), TRUE, fr[[l]])
      geno[, , l] <- calls
    }
  }
  geno
}
transmit <- function(parent) {
  n <- dim(parent)[1]; n_loc <- dim(parent)[3]
  pick <- sample.int(2, n * n_loc, TRUE)
  matrix(parent[cbind(rep(seq_len(n), n_loc), pick,
                      rep(seq_len(n_loc), each = n))], n, n_loc)
}
make_chick <- function(mum, dad) {
  chick <- array(0L, dim(mum))
  chick[, 1, ] <- transmit(mum)
  chick[, 2, ] <- transmit(dad)
  chick
}
locus_mismatch <- function(chick, mum, soc) {
  a1 <- chick[, 1, ]; a2 <- chick[, 2, ]
  in_mum <- function(a) a == mum[, 1, ] | a == mum[, 2, ]
  in_soc <- function(a) a == soc[, 1, ] | a == soc[, 2, ]
  !((in_mum(a1) & in_soc(a2)) | (in_mum(a2) & in_soc(a1)))
}
nested_counts <- function(mis, panels) {
  cum <- t(apply(mis, 1, cumsum))
  cum[, panels, drop = FALSE]
}
brood_set <- function(n, fr, e_rate, epp, sire) {
  mum <- draw_geno(n, fr)
  if (sire == "unrelated") {
    soc <- draw_geno(n, fr); other <- draw_geno(n, fr)
  } else {
    gran_f <- draw_geno(n, fr); gran_m <- draw_geno(n, fr)
    soc <- make_chick(gran_f, gran_m); other <- make_chick(gran_f, gran_m)
  }
  is_ep <- runif(n) < epp
  sire_g <- soc
  sire_g[is_ep, , ] <- other[is_ep, , ]
  chick <- make_chick(mum, sire_g)
  list(chick = add_error(chick, e_rate, fr), mum = add_error(mum, e_rate, fr),
       soc = add_error(soc, e_rate, fr), is_ep = is_ep)
}

## ---- exact ----
exact_locus <- function(f, e_rate) {
  k <- length(f)
  tr_mat <- (1 - e_rate) * diag(k) + e_rate * matrix(f, k, k, byrow = TRUE)
  q_mat  <- t(tr_mat) %*% (f * tr_mat)
  cells  <- expand.grid(h1 = 1:k, h2 = 1:k, o = 1:k)
  p_par  <- 0.5 * (q_mat[cbind(cells$h1, cells$o)] * f[cells$h2] +
                   f[cells$h1] * q_mat[cbind(cells$h2, cells$o)])
  p_rand <- f[cells$h1] * f[cells$h2] * f[cells$o]
  n_cell <- nrow(cells)
  i_m <- rep(seq_len(n_cell), times = n_cell)
  i_s <- rep(seq_len(n_cell), each = n_cell)
  o1 <- cells$o[i_m]; o2 <- cells$o[i_s]
  in_m <- function(a) a == cells$h1[i_m] | a == cells$h2[i_m]
  in_s <- function(a) a == cells$h1[i_s] | a == cells$h2[i_s]
  bad <- !((in_m(o1) & in_s(o2)) | (in_m(o2) & in_s(o1)))
  c(within = sum(p_par[i_m] * p_par[i_s] * bad),
    extra  = sum(p_par[i_m] * p_rand[i_s] * bad))
}
tail_prob <- function(p_loc, k_rule) {
  pmf <- 1
  for (q in p_loc) pmf <- c(pmf * (1 - q), 0) + c(0, pmf * q)
  sum(pmf[(k_rule + 1):length(pmf)])
}

set.seed(3107)
max_loc  <- 25
fr_ms    <- draw_freqs(max_loc, n_allele, dir_shape)
panel_ms <- c(8, 15, 25)
err_grid <- c(0, 0.005, 0.01, 0.02)
epp_true <- 0.10

exact_tab <- do.call(rbind, lapply(err_grid, function(e_rate) {
  p_loc <- t(vapply(fr_ms, exact_locus, numeric(2), e_rate = e_rate))
  do.call(rbind, lapply(panel_ms, function(n_loc) {
    do.call(rbind, lapply(1:2, function(k_rule) {
      fe <- tail_prob(p_loc[1:n_loc, "within"], k_rule)
      ce <- tail_prob(p_loc[1:n_loc, "extra"], k_rule)
      data.frame(e_rate = e_rate, n_loc = n_loc, k_rule = k_rule,
                 false_ex = fe, catch = ce,
                 rate = epp_true * ce + (1 - epp_true) * fe,
                 p_within = mean(p_loc[1:n_loc, "within"]))
    }))
  }))
}))
pick_ex <- function(e_rate, n_loc, k_rule, col) {
  exact_tab[exact_tab$e_rate == e_rate & exact_tab$n_loc == n_loc &
              exact_tab$k_rule == k_rule, col]
}
w_one   <- pick_ex(0.01, 25, 1, "p_within")
c_fac   <- w_one / 0.01
w_loc_fac <- vapply(fr_ms, function(f) exact_locus(f, 0.01)[["within"]], numeric(1)) / 0.01
c_even  <- exact_locus(rep(1 / n_allele, n_allele), 0.01)[["within"]] / 0.01
approx8 <- 1 - (1 - w_one)^8
approx25 <- 1 - (1 - w_one)^25

## ---- simcheck ----
n_brood <- 4000
set.seed(3108)
sim_tab <- do.call(rbind, lapply(err_grid, function(e_rate) {
  bs  <- brood_set(n_brood, fr_ms, e_rate, epp_true, "unrelated")
  cnt <- nested_counts(locus_mismatch(bs$chick, bs$mum, bs$soc), panel_ms)
  do.call(rbind, lapply(1:2, function(k_rule) {
    data.frame(e_rate = e_rate, n_loc = panel_ms, k_rule = k_rule,
               sim_rate = colMeans(cnt >= k_rule))
  }))
}))
check_tab <- merge(exact_tab, sim_tab)
check_tab$z <- (check_tab$sim_rate - check_tab$rate) /
  sqrt(check_tab$rate * (1 - check_tab$rate) / n_brood)
max_z <- max(abs(check_tab$z))

## ---- fig-rule-inflation ----
plot_ex <- exact_tab
plot_ex$rule <- ifelse(plot_ex$k_rule == 1, "one mismatch excludes", "two mismatches exclude")
plot_sim <- check_tab
plot_sim$rule <- ifelse(plot_sim$k_rule == 1, "one mismatch excludes", "two mismatches exclude")
plot_ex$err  <- factor(plot_ex$e_rate)
plot_sim$err <- factor(plot_sim$e_rate)
ggplot(plot_ex, aes(n_loc, rate, colour = err)) +
  geom_line(linewidth = 0.9) +
  geom_point(data = plot_sim, aes(y = sim_rate), size = 2.2, shape = 21,
             fill = te_paper, stroke = 1) +
  facet_wrap(~ rule) +
  scale_colour_manual(values = c(te_ink, te_forest, te_gold, te_rust),
                      name = "allele error rate") +
  scale_x_continuous(breaks = panel_ms) +
  labs(x = "microsatellite loci", y = "reported extra-pair rate",
       title = "More loci, more extra-pair paternity",
       subtitle = "the zero error line sits on the true rate, 0.10") +
  theme_datasheet() +
  theme(legend.position = "bottom")

## ---- brother ----
fr_even <- replicate(max_loc, rep(1 / n_allele, n_allele), simplify = FALSE)
spectra <- list(skewed = fr_ms, even = fr_even)
het_of  <- function(fr) mean(vapply(fr, function(f) 1 - sum(f^2), numeric(1)))
neff_of <- function(fr) mean(vapply(fr, function(f) 1 / sum(f^2), numeric(1)))
excl_of <- function(fr) mean(vapply(fr, function(f) exact_locus(f, 0)[["extra"]], numeric(1)))
spec_tab <- data.frame(spectrum = names(spectra),
                       he   = vapply(spectra, function(fr) het_of(fr[1:8]), numeric(1)),
                       n_eff = vapply(spectra, function(fr) neff_of(fr[1:8]), numeric(1)),
                       excl = vapply(spectra, function(fr) excl_of(fr[1:8]), numeric(1)))

set.seed(3109)
bro_tab <- do.call(rbind, lapply(names(spectra), function(sp) {
  do.call(rbind, lapply(c(0, 0.01), function(e_rate) {
    do.call(rbind, lapply(c("brother", "unrelated"), function(sire) {
      bs  <- brood_set(n_brood, spectra[[sp]], e_rate, 1, sire)
      cnt <- nested_counts(locus_mismatch(bs$chick, bs$mum, bs$soc), panel_ms)
      data.frame(spectrum = sp, e_rate = e_rate, sire = sire, n_loc = panel_ms,
                 miss1 = colMeans(cnt < 1), miss2 = colMeans(cnt < 2),
                 med = apply(cnt, 2, median))
    }))
  }))
}))
pick_bro <- function(sp, e_rate, sire, n_loc, col) {
  bro_tab[bro_tab$spectrum == sp & bro_tab$e_rate == e_rate &
            bro_tab$sire == sire & bro_tab$n_loc == n_loc, col]
}

## ---- fig-brother ----
bro_plot <- bro_tab[bro_tab$e_rate == 0 & bro_tab$sire == "brother", ]
bro_long <- rbind(
  data.frame(bro_plot[, c("spectrum", "n_loc")], miss = bro_plot$miss1,
             rule = "one mismatch excludes"),
  data.frame(bro_plot[, c("spectrum", "n_loc")], miss = bro_plot$miss2,
             rule = "two mismatches exclude"))
bro_long$spectrum <- ifelse(bro_long$spectrum == "skewed",
                            "skewed frequencies (Dirichlet 0.8)", "even frequencies")
ggplot(bro_long, aes(n_loc, miss, colour = spectrum)) +
  geom_line(linewidth = 0.9) +
  geom_point(size = 2.4) +
  facet_wrap(~ rule) +
  scale_colour_manual(values = c(te_gold, te_forest), name = NULL) +
  scale_x_continuous(breaks = panel_ms) +
  labs(x = "microsatellite loci", y = "brother-sired chicks not excluded",
       title = "A brother escapes a small panel",
       subtitle = "every chick here is extra-pair; no genotyping error") +
  theme_datasheet() +
  theme(legend.position = "bottom")

## ---- snp ----
snp_loc   <- 240
panel_snp <- c(60, 120, 240)
set.seed(3110)
fr_snp <- lapply(runif(snp_loc, 0.2, 0.5), function(q) c(q, 1 - q))
c_snp  <- mean(vapply(fr_snp[1:60], function(f) exact_locus(f, 0.01)[["within"]], numeric(1))) / 0.01

em_binmix <- function(x, n_loc, iter = 5000, tol = 1e-10) {
  pi_ep <- 0.2; p_wp <- 0.01; p_ep <- 0.2; ll_old <- -Inf
  for (i in seq_len(iter)) {
    d_ep <- pi_ep * dbinom(x, n_loc, p_ep)
    d_wp <- (1 - pi_ep) * dbinom(x, n_loc, p_wp)
    ll <- sum(log(d_ep + d_wp))
    if (abs(ll - ll_old) < tol) break    # stop once the log-likelihood has settled
    ll_old <- ll
    w_ep <- d_ep / (d_ep + d_wp)
    pi_ep <- mean(w_ep)
    p_ep  <- sum(w_ep * x) / (n_loc * max(sum(w_ep), 1e-9))
    p_wp  <- sum((1 - w_ep) * x) / (n_loc * sum(1 - w_ep))
  }
  c(pi_ep = pi_ep, p_wp = p_wp, p_ep = p_ep)
}

e_mix <- 0.01
mix_runs <- list()
mix_tab <- do.call(rbind, lapply(c("unrelated", "brother"), function(sire) {
  do.call(rbind, lapply(c("microsatellite", "SNP"), function(kind) {
    fr   <- if (kind == "SNP") fr_snp else fr_ms
    pans <- if (kind == "SNP") panel_snp else panel_ms
    bs  <- brood_set(n_brood, fr, e_mix, epp_true, sire)
    cnt <- nested_counts(locus_mismatch(bs$chick, bs$mum, bs$soc), pans)
    mix_runs[[paste(sire, kind)]] <<- list(cnt = cnt, is_ep = bs$is_ep)
    do.call(rbind, lapply(seq_along(pans), function(j) {
      x <- cnt[, j]; fit <- em_binmix(x, pans[j])
      data.frame(sire = sire, kind = kind, n_loc = pans[j], true = mean(bs$is_ep),
                 rule1 = mean(x >= 1), rule2 = mean(x >= 2),
                 ep_med = median(x[bs$is_ep]),
                 wp_q99 = unname(quantile(x[!bs$is_ep], 0.99, type = 1)),
                 em = fit[["pi_ep"]], p_wp = fit[["p_wp"]], p_ep = fit[["p_ep"]])
    }))
  }))
}))
pick_mix <- function(sire, n_loc, col) {
  mix_tab[mix_tab$sire == sire & mix_tab$n_loc == n_loc, col]
}
mix_gap <- max(abs(mix_tab$em - mix_tab$true))

## ---- fig-mixture ----
hist_panel <- function(key, j, n_loc, lab, y_lab = NULL) {
  run <- mix_runs[[key]]
  x   <- run$cnt[, j]
  fit <- em_binmix(x, n_loc)
  x_max <- max(x)
  obs <- rbind(
    data.frame(k = 0:x_max, n = tabulate(x[!run$is_ep] + 1, x_max + 1), who = "within-pair"),
    data.frame(k = 0:x_max, n = tabulate(x[run$is_ep] + 1, x_max + 1), who = "extra-pair"))
  fitted <- data.frame(k = 0:x_max,
                       n = length(x) * (fit[["pi_ep"]] * dbinom(0:x_max, n_loc, fit[["p_ep"]]) +
                                        (1 - fit[["pi_ep"]]) * dbinom(0:x_max, n_loc, fit[["p_wp"]])))
  obs$who <- factor(obs$who, levels = c("within-pair", "extra-pair"))
  n_wp <- obs$n[obs$who == "within-pair"]
  obs$ymin <- ifelse(obs$who == "within-pair", 0, n_wp)
  obs$ymax <- obs$ymin + obs$n
  ggplot(obs) +
    geom_rect(aes(xmin = k - 0.45, xmax = k + 0.45, ymin = ymin, ymax = ymax, fill = who)) +
    geom_point(data = fitted, aes(k, n), colour = te_ink, size = 1.3) +
    scale_fill_manual(values = c("extra-pair" = te_rust, "within-pair" = te_line), name = NULL) +
    scale_y_sqrt() +
    labs(x = "mismatching loci", y = y_lab, title = lab) +
    theme_datasheet() +
    theme(plot.title = element_text(size = 11))
}
hist_panel("brother microsatellite", 1, 8, "8 loci, brother sires",
           "chicks (square root scale)") +
  hist_panel("brother SNP", 1, 60, "60 SNPs, brother sires") +
  hist_panel("unrelated SNP", 2, 120, "120 SNPs, unrelated") +
  plot_layout(guides = "collect") +
  plot_annotation(title = "Two humps, and where they meet",
                  subtitle = "black points: expected counts under the fitted mixture",
                  theme = theme_datasheet()) &
  theme(legend.position = "bottom")

## ---- lod ----
lod_social <- function(chick, mum, soc, fr, eps) {
  lod <- numeric(dim(chick)[1])
  for (l in seq_along(fr)) {
    f <- fr[[l]]
    o1 <- chick[, 1, l]; o2 <- chick[, 2, l]
    m1 <- mum[, 1, l]; m2 <- mum[, 2, l]; s1 <- soc[, 1, l]; s2 <- soc[, 2, l]
    hom  <- o1 == o2
    p_hw <- ifelse(hom, f[o1]^2, 2 * f[o1] * f[o2])
    gives <- function(a, b) ifelse(hom, (a == o1) * (b == o1),
                                   (a == o1) * (b == o2) + (a == o2) * (b == o1))
    t_soc <- 0.25 * (gives(m1, s1) + gives(m1, s2) + gives(m2, s1) + gives(m2, s2))
    from_m1 <- 0.5 * ((m1 == o1) + (m2 == o1))
    from_m2 <- 0.5 * ((m1 == o2) + (m2 == o2))
    t_rand <- ifelse(hom, from_m1 * f[o1], from_m1 * f[o2] + from_m2 * f[o1])
    lod <- lod + log((1 - eps) * t_soc + eps * p_hw) -
      log((1 - eps) * t_rand + eps * p_hw)
  }
  lod
}

eps_assumed <- 0.02
e_assumed   <- 0.01
n_cal       <- 20000
alpha_cal   <- 0.01
set.seed(3111)
cal_tab <- as.data.frame(do.call(rbind, lapply(panel_ms[1:2], function(n_loc) {
  fr <- fr_ms[1:n_loc]
  wp <- brood_set(n_cal, fr, e_assumed, 0, "unrelated")
  ep <- brood_set(n_cal, fr, e_assumed, 1, "unrelated")
  lod_wp <- lod_social(wp$chick, wp$mum, wp$soc, fr, eps_assumed)
  lod_ep <- lod_social(ep$chick, ep$mum, ep$soc, fr, eps_assumed)
  thr <- unname(quantile(lod_wp, alpha_cal))
  c(n_loc = n_loc, thr = thr, false_ex = mean(lod_wp < thr),
    miss = mean(lod_ep >= thr))
})))

## ---- study ----
n_study <- 200
n_rep   <- 300
scen <- data.frame(label = c("as assumed", "error doubled", "brother sires"),
                   e_rate = c(e_assumed, 2 * e_assumed, e_assumed),
                   sire = c("unrelated", "unrelated", "brother"))
method_lab <- c("likelihood ratio, corrected", "mixture", "two mismatch rule")
set.seed(3112)
study_tab <- do.call(rbind, lapply(seq_len(nrow(cal_tab)), function(j) {
  n_loc <- cal_tab$n_loc[j]; fr <- fr_ms[1:n_loc]
  do.call(rbind, lapply(seq_len(nrow(scen)), function(s) {
    est <- replicate(n_rep, {
      bs  <- brood_set(n_study, fr, scen$e_rate[s], epp_true, scen$sire[s])
      lod <- lod_social(bs$chick, bs$mum, bs$soc, fr, eps_assumed)
      cnt <- rowSums(locus_mismatch(bs$chick, bs$mum, bs$soc))
      raw <- mean(lod < cal_tab$thr[j])
      c(truth = mean(bs$is_ep),
        lr = (raw - cal_tab$false_ex[j]) / (1 - cal_tab$false_ex[j] - cal_tab$miss[j]),
        mix = em_binmix(cnt, n_loc)[["pi_ep"]],
        rule2 = mean(cnt >= 2))
    })
    err <- est[c("lr", "mix", "rule2"), ] - rep(est["truth", ], each = 3)
    data.frame(n_loc = n_loc, scenario = scen$label[s], method = method_lab,
               bias = rowMeans(err), sd_err = apply(err, 1, sd),
               mc_se = apply(err, 1, sd) / sqrt(n_rep))
  }))
}))
pick_st <- function(n_loc, scenario, method, col) {
  study_tab[study_tab$n_loc == n_loc & study_tab$scenario == scenario &
              study_tab$method == method, col]
}
lr_lab <- method_lab[1]; mx_lab <- method_lab[2]; r2_lab <- method_lab[3]
max_mcse <- max(study_tab$mc_se)

## ---- fig-study ----
study_plot <- study_tab
study_plot$scenario <- factor(study_plot$scenario, levels = scen$label)
study_plot$panel <- factor(paste(study_plot$n_loc, "microsatellites"),
                           levels = paste(panel_ms[1:2], "microsatellites"))
ggplot(study_plot, aes(scenario, bias, colour = method)) +
  geom_hline(yintercept = 0, linetype = "dashed", colour = te_body, linewidth = 0.6) +
  geom_errorbar(aes(ymin = bias - sd_err, ymax = bias + sd_err),
                width = 0.25, linewidth = 0.7, position = position_dodge(width = 0.6)) +
  geom_point(size = 2.6, position = position_dodge(width = 0.6)) +
  facet_wrap(~ panel) +
  scale_colour_manual(values = c(te_forest, te_gold, te_rust), name = NULL) +
  labs(x = NULL, y = "estimate minus realised rate",
       title = "Each repair fails where its assumption does",
       subtitle = "points: mean error; bars: one standard deviation; true rate 0.10") +
  theme_datasheet() +
  theme(legend.position = "bottom")
