# Phylogenetic signal: Blomberg’s K and Pagel’s lambda
# https://tidyecology.com/posts/phylogenetic-signal-k-lambda/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-08

## ---- setup ----
library(ape)
library(nlme)
library(ggplot2)

te_paper <- "#f5f4ee"; te_ink   <- "#16241d"; te_body <- "#2c3a31"
te_faint <- "#5d6b61"; te_sage  <- "#93a87f"; te_rust <- "#b5534e"
te_line  <- "#dad9ca"; te_low   <- "#c9b458"; te_high <- "#1d5b4e"

theme_te <- function(base_size = 12) {
  theme_minimal(base_size = base_size) +
    theme(
      plot.background  = element_rect(fill = te_paper, colour = NA),
      panel.background = element_rect(fill = te_paper, colour = NA),
      panel.grid.major = element_line(colour = te_line, linewidth = 0.3),
      panel.grid.minor = element_blank(),
      axis.text  = element_text(colour = te_body),
      axis.title = element_text(colour = te_ink),
      plot.title = element_text(colour = te_ink, face = "bold"),
      strip.text = element_text(colour = te_ink, face = "bold")
    )
}

## ---- data ----
set.seed(414)
tree <- rcoal(40)
tree$tip.label <- sprintf("sp%02d", 1:40)

# simulate a trait with a given Pagel's lambda: scale off-diagonal covariances
sim_pagel <- function(phy, lambda, sigma2 = 1) {
  V <- vcv(phy); Vl <- V * lambda; diag(Vl) <- diag(V)
  x <- MASS::mvrnorm(1, mu = rep(0, nrow(V)), Sigma = sigma2 * Vl)
  setNames(x, rownames(V))
}
set.seed(2)   ; body_mass    <- sim_pagel(tree, lambda = 1.0)
set.seed(291) ; labile_trait <- sim_pagel(tree, lambda = 0.1)

## ---- k-function ----
blomberg_K <- function(x, phy) {
  x  <- x[phy$tip.label]; n <- length(x)
  V  <- vcv(phy); Vi <- solve(V); one <- rep(1, n)
  a  <- as.numeric((t(one) %*% Vi %*% x) / (t(one) %*% Vi %*% one))  # phylo mean
  d  <- x - a
  mse_star <- as.numeric(t(d) %*% d) / (n - 1)          # ignores the tree
  mse_phy  <- as.numeric(t(d) %*% Vi %*% d) / (n - 1)    # corrects for the tree
  expected <- (sum(diag(V)) - n / sum(Vi)) / (n - 1)     # value under Brownian motion
  (mse_star / mse_phy) / expected
}

K_mass   <- blomberg_K(body_mass, tree)
K_labile <- blomberg_K(labile_trait, tree)
c(body_mass = K_mass, labile = K_labile)

## ---- k-test ----
set.seed(303); K_mass_null   <- replicate(999, blomberg_K(setNames(sample(body_mass), tree$tip.label), tree))
set.seed(304); K_labile_null <- replicate(999, blomberg_K(setNames(sample(labile_trait), tree$tip.label), tree))
p_mass   <- (1 + sum(K_mass_null   >= K_mass))   / 1000
p_labile <- (1 + sum(K_labile_null >= K_labile)) / 1000
c(p_mass = p_mass, p_labile = p_labile)

## ---- lambda ----
est_lambda <- function(x, phy) {
  d  <- data.frame(sp = phy$tip.label, y = x[phy$tip.label])
  m1 <- gls(y ~ 1, data = d, correlation = corPagel(0.3, phy, form = ~sp), method = "ML")
  m0 <- gls(y ~ 1, data = d, method = "ML")                       # lambda = 0 (star)
  lam <- as.numeric(coef(m1$modelStruct$corStruct, unconstrained = FALSE))
  lr  <- as.numeric(2 * (logLik(m1) - logLik(m0)))
  c(lambda = lam, LR = lr, p = pchisq(lr, df = 1, lower.tail = FALSE))
}
round(est_lambda(body_mass, tree), 4)
round(est_lambda(labile_trait, tree), 4)

## ---- fig-signal-trees ----
ramp <- colorRampPalette(c(te_low, te_high))
col_from <- function(v) ramp(100)[as.integer(cut(v, breaks = seq(min(v), max(v), length.out = 101), include.lowest = TRUE))]

op <- par(mfrow = c(1, 2), mar = c(1, 0.5, 2.5, 0.5), bg = te_paper, fg = te_body, col.main = te_ink)
plot(tree, show.tip.label = FALSE, edge.color = te_faint, edge.width = 1.2)
tiplabels(pch = 21, bg = col_from(body_mass[tree$tip.label]), col = te_ink, cex = 1.1)
title(main = sprintf("body mass  (K = %.2f, p = %.3f)", K_mass, p_mass), cex.main = 0.95, font.main = 1)
plot(tree, show.tip.label = FALSE, edge.color = te_faint, edge.width = 1.2)
tiplabels(pch = 21, bg = col_from(labile_trait[tree$tip.label]), col = te_ink, cex = 1.1)
title(main = sprintf("labile trait  (K = %.2f, p = %.2f)", K_labile, p_labile), cex.main = 0.95, font.main = 1)
par(op)

## ---- fig-k-null ----
dfn <- rbind(data.frame(trait = "body mass",    K = K_mass_null,   obs = K_mass),
             data.frame(trait = "labile trait", K = K_labile_null, obs = K_labile))
dfn$trait <- factor(dfn$trait, levels = c("body mass", "labile trait"))
ggplot(dfn, aes(K)) +
  geom_histogram(bins = 40, fill = te_sage, colour = te_paper, linewidth = 0.2) +
  geom_vline(aes(xintercept = obs), colour = te_rust, linewidth = 1) +
  facet_wrap(~trait, scales = "free_x") +
  labs(x = "Blomberg's K under tip randomisation", y = "null replicates",
       title = "Observed K (red) against a phylogeny-free null") +
  theme_te(12)

## ---- missing-tips ----
lambda_grid <- function(x, phy) {   # profile likelihood for lambda, grid of 0.02
  V <- vcv(phy)[names(x), names(x)]; n <- length(x); one <- rep(1, n)
  ll <- sapply(seq(0, 1, 0.02), function(l) {
    Vl <- V * l; diag(Vl) <- diag(V); R <- chol(Vl)
    z1 <- backsolve(R, one, transpose = TRUE); zx <- backsolve(R, x, transpose = TRUE)
    a <- sum(z1 * zx) / sum(z1^2); r <- zx - a * z1
    -0.5 * (n * log(sum(r^2) / n) + 2 * sum(log(diag(R))))
  })
  seq(0, 1, 0.02)[which.max(ll)]
}
fill_tree <- function(x, phy, gone) {   # Brownian conditional mean from the measured tips
  V <- vcv(phy)[names(x), names(x)]; keep <- !gone
  Vi <- solve(V[keep, keep]); one <- rep(1, sum(keep))
  a <- as.numeric((one %*% Vi %*% x[keep]) / (one %*% Vi %*% one))
  x[gone] <- a + V[gone, keep] %*% Vi %*% (x[keep] - a); x
}
one_tree <- function(lambda_true, gaps) {
  tr <- rcoal(40); tr$tip.label <- sprintf("sp%02d", 1:40)
  x  <- sim_pagel(tr, lambda = lambda_true)
  if (gaps == "random") {
    gone <- tr$tip.label %in% sample(tr$tip.label, 10)
  } else {
    clades <- prop.part(tr)
    gone <- tr$tip.label %in% tr$tip.label[clades[[which.min(abs(lengths(clades) - 10))]]]
  }
  gone <- setNames(gone, tr$tip.label)[names(x)]
  x_tree <- fill_tree(x, tr, gone)
  x_mean <- x; x_mean[gone] <- mean(x[!gone])
  kept <- drop.tip(tr, names(x)[gone])
  c(K_full = blomberg_K(x, tr), K_tree = blomberg_K(x_tree, tr),
    K_mean = blomberg_K(x_mean, tr), K_drop = blomberg_K(x[!gone], kept),
    L_full = lambda_grid(x, tr), L_tree = lambda_grid(x_tree, tr),
    L_mean = lambda_grid(x_mean, tr), L_drop = lambda_grid(x[!gone], kept))
}
set.seed(163)
n_tree <- 200
cells <- expand.grid(lambda = c(1, 0.5), gaps = c("random", "clade"), stringsAsFactors = FALSE)
runs  <- lapply(seq_len(nrow(cells)), function(i)
  replicate(n_tree, one_tree(cells$lambda[i], cells$gaps[i])))
fills <- do.call(rbind, lapply(seq_len(nrow(cells)), function(i) {
  m <- runs[[i]]
  data.frame(cells[i, ],
             K_tree = median(m["K_tree", ] / m["K_full", ]),
             K_mean = median(m["K_mean", ] / m["K_full", ]),
             K_drop = median(m["K_drop", ] / m["K_full", ]),
             tree_up = mean(m["K_tree", ] > m["K_full", ]),
             lam_full = mean(m["L_full", ]), lam_tree = mean(m["L_tree", ]),
             lam_mean = mean(m["L_mean", ]), lam_drop = mean(m["L_drop", ]))
}))
print(fills, digits = 3, row.names = FALSE)
cell <- function(l, g) which(fills$lambda == l & fills$gaps == g)
share_se <- sqrt(0.25 / n_tree)
# how many cells each arm keeps within the largest drop-tips miss of the other three
drop_dev <- max(abs(fills$K_drop[-cell(0.5, "random")] - 1))
n_close  <- sapply(fills[c("K_tree", "K_mean", "K_drop")], function(r) sum(abs(r - 1) <= drop_dev))
worst_miss <- min(abs(unlist(fills[cell(0.5, "random"), c("K_tree", "K_mean", "K_drop")]) - 1))
