# Errors-in-variables and Deming regression
# https://tidyecology.com/posts/errors-in-variables-deming-regression/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-11-04

## ---- setup ----
suppressMessages({library(ggplot2); library(dplyr); library(tidyr)})
theme_te <- theme_minimal(base_size = 12) +
  theme(
        plot.background  = element_rect(fill = "#f5f4ee", colour = NA),
        panel.background = element_rect(fill = "#f5f4ee", colour = NA),
        panel.grid.minor = element_blank(),
        plot.title = element_text(face = "bold"), legend.position = "top")
col_true <- "#2f8f63"; col_fwd <- "#b5534e"; col_inv <- "#4f6d8c"; col_dem <- "#cda23f"

deming <- function(w, z, delta) {
  sxx <- var(w); syy <- var(z); sxy <- cov(w, z)
  b <- ((syy - delta*sxx) + sqrt((syy - delta*sxx)^2 + 4*delta*sxy^2)) / (2*sxy)
  c(intercept = mean(z) - b*mean(w), slope = b)
}

## ---- sci-tex-helper ----
sci_tex <- function(x, digits = 2) {
  s <- formatC(x, format = "e", digits = digits)
  paste0("$", sub("e.*", "", s), " \\times 10^{", as.integer(sub(".*e", "", s)), "}$")
}

## ---- simulate ----
set.seed(5231)
n   <- 200
a0  <- 2; b1 <- 1.5
xt  <- rnorm(n, mean = 12, sd = 4)      # true, unobserved
yt  <- a0 + b1 * xt                      # exact functional relationship
w   <- xt + rnorm(n, 0, sd = 2)          # recorded x, error variance 4
z   <- yt + rnorm(n, 0, sd = 4)          # recorded y, error variance 16

## ---- ols-both ----
f_yx  <- lm(z ~ w)                       # forward: attenuated
f_xy  <- lm(w ~ z)                       # inverse
b_fwd <- unname(coef(f_yx)[2])
b_inv <- unname(1 / coef(f_xy)[2])       # inverse fit expressed as z-on-w slope

## ---- deming-fit ----
delta_true <- 4
dem <- deming(w, z, delta_true)

jk     <- sapply(seq_len(n), function(i) deming(w[-i], z[-i], delta_true)["slope"])
se_dem <- sqrt((n - 1) / n * sum((jk - mean(jk))^2))

## ---- deming-numbers ----
b_dem <- unname(dem["slope"])
b_ma  <- unname(deming(w, z, 1)["slope"])
b_sma <- sign(cov(w, z)) * sd(z) / sd(w)

## ---- fig-bracket ----
inv_a <- mean(z) - b_inv * mean(w)
lines <- data.frame(
  method    = factor(c("true", "OLS z on w", "OLS w on z", "Deming"),
                     levels = c("true", "OLS z on w", "OLS w on z", "Deming")),
  intercept = c(a0, coef(f_yx)[1], inv_a, unname(dem["intercept"])),
  slope     = c(b1, b_fwd, b_inv, b_dem)
)
ggplot(data.frame(w = w, z = z), aes(w, z)) +
  geom_point(alpha = 0.35, size = 1.1, colour = "#5d6b61") +
  geom_abline(data = lines, aes(intercept = intercept, slope = slope, colour = method),
              linewidth = 1.0) +
  scale_colour_manual(values = c("true" = col_true, "OLS z on w" = col_fwd,
                                 "OLS w on z" = col_inv, "Deming" = col_dem), name = NULL) +
  labs(title = "Ordinary regression brackets the truth; Deming recovers it",
       x = "w (recorded x, with error)", y = "z (recorded y, with error)") +
  theme_te

## ---- fig-sensitivity ----
delta_grid <- 10^seq(-1.5, 1.5, length.out = 60)
d2 <- data.frame(delta = delta_grid,
                 slope = sapply(delta_grid, function(d) deming(w, z, d)["slope"]))
marks <- data.frame(delta = c(1, 4),
                    slope = c(b_ma, b_dem),
                    lab   = c("major axis (ratio 1)", "correct ratio 4"))
ggplot(d2, aes(delta, slope)) +
  geom_hline(yintercept = b1, colour = col_true, linewidth = 0.5) +
  geom_line(colour = col_dem, linewidth = 1.1) +
  geom_point(data = marks, size = 3, colour = "#16241d") +
  geom_text(data = marks, aes(label = lab), vjust = -1, size = 3.3, colour = "#16241d") +
  scale_x_log10() +
  labs(title = "The Deming slope depends entirely on the assumed error ratio",
       x = "assumed error-variance ratio (log scale)", y = "Deming slope") +
  theme_te

## ---- ba-one ----
set.seed(20260927)
ba_n  <- 200; ba_xt <- rnorm(ba_n, mean = 12, sd = 4)
ba_a  <- ba_xt + rnorm(ba_n, 0, sd = 2)     # method A, error variance 4
ba_b  <- ba_xt + rnorm(ba_n, 0, sd = 4)     # method B, error variance 16
ba_d  <- ba_b - ba_a; ba_m <- (ba_a + ba_b) / 2
ba_co <- summary(lm(ba_d ~ ba_m))$coefficients
ba_closed <- (16 - 4) / (2 * (16 + (4 + 16) / 4))   # slope of D on M, no bias at all
ba_flat <- sqrt(1 - (16 - 4) / 16)   # true slope of B on A that makes the numerator zero
ba_loa <- mean(ba_d) + c(-1.96, 1.96) * sd(ba_d)
ba_p_lm  <- ba_co[2, 4]; ba_p_cor <- cor.test(ba_d, ba_m)$p.value; ba_p_pit <- cor.test(ba_b - 1 * ba_a, ba_b + 1 * ba_a)$p.value   # Pitman test of slope 1
stopifnot(abs(ba_p_lm - ba_p_cor) < 1e-12, abs(ba_p_lm - ba_p_pit) < 1e-12)

## ---- fig-bland-altman ----
ggplot(data.frame(m = ba_m, d = ba_d), aes(m, d)) +
  geom_point(alpha = 0.35, size = 1.1, colour = "#5d6b61") +
  geom_hline(yintercept = mean(ba_d), colour = col_dem, linewidth = 0.6) +
  geom_hline(yintercept = ba_loa, colour = col_dem, linewidth = 0.6, linetype = "dashed") +
  geom_abline(intercept = ba_co[1, 1], slope = ba_co[2, 1], colour = col_fwd, linewidth = 1.0) +
  labs(title = "A tilted Bland-Altman cloud from two unbiased methods",
       x = "mean of the two methods", y = "method B minus method A") +
  theme_te

## ---- ba-sim ----
set.seed(4527); ba_reps <- 1000; ba_boot <- 199; ba_ndup <- 20
ba_dem <- function(sxx, syy, sxy, d) ((syy - d*sxx) + sqrt((syy - d*sxx)^2 + 4*d*sxy^2)) / (2*sxy)
ba_one <- function(n) {
  xt_i <- rnorm(n, 12, 4); a <- xt_i + rnorm(n, 0, 2); b <- xt_i + rnorm(n, 0, 4)
  p_ba <- summary(lm(I(b - a) ~ I((a + b) / 2)))$coefficients[2, 4]
  dup_a <- rnorm(ba_ndup, 0, 2) - rnorm(ba_ndup, 0, 2); dup_b <- rnorm(ba_ndup, 0, 4) - rnorm(ba_ndup, 0, 4)
  d_hat <- sum(dup_b^2) / sum(dup_a^2)
  ui <- matrix(sample.int(n, n * ba_boot, replace = TRUE), n); di <- matrix(sample.int(ba_ndup, ba_ndup * ba_boot, replace = TRUE), ba_ndup)
  ab <- matrix(a[ui], n); bb <- matrix(b[ui], n)
  sxx <- apply(ab, 2, var); syy <- apply(bb, 2, var)
  sxy <- (colSums(ab * bb) - n * colMeans(ab) * colMeans(bb)) / (n - 1)
  d_boot <- colSums(matrix(dup_b[di], ba_ndup)^2) / colSums(matrix(dup_a[di], ba_ndup)^2)
  est <- sapply(c(ma = 1, known = delta_true, fixed = d_hat), function(r) deming(a, b, r)[["slope"]])
  se  <- sapply(list(ma = 1, known = delta_true, fixed = d_hat, resampled = d_boot),
                function(r) sd(ba_dem(sxx, syy, sxy, r)))
  est <- c(est, resampled = est[["fixed"]])
  c(bland_altman = p_ba < 0.05, abs(est - 1) / se > qnorm(0.975), ma_slope = est[["ma"]],
    ma_same = (est[["ma"]] > 1) == (var(b) > var(a)))
}
ba_ns  <- c(30, 100, 400); ba_res <- sapply(ba_ns, function(n) rowMeans(replicate(ba_reps, ba_one(n))))
ba_pow <- pnorm(atanh(6 / sqrt(20 * 21)) * sqrt(ba_ns - 3) - qnorm(0.975))   # Fisher z power
ba_mcse <- sqrt(c(0.05 * 0.95, 0.25) / ba_reps); ba_f3 <- function(row) paste(sprintf("%.3f", ba_res[row, ]), collapse = ", ")
ba_ma_pop <- (12 + sqrt(12^2 + 4 * 16^2)) / (2 * 16)   # major axis from population moments
stopifnot(all(ba_res["ma_same", ] == 1))                # MA slope > 1 exactly when var(B) > var(A), given cov(A, B) > 0
