# Instrumental variables and 2SLS
# https://tidyecology.com/posts/instrumental-variables-2sls/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-11-14

## ---- setup ----
library(ggplot2); library(dplyr); library(tidyr)
paper<-"#f5f4ee"; ink<-"#16241d"; body<-"#2c3a31"; forest<-"#275139"
sage<-"#93a87f"; line<-"#dad9ca"; faint<-"#5d6b61"; brick<-"#b5534e"
theme_te <- function(base=12) theme_minimal(base_size=base) +
  theme(plot.background=element_rect(fill=paper,colour=NA),
        panel.background=element_rect(fill=paper,colour=NA),
        panel.grid.major=element_line(colour=line,linewidth=0.3),
        panel.grid.minor=element_blank(),
        axis.text=element_text(colour=body), axis.title=element_text(colour=ink),
        plot.title=element_text(colour=ink,face="bold"),
        plot.subtitle=element_text(colour=faint),
        legend.position="top")

## ---- data ----
set.seed(4163)
n <- 800; b_true <- 0.5; a1 <- 0.8
U <- rnorm(n)                          # unobserved confounder
Z <- rnorm(n)                          # instrument, independent of U
X <- 0.2 + a1*Z + 0.9*U + rnorm(n, 0, 0.7)
Y <- 1.0 + b_true*X + 0.8*U + rnorm(n, 0, 0.7)

## ---- ols ----
ols   <- lm(Y ~ X)
b_ols <- coef(ols)["X"]
ci_ols<- confint(ols)["X", ]

## ---- tsls ----
s1    <- lm(X ~ Z); Xhat <- fitted(s1)   # first stage
s2    <- lm(Y ~ Xhat)                     # second stage
b_2sls<- coef(s2)["Xhat"]

## ---- wald ----
b_wald <- cov(Y, Z) / cov(X, Z)
gap    <- b_2sls - b_wald

## ---- fp-guard-1 ----
stopifnot(abs(gap) < 1e-10)

## ---- se ----
uhat  <- Y - (coef(s2)[1] + b_2sls*X)     # structural residual
sig2  <- sum(uhat^2)/(n-2)
Xd    <- cbind(1, Xhat)
se_2sls<- sqrt((sig2 * solve(crossprod(Xd)))[2,2])
se_naive<- summary(s2)$coef["Xhat","Std. Error"]
ci_2sls<- b_2sls + c(-1,1)*qnorm(0.975)*se_2sls

## ---- fig-estimates ----
est <- data.frame(
  method = factor(c("OLS (naive)","2SLS (instrument)"),
                  levels=c("OLS (naive)","2SLS (instrument)")),
  est = c(b_ols,b_2sls), lo = c(ci_ols[1],ci_2sls[1]), hi = c(ci_ols[2],ci_2sls[2]))
ggplot(est, aes(method, est, colour=method)) +
  geom_hline(yintercept=b_true, linetype="dashed", colour=ink, linewidth=0.5) +
  geom_errorbar(aes(ymin=lo, ymax=hi), width=0.12, linewidth=0.8) +
  geom_point(size=3.4) +
  scale_colour_manual(values=c("OLS (naive)"=brick,"2SLS (instrument)"=forest), guide="none") +
  labs(x=NULL, y="estimated effect of X on Y",
       title="Instrument recovers the effect an unmeasured confounder hides",
       subtitle="Dashed line: true effect (0.5). Naive OLS is biased upward; 2SLS straddles the truth") +
  theme_te()

## ---- firststage ----
a1_hat <- coef(s1)["Z"]
Fstat  <- summary(s1)$coef["Z","t value"]^2
r2s1   <- summary(s1)$r.squared

## ---- sweep ----
strengths <- c(0.05,0.1,0.2,0.4,0.8); nsim <- 1000
set.seed(50)
sweep <- lapply(strengths, function(a){
  bb <- Fv <- bo <- numeric(nsim)
  for (i in 1:nsim){
    U <- rnorm(n); Z <- rnorm(n)
    X <- 0.2 + a*Z + 0.9*U + rnorm(n,0,0.7)
    Y <- 1.0 + b_true*X + 0.8*U + rnorm(n,0,0.7)
    bb[i] <- cov(Y,Z)/cov(X,Z)
    bo[i] <- cov(Y,X)/var(X)                # OLS slope, same data
    Fv[i] <- summary(lm(X~Z))$coef["Z","t value"]^2
  }
  q <- quantile(bb, c(.25,.5,.75))
  data.frame(strength=a, medF=median(Fv), p25=q[1], med=q[2], p75=q[3], sdEst=sd(bb),
             ols=median(bo))
}) %>% bind_rows()

## ---- fig-weak ----
ggplot(sweep, aes(medF, med)) +
  geom_ribbon(aes(ymin=p25, ymax=p75), fill=sage, alpha=0.35) +
  geom_hline(yintercept=b_true, linetype="dashed", colour=forest, linewidth=0.5) +
  geom_line(aes(y=ols), linetype="dotted", colour=brick, linewidth=0.5) +
  geom_vline(xintercept=10, linetype="dashed", colour=faint, linewidth=0.4) +
  geom_line(colour=ink, linewidth=0.6) + geom_point(colour=forest, size=2.6) +
  annotate("text", x=10, y=1.35, label="F = 10", hjust=-0.1, colour=faint, size=3.2) +
  annotate("text", x=1.5, y=sweep$ols[1]+0.07, label="OLS slope", hjust=0, colour=brick, size=3.2) +
  annotate("text", x=200, y=b_true-0.09, label="truth", hjust=1, colour=forest, size=3.2) +
  scale_x_log10() + coord_cartesian(ylim=c(-0.2,1.4)) +
  labs(x="first-stage F statistic (log scale)", y="2SLS estimate (median, 25-75%)",
       title="Weak instruments bias 2SLS toward OLS and widen its spread",
       subtitle="As the instrument weakens, the estimate drifts toward the confounded slope") +
  theme_te()

## ---- exclusion ----
set.seed(9163); gamma <- 0.15
excl <- lapply(c(0.8,0.2), function(a){
  bb <- numeric(600)
  for (i in 1:600){
    U <- rnorm(n); Z <- rnorm(n)
    X <- 0.2 + a*Z + 0.9*U + rnorm(n,0,0.7)
    Y <- 1.0 + b_true*X + 0.8*U + gamma*Z + rnorm(n,0,0.7)   # direct Z to Y
    bb[i] <- cov(Y,Z)/cov(X,Z)
  }
  data.frame(strength=a, bias=mean(bb)-b_true, approx=gamma/a)
}) %>% bind_rows()

## ---- coverage ----
set.seed(7163); ncov <- 1200; cov_ols <- cov_iv <- logical(ncov)
for (i in 1:ncov){
  U <- rnorm(n); Z <- rnorm(n)
  X <- 0.2 + 0.8*Z + 0.9*U + rnorm(n,0,0.7)
  Y <- 1.0 + b_true*X + 0.8*U + rnorm(n,0,0.7)
  o <- lm(Y~X); ci <- confint(o)["X",]; cov_ols[i] <- b_true>=ci[1] & b_true<=ci[2]
  Xh <- fitted(lm(X~Z)); s <- lm(Y~Xh); bh <- coef(s)[2]
  uh <- Y-(coef(s)[1]+bh*X); v <- sum(uh^2)/(n-2)
  se <- sqrt((v*solve(crossprod(cbind(1,Xh))))[2,2])
  cih <- bh+c(-1,1)*qnorm(0.975)*se; cov_iv[i] <- b_true>=cih[1] & b_true<=cih[2]
}
