# Community covariates and species traits
# https://tidyecology.com/posts/community-covariates-and-traits/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-07

## ---- setup ----
library(ggplot2); library(dplyr)
logit <- qlogis; expit <- plogis

te_ink<-"#16241d"; te_body<-"#2c3a31"; te_forest<-"#275139"; te_faint<-"#5d6b61"
te_sage<-"#93a87f"; te_paper<-"#f5f4ee"; te_line<-"#dad9ca"; te_brick<-"#b5534e"; te_gold<-"#cda23f"
theme_te <- function(base_size=12){
  theme_minimal(base_size=base_size) +
    theme(plot.background=element_rect(fill=te_paper,colour=NA),
          panel.background=element_rect(fill="#f5f4ee",colour=NA),
          panel.grid.minor=element_blank(),
          panel.grid.major=element_line(colour=te_line,linewidth=0.3),
          axis.title=element_text(colour=te_body), axis.text=element_text(colour=te_faint),
          plot.title=element_text(colour=te_ink,face="bold"),
          plot.subtitle=element_text(colour=te_faint,size=rel(0.9)),
          legend.text=element_text(colour=te_faint,size=rel(0.8)))
}

## ---- simulate ----
set.seed(145)
S <- 34L; R <- 95L; K <- 4L
x  <- as.numeric(scale(rnorm(R)))          # site covariate (standardised)
tr <- as.numeric(scale(rnorm(S)))          # species trait (standardised)
mu_a <- logit(0.35); sd_a <- 0.85          # community intercept
beta0 <- 0.20; beta1 <- 0.90; sd_b <- 0.50 # slope depends on trait: b ~ N(beta0 + beta1*tr, sd_b)
mu_p <- logit(0.30); sd_p <- 0.60          # community detection

a_t <- rnorm(S, mu_a, sd_a)                          # species intercepts
b_t <- rnorm(S, beta0 + beta1*tr, sd_b)              # species slopes, tied to the trait
lp_t <- rnorm(S, mu_p, sd_p); p_t <- expit(lp_t)
eta <- matrix(a_t, S, R) + outer(b_t, x); psi <- expit(eta)
z <- matrix(rbinom(S*R, 1, psi), S, R)
y <- matrix(rbinom(S*R, K, ifelse(z==1, p_t, 0)), S, R)
det_sites <- rowSums(y > 0)
c(species=S, sites=R, det_min=min(det_sites), det_max=max(det_sites), total_det=sum(y))

## ---- independent-mle ----
one <- function(yi){
  nll <- function(par){ a<-par[1]; b<-par[2]; p<-expit(par[3]); ps<-expit(a+b*x); d<-yi>0
    -(sum(log(ps[d]) + dbinom(yi[d],K,p,log=TRUE)) + sum(log(ps[!d]*(1-p)^K + (1-ps[!d])))) }
  op <- try(optim(c(logit(mean(yi>0)+.01),0,0), nll, method="BFGS", hessian=TRUE), silent=TRUE)
  if (inherits(op,"try-error")) return(c(b=NA, se=NA))
  se <- try(sqrt(diag(solve(op$hessian)))[2], silent=TRUE)
  c(b=op$par[2], se=ifelse(inherits(se,"try-error")||is.na(se), NA, se))
}
mle <- t(sapply(1:S, function(s) one(y[s,]))); colnames(mle) <- c("b","se")
round(c(median_SE=median(mle[,"se"]), max_SE=max(mle[,"se"])), 2)

## ---- naive-two-stage ----
ok <- is.finite(mle[,"b"]) & is.finite(mle[,"se"])   # drop any species without a finite estimate
naive <- lm(mle[ok,"b"] ~ tr[ok])
c(beta1_naive = round(coef(naive)[2],3), true = beta1)

## ---- hier-mcmc ----
llmat <- function(a, b, lp){                       # per-species log-likelihood, z summed out
  e <- matrix(a,S,R) + outer(b,x); ps <- expit(e); p <- expit(lp); Pm <- matrix(p,S,R)
  D <- y>0; term <- matrix(0,S,R)
  term[D]  <- log(ps[D]) + dbinom(y[D],K,Pm[D],log=TRUE)
  term[!D] <- log(ps[!D]*(1-Pm[!D])^K + (1-ps[!D]))
  rowSums(term)
}
ni <- 16000L; nb <- 6000L; nt <- 2L
a <- rep(logit(0.35),S); b <- rep(0,S); lp <- rep(logit(0.3),S)
m_a<-0; s_a<-1; B0<-0; B1<-0; s_b<-1; m_p<-logit(0.3); s_p<-1
t_a<-0.22; t_b<-0.22; t_lp<-0.22; t_h<-0.13; t_lsd<-0.13
keep <- seq(nb+1,ni,by=nt); nk <- length(keep); kk <- 0L
Bpost <- matrix(NA,nk,S); HYP <- matrix(NA,nk,7)    # m_a,s_a,B0,B1,s_b,m_p,s_p

for (it in 1:ni){
  ll <- llmat(a,b,lp)
  prop <- a + rnorm(S,0,t_a); llp <- llmat(prop,b,lp)                    # intercepts
  au <- log(runif(S)) < (llp+dnorm(prop,m_a,s_a,log=TRUE) - (ll+dnorm(a,m_a,s_a,log=TRUE))); a[au]<-prop[au]
  ll <- llmat(a,b,lp)
  mb <- B0 + B1*tr
  prop <- b + rnorm(S,0,t_b); llp <- llmat(a,prop,lp)                    # slopes (prior mean B0+B1*trait)
  au <- log(runif(S)) < (llp+dnorm(prop,mb,s_b,log=TRUE) - (ll+dnorm(b,mb,s_b,log=TRUE))); b[au]<-prop[au]
  ll <- llmat(a,b,lp)
  prop <- lp + rnorm(S,0,t_lp); llp <- llmat(a,b,prop)                   # detection
  au <- log(runif(S)) < (llp+dnorm(prop,m_p,s_p,log=TRUE) - (ll+dnorm(lp,m_p,s_p,log=TRUE))); lp[au]<-prop[au]
  # hyperparameters
  pr<-m_a+rnorm(1,0,t_h); if(log(runif(1))<sum(dnorm(a,pr,s_a,log=TRUE))-sum(dnorm(a,m_a,s_a,log=TRUE)))m_a<-pr
  pr<-exp(log(s_a)+rnorm(1,0,t_lsd)); if(log(runif(1))<sum(dnorm(a,m_a,pr,log=TRUE))+dnorm(pr,0,2.5,log=TRUE)+log(pr)-sum(dnorm(a,m_a,s_a,log=TRUE))-dnorm(s_a,0,2.5,log=TRUE)-log(s_a))s_a<-pr
  mb<-B0+B1*tr; pr<-B0+rnorm(1,0,t_h); if(log(runif(1))<sum(dnorm(b,pr+B1*tr,s_b,log=TRUE))-sum(dnorm(b,mb,s_b,log=TRUE)))B0<-pr
  mb<-B0+B1*tr; pr<-B1+rnorm(1,0,t_h); if(log(runif(1))<sum(dnorm(b,B0+pr*tr,s_b,log=TRUE))-sum(dnorm(b,mb,s_b,log=TRUE)))B1<-pr
  mb<-B0+B1*tr; pr<-exp(log(s_b)+rnorm(1,0,t_lsd)); if(log(runif(1))<sum(dnorm(b,mb,pr,log=TRUE))+dnorm(pr,0,2.5,log=TRUE)+log(pr)-sum(dnorm(b,mb,s_b,log=TRUE))-dnorm(s_b,0,2.5,log=TRUE)-log(s_b))s_b<-pr
  pr<-m_p+rnorm(1,0,t_h); if(log(runif(1))<sum(dnorm(lp,pr,s_p,log=TRUE))-sum(dnorm(lp,m_p,s_p,log=TRUE)))m_p<-pr
  pr<-exp(log(s_p)+rnorm(1,0,t_lsd)); if(log(runif(1))<sum(dnorm(lp,m_p,pr,log=TRUE))+dnorm(pr,0,2.5,log=TRUE)+log(pr)-sum(dnorm(lp,m_p,s_p,log=TRUE))-dnorm(s_p,0,2.5,log=TRUE)-log(s_p))s_p<-pr
  if (it>nb && ((it-nb)%%nt==0)){ kk<-kk+1L; Bpost[kk,]<-b; HYP[kk,]<-c(m_a,s_a,B0,B1,s_b,m_p,s_p) }
}
hp <- colMeans(HYP); bhat <- colMeans(Bpost)
round(c(beta0=hp[3], beta1=hp[4], sd_b=hp[5]), 3)

## ---- fig-shrinkage ----
df <- data.frame(det=det_sites, mle=mle[,"b"], se=mle[,"se"], hier=bhat)
df <- df[order(df$det),]; df$rank <- 1:S; comm_mean <- mean(bhat)
ggplot(df, aes(rank)) +
  geom_hline(yintercept=comm_mean, linetype="dashed", colour=te_faint) +
  geom_errorbar(aes(ymin=mle-se, ymax=mle+se), colour=te_brick, width=0, linewidth=0.4, alpha=0.7) +
  geom_point(aes(y=mle, shape="Independent (per species) +/-1 SE"), colour=te_brick, size=1.7) +
  geom_point(aes(y=hier, shape="Hierarchical (partially pooled)"), colour=te_forest, size=1.7) +
  scale_shape_manual(NULL, values=c("Independent (per species) +/-1 SE"=1,"Hierarchical (partially pooled)"=16)) +
  coord_cartesian(ylim=c(-4,4)) +
  labs(x="species ordered by number of detections (fewest at left)", y="occupancy-covariate slope",
       title="Ill-identified slopes are noisy; pooling reins them in",
       subtitle="dashed: community mean; the vertical scale is clipped at plus and minus 4") +
  theme_te() + theme(legend.position = "inside", legend.position.inside = c(0.72,0.13),
                     legend.background=element_rect(fill="#ffffff",colour=te_line))

## ---- fig-trait ----
b1h <- hp[4]; b0h <- hp[3]; xr <- range(tr)
lh <- data.frame(tr=seq(xr[1],xr[2],length=50)); lh$y <- b0h + b1h*lh$tr
ln <- data.frame(tr=lh$tr, y=coef(naive)[1] + coef(naive)[2]*lh$tr)
dpt <- data.frame(tr=tr, mle=mle[,"b"], hier=bhat)
ggplot() +
  geom_point(data=dpt, aes(tr, mle), colour=te_brick, shape=1, size=1.8) +
  geom_point(data=dpt, aes(tr, hier), colour=te_forest, size=1.8) +
  geom_line(data=ln, aes(tr, y, linetype="Naive two-stage (OLS on MLE slopes)"), colour=te_gold, linewidth=0.8) +
  geom_line(data=lh, aes(tr, y, linetype="Hierarchical community relationship"), colour=te_forest, linewidth=0.8) +
  scale_linetype_manual(NULL, values=c("Naive two-stage (OLS on MLE slopes)"="dashed","Hierarchical community relationship"="solid")) +
  coord_cartesian(ylim=c(-4,4)) +
  labs(x="species trait (standardised)", y="occupancy-covariate slope",
       title="A trait explains how species respond to the gradient",
       subtitle="hollow: independent slopes; filled: pooled") +
  theme_te() + theme(legend.position = "inside", legend.position.inside = c(0.35,0.9),
                     legend.background=element_rect(fill="#ffffff",colour=te_line))
