# Offsets for rates and densities in Poisson GLMs
# https://tidyecology.com/posts/offsets-for-rates-and-densities/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-11-04

## ---- data ----
set.seed(1)
n <- 120
cover       <- runif(n, 0, 1)                                     # forest cover gradient, 0 to 1
trap_nights <- round(exp(1.9 + 0.9 * cover + rnorm(n, 0, 0.25)))  # effort rises with cover
trap_nights[trap_nights < 1] <- 1
b0 <- log(2)                                                      # true density: 2 beetles per night at cover 0
b1 <- -1.5                                                        # density declines with cover
log_density <- b0 + b1 * cover
mu          <- exp(log_density) * trap_nights                    # expected count = density times effort
count       <- rpois(n, mu)
dat <- data.frame(count, cover, trap_nights)

## ---- describe ----
round(cor(dat$cover, dat$trap_nights), 3)   # effort tied to the predictor
range(dat$trap_nights)                       # effort varies five- to six-fold
range(dat$count)

## ---- raw-vs-rate ----
library(ggplot2)
d1 <- data.frame(
  cover  = rep(dat$cover, 2),
  effort = rep(dat$trap_nights, 2),
  value  = c(dat$count, dat$count / dat$trap_nights),
  metric = rep(c("raw count", "density (count / trap-night)"), each = nrow(dat)))
d1$metric <- factor(d1$metric, levels = c("raw count", "density (count / trap-night)"))
ggplot(d1, aes(cover, value)) +
  geom_point(aes(size = effort), colour = "#275139", alpha = 0.55) +
  geom_smooth(method = "loess", formula = y ~ x, se = FALSE,
              colour = "#b5534e", linewidth = 0.9) +
  facet_wrap(~metric, scales = "free_y") +
  scale_size_continuous(range = c(0.6, 3.6), name = "trap-nights") +
  labs(x = "forest cover", y = NULL) +
  theme_minimal(base_size = 12) +
  theme(panel.grid.minor = element_blank(),
        strip.text = element_text(colour = "#16241d", face = "bold"),
        plot.background = element_rect(fill = "#f5f4ee", colour = NA),
        panel.background = element_rect(fill = "#f5f4ee", colour = NA))

## ---- naive ----
m_naive <- glm(count ~ cover, family = poisson, data = dat)
round(summary(m_naive)$coefficients, 3)

## ---- offset ----
m_off <- glm(count ~ cover + offset(log(trap_nights)),
             family = poisson, data = dat)
round(summary(m_off)$coefficients, 3)
c(true_intercept = round(b0, 3), true_slope = b1)

## ---- coef-fig ----
ci_n <- suppressMessages(confint(m_naive))["cover", ]; est_n <- coef(m_naive)["cover"]
ci_o <- suppressMessages(confint(m_off))["cover", ];   est_o <- coef(m_off)["cover"]
fd <- data.frame(
  model = factor(c("naive (count ~ cover)", "offset (rate ~ cover)"),
                 levels = c("offset (rate ~ cover)", "naive (count ~ cover)")),
  est = c(est_n, est_o), lo = c(ci_n[1], ci_o[1]), hi = c(ci_n[2], ci_o[2]))
ggplot(fd, aes(est, model)) +
  geom_vline(xintercept = b1, linetype = "dashed", colour = "#93a87f", linewidth = 0.7) +
  geom_errorbar(aes(xmin = lo, xmax = hi),
                orientation = "y", width = 0.16, colour = "#275139", linewidth = 0.9) +
  geom_point(size = 3.4, colour = "#275139") +
  annotate("text", x = b1, y = 2.42, label = "  true slope = -1.5",
           colour = "#46604a", size = 3.3, family = "mono", hjust = 0) +
  scale_x_continuous(expand = expansion(mult = c(0.08, 0.05))) +
  labs(x = "cover slope (95% CI)", y = NULL) +
  theme_minimal(base_size = 12) +
  theme(panel.grid.minor = element_blank(),
        plot.background = element_rect(fill = "#f5f4ee", colour = NA),
        panel.background = element_rect(fill = "#f5f4ee", colour = NA))

## ---- pred-rate ----
nd <- data.frame(cover = c(0, 1), trap_nights = 1)
pred <- predict(m_off, newdata = nd, type = "response")
round(pred, 3)                                  # density per trap-night at cover 0 and 1
round(c(true_0 = exp(b0), true_1 = exp(b0 + b1)), 3)

## ---- curve-fig ----
grid <- data.frame(cover = seq(0, 1, length.out = 100), trap_nights = 1)
pr <- predict(m_off, newdata = grid, type = "link", se.fit = TRUE)
grid$fit <- exp(pr$fit)
grid$lo  <- exp(pr$fit - 1.96 * pr$se.fit)
grid$hi  <- exp(pr$fit + 1.96 * pr$se.fit)
grid$tru <- exp(b0 + b1 * grid$cover)
obs <- data.frame(cover = dat$cover, rate = dat$count / dat$trap_nights,
                  effort = dat$trap_nights)
ggplot() +
  geom_ribbon(data = grid, aes(cover, ymin = lo, ymax = hi),
              fill = "#275139", alpha = 0.16) +
  geom_point(data = obs, aes(cover, rate, size = effort),
             colour = "#46604a", alpha = 0.5) +
  geom_line(data = grid, aes(cover, fit), colour = "#275139", linewidth = 1) +
  geom_line(data = grid, aes(cover, tru), colour = "#b5534e",
            linetype = "dashed", linewidth = 0.8) +
  scale_size_continuous(range = c(0.6, 3.6), name = "trap-nights") +
  labs(x = "forest cover", y = "density (beetles per trap-night)") +
  theme_minimal(base_size = 12) +
  theme(panel.grid.minor = element_blank(),
        plot.background = element_rect(fill = "#f5f4ee", colour = NA),
        panel.background = element_rect(fill = "#f5f4ee", colour = NA))

## ---- ratio-poisson ----
dens_warn <- character(0)
m_dens <- withCallingHandlers(
  glm(count / trap_nights ~ cover, family = poisson, data = dat),
  warning = function(w) {
    dens_warn <<- c(dens_warn, conditionMessage(w))
    invokeRestart("muffleWarning")
  })
n_nonint_warn <- length(dens_warn)
stopifnot(n_nonint_warn > 0)                  # the Poisson family warns on a non-integer response
c(warnings = n_nonint_warn, non_integer_densities = sum(dat$count %% dat$trap_nights != 0))
dens_warn[1]
m_dens100 <- suppressWarnings(                # the same density per 100 trap-nights
  glm(100 * count / trap_nights ~ cover, family = poisson, data = dat))
m_quasi    <- glm(count / trap_nights ~ cover, family = quasipoisson, data = dat)
m_quasi100 <- glm(100 * count / trap_nights ~ cover, family = quasipoisson, data = dat)
se_tab <- rbind(
  "Poisson, per trap-night"      = summary(m_dens)$coefficients["cover", 1:2],
  "Poisson, per 100 trap-nights" = summary(m_dens100)$coefficients["cover", 1:2],
  "quasi-Poisson, either unit"   = summary(m_quasi)$coefficients["cover", 1:2],
  "count with offset"            = summary(m_off)$coefficients["cover", 1:2])
round(se_tab, 3)
stopifnot(abs(se_tab[2, 2] / se_tab[1, 2] - 1 / sqrt(100)) < 1e-4,  # Poisson SE scales as 1 / sqrt(c)
          abs(summary(m_quasi100)$coefficients["cover", 2] - se_tab[3, 2]) < 1e-6)
phi_night <- summary(m_quasi)$dispersion
round(c(dispersion_per_night = phi_night, mean_inverse_effort = mean(1 / dat$trap_nights)), 3)

## ---- ratio-sim ----
sim_dat <- function(slope) {                  # the generator of the data chunk, as a function
  cv <- runif(n, 0, 1)
  tn <- round(exp(1.9 + 0.9 * cv + rnorm(n, 0, 0.25))); tn[tn < 1] <- 1
  data.frame(count = rpois(n, exp(b0 + slope * cv) * tn), cover = cv, trap_nights = tn)
}
p_cover <- function(fit) summary(fit)$coefficients["cover", 4]
one_rep <- function(slope) {
  d <- sim_dat(slope); d$dens <- d$count / d$trap_nights
  q <- glm(dens ~ cover, family = quasipoisson, data = d)
  c(per_night = suppressWarnings(p_cover(glm(dens ~ cover, family = poisson, data = d))),
    per_100   = suppressWarnings(p_cover(glm(100 * dens ~ cover, family = poisson, data = d))),
    quasi     = p_cover(q),
    offset    = p_cover(glm(count ~ cover + offset(log(trap_nights)), family = poisson, data = d)),
    gaussian  = p_cover(lm(dens ~ cover, data = d)),
    phi       = summary(q)$dispersion,
    inv_eff   = mean(1 / d$trap_nights))
}
set.seed(4401); n_rep <- c(null = 1000, weak = 4000)   # own seed; more runs where powers are close
sims <- Map(function(s, k) t(replicate(k, one_rep(s))), c(null = 0, weak = -0.3), n_rep)
reject <- sapply(sims, function(m) colMeans(m[, 1:5] < 0.05))
round(reject, 3)
mc_se <- sqrt(sweep(reject * (1 - reject), 2, n_rep, "/"))
phi_sim <- mean(sims$null[, "phi"]); phi_100 <- 100 * phi_sim   # dispersion per trap-night, per 100
pred_100 <- 2 * pnorm(-1.96 / sqrt(phi_100))  # null rejection if every site had the harmonic-mean effort
edge <- (sims$weak[, "offset"] < 0.05) - (sims$weak[, "quasi"] < 0.05)   # paired, same data sets
edge_se <- sd(edge) / sqrt(n_rep[["weak"]]); stopifnot(mean(edge) > 2 * edge_se, mean(edge) < 0.03)  # real, small
edge_g <- (sims$weak[, "offset"] < 0.05) - (sims$weak[, "gaussian"] < 0.05)   # the same, against the Gaussian
round(c(mean_phi = phi_sim, mean_inverse_effort = mean(sims$null[, "inv_eff"]),
        predicted_null_per_100 = pred_100, offset_edge = mean(edge), edge_mc_se = edge_se,
        gaussian_gap = mean(edge_g), gaussian_gap_se = sd(edge_g) / sqrt(n_rep[["weak"]])), 3)

## ---- dispersion ----
round(deviance(m_off) / df.residual(m_off), 3)
