# Flight-season shifts and false occupancy trends
# https://tidyecology.com/posts/flight-season-shifts-and-occupancy-trends/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-11-04

## ---- setup ----
library(ggplot2)
library(patchwork)

te_paper  <- "#f5f4ee"
te_ink    <- "#16241d"
te_body   <- "#2c3a31"
te_forest <- "#275139"
te_rust   <- "#b5534e"
te_gold   <- "#c9b458"
te_line   <- "#dad9ca"

theme_datasheet <- function() {
  theme_minimal(base_size = 12) +
    theme(plot.background  = element_rect(fill = te_paper, colour = NA),
          panel.background = element_rect(fill = te_paper, colour = NA),
          panel.grid.major = element_line(colour = te_line, linewidth = 0.3),
          panel.grid.minor = element_blank(),
          text             = element_text(colour = te_body),
          plot.title       = element_text(colour = te_ink, face = "bold"),
          plot.subtitle    = element_text(colour = te_body),
          axis.text        = element_text(colour = te_body),
          strip.text       = element_text(colour = te_ink))
}

## ---- sci-tex-helper ----
sci_tex <- function(x, digits = 2) {
  s <- formatC(x, format = "e", digits = digits)
  paste0("$", sub("e.*", "", s), " \\times 10^{", as.integer(sub(".*e", "", s)), "}$")
}

## ---- design ----
n_site    <- 300
n_year    <- 10
psi_true  <- 0.4
p_max     <- 0.6
season_sd <- 12
win_lo    <- 150
win_hi    <- 230
shift_use <- -3
mid_central <- 190
mid_edge    <- 172
n_visit_use <- 4

year_c   <- seq_len(n_year) - (n_year + 1) / 2
unit_yc  <- rep(year_c, each = n_site)
unit_yr  <- rep(seq_len(n_year), each = n_site)

sim_records <- function(peak_mid, shift, clus_sd, n_visit, trend = 0) {
  n_unit <- n_site * n_year
  z_unit <- rbinom(n_unit, 1, plogis(qlogis(psi_true) + trend * unit_yc))
  if (clus_sd == 0) {
    days <- matrix(runif(n_unit * n_visit, win_lo, win_hi), n_unit)
  } else {
    centre <- runif(n_unit, win_lo, win_hi)
    days <- centre + matrix(rnorm(n_unit * n_visit, 0, clus_sd), n_unit)
    days <- pmin(pmax(days, win_lo), win_hi)
  }
  peak  <- peak_mid + shift * unit_yc
  p_mat <- p_max * exp(-(days - peak)^2 / (2 * season_sd^2))
  y_mat <- matrix(rbinom(n_unit * n_visit, 1, z_unit * p_mat), n_unit)
  list(days = days, y = y_mat)
}

## ---- arithmetic ----
visit_p_mean <- function(peak) {
  p_max * season_sd * sqrt(2 * pi) / (win_hi - win_lo) *
    (pnorm((win_hi - peak) / season_sd) - pnorm((win_lo - peak) / season_sd))
}
geom_lab <- c(central = "season centred in the window",
              edge = "season running towards the early edge")
arith_tab <- do.call(rbind, lapply(names(geom_lab), function(g) {
  peak_t <- (if (g == "central") mid_central else mid_edge) + shift_use * year_c
  p_bar  <- visit_p_mean(peak_t)
  share  <- psi_true * (1 - (1 - p_bar)^n_visit_use)
  data.frame(geometry = g, year = seq_len(n_year), peak = peak_t,
             p_bar = p_bar, share = share)
}))
share_slope <- sapply(names(geom_lab), function(g) {
  s_g <- arith_tab$share[arith_tab$geometry == g]
  unname(coef(lm(qlogis(s_g) ~ year_c))[2])
})
pbar_c <- arith_tab$p_bar[arith_tab$geometry == "central"]
pbar_e <- arith_tab$p_bar[arith_tab$geometry == "edge"]
peak_e <- arith_tab$peak[arith_tab$geometry == "edge"]
peak_c <- arith_tab$peak[arith_tab$geometry == "central"]

## ---- fig-window ----
day_seq <- seq(120, 250, by = 1)
curve_df <- do.call(rbind, lapply(names(geom_lab), function(g) {
  mid_g <- if (g == "central") mid_central else mid_edge
  do.call(rbind, lapply(c(1, n_year), function(yr) {
    data.frame(geometry = unname(geom_lab[g]), day = day_seq,
               year = paste("year", yr),
               p = p_max * exp(-(day_seq - mid_g - shift_use * year_c[yr])^2 /
                                 (2 * season_sd^2)))
  }))
}))
curve_df$geometry <- factor(curve_df$geometry, levels = geom_lab)
ggplot(curve_df, aes(day, p, colour = year)) +
  annotate("rect", xmin = win_lo, xmax = win_hi, ymin = -Inf, ymax = Inf,
           fill = te_line, alpha = 0.6) +
  geom_line(linewidth = 0.9) +
  facet_wrap(~ geometry) +
  scale_colour_manual(values = c(te_gold, te_rust), name = NULL) +
  labs(x = "day of year", y = "detection per visit",
       title = "The same shift, two positions",
       subtitle = "grey band: the recording window") +
  theme_datasheet() + theme(legend.position = "bottom")

## ---- fitters ----
curve_parts <- function(dat, move) {
  y_mat <- dat$y
  none  <- rowSums(y_mat) == 0
  pieces <- function(th) {
    s_u   <- plogis(th[1] + th[2] * unit_yc)
    peak  <- th[4] + (if (move) th[6] else 0) * unit_yc
    sig2  <- exp(2 * th[5])
    dev   <- dat$days - peak
    log_p <- plogis(th[3], log.p = TRUE) - dev^2 / (2 * sig2)
    p_mat <- exp(log_p)
    l_occ <- exp(rowSums(y_mat * log_p + (1 - y_mat) * log1p(-p_mat)))
    list(s_u = s_u, sig2 = sig2, dev = dev, p_mat = p_mat, l_occ = l_occ,
         lik = s_u * l_occ + (1 - s_u) * none)
  }
  nll <- function(th) -sum(log(pieces(th)$lik))
  grad <- function(th) {
    pc    <- pieces(th)
    w_occ <- pc$s_u * pc$l_occ / pc$lik
    d_psi <- (pc$l_occ - none) / pc$lik * pc$s_u * (1 - pc$s_u)
    resid <- (y_mat - pc$p_mat) / (1 - pc$p_mat)
    g_m   <- rowSums(resid * pc$dev) / pc$sig2
    -c(sum(d_psi), sum(d_psi * unit_yc),
       sum(w_occ * rowSums(resid) * (1 - plogis(th[3]))),
       sum(w_occ * g_m), sum(w_occ * rowSums(resid * pc$dev^2) / pc$sig2),
       if (move) sum(w_occ * g_m * unit_yc))
  }
  det_days <- dat$days[y_mat == 1]
  list(nll = nll, grad = grad,
       start = c(psi_start(dat), 0, 0, mean(det_days), log(sd(det_days)),
                 if (move) 0))
}

psi_start <- function(dat) qlogis(max(mean(rowSums(dat$y) > 0), 0.05))

wald_out <- function(fit, hess, extra = NA) {
  cov_m <- tryCatch(solve(hess), error = function(e) NULL)
  se_b  <- if (is.null(cov_m) || cov_m[2, 2] <= 0) NA else sqrt(cov_m[2, 2])
  c(slope = fit$par[2], se = se_b, psi = plogis(fit$par[1]),
    extra = extra, conv = fit$convergence)
}

fit_curve <- function(dat, move) {
  cp  <- curve_parts(dat, move)
  fit <- optim(cp$start, cp$nll, cp$grad, method = "BFGS",
               control = list(maxit = 500))
  wald_out(fit, optimHess(fit$par, cp$nll, cp$grad),
           if (move) fit$par[6] else NA)
}

gh_nodes <- c(-3.436159, -2.532732, -1.756684, -1.036611, -0.342901,
              0.342901, 1.036611, 1.756684, 2.532732, 3.436159)
gh_wts <- c(7.640433e-06, 1.343646e-03, 3.387439e-02, 2.401386e-01,
            6.108626e-01, 6.108626e-01, 2.401386e-01, 3.387439e-02,
            1.343646e-03, 7.640433e-06) / sqrt(pi)

fit_year_p <- function(dat, site_re = FALSE) {
  n_vis <- ncol(dat$y)
  n_det <- 0:n_vis
  tab   <- matrix(table(factor(unit_yr, seq_len(n_year)),
                        factor(rowSums(dat$y), n_det)), n_year)
  nll <- function(th, re) {
    s_t <- plogis(th[1] + th[2] * year_c)
    tot <- 0
    for (yr in seq_len(n_year)) {
      if (re) {
        re_sd <- exp(th[n_year + 3] + th[n_year + 4] * year_c[yr])
        p_q   <- plogis(th[2 + yr] + sqrt(2) * re_sd * gh_nodes)
        l_occ <- vapply(n_det, function(k) sum(gh_wts * p_q^k *
                                               (1 - p_q)^(n_vis - k)), 0)
      } else {
        p_y   <- plogis(th[2 + yr])
        l_occ <- p_y^n_det * (1 - p_y)^(n_vis - n_det)
      }
      lik <- s_t[yr] * l_occ + (1 - s_t[yr]) * (n_det == 0)
      tot <- tot + sum(tab[yr, ] * log(lik))
    }
    -tot
  }
  run <- function(st, re) {
    optim(st, nll, re = re, method = "BFGS", control = list(maxit = 2000))
  }
  base_th <- c(psi_start(dat), 0, rep(0, n_year))
  fit <- run(base_th, re = FALSE)
  if (site_re) {
    # two starts, the detected share and the year-specific p optimum; keep the better
    fits <- list(run(c(base_th, log(0.5), 0), re = TRUE),
                 run(c(fit$par, log(0.5), 0), re = TRUE))
    fit <- fits[[which.min(sapply(fits, function(f) f$value))]]
  }
  # extra: for the site effect, the ratio of its sd in year 10 to year 1
  wald_out(fit, optimHess(fit$par, nll, re = site_re),
           if (site_re) exp(fit$par[n_year + 4] * (year_c[n_year] - year_c[1])) else NA)
}

fit_naive <- function(dat) {
  seen <- rowSums(dat$y) > 0
  unname(coef(glm(seen ~ unit_yc, family = binomial))[2])
}

set.seed(4101)
check_dat <- sim_records(mid_edge, shift_use, 5, n_visit_use)
check_cp  <- curve_parts(check_dat, move = TRUE)
check_th  <- c(qlogis(0.35), -0.05, qlogis(0.55), 175, log(13), -2)
num_grad  <- vapply(seq_along(check_th), function(j) {
  step <- rep(0, length(check_th))
  step[j] <- 1e-5
  (check_cp$nll(check_th + step) - check_cp$nll(check_th - step)) / 2e-5
}, 0)
grad_gap <- max(abs(num_grad - check_cp$grad(check_th)) / pmax(1, abs(num_grad)))

## ---- grid ----
run_cell <- function(peak_mid, clus_sd, n_rep, seed, shift = shift_use,
                     n_visit = n_visit_use, trend = 0, with_re = TRUE) {
  set.seed(seed)
  out <- lapply(seq_len(n_rep), function(r) {
    dat <- sim_records(peak_mid, shift, clus_sd, n_visit, trend)
    rbind(naive   = c(fit_naive(dat), NA, NA, NA, 0),
          year_p  = fit_year_p(dat),
          site_re = if (with_re) fit_year_p(dat, site_re = TRUE) else rep(NA, 5),
          fixed   = fit_curve(dat, move = FALSE),
          moving  = fit_curve(dat, move = TRUE))
  })
  arr <- simplify2array(out)
  dimnames(arr)[[2]] <- c("slope", "se", "psi", "extra", "conv")
  arr
}

summarise_cell <- function(arr, trend = 0) {
  n_r <- dim(arr)[3]
  data.frame(estimator = dimnames(arr)[[1]],
    slope = apply(arr[, "slope", , drop = FALSE], 1, mean),
    slope_mcse = apply(arr[, "slope", , drop = FALSE], 1, sd) / sqrt(n_r),
    psi = apply(arr[, "psi", , drop = FALSE], 1, mean),
    cover = apply(abs(arr[, "slope", , drop = FALSE] - trend) <
                    1.96 * arr[, "se", , drop = FALSE], 1, mean, na.rm = TRUE),
    row.names = NULL)
}

n_rep_grid <- 20
grid_design <- expand.grid(geometry = c("central", "edge"),
                           clus_sd = c(0, 5, 14), stringsAsFactors = FALSE)
grid_arrays <- lapply(seq_len(nrow(grid_design)), function(i) {
  run_cell(if (grid_design$geometry[i] == "central") mid_central else mid_edge,
           grid_design$clus_sd[i], n_rep_grid, seed = 5200 + i)
})
grid_tab <- do.call(rbind, lapply(seq_len(nrow(grid_design)), function(i) {
  cbind(grid_design[i, ], summarise_cell(grid_arrays[[i]]), row.names = NULL)
}))
cell_val <- function(g, cs, est, what) {
  grid_tab[grid_tab$geometry == g & grid_tab$clus_sd == cs &
             grid_tab$estimator == est, what]
}
shift_draws <- function(g, cs) {
  grid_arrays[[which(grid_design$geometry == g &
                       grid_design$clus_sd == cs)]]["moving", "extra", ]
}
n_nonconv <- sum(sapply(grid_arrays, function(a) sum(a[-1, "conv", ] != 0)))
n_se_missing <- sum(sapply(grid_arrays, function(a) sum(is.na(a[-1, "se", ]))))
n_fits_se <- 4 * n_rep_grid * nrow(grid_design)
cell_draws <- function(g, cs, est, what) {
  grid_arrays[[which(grid_design$geometry == g &
                     grid_design$clus_sd == cs)]][est, what, ]
}

## ---- fig-grid ----
est_lab <- c(naive = "detected share", year_p = "year-specific p",
             site_re = "year p + site effect", fixed = "fixed curve",
             moving = "moving curve")
vis_lab <- c("visit days independent", "visits bunched, sd 5 days",
             "visits bunched, sd 14 days")
plot_tab <- grid_tab
plot_tab$estimator <- factor(est_lab[plot_tab$estimator], levels = rev(est_lab))
plot_tab$geometry  <- factor(geom_lab[plot_tab$geometry], levels = geom_lab)
plot_tab$visits    <- factor(vis_lab[match(plot_tab$clus_sd, c(0, 5, 14))],
                             levels = vis_lab)
dodge <- position_dodge(width = 0.6)
grid_slope <- ggplot(plot_tab, aes(slope, estimator, colour = visits)) +
  geom_vline(xintercept = 0, linetype = "dashed", colour = te_body) +
  geom_errorbar(aes(xmin = slope - 2 * slope_mcse, xmax = slope + 2 * slope_mcse),
                orientation = "y", width = 0, position = dodge, linewidth = 0.6) +
  geom_point(size = 2.2, position = dodge) +
  facet_wrap(~ geometry) +
  scale_x_continuous(breaks = c(-0.1, -0.05, 0)) +
  scale_colour_manual(values = c(te_gold, te_forest, te_rust), name = NULL) +
  labs(x = "mean fitted trend in logit occupancy per year (truth 0)", y = NULL,
       title = "At the edge, bunched visits defeat the year term",
       subtitle = "bars: two Monte Carlo standard errors either side") +
  theme_datasheet() + theme(legend.position = "bottom")
grid_psi <- ggplot(plot_tab[plot_tab$estimator != "detected share", ],
                   aes(psi, estimator, colour = visits)) +
  geom_vline(xintercept = psi_true, linetype = "dashed", colour = te_body) +
  geom_point(size = 2.2, position = dodge) +
  facet_wrap(~ geometry) +
  scale_colour_manual(values = c(te_gold, te_forest, te_rust), guide = "none") +
  labs(x = "mean fitted occupancy, middle of the series (truth 0.4)", y = NULL) +
  theme_datasheet()
(grid_slope / grid_psi) + plot_layout(heights = c(1.3, 1)) +
  plot_annotation(theme = theme_datasheet())

## ---- heterogeneity ----
centre_grid <- seq(win_lo, win_hi, by = 0.5)
set.seed(6130)
offset_draws <- rnorm(4000, 0, 5)
het_tab <- do.call(rbind, lapply(names(geom_lab), function(g) {
  mid_g <- if (g == "central") mid_central else mid_edge
  do.call(rbind, lapply(seq_len(n_year), function(yr) {
    peak <- mid_g + shift_use * year_c[yr]
    site_p <- vapply(centre_grid, function(cc) {
      d_v <- pmin(pmax(cc + offset_draws, win_lo), win_hi)
      mean(p_max * exp(-(d_v - peak)^2 / (2 * season_sd^2)))
    }, 0)
    p_pool <- mean(site_p)
    data.frame(geometry = g, year = yr, p_mean = p_pool,
               p_cv = sd(site_p) / p_pool,
               det_true = mean(1 - (1 - site_p)^n_visit_use),
               det_homog = 1 - (1 - p_pool)^n_visit_use)
  }))
}))
het_tab$det_ratio <- het_tab$det_true / het_tab$det_homog
het_e <- het_tab[het_tab$geometry == "edge", ]
het_c <- het_tab[het_tab$geometry == "central", ]

## ---- fig-heterogeneity ----
het_plot <- het_tab
het_plot$geometry <- factor(geom_lab[het_plot$geometry], levels = geom_lab)
ggplot(het_plot, aes(year, det_ratio, colour = geometry)) +
  geom_hline(yintercept = 1, linetype = "dashed", colour = te_body) +
  geom_line(linewidth = 0.9) + geom_point(size = 2) +
  scale_colour_manual(values = c(te_forest, te_rust), name = NULL) +
  scale_x_continuous(breaks = 1:n_year) +
  labs(x = "year", y = "true over one-p value", title = "Detected share, occupied",
       subtitle = "true share over what one p implies, visits bunched, sd 5 days") +
  theme_datasheet() + theme(legend.position = "bottom")

## ---- arms ----
n_rep_arm <- 20
edge5 <- grid_arrays[[which(grid_design$geometry == "edge" &
                              grid_design$clus_sd == 5)]]
arm_visits <- lapply(c(2, 8), function(k)
  run_cell(mid_edge, 5, n_rep_arm, seed = 7300 + k, n_visit = k, with_re = FALSE))
arm_shift <- lapply(c(0, -1.5), function(sh)
  run_cell(mid_edge, 5, n_rep_arm, seed = 7400 + 10 * abs(sh), shift = sh,
           with_re = FALSE))
trend_decl <- -0.04
arm_trend <- run_cell(mid_edge, 5, n_rep_arm, seed = 7500, trend = trend_decl,
                      with_re = FALSE)

arm_tab <- rbind(
  cbind(arm = "visits", level = 2, summarise_cell(arm_visits[[1]])),
  cbind(arm = "visits", level = 4, summarise_cell(edge5)),
  cbind(arm = "visits", level = 8, summarise_cell(arm_visits[[2]])),
  cbind(arm = "shift", level = 0, summarise_cell(arm_shift[[1]])),
  cbind(arm = "shift", level = -1.5, summarise_cell(arm_shift[[2]])),
  cbind(arm = "shift", level = -3, summarise_cell(edge5)))
trend_tab <- summarise_cell(arm_trend, trend = trend_decl)
arm_val <- function(a, lv, est, what) {
  arm_tab[arm_tab$arm == a & arm_tab$level == lv & arm_tab$estimator == est, what]
}
tr_val <- function(est, what) trend_tab[trend_tab$estimator == est, what]
shift_trend <- arm_trend["moving", "extra", ]
