# Gompertz state-space model in R
# https://tidyecology.com/posts/gompertz-state-space-model/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-05-15

## ---- setup ----
suppressPackageStartupMessages({library(ggplot2); library(dplyr); library(tidyr)})
ink<-"#16241d"; body<-"#2c3a31"; forest<-"#275139"; gold<-"#c9b458"
paper<-"#f5f4ee"; line<-"#dad9ca"; faint<-"#5d6b61"; aban<-"#b5534e"; mown<-"#2f8f63"
theme_te <- function(){
  theme_minimal(base_size=12)+
  theme(plot.background=element_rect(fill=paper,colour=NA),
        panel.background=element_rect(fill=paper,colour=NA),
        panel.grid.minor=element_blank(),
        panel.grid.major=element_line(colour=line,linewidth=.3),
        axis.title=element_text(colour=ink), axis.text=element_text(colour=faint),
        plot.title=element_text(colour=ink,size=12,face="bold"),
        strip.text=element_text(colour=ink,face="bold"),
        legend.position="bottom", legend.title=element_text(colour=ink),
        legend.text=element_text(colour=body))
}

## ---- helpers ----
kf_nll <- function(par, y, fixed_so2=NA){
  a<-par[1]; b<-tanh(par[2]); sp2<-exp(par[3])
  so2 <- if(is.na(fixed_so2)) exp(par[4]) else fixed_so2
  n_t<-length(y); xp<-a/(1-b); Pp<-sp2/(1-b^2); ll<-0
  for(t in 1:n_t){
    v<-y[t]-xp; S<-Pp+so2; K<-Pp/S      # innovation, its variance, Kalman gain
    xf<-xp+K*v; Pf<-(1-K)*Pp            # filtered state and variance
    ll<-ll-0.5*(log(2*pi*S)+v*v/S)      # log-likelihood contribution
    xp<-a+b*xf; Pp<-b^2*Pf+sp2          # predict the next state
  }
  -ll
}
fit_gss <- function(y){
  n<-length(y); x0<-y[-n]; x1<-y[-1]; b0<-cov(x0,x1)/var(x0); a0<-mean(x1)-b0*mean(x0)
  vr<-var(x1-(a0+b0*x0)); b0<-min(max(b0,-0.9),0.9)
  st<-c(a0, atanh(b0), log(0.5*vr), log(0.5*vr))
  op<-optim(st, kf_nll, y=y, method="BFGS", hessian=TRUE, control=list(maxit=1000))
  b<-tanh(op$par[2]); V<-tryCatch(solve(op$hessian), error=function(e) matrix(NA,4,4))
  se_tb<-sqrt(V[2,2])
  list(a=op$par[1], b=b, sp=sqrt(exp(op$par[3])), so=sqrt(exp(op$par[4])),
       theta_b=op$par[2], se_tb=se_tb, se_b=(1-b^2)*se_tb, conv=op$convergence, ll=-op$value)
}
naive_b <- function(y){ n<-length(y); cov(y[-n],y[-1])/var(y[-n]) }
kf_smooth <- function(y,a,b,sp2,so2){ n_t<-length(y)
  xp<-xf<-Pp<-Pf<-numeric(n_t); m<-a/(1-b); P<-sp2/(1-b^2)
  for(t in 1:n_t){ xp[t]<-m; Pp[t]<-P; v<-y[t]-m; S<-P+so2; K<-P/S
    xf[t]<-m+K*v; Pf[t]<-(1-K)*P; m<-a+b*xf[t]; P<-b^2*Pf[t]+sp2 }
  xs<-numeric(n_t); xs[n_t]<-xf[n_t]
  for(t in (n_t-1):1){ C<-Pf[t]*b/Pp[t+1]; xs[t]<-xf[t]+C*(xs[t+1]-xp[t+1]) }
  xs
}

## ---- simulate ----
b_true<-0.70; sp_true<-0.15; so_true<-0.15
Xstar<-log(200); a_true<-(1-b_true)*Xstar; Tlen<-50

set.seed(201)
Xstate <- numeric(Tlen); Xstate[1]<-Xstar
for(t in 1:(Tlen-1)) Xstate[t+1] <- a_true + b_true*Xstate[t] + rnorm(1,0,sp_true)
Yobs <- Xstate + rnorm(Tlen, 0, so_true)     # observed counts on the log scale

## ---- fit ----
nb <- naive_b(Yobs)
f  <- fit_gss(Yobs)
ci_b <- tanh(f$theta_b + c(-1,1)*1.96*f$se_tb)

round(c(naive_slope=nb, gss_b=f$b, ci_lo=ci_b[1], ci_hi=ci_b[2],
        gss_sp=f$sp, gss_so=f$so), 3)

## ---- fig-observed ----
xs <- kf_smooth(Yobs, f$a, f$b, f$sp^2, f$so^2)
d1 <- data.frame(year=1:Tlen, true=Xstate, obs=Yobs, smooth=xs)
ggplot(d1, aes(year))+
  geom_line(aes(y=true, colour="True log abundance"), linewidth=.9)+
  geom_point(aes(y=obs, colour="Observed count"), size=1.8, alpha=.9)+
  geom_line(aes(y=smooth, colour="State-space estimate"), linewidth=.9, linetype="22")+
  scale_colour_manual(values=c("True log abundance"=forest,"Observed count"=gold,"State-space estimate"=aban))+
  labs(x="year", y="log abundance", colour=NULL,
       title="Observed counts are a noisy view of the true state")+
  theme_te()

## ---- mc ----
set.seed(7100); M<-400; nbv<-numeric(M); gbv<-numeric(M); cv<-integer(M)
for(i in 1:M){ x<-numeric(Tlen); x[1]<-Xstar
  for(t in 1:(Tlen-1)) x[t+1]<-a_true+b_true*x[t]+rnorm(1,0,sp_true)
  y<-x+rnorm(Tlen,0,so_true); nbv[i]<-naive_b(y); ff<-fit_gss(y); gbv[i]<-ff$b; cv[i]<-ff$conv }
round(c(naive_mean=mean(nbv), gss_mean=mean(gbv[cv==0]),
        gss_sd=sd(gbv[cv==0])), 3)

## ---- profile ----
so_grid <- seq(0.02, 0.34, by=0.01)
prof <- sapply(so_grid, function(so){
  st<-c(f$a, atanh(min(max(nb,-0.9),0.9)), log(0.02))
  op<-optim(st, kf_nll, y=Yobs, fixed_so2=so^2, method="BFGS", control=list(maxit=1000))
  -op$value })
prof_rel <- prof - max(prof)
range(so_grid[prof_rel > -1.92])   # observation SD values within a 95% likelihood interval

## ---- fig-profile ----
d2 <- data.frame(so=so_grid, rel=prof_rel)
ggplot(d2, aes(so, rel))+
  geom_hline(yintercept=-1.92, colour=faint, linetype="22", linewidth=.4)+
  geom_line(colour=forest, linewidth=1)+
  geom_vline(xintercept=f$so, colour=aban, linewidth=.7)+
  geom_vline(xintercept=so_true, colour=gold, linewidth=.7, linetype="22")+
  annotate("text", x=f$so, y=0, label="MLE ", hjust=1, vjust=1.4, colour=aban, size=3.3)+
  annotate("text", x=so_true, y=-3.2, label=" truth", hjust=0, colour="#8a7a1e", size=3.3)+
  labs(x="observation standard deviation (fixed)", y="profile log-likelihood, relative to maximum",
       title="The observation-error variance is barely identified")+
  theme_te()
