# Cleaning GBIF and iNaturalist records in R
# https://tidyecology.com/posts/cleaning-gbif-occurrence-data/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-14

## ----
# library(rgbif)

# key <- name_backbone("Pulsatilla patens")$usageKey
# raw <- occ_search(taxonKey = key, hasCoordinate = TRUE, limit = 2000)$data

# # iNaturalist records directly, through the spocc package:
# # spocc::occ(query = "Pulsatilla patens", from = "inat", limit = 2000)

## ---- setup ----
library(dplyr)
library(sf)
library(ggplot2)

paper <- "#f5f4ee"; ink <- "#16241d"; forest <- "#275139"
gold  <- "#cda23f"; abandoned <- "#b5534e"; faint <- "#5d6b61"; line <- "#dad9ca"

theme_te <- theme_minimal(base_size = 12) +
  theme(panel.grid.minor = element_blank(),
        panel.grid.major = element_line(colour = line, linewidth = 0.3),
        plot.background = element_rect(fill = paper, colour = NA),
        panel.background = element_rect(fill = paper, colour = NA),
        axis.title = element_text(colour = ink),
        axis.text = element_text(colour = faint),
        plot.title = element_text(colour = ink, face = "bold"))

## ---- read ----
records <- read.csv("occurrences_raw.csv", stringsAsFactors = FALSE,
                    na.strings = "")
nrow(records)

## ---- peek ----
head(records[, c("decimalLatitude", "decimalLongitude",
                 "coordinateUncertaintyInMeters", "basisOfRecord",
                 "occurrenceStatus")], 4)

## ---- step-coords ----
clean <- records |>
  filter(!is.na(decimalLatitude), !is.na(decimalLongitude)) |>
  filter(decimalLatitude != 0 | decimalLongitude != 0) |>
  filter(between(decimalLatitude, -90, 90),
         between(decimalLongitude, -180, 180))
nrow(clean)

## ---- step-basis ----
clean <- clean |>
  filter(is.na(occurrenceStatus) | occurrenceStatus == "PRESENT") |>
  filter(basisOfRecord %in% c("HUMAN_OBSERVATION", "PRESERVED_SPECIMEN",
                              "MACHINE_OBSERVATION", "OBSERVATION"))
nrow(clean)

## ---- step-precision ----
clean <- clean |>
  filter(is.na(coordinateUncertaintyInMeters) |
         coordinateUncertaintyInMeters <= 10000)
nrow(clean)

## ---- step-dupes ----
clean <- clean |>
  distinct(species, decimalLatitude, decimalLongitude, .keep_all = TRUE)
nrow(clean)

## ---- step-region ----
pts <- st_as_sf(clean, coords = c("decimalLongitude", "decimalLatitude"),
                crs = 4326, remove = FALSE)

region <- st_as_sfc(st_bbox(c(xmin = 20, ymin = 44, xmax = 28, ymax = 49),
                            crs = st_crs(4326)))
inside <- st_within(pts, region, sparse = FALSE)[, 1]
pts    <- pts[inside, ]
nrow(pts)

## ---- fig-clean ----
ggplot() +
  geom_sf(data = region, fill = NA, colour = faint, linewidth = 0.4) +
  geom_sf(data = pts, colour = forest, size = 2.4, alpha = 0.9) +
  labs(x = NULL, y = NULL) +
  theme_te

## ----
# library(CoordinateCleaner)

# valid <- cc_val(records, lon = "decimalLongitude", lat = "decimalLatitude")
# flags <- clean_coordinates(
#   valid,
#   lon = "decimalLongitude", lat = "decimalLatitude", species = "species",
#   tests = c("zeros", "equal", "duplicates", "seas", "centroids", "institutions"))

# clean <- valid[flags$.summary, ]   # keep records that passed every test

## ---- centroid-sim ----
cen_avail <- seq(-4, 4, length.out = 2001)   # flat availability: quadrature nodes
fit_niche <- function(x_rec) {                # log-quadratic point process by optim
  nll <- function(b) { eta <- b[1] * cen_avail + b[2] * cen_avail^2; top <- max(eta)
    -(sum(b[1] * x_rec + b[2] * x_rec^2) -
        length(x_rec) * (top + log(mean(exp(eta - top))))) }
  b <- optim(c(0, -1), nll, method = "BFGS")$par
  c(opt = -b[1] / (2 * b[2]), tol = sqrt(-1 / (2 * b[2])))
}
tol_true <- 0.8; n_rec <- 400; n_cen_rep <- 100
mix_tol <- function(e, d) sqrt((1 - e) * tol_true^2 + e * (1 - e) * d^2)  # the formula
cen_design <- expand.grid(share = c(0.02, 0.05, 0.10), cen = c(0, 1.5, 3))
set.seed(348)
cen_fits <- t(sapply(seq_len(nrow(cen_design)), function(i) {
  e <- cen_design$share[i]; d <- cen_design$cen[i]
  fits <- replicate(n_cen_rep, {
    x_rec <- rnorm(n_rec, 0, tol_true); x_rec <- x_rec[abs(x_rec) < 4]
    k <- round(e * length(x_rec))                # records moved to the centroid
    x_bad <- c(x_rec[seq_len(length(x_rec) - k)], rep(d, k))
    c(fit_niche(x_bad), sd_rec = sqrt(mean((x_bad - mean(x_bad))^2)),  # records' own SD
      thin = fit_niche(unique(x_bad)))
  })
  c(rowMeans(fits), se_tol = sd(fits["tol", ]) / sqrt(n_cen_rep))
}))
cen_tab <- data.frame(cen_design, cen_fits,
                      cf_tol = mix_tol(cen_design$share, cen_design$cen),
                      cf_opt = cen_design$share * cen_design$cen)
round(cen_tab, 3)
cen_row <- function(e, d) cen_tab[cen_tab$share == e & cen_tab$cen == d, ]
cen_edge <- cen_tab$share == 0.10 & cen_tab$cen == 3     # the widest curve
stopifnot(which.max(abs(cen_tab$tol - cen_tab$cf_tol)) == which(cen_edge))

## ---- coord-repeats ----
coord_counts <- count(records, decimalLatitude, decimalLongitude, sort = TRUE)
head(coord_counts, 3)
records[records$countryCode %in% "PT",
        c("decimalLatitude", "decimalLongitude", "countryCode")]
