# Cleaning species names before you count
# https://tidyecology.com/posts/cleaning-species-names/
#
# The R code of this Tidy Ecology tutorial, chunk by chunk, in the order of the post.
# Run it top to bottom in a fresh R session. Code the post shows but does not run
# is commented out. The prose, and the numbers it quotes, are in the post itself.
# Post last modified: 2026-10-14

## ---- make-nb ----
set.seed(317)

pool <- c("Festuca rubra", "Trifolium pratense", "Plantago lanceolata",
          "Achillea millefolium", "Carex flacca", "Carex flava")
authority <- setNames(c("L.", "L.", "L.", "L.", "Schreb.", "L."), pool)
prob <- c(0.34, 0.20, 0.16, 0.14, 0.09, 0.07)

swap_letters <- function(x) {
  ch <- strsplit(x, "")[[1]]
  ok <- which(ch != " " & c(ch[-1], "") != " ")
  ok <- ok[ok < length(ch)]
  i <- sample(ok, 1)
  ch[c(i, i + 1)] <- ch[c(i + 1, i)]
  paste(ch, collapse = "")
}
mistype <- function(x) {
  switch(sample(6, 1),
         tolower(x),
         paste0(x, " "),
         sub(" ", "  ", x),
         swap_letters(x),
         sub("^(.)[a-z]+ ", "\\1. ", x),
         paste(x, authority[[x]]))
}

effort <- c(5, 7, 9, 12, 15, 18, 22, 27, 33, 40, 50, 60)
plots <- paste0("P", sprintf("%02d", seq_along(effort)))

nb <- do.call(rbind, lapply(seq_along(effort), function(i) {
  sp <- sample(pool, effort[i], replace = TRUE, prob = prob)
  typed <- vapply(sp, function(s) if (runif(1) < 0.30) mistype(s) else s, character(1))
  data.frame(plot = plots[i], recorded = unname(typed), true_name = sp)
}))

## ---- peek ----
head(nb[, c("plot", "recorded")], 8)
nrow(nb)

## ---- headline ----
length(unique(nb$recorded))

## ---- all-variants ----
sort(unique(nb$recorded))

## ---- chain ----
x <- nb$recorded
step <- c(raw = length(unique(x)))

x <- trimws(x)
step["trim whitespace"] <- length(unique(x))

x <- gsub("\\s+", " ", x)
step["squish doubles"] <- length(unique(x))

x <- tolower(x)
step["lower case"] <- length(unique(x))

step

## ---- fig-funnel ----
library(ggplot2)
te_canvas <- theme(plot.background  = element_rect(fill = "#f5f4ee", colour = NA),
                   panel.background = element_rect(fill = "#f5f4ee", colour = NA))

steps <- data.frame(
  stage = factor(c(names(step), "lookup table"), levels = c(names(step), "lookup table")),
  n = c(step, 6),
  kind = c("string", "string", "string", "string", "decision")
)

ggplot(steps, aes(stage, n, group = 1)) +
  geom_hline(yintercept = 6, linetype = "dashed", colour = "#b5534e") +
  geom_step(colour = "#93a87f", linewidth = 1) +
  geom_point(aes(colour = kind), size = 3.6) +
  geom_text(aes(label = n, hjust = ifelse(stage == "lookup table", 1.3, -0.4)),
            vjust = -0.5, size = 3.4, colour = "#16241d") +
  annotate("text", x = 1.4, y = 6, label = "true richness", vjust = 1.6,
           colour = "#b5534e", size = 3.2, hjust = 0) +
  scale_colour_manual(values = c(string = "#275139", decision = "#b5534e")) +
  scale_y_log10(limits = c(5, 60)) +
  labs(x = NULL, y = "distinct names (log scale)", colour = NULL) +
  theme_minimal(base_size = 12) + te_canvas +
  theme(legend.position = "top", panel.grid.minor = element_blank(),
        axis.text.x = element_text(angle = 20, hjust = 1))

## ---- lookup-build ----
clean <- tolower(gsub("\\s+", " ", trimws(nb$recorded)))

lookup <- data.frame(recorded = sort(unique(clean)))
head(lookup, 6)

## ---- lookup-apply ----
key <- tapply(tolower(nb$true_name), clean, function(v) v[1])
accepted <- unname(key[clean])
length(unique(accepted))

## ---- per-plot ----
per <- do.call(rbind, lapply(plots, function(p) {
  s <- nb[nb$plot == p, ]
  data.frame(plot = p, records = nrow(s),
             apparent = length(unique(s$recorded)),
             true = length(unique(s$true_name)))
}))
per$inflation <- per$apparent - per$true
per

## ---- effort-fit ----
fit <- lm(inflation ~ records, data = per)
c(slope = unname(round(coef(fit)[2], 3)),
  p_value = signif(summary(fit)$coefficients[2, 4], 2),
  cor_apparent = round(cor(per$records, per$apparent), 2),
  cor_true = round(cor(per$records, per$true), 2))

## ---- fig-effort ----
long <- rbind(
  data.frame(records = per$records, richness = per$apparent, count = "as typed"),
  data.frame(records = per$records, richness = per$true, count = "cleaned")
)

ggplot(long, aes(records, richness, colour = count)) +
  geom_smooth(method = "lm", formula = y ~ x, se = FALSE, linewidth = 0.8) +
  geom_point(size = 3) +
  scale_colour_manual(values = c("as typed" = "#b5534e", "cleaned" = "#275139")) +
  labs(x = "records in plot (sampling effort)", y = "species richness", colour = NULL) +
  theme_minimal(base_size = 12) + te_canvas +
  theme(legend.position = "top", panel.grid.minor = element_blank())

## ---- adist-typo ----
adist("festuca rubra", "fetsuca rubra")

## ---- adist-real ----
adist("carex flacca", "carex flava")

## ---- fuzzy-fail ----
reference <- setdiff(tolower(pool), "carex flava")

snap <- function(s, ref, max_edit = 2) {
  d <- adist(s, ref)
  i <- which.min(d)
  if (d[i] <= max_edit) ref[i] else NA_character_
}
matched <- vapply(clean, snap, character(1), ref = reference)

flava_records <- sum(nb$true_name == "Carex flava")
flava_to_flacca <- sum(nb$true_name == "Carex flava" & matched %in% "carex flacca")
c(flava_records = flava_records, silently_relabelled = flava_to_flacca,
  richness_after = length(unique(na.omit(matched))))

## ---- fuzzy-abundance ----
before <- sum(nb$true_name == "Carex flacca")
after <- sum(matched == "carex flacca", na.rm = TRUE)
c(flacca_before = before, flacca_after = after,
  per_cent_change = round(100 * (after / before - 1)))
