# POL 211 Week 7: Which estimator should we use? -----------------------------
#
# This script repeats the analysis from class, in the order of the slides.
# Save it in your POL 211 project's notes/ folder and run it one chunk at a
# time.
#
# No data to download this week: every dataset here is made up, so we know
# the truth and can judge each estimator against it.


# Packages -------------------------------------------------------------------

library(tidyverse)   # wrangling + plotting


# 1. The estimand: the average number of protests per week --------------------

# a made-up population of 10,000 city-weeks. Most weeks are quiet and a few
# are very busy (a right-skewed shape, like most counts). We wrote it, so we
# know the truth: mu, the average, is 12, and sigma, the SD, is 6
set.seed(211)
protests = round(sample(qgamma(ppoints(10000), shape = 4, scale = 3)))
mu = mean(protests)
sigma = sqrt(mean((protests - mu)^2))   # divide by N: the population's SD
mu
sigma

ggplot(tibble(protests = protests), aes(x = protests)) +
  geom_bar() +
  geom_vline(xintercept = mu, color = "darkorange3", linewidth = 1.2) +
  coord_cartesian(xlim = c(0, 40)) +
  labs(x = "Protests in one city-week", y = "City-weeks (out of 10,000)")


# 2. Four candidate estimators of the average ---------------------------------

# a function that takes one sample of weeks, y, and applies four rules to it:
# one row per estimator, one estimate each. The factor() keeps them in order
est_names = c("1. First week only", "2. Always say 2", "3. Sample average", "4. Sum ÷ (n + 5)")
four = function(y) {                                   # y: the weeks we sampled
  tibble(estimator = factor(est_names, levels = est_names),
         estimate = c(y[1],                            # 1. the first value in the sample
                      2,                               # 2. ignore the data
                      mean(y),                         # 3. add up, divide by n
                      sum(y) / (length(y) + 5)))       # 4. add up, divide by n + 5
}

# same data in, four different answers out. The truth is 12
set.seed(211)
one_sample = sample(protests, size = 20, replace = TRUE)   # 20 weeks, drawn at random
four(one_sample)


# 3. Rerun the study: each estimator has its own sampling distribution -------

# 2,000 reruns at each of three sample sizes. Each rerun draws a fresh sample
# and applies four() to it. map() returns a small tibble per row; unnest()
# spreads them out, so we get one row per rerun per estimator
set.seed(211)
sims = expand_grid(n = c(20, 100, 3000), run = 1:2000) |>
  mutate(est = map(n, \(k) four(sample(protests, size = k, replace = TRUE)))) |>
  unnest(est)

# the four sampling distributions with 20 weeks each. Orange: the truth
sims |>
  filter(n == 20) |>
  ggplot(aes(x = estimate)) +
  geom_histogram(binwidth = 1, boundary = 0) +
  geom_vline(xintercept = mu, color = "darkorange3", linewidth = 1.2) +
  facet_wrap(~estimator, nrow = 1, scales = "free_y") +
  coord_cartesian(xlim = c(-1, 30)) +
  labs(x = "Estimate of the average weekly protests", y = "Reruns (out of 2,000)")


# 4. Bias: where is each sampling distribution centered? -------------------

# the average of the 2,000 reruns at n = 20. By week 5's rules, the centers
# are mu, 2, mu, and n * mu / (n + 5) = 20 * 12 / 25 = 9.6
sims |>
  filter(n == 20) |>
  group_by(estimator) |>
  summarize(average = mean(estimate))


# 5. Variance: how spread out is each sampling distribution? ----------------

# by week 5's rules: sigma^2, 0, sigma^2 / n, and n * sigma^2 / (n + 5)^2.
# A helper that gives each estimator's center, variance, bias, and MSE at
# sample size n, from those formulas
theory = function(n) {
  tibble(estimator = factor(est_names, levels = est_names),
         center = c(mu, 2, mu, n * mu / (n + 5)),
         variance = c(sigma^2, 0, sigma^2 / n, n * sigma^2 / (n + 5)^2)) |>
    mutate(bias = center - mu,
           mse = bias^2 + variance)
}
th20 = theory(20)

# the spread (SD = square root of the variance) at n = 20: 6, 0, 1.34, 1.07
th20 |>
  mutate(sd = sqrt(variance)) |>
  select(estimator, variance, sd)

# check against the simulation: the SD of the 2,000 reruns
sims |>
  filter(n == 20) |>
  group_by(estimator) |>
  summarize(sd = sd(estimate))

# efficiency: among the unbiased estimators (1 and 3), the sample average has
# the smaller spread
th20 |>
  mutate(unbiased = if_else(bias == 0, "Unbiased", "Biased")) |>
  ggplot(aes(x = fct_rev(estimator), y = sqrt(variance), fill = unbiased)) +
  geom_col(width = 0.7) +
  coord_flip() +
  labs(x = NULL, y = "Spread of the estimator (SD), n = 20", fill = NULL)


# 6. Bias shifts the cloud of estimates; variance spreads it out --------------

# week 2's picture: simulated throws at a target, one dot per estimate. The
# bullseye (0, 0) is the truth
set.seed(211)
panels = tribble(
  ~panel,                     ~shift, ~spread,
  "Low bias, low variance",   0,      0.25,
  "Low bias, high variance",  0,      0.85,
  "High bias, low variance",  1.4,    0.25,
  "High bias, high variance", 1.4,    0.85
) |>
  mutate(panel = fct_inorder(panel))

throws = panels |>
  slice(rep(1:4, each = 15)) |>                        # 15 throws per panel
  mutate(x = shift + rnorm(n(), 0, spread),
         y = 0.6 * shift + rnorm(n(), 0, spread))

ggplot(throws, aes(x, y)) +
  annotate("point", x = 0, y = 0, color = "darkorange3", size = 5) +
  geom_point(size = 3, alpha = 0.8) +
  facet_wrap(~panel, nrow = 1) +
  coord_fixed(xlim = c(-3.2, 3.2), ylim = c(-3.2, 3.2))


# 7. Mean squared error: bias and variance together --------------------------

# MSE = bias^2 + variance. Its square root is a typical miss, in protests
th20 |>
  transmute(estimator,
            bias_squared = bias^2,
            variance,
            mse,
            typical_miss = sqrt(mse))


# 8. Which estimator wins depends on the truth ---------------------------------

# each estimator's MSE (n = 20) in possible worlds with different true
# averages, holding sigma at 6. The dashed line is our world (12)
tibble(truth = seq(0, 20, by = 0.05)) |>
  mutate(`1. First week only` = sigma^2,
         `2. Always say 2` = (2 - truth)^2,
         `3. Sample average` = sigma^2 / 20,
         `4. Sum ÷ (n + 5)` = (truth / 5)^2 + 20 * sigma^2 / 25^2) |>
  pivot_longer(-truth, names_to = "estimator", values_to = "mse") |>
  ggplot(aes(x = truth, y = mse, color = estimator)) +
  geom_vline(xintercept = mu, linetype = "dashed") +
  geom_line(linewidth = 1.2) +
  coord_cartesian(ylim = c(0, 40)) +
  labs(x = "A possible true average number of protests per week",
       y = "MSE at n = 20", color = NULL)


# 9. Consistent: do the estimators close in on the truth as n grows? ---------

# all four estimators at n = 20, 100, and 3,000. The first week and "always
# say 2" never improve; the other two close in on 12
sims |>
  ggplot(aes(x = estimate)) +
  geom_histogram(binwidth = 0.25, boundary = 0) +
  geom_vline(xintercept = mu, color = "darkorange3", linewidth = 1) +
  facet_wrap(~ n + estimator, scales = "free_y", ncol = 4, labeller = label_both) +
  coord_cartesian(xlim = c(0, 24)) +
  labs(x = "Estimate", y = NULL)

# the spread of each estimator at each sample size
sims |>
  group_by(n, estimator) |>
  summarize(average = mean(estimate), sd = sd(estimate), .groups = "drop")


# 10. Counting the dead: a world where we know the truth -----------------------

# 1,200 people were killed. A truth commission records each death with
# chance 1/2, a human-rights NGO with chance 1/3, independently. rbinom()
# gives each death a 1 (on the list) or a 0 (not on it)
set.seed(211)
lists = tibble(run = 1:2000) |>
  mutate(res = map(run, \(i) {
    commission = rbinom(1200, size = 1, prob = 1 / 2)
    ngo = rbinom(1200, size = 1, prob = 1 / 3)
    tibble(documented = sum(commission | ngo),                # names on either list
           two_lists = sum(commission) * sum(ngo) /
                       sum(commission & ngo))                 # n_A x n_B / (names on both)
  })) |>
  unnest(res)

# the names we have average about 800, a third short of the truth (1,200).
# The two-list estimate is centered on the truth
lists |>
  summarize(names_we_have = mean(documented),
            two_lists = mean(two_lists))
