# POL 211 Week 8: Confidence intervals -----------------------------------------
#
# This script repeats the analysis from class, in the order of the slides.
# Save it in your POL 211 project's notes/ folder and run it one chunk at a
# time.
#
# Data: resume.csv (the résumé experiment from week 1), in your project's
# data/ folder. Everything else is simulated, so we know the truth.


# Packages -------------------------------------------------------------------

library(tidyverse)   # wrangling + plotting
library(here)        # file paths that work on any computer


# 1. The résumé study: the gap, its standard error, its 95% interval ----------

resume = read_csv(here("data", "resume.csv"))

# callback rate and number of résumés for each race of name
groups = resume |>
  group_by(race) |>
  summarize(n = n(), p_hat = mean(call))
groups

p_aa = groups$p_hat[groups$race == "black"]   # AA-named résumés
p_w = groups$p_hat[groups$race == "white"]    # white-named résumés
n_aa = groups$n[groups$race == "black"]
n_w = groups$n[groups$race == "white"]

# the estimate: the difference in callback rates
gap = p_aa - p_w

# its standard error (week 5's formula, with the p-hats plugged in: week 6)
se = sqrt(p_aa * (1 - p_aa) / n_aa + p_w * (1 - p_w) / n_w)

# the 95% interval: estimate plus or minus 1.96 standard errors
ci = gap + c(-1.96, 1.96) * se

# in percentage points: about -3.2, with an interval of about -4.7 to -1.7
100 * gap
100 * se
100 * ci


# 2. Rerun the study: one rerun, one interval -----------------------------------

# week 4's DGP for the résumé study. We wrote it, so we know the truth:
# AA-named résumés get callbacks 3.2 points less often
make_resumes = function() {
  tibble(name = sample(c("white", "black"), size = 4870, replace = TRUE),
         exper = sample(0:10, size = 4870, replace = TRUE),
         age = sample(22:55, size = 4870, replace = TRUE),
         distance = sample(0:30, size = 4870, replace = TRUE)) |>
    mutate(p = 0.099 - 0.032 * (name == "black") + 0.004 * exper -
                 0.0005 * (age - 22) - 0.001 * distance,
           call = rbinom(4870, size = 1, prob = p))
}
true_gap = -0.032

# one run of the study: the gap and its plug-in standard error
gap_and_se = function() {
  make_resumes() |>
    group_by(name) |>
    summarize(p_hat = mean(call), n = n()) |>
    summarize(gap = p_hat[name == "black"] - p_hat[name == "white"],
              se = sqrt(sum(p_hat * (1 - p_hat) / n)))
}

# one rerun, with its interval. Does it contain the truth?
set.seed(211)
one_run = gap_and_se() |>
  mutate(lower = gap - 1.96 * se,
         upper = gap + 1.96 * se,
         catches = lower <= true_gap & upper >= true_gap)
one_run


# 3. Ten reruns, then 1,000: coverage ------------------------------------------

# 1,000 reruns, each with its own estimate, its own SE, and its own interval.
# (Same seed as above, so rerun 1 is the one from section 2.) Takes a few seconds
set.seed(211)
reruns = map(1:1000, \(i) gap_and_se()) |>
  list_rbind() |>
  mutate(run = row_number(),
         lower = gap - 1.96 * se,
         upper = gap + 1.96 * se,
         catches = lower <= true_gap & upper >= true_gap)

# the first ten: the estimates vary a lot; the SEs barely move
reruns |>
  slice_head(n = 10)

# coverage: the share of reruns whose interval contains the truth (about 95%)
mean(reruns$catches)

# the first 100 intervals; orange ones miss the truth
reruns |>
  slice_head(n = 100) |>
  ggplot(aes(y = run, x = 100 * gap, xmin = 100 * lower, xmax = 100 * upper, color = catches)) +
  geom_vline(xintercept = 100 * true_gap, linewidth = 1.2) +
  geom_linerange() +
  geom_point(size = 1) +
  scale_color_manual(values = c(`TRUE` = "steelblue", `FALSE` = "darkorange3")) +
  labs(x = "Callback gap, with its 95% interval (percentage points)", y = "Rerun",
       color = "Catches the truth?")


# 🚨 Your turn: what does [1, 9] mean? ----------------------------------------

# A get-out-the-vote study finds an effect on turnout of 5 points, with a
# standard error of 2 points.
#
# 1. By hand: the 95% confidence interval (use 2 in place of 1.96)
# 2. Which of these are right?
#    a. "There's a 95% probability that the true effect is between 1 and 9 points"
#    b. "The interval [1, 9] came from a method that catches the true effect in 95% of studies"
#    c. "95% of studies like this one would find an effect between 1 and 9 points"
#    d. "Betting that the true effect is between 1 and 9 points is a sensible bet:
#        the method behind it wins in 95% of studies"


# 4. A 95% interval is not a forecast of the next study ------------------------

# for each rerun's interval, the share of the OTHER 999 reruns' estimates that
# land inside it (the "capture percentage")
capture_each = map_dbl(1:nrow(reruns), \(i) {
  others = reruns$gap[-i]
  mean(others >= reruns$lower[i] & others <= reruns$upper[i])
})

# on average, about 83%, not 95%
mean(capture_each)

# one rerun whose estimate landed high (about 1.5 SEs above the truth)
example = reruns |>
  slice_min(abs(gap - (true_gap + 1.5 * se)), n = 1, with_ties = FALSE)
example
capture_each[example$run]   # its interval holds only about 68% of the others


# 5. Where the multipliers come from: other confidence levels ----------------

# the multiplier is where the middle share of a standard normal ends.
# qnorm() gives the point with a given share BELOW it, so for the middle 95%,
# ask for 0.95 + 0.025 = 0.975
qnorm(0.975)

# alpha is the leftover share; the multiplier for any level is qnorm(1 - alpha / 2)
tibble(level = c(0.50, 0.80, 0.90, 0.95, 0.99)) |>
  mutate(alpha = 1 - level,
         z = qnorm(1 - alpha / 2),
         lower = 100 * (gap - z * se),     # the résumé gap's interval at each level
         upper = 100 * (gap + z * se))


# 6. Estimating the SE: the t distribution -------------------------------------

# the 95% multiplier from the t distribution, for samples of different sizes:
# bigger for small samples, 1.96 for big ones
tibble(n = c(3, 5, 10, 30, 100, 1000)) |>
  mutate(df = n - 1,
         multiplier = qt(0.975, df = df))

# check it: a normal population (week 6's canvass, mean 40, SD 20). Draw
# samples, plug in the sample SD, and build intervals with 1.96 and with t.
# 10,000 reruns at each size
set.seed(211)
t_vs_z = expand_grid(n = c(5, 30), run = 1:10000) |>
  mutate(stats = map(n, \(k) {
    y = rnorm(k, mean = 40, sd = 20)
    tibble(ybar = mean(y), s = sd(y))
  })) |>
  unnest(stats) |>
  mutate(z_catches = abs(ybar - 40) <= 1.96 * s / sqrt(n),
         t_catches = abs(ybar - 40) <= qt(0.975, df = n - 1) * s / sqrt(n))

# coverage: with 5 observations, 1.96 falls short of 95%; t gets it right
t_vs_z |>
  group_by(n) |>
  summarize(coverage_z = mean(z_catches), coverage_t = mean(t_catches))


# 7. When the CLT carries it: a skewed population --------------------------------

# week 7's weekly protests: right-skewed, nothing like a normal
set.seed(211)
protests = round(sample(qgamma(ppoints(10000), shape = 4, scale = 3)))
mu_protests = mean(protests)

set.seed(211)
skewed = expand_grid(n = c(5, 30, 200), run = 1:10000) |>
  mutate(stats = map(n, \(k) {
    y = sample(protests, size = k, replace = TRUE)
    tibble(ybar = mean(y), s = sd(y))
  })) |>
  unnest(stats) |>
  mutate(t_catches = abs(ybar - mu_protests) <= qt(0.975, df = n - 1) * s / sqrt(n))

# a little short of 95% for small samples; right on by a few hundred
skewed |>
  group_by(n) |>
  summarize(coverage = mean(t_catches))


# 8. Using the résumé interval: read both ends ----------------------------------

# each end, as a share of the white-named callback rate (about 9.7%)
-ci[2] / p_w   # the near end: the smallest gap the data allow (about 17% fewer callbacks)
-ci[1] / p_w   # the far end: the largest gap the data allow (about 49% fewer)

# how tight is it? the far end divided by the near end
ci[1] / ci[2]


# 9. Comparing two estimates: overlap vs. the difference ------------------------

# two made-up states: 52% and 46%, each with a standard error of 2 points
est_a = 52
est_b = 46
se_a = 2
se_b = 2

# each state's interval (using 2 in place of 1.96): they overlap
est_a + c(-2, 2) * se_a
est_b + c(-2, 2) * se_b

# the difference, its SE (week 5: variances add), and its interval: excludes zero
diff = est_a - est_b
se_diff = sqrt(se_a^2 + se_b^2)
diff + c(-2, 2) * se_diff


# 🚨 Your turn: the race gap, by the name's sex ---------------------------------

# Is the race gap the same for women's and men's names? Estimate the gap
# (AA - white) separately for women's names and men's names, each with a 95%
# interval. Start here:
resume |>
  group_by(sex, race) |>
  summarize(n = n(), p_hat = mean(call))

# 1. Read both ends of each interval. What does each one tell you?
# 2. Why is one interval so much wider than the other?
# 3. Is the race gap different for women's and men's names? (Don't eyeball the overlap)


# 10. Use confidence intervals for planning --------------------------------------

# if our own résumé study found the same gap, how wide would its interval be?
# The margin is 1.96 * SE, and the SE shrinks with sqrt(n), so
# margin(n) = margin with 4,870 résumés * sqrt(4870 / n)
margin_now = 100 * 1.96 * se
tibble(n = c(500, 1000, 2000, 4870, 20000, 45000)) |>
  mutate(margin = margin_now * sqrt(4870 / n),
         lower = 100 * gap - margin,
         upper = 100 * gap + margin)


# 11. R does all of this for you ------------------------------------------------

# lm() with confint(): the second row is the gap's 95% interval
# (lm() uses the t multiplier, which with 4,868 degrees of freedom is 1.96)
lm(call ~ I(race == "black"), data = resume) |>
  confint()
