# POL 211 Week 6: The shape of the sampling distribution ---------------------
#
# This script repeats the analysis from class, in the order of the slides.
# Save it in your POL 211 project's notes/ folder and run it one chunk at a
# time.
#
# Before you start:
#   - resume.csv (from week 3 on the course schedule) should already be in
#     your project's data/ folder
#
# Today has two activities (the canvass with pnorm() and qnorm(), and the
# average size of protests). Each has its own 🚨 section with the questions;
# the answers are yours to work out.


# Packages -------------------------------------------------------------------

library(tidyverse)   # wrangling + plotting
library(here)        # file paths that work on any computer


# 1. Recap: the sampling distribution is centered on the truth ---------------

# week 4's made-up résumé study. The systematic part sets each résumé's chance
# of a call (p); the random part is each employer's weighted coin flip. An
# African American-sounding name (AA; labeled "black" in the data) lowers the
# chance of a call by 3.2 points: that's the true gap we built in
make_resumes = function() {
  tibble(name = sample(c("white", "black"), size = 4870, replace = TRUE),
         exper = sample(0:10, size = 4870, replace = TRUE),
         age = sample(22:55, size = 4870, replace = TRUE),
         distance = sample(0:30, size = 4870, replace = TRUE)) |>
    mutate(p = 0.099 - 0.032 * (name == "black") + 0.004 * exper -
                 0.0005 * (age - 22) - 0.001 * distance,      # systematic part
           call = rbinom(4870, size = 1, prob = p))            # random part
}

# run the study once and return just the estimate: the gap in callback rates
run_resumes = function() {
  make_resumes() |>
    summarize(gap = mean(call[name == "black"]) - mean(call[name == "white"])) |>
    pull(gap)
}

# 1,000 reruns (same seed as weeks 4 and 5, so the same 1,000 studies)
set.seed(211)
gaps = tibble(run = 1:1000) |>
  mutate(gap = map_dbl(run, \(i) run_resumes()))

# the sampling distribution, centered on the true gap (orange line)
ggplot(gaps, aes(x = 100 * gap)) +
  geom_histogram(binwidth = 0.25, center = 0, color = "white") +
  geom_vline(xintercept = -3.2, color = "darkorange3", linewidth = 1.2) +
  labs(x = "Callback gap in one study (percentage points)", y = "Studies (out of 1,000)")


# 2. Recap: the formula needs p, and all we have is p-hat ---------------------

# the real study: Bertrand and Mullainathan (2004), via Imai
resume = read_csv(here("data", "resume.csv"))

# each group's callback share (p-hat) and size
groups = resume |>
  group_by(race) |>
  summarize(n = n(), p_hat = mean(call))
groups

# week 5's standard error formula, with p-hat plugged in for p. This is an
# estimate of the sampling distribution's width, from one run of data
groups |>
  summarize(se_plug_in = sqrt(sum(p_hat * (1 - p_hat) / n)))

# week 1's regression gives (almost) the same number
m_resume = lm(call ~ I(race == "black"), data = resume)
summary(m_resume)


# 3. Every sampling distribution we've built looks like a bell ---------------

# Problem Set 5's canvass: does knocking on doors raise a voter's feeling
# thermometer toward the candidate? (We build it step by step in section 8)
make_canvass = function(n) {
  tibble(dem = rbinom(n, size = 1, prob = 0.5),
         canvassed = rbinom(n, size = 1, prob = 0.5)) |>
    mutate(therm = 40 + 5 * canvassed + 20 * dem + rnorm(n, mean = 0, sd = 20))
}
run_canvass = function(n) {
  make_canvass(n) |>
    summarize(gap = mean(therm[canvassed == 1]) - mean(therm[canvassed == 0])) |>
    pull(gap)
}

# 1,000 canvass studies with 1,000 voters each
set.seed(211)
canvass_gaps = tibble(run = 1:1000) |>
  mutate(gap = map_dbl(run, \(i) run_canvass(1000)))

# both sampling distributions are bells, even though one callback is just 0 or 1
bind_rows(
  gaps |> transmute(study = "Résumé study: callback gap (points)", estimate = 100 * gap),
  canvass_gaps |> transmute(study = "Canvass: thermometer gap", estimate = gap)
) |>
  ggplot(aes(x = estimate)) +
  geom_histogram(bins = 30, color = "white") +
  facet_wrap(~study, scales = "free_x") +
  labs(x = "Estimate in one study", y = "Studies (out of 1,000)")


# 4. The law of large numbers: twenty people roll a die ----------------------

# 20 people each roll a die 2,000 times. After every roll, each one computes
# the average of all their rolls so far (cumsum() / roll)
set.seed(211)
rolls = expand_grid(person = 1:20, roll = 1:2000) |>
  mutate(value = sample(1:6, size = n(), replace = TRUE)) |>
  group_by(person) |>
  mutate(running_average = cumsum(value) / roll) |>
  ungroup()

# every running average closes in on the expectation of one roll, 3.5.
# The x-axis is on a log scale so the early, wild part is visible
ggplot(rolls, aes(x = roll, y = running_average, group = person)) +
  geom_hline(yintercept = 3.5, color = "darkorange3", linewidth = 1.2) +
  geom_line(alpha = 0.5) +
  scale_x_log10() +
  labs(x = "Number of rolls so far (log scale)", y = "Average of the rolls so far")


# 5. The funnel's width is week 5's formula: sigma / sqrt(n) ------------------

# one roll's SD is sqrt(35/12), about 1.71 (Problem Set 5). Week 5: the SD of
# an average of n rolls is sigma / sqrt(n)
sigma_die = sqrt(35 / 12)

# the band 3.5 plus or minus 2 SDs of the average, at every number of rolls
band = tibble(roll = 1:2000) |>
  mutate(sd_average = sigma_die / sqrt(roll),
         upper = 3.5 + 2 * sd_average,
         lower = 3.5 - 2 * sd_average)

# the SD of the average at 10, 100, and 1,000 rolls: ten times the rolls,
# about a third of the width
band |>
  filter(roll %in% c(10, 100, 1000))

ggplot(rolls, aes(x = roll)) +
  geom_line(aes(y = running_average, group = person), alpha = 0.25) +
  geom_hline(yintercept = 3.5, color = "darkorange3", linewidth = 1) +
  geom_line(data = band, aes(y = upper), color = "goldenrod", linewidth = 1.2, linetype = "dashed") +
  geom_line(data = band, aes(y = lower), color = "goldenrod", linewidth = 1.2, linetype = "dashed") +
  scale_x_log10() +
  coord_cartesian(ylim = c(1, 6.5)) +
  labs(x = "Number of rolls so far (log scale)", y = "Average of the rolls so far")


# 6. Check it: the plug-in standard error closes in on the true width ---------

# the résumé DGP with the number of résumés as an input
make_resumes_n = function(n) {
  tibble(name = sample(c("white", "black"), size = n, replace = TRUE),
         exper = sample(0:10, size = n, replace = TRUE),
         age = sample(22:55, size = n, replace = TRUE),
         distance = sample(0:30, size = n, replace = TRUE)) |>
    mutate(p = 0.099 - 0.032 * (name == "black") + 0.004 * exper -
                 0.0005 * (age - 22) - 0.001 * distance,
           call = rbinom(n, size = 1, prob = p))
}

# one study: its gap, and its plug-in SE (week 5's formula with its own p-hats)
gap_and_se = function(n) {
  make_resumes_n(n) |>
    group_by(name) |>
    summarize(p_hat = mean(call), n = n()) |>
    summarize(gap = p_hat[name == "black"] - p_hat[name == "white"],
              se_hat = sqrt(sum(p_hat * (1 - p_hat) / n)))
}

# the true width: the same formula with the DGP's true callback chances, which
# we know because we wrote the DGP (experience averages 5 years, age 38.5,
# distance 15 miles)
p_aa_true = 0.099 - 0.032 + 0.004 * 5 - 0.0005 * 16.5 - 0.001 * 15
p_w_true = p_aa_true + 0.032
true_width = function(n) {
  sqrt(p_aa_true * (1 - p_aa_true) / (n / 2) + p_w_true * (1 - p_w_true) / (n / 2))
}

# 1,000 studies at each size. map() runs one study per row and returns a
# one-row tibble; unnest() spreads it into columns. The ratio is 1 when a
# study's plug-in SE is exactly right
set.seed(211)
plug_in = expand_grid(n = c(200, 1000, 4870), run = 1:1000) |>
  mutate(out = map(n, gap_and_se)) |>
  unnest(out) |>
  mutate(ratio = se_hat / true_width(n))

ggplot(plug_in, aes(x = ratio)) +
  geom_histogram(binwidth = 0.0125, boundary = 1, color = "white") +
  geom_vline(xintercept = 1, color = "darkorange3", linewidth = 1.2) +
  facet_wrap(~n, labeller = label_both) +
  coord_cartesian(xlim = c(0.5, 1.5)) +
  labs(x = "Plug-in standard error ÷ the true width", y = "Studies (out of 1,000)")

# the share of studies whose plug-in SE lands within 10% of the truth
plug_in |>
  group_by(n) |>
  summarize(share_within_10_percent = mean(abs(ratio - 1) <= 0.10))


# 7. The LLN is also why every simulation works ------------------------------

# week 4's 1,000 reruns: the average of the first k estimates settles on the
# true gap as k grows. cummean() is the running average
gaps |>
  mutate(reruns = row_number(),
         center_so_far = cummean(100 * gap)) |>
  ggplot(aes(x = reruns, y = center_so_far)) +
  geom_hline(yintercept = -3.2, color = "darkorange3", linewidth = 1.2) +
  geom_line(linewidth = 1) +
  scale_x_log10() +
  labs(x = "Reruns so far (log scale)", y = "Average estimate so far (points)")


# 8. The canvass, Step 1: the systematic part --------------------------------

# 1,000 voters; coin flips decide who's a Democrat and who gets a knock. The
# systematic part is the average score for voters with these inputs: it takes
# just four values (40, 45, 60, 65)
set.seed(211)
canvass = tibble(dem = rbinom(1000, size = 1, prob = 0.5),          # a coin flip: Democrat or not
                 canvassed = rbinom(1000, size = 1, prob = 0.5)) |> # a coin flip: canvassed or not
  mutate(systematic = 40 +              # an uncanvassed non-Democrat
                      5 * canvassed +   # a knock on the door: +5 points (the effect)
                      20 * dem)         # a registered Democrat: +20 points
canvass


# 9. The canvass, Step 2: the random part -------------------------------------

# each voter's noise is a draw from a normal with mean 0 and SD 20. A real
# researcher sees dem, canvassed, and therm, but not the two parts
canvass = canvass |>
  mutate(noise = rnorm(1000, mean = 0, sd = 20),   # any number, centered on 0
         therm = systematic + noise)               # the score we'd actually see
canvass


# 10. One run of the study gives one estimate --------------------------------

# the difference in mean thermometer scores; the true effect we built in is 5
canvass |>
  group_by(canvassed) |>
  summarize(average_therm = mean(therm))


# 11. Continuous: make the bars thinner and they become a curve --------------

# 100,000 draws of the canvass noise
set.seed(211)
noise = tibble(x = rnorm(100000, mean = 0, sd = 20))

# a helper: a histogram with bars of a given width, on the density scale, with
# the normal's density curve (dnorm) on top
plot_bars = function(width) {
  ggplot(noise, aes(x = x)) +
    geom_histogram(aes(y = after_stat(density)), binwidth = width, boundary = 0, color = "white") +
    stat_function(fun = dnorm, args = list(mean = 0, sd = 20), color = "darkorange3", linewidth = 1.2) +
    coord_cartesian(xlim = c(-70, 70)) +
    labs(x = "Noise (thermometer points)", y = "Density",
         title = paste("Bars", width, "points wide"))
}
plot_bars(20)
plot_bars(5)
plot_bars(1)


# 12. Probability is the area under the curve --------------------------------

# the shaded area is P(0 < noise < 20)
tibble(x = seq(-70, 70, by = 0.1)) |>
  mutate(density = dnorm(x, mean = 0, sd = 20)) |>
  ggplot(aes(x = x, y = density)) +
  geom_area(data = \(d) filter(d, x >= 0, x <= 20), fill = "goldenrod", alpha = 0.8) +
  geom_line(linewidth = 1.2) +
  labs(x = "Noise (thermometer points)", y = "Density")

# the share of the 100,000 draws that land between 0 and 20
noise |> summarize(share = mean(x > 0 & x < 20))


# 13. Any one exact value has probability zero -------------------------------

# how many of the 100,000 draws are exactly 3.7?
noise |> summarize(exactly_3.7 = sum(x == 3.7))


# 14. The normal is a bell, set by its mean and its SD ------------------------

# blue: uncanvassed non-Democrats, N(40, 20^2). Orange: canvassed Democrats,
# N(65, 20^2). Gold: the same center as blue, with half the SD
expand_grid(x = seq(-40, 120, by = 0.25),
            curve = c("Mean 40, SD 20", "Mean 65, SD 20", "Mean 40, SD 10")) |>
  mutate(mu = parse_number(str_extract(curve, "Mean \\d+")),
         sigma = parse_number(str_extract(curve, "SD \\d+")),
         density = dnorm(x, mean = mu, sd = sigma)) |>
  ggplot(aes(x = x, y = density, color = curve)) +
  geom_line(linewidth = 1.3) +
  labs(x = "Thermometer score", y = "Density", color = NULL)


# 15. The 68-95-99.7 rule ----------------------------------------------------

# the standard normal (mean 0, SD 1), with 1, 2, and 3 SDs on each side shaded
tibble(z = seq(-4, 4, by = 0.01)) |>
  mutate(density = dnorm(z)) |>
  ggplot(aes(x = z, y = density)) +
  geom_area(data = \(d) filter(d, abs(z) <= 3), fill = "goldenrod", alpha = 0.25) +
  geom_area(data = \(d) filter(d, abs(z) <= 2), fill = "goldenrod", alpha = 0.45) +
  geom_area(data = \(d) filter(d, abs(z) <= 1), fill = "goldenrod", alpha = 0.8) +
  geom_line(linewidth = 1.2) +
  labs(x = "SDs from the mean", y = "Density")


# 16. pnorm() gives the area to the left -------------------------------------

# the share of uncanvassed non-Democrats, N(40, 20^2), scoring 60 or below
pnorm(60, mean = 40, sd = 20)

# the share scoring between 20 and 60: the area left of 60, minus the area
# left of 20
pnorm(60, 40, 20) - pnorm(20, 40, 20)


# 17. qnorm() runs pnorm() backward ------------------------------------------

# the score with 97.5% of the area to its left: the top 2.5% start here
qnorm(0.975, mean = 40, sd = 20)

# the standard normal, in SDs from the mean: the famous 1.96
qnorm(0.975)


# 🚨 Your turn: the canvass, with pnorm() and qnorm() ----

# In the canvass DGP:
#   - Uncanvassed non-Democrats score N(40, 20^2)
#   - Canvassed Democrats score N(65, 20^2)
#
# 1. What share of uncanvassed non-Democrats score above 50?
# 2. Problem Set 5 warned that a few scores land below 0. What share of
#    uncanvassed non-Democrats?
# 3. What score marks the warmest 10% of uncanvassed non-Democrats?
# 4. What share of canvassed Democrats score above 50?
#
# Check one of your answers with rnorm() draws.



# 18. The CLT: average more draws, and a bell appears -------------------------

# three starting distributions that look nothing like a bell: a callback (an
# 8% chance), a die roll, and a protest's size (week 5's activity)
draw_callback = function(n) rbinom(n, size = 1, prob = 0.08)
draw_die = function(n) sample(1:6, size = n, replace = TRUE)
draw_protest = function(n) sample(c(100, 1000, 10000), size = n, replace = TRUE,
                                  prob = c(0.6, 0.3, 0.1))

# each start's mean and SD for one draw
starts = tibble(start = c("A callback (8% chance)", "A die roll", "A protest's size"),
                mu = c(0.08, 3.5, 1360),
                sigma = c(sqrt(0.08 * 0.92), sqrt(35 / 12), sqrt(8456400)))

# a helper that draws n values from whichever start we name
draw_one = function(start, n) {
  switch(start,
         "A callback (8% chance)" = draw_callback(n),
         "A die roll" = draw_die(n),
         "A protest's size" = draw_protest(n))
}

# the average of n draws, rerun 1,000 times, for each start and each n.
# map2_dbl() is map_dbl() with two inputs per row
set.seed(211)
clt = expand_grid(start = starts$start, n = c(1, 5, 30, 500), run = 1:1000) |>
  mutate(average = map2_dbl(start, n, \(s, k) mean(draw_one(s, k)))) |>
  left_join(starts, by = "start")

# the normal curve the CLT predicts for each panel: week 5's center (mu) and
# width (sigma / sqrt(n))
clt_curves = clt |>
  distinct(start, n, mu, sigma) |>
  mutate(x = map2(mu, sigma / sqrt(n), \(m, s) seq(m - 4 * s, m + 4 * s, length.out = 200))) |>
  unnest(x) |>
  mutate(density = dnorm(x, mean = mu, sd = sigma / sqrt(n)))

# read across each row: by 500 draws, every start matches its curve
ggplot(clt, aes(x = average)) +
  geom_histogram(aes(y = after_stat(density)), bins = 30, color = "white") +
  geom_line(data = clt_curves, aes(x = x, y = density), color = "darkorange3", linewidth = 1) +
  facet_wrap(~start + n, scales = "free", ncol = 4, labeller = label_both) +
  labs(x = "Average of n draws, in 1,000 reruns", y = "Density")


# 19. Why a bell? Extremes need everything to go the same way ----------------

# the sum of 1, 2, 3, and 10 dice, 20,000 times each
set.seed(211)
expand_grid(dice = c(1, 2, 3, 10), roll = 1:20000) |>
  mutate(sum = map_dbl(dice, \(k) sum(sample(1:6, size = k, replace = TRUE)))) |>
  ggplot(aes(x = sum)) +
  geom_bar() +
  facet_wrap(~dice, scales = "free", nrow = 1, labeller = label_both) +
  labs(x = "Sum of the dice", y = "Rolls (out of 20,000)")


# 20. Zoom out and the average collapses onto 3.5. Zoom in and it's a bell ----

# the die averages from section 18, at 5, 30, and 500 rolls. z measures each
# average in standard errors from 3.5 (a z-score, with week 5's sigma / sqrt(n))
die_avgs = clt |>
  filter(start == "A die roll", n > 1) |>
  mutate(z = (average - mu) / (sigma / sqrt(n)))

# zoomed out, on the scale of one roll: the averages pile onto 3.5 (the LLN)
ggplot(die_avgs, aes(x = average)) +
  geom_histogram(bins = 40, color = "white") +
  geom_vline(xintercept = 3.5, color = "darkorange3", linewidth = 1) +
  facet_wrap(~n, labeller = label_both) +
  coord_cartesian(xlim = c(1, 6)) +
  labs(x = "The average, on the scale of one roll", y = "Reruns")

# zoomed in, in standard errors: the same standard normal bell at every n
# (the CLT)
ggplot(die_avgs, aes(x = z)) +
  geom_histogram(aes(y = after_stat(density)), bins = 30, color = "white") +
  stat_function(fun = dnorm, color = "darkorange3", linewidth = 1) +
  facet_wrap(~n, labeller = label_both) +
  coord_cartesian(xlim = c(-4, 4)) +
  labs(x = "The average, in standard errors from 3.5", y = "Density")


# 21. The résumé gap is a difference of averages, so it's normal too ---------

# week 4's 1,000 reruns, with the normal curve centered on -3.2 points and
# the reruns' own SD
gap_sd = sd(gaps$gap)

ggplot(gaps, aes(x = 100 * gap)) +
  geom_histogram(aes(y = after_stat(density)), binwidth = 0.25, center = 0, color = "white") +
  stat_function(fun = \(x) dnorm(x, mean = -3.2, sd = 100 * gap_sd),
                color = "darkorange3", linewidth = 1.3) +
  labs(x = "Callback gap in one study (percentage points)", y = "Density")


# 🚨 Your turn: protest sizes ----

# Week 5's protest: 100 people (probability 0.6), 1,000 (0.3), or 10,000
# (0.1). Mean 1,360, SD about 2,908. This draws 5 protests:
#   sample(c(100, 1000, 10000), size = 5, replace = TRUE, prob = c(0.6, 0.3, 0.1))
#
# 1. Simulate the average size of 5 protests, 1,000 times, and plot a
#    histogram. A bell?
# 2. Do the same for 30 protests and for 200 protests. Where does it start to
#    look like a bell?
# 3. Compare the SD of your 200-protest averages to what week 5's formula
#    says it should be



# 22. Linked draws: the bell is wider than the formula says ------------------

# 1,000 résumés, each with an 8% chance of a call on average. Résumés at the
# same employer share its mood: a 2% chance at a firm that already filled the
# job, 14% at one that badly needs to hire. per_employer = "1" makes the
# résumés independent; "all" makes one decision for all 1,000
share_linked = function(per_employer, n = 1000) {
  if (per_employer == "all") return(mean(rep(rbinom(1, size = 1, prob = 0.08), n)))
  m = as.numeric(per_employer)
  if (m == 1) return(mean(rbinom(n, size = 1, prob = 0.08)))
  p_employer = sample(c(0.02, 0.14), size = n / m, replace = TRUE)   # one mood per employer
  mean(rbinom(n, size = 1, prob = rep(p_employer, each = m)))        # m résumés share it
}

# 1,000 studies under each design
set.seed(211)
linked = expand_grid(per_employer = c("1", "4", "50", "all"), run = 1:1000) |>
  mutate(share = map_dbl(per_employer, share_linked))

# week 5's formula assumes independence: SD = sqrt(p(1 - p) / n)
sd_formula = sqrt(0.08 * 0.92 / 1000)

# the formula's bell (orange) against the real spread. A share among 1,000
# résumés is a multiple of 0.1 points, so one bar per possible share
linked |>
  filter(per_employer != "all") |>
  ggplot(aes(x = 100 * share)) +
  geom_histogram(aes(y = after_stat(density)), binwidth = 0.1, center = 8, color = "white") +
  stat_function(fun = \(x) dnorm(x, mean = 8, sd = 100 * sd_formula),
                color = "darkorange3", linewidth = 1.2) +
  facet_wrap(~per_employer, labeller = label_both) +
  coord_cartesian(xlim = c(4, 12)) +
  labs(x = "Callback share among 1,000 résumés (%)", y = "Density")


# 23. Why independence is key: each linked draw adds less new information ----

# the real SD of the share, against the formula's. "worth" is the effective
# sample size: how many independent résumés the 1,000 linked ones are worth
linked |>
  group_by(per_employer) |>
  summarize(real_sd = sd(share)) |>
  mutate(formula_sd = sd_formula,
         real_over_formula = real_sd / formula_sd,
         worth = 1000 / real_over_formula^2)


# 24. The payoff: center, width, and shape from one run of data --------------

# the sampling distribution the real study implies: centered on the truth,
# with week 1's standard error (from lm()), and normal. The shaded middle is
# 1.96 SEs on each side
se_lm = summary(m_resume)$coefficients[2, "Std. Error"]

tibble(x = seq(-6, -0.5, by = 0.01)) |>
  mutate(density = dnorm(x, mean = -3.2, sd = 100 * se_lm)) |>
  ggplot(aes(x = x, y = density)) +
  geom_area(data = \(d) filter(d, abs(x + 3.2) <= 1.96 * 100 * se_lm), fill = "goldenrod", alpha = 0.8) +
  geom_line(linewidth = 1.2) +
  labs(x = "Callback gap in one study (percentage points)", y = "Density")


# 25. Check it: 95% of week 4's reruns land within 1.96 SEs -----------------

# the simulation
gaps |>
  summarize(share_within = mean(abs(gap - (-0.032)) <= 1.96 * sd(gap)))

# the normal, without simulating
pnorm(1.96) - pnorm(-1.96)


# 26. The payoff: -3.2 points, give or take one standard error ---------------

# week 1's estimate and standard error, in percentage points
gap_real = coef(m_resume)[2]
100 * gap_real
100 * se_lm

# 95% of studies like this one land within 1.96 SEs of the true gap. So
# unless ours is one of the unlucky 5%, the true gap is in this range:
100 * (gap_real + c(-1.96, 1.96) * se_lm)
