# POL 211 Week 5: Expectations, variance, and the analytic path to the standard error -----
#
# This script repeats the analysis from class, in the order of the slides.
# Save it in your POL 211 project's notes/ folder and run it one chunk at a
# time.
#
# Before you start:
#   - resume.csv (from week 3 on the course schedule) should already be in
#     your project's data/ folder
#
# Today has two activities (the expected sum of two dice, and a protest's size
# as a random variable). Each has its own 🚨 section with the questions; the
# answers are yours to work out.


# Packages -------------------------------------------------------------------

library(tidyverse)   # wrangling + plotting
library(here)        # file paths that work on any computer


# 1. Last week: a DGP that mimics the résumé study ----------------------------

# week 4's made-up data-generating process, shaped like Bertrand and
# Mullainathan (2004). The systematic part sets each résumé's chance of a call
# (p); the random part is each employer's weighted coin flip. We chose the
# effect ourselves: an African American-sounding name (AA for short) lowers
# the chance of a call by 3.2 points. That's the estimand. (The DGP labels AA
# names "black", like the race column in the real data.)
make_resumes = function() {
  tibble(name = sample(c("white", "black"), size = 4870, replace = TRUE),   # a coin flip picks each name
         exper = sample(0:10, size = 4870, replace = TRUE),
         age = sample(22:55, size = 4870, replace = TRUE),
         distance = sample(0:30, size = 4870, replace = TRUE)) |>
    mutate(p = 0.099 - 0.032 * (name == "black") + 0.004 * exper -
                 0.0005 * (age - 22) - 0.001 * distance,      # systematic: each résumé's chance of a call
           call = rbinom(4870, size = 1, prob = p))            # random: each employer's weighted coin flip
}

# run the study once and return just the estimate: the gap in callback rates
run_resumes = function() {
  make_resumes() |>
    summarize(gap = mean(call[name == "black"]) - mean(call[name == "white"])) |>
    pull(gap)
}


# 2. Rerun the study 1,000 times: the sampling distribution ------------------

# same seed as week 4, so these are the same 1,000 studies
set.seed(211)
gaps = tibble(run = 1:1000) |>
  mutate(gap = map_dbl(run, \(i) run_resumes()))   # one whole study per row

# every gap the study could produce, and how often. The orange line is the
# -3.2 points we built in
ggplot(gaps, aes(x = 100 * gap)) +
  geom_histogram(binwidth = 0.25, center = 0, color = "white") +
  geom_vline(xintercept = -3.2, color = "darkorange3", linewidth = 1.2) +
  labs(x = "Callback gap in one study (percentage points)", y = "Studies (out of 1,000)")


# 3. The SD of the sampling distribution is the standard error ---------------

# center: where the estimates land on average. Spread: how far a typical study
# lands from the truth. That spread is the standard error (in proportions;
# times 100 gives points)
gaps |>
  summarize(center = mean(gap), spread = sd(gap))


# 4. What if the study had been smaller? 500 résumés instead of 4,870 ---------

# the same DGP with the number of résumés as an input
make_resumes_n = function(n) {
  tibble(name = sample(c("white", "black"), size = n, replace = TRUE),
         exper = sample(0:10, size = n, replace = TRUE),
         age = sample(22:55, size = n, replace = TRUE),
         distance = sample(0:30, size = n, replace = TRUE)) |>
    mutate(p = 0.099 - 0.032 * (name == "black") + 0.004 * exper -
                 0.0005 * (age - 22) - 0.001 * distance,
           call = rbinom(n, size = 1, prob = p))
}
run_resumes_n = function(n) {
  make_resumes_n(n) |>
    summarize(gap = mean(call[name == "black"]) - mean(call[name == "white"])) |>
    pull(gap)
}

# 1,000 studies at each size. expand_grid() makes one row per size per run;
# map_dbl() runs one study per row, with that row's n
set.seed(211)
gaps_by_n = expand_grid(n = c(500, 4870), run = 1:1000) |>
  mutate(gap = map_dbl(n, run_resumes_n))

ggplot(gaps_by_n, aes(x = 100 * gap)) +
  geom_histogram(binwidth = 0.5, boundary = 0, color = "white") +
  geom_vline(xintercept = -3.2, color = "darkorange3", linewidth = 1.2) +
  facet_wrap(~n, labeller = label_both) +
  labs(x = "Callback gap in one study (percentage points)", y = "Studies (out of 1,000)")

# both center on the truth, but the smaller study's standard error is about
# three times bigger (in percentage points)
gaps_by_n |>
  group_by(n) |>
  summarize(standard_error = 100 * sd(gap))


# 5. Week 3's formula gives week 1's standard error --------------------------

# the real résumé study: 4,870 résumés, each randomly given a white- or
# AA-sounding name (race == "black" in the data). call = 1 if the employer
# called back
resume = read_csv(here("data", "resume.csv"))

# week 1's regression: the slope is the gap, and its standard error is the
# number we're explaining today
m_resume = lm(call ~ I(race == "black"), data = resume)
se_lm = summary(m_resume)$coefficients[2, "Std. Error"]
se_lm

# week 3's formula: each group's variance over its size, added, square-rooted
groups = resume |>
  group_by(race) |>
  summarize(n = n(), var = var(call))
groups

sqrt(sum(groups$var / groups$n))   # the same number


# 6. Recap: résumé #1's probability distribution -----------------------------

# week 4's one run of the made-up study (same seed), and its first résumé.
# Y_1, whether résumé #1 gets a call, is a Bernoulli random variable:
# P(Y_1 = 1) = p_1 and P(Y_1 = 0) = 1 - p_1
set.seed(211)
resumes = make_resumes()
r1 = resumes |> slice(1)
r1

p1 = r1$p
p1          # P(Y_1 = 1)
1 - p1      # P(Y_1 = 0)


# 7. The expected value of a coin flip is its probability ---------------------

# E[X] weights each possible value by its probability and adds:
# E[Y_1] = 1 * P(Y_1 = 1) + 0 * P(Y_1 = 0) = p_1
1 * p1 + 0 * (1 - p1)


# 8. Check it by simulation: the long-run average of draws is the expectation -

# flip résumé #1's coin 100,000 times (its p, rounded to 0.0545, as on the
# slide) and average the draws
set.seed(211)
flips = tibble(draw = rbinom(100000, size = 1, prob = 0.0545))
flips |>
  summarize(average = mean(draw))

# the running average after each flip: it swings around early, then settles on
# E[Y_1] = 0.0545. cumsum() adds up the draws so far
flips |>
  mutate(n = row_number(),
         running_average = cumsum(draw) / n) |>
  filter(n >= 10) |>
  ggplot(aes(x = n, y = running_average)) +
  geom_hline(yintercept = 0.0545, color = "darkorange3", linewidth = 1.2) +
  geom_line() +
  scale_x_log10() +
  coord_cartesian(ylim = c(0, 0.2)) +
  labs(x = "Number of flips (log scale)", y = "Average of the flips so far")


# 9. The rules for expectations ----------------------------------------------

# the same rules as week 3's mean, for random variables:
#   E[kX] = k E[X]           multiply by a constant
#   E[X + k] = E[X] + k      add a constant
#   E[k] = k                 a constant's expectation is itself
#   E[X + Y] = E[X] + E[Y]   the sum rule (and E[X - Y] = E[X] - E[Y])
#
# The sum rule holds ALWAYS: whether or not X and Y are related. So the
# expected number of callbacks in a study is the sum of each résumé's p:
# E[Y_1 + Y_2 + ... + Y_n] = E[Y_1] + E[Y_2] + ... + E[Y_n]


# 🚨 Your turn: the expected sum of two dice ----

# Roll two fair dice, D_1 and D_2
#
# 1. By hand: E[D_1], E[D_2], and E[D_1 + D_2]
# 2. Simulate 100,000 rolls of each die with
#    sample(1:6, size = 100000, replace = TRUE). Is mean(d1 + d2) close?
# 3. The rule says "always," even when the two numbers are related. Roll one
#    die and count it twice: d1 + d1. Is E[D_1 + D_1] still E[D_1] + E[D_1]?



# 10. Check it: the share centers on the average p_i -------------------------

# each AA résumé in week 4's run has its own p, set by its experience, age,
# and distance. The rules say the share called back is centered on their
# average
aa = resumes |> filter(name == "black")   # week 4's AA-named résumés
mean(aa$p)                                # the average p_i
range(aa$p)                               # the p_i differ quite a bit

# keep the same résumés, re-flip every employer's coin 1,000 times, and record
# the share called back each time
set.seed(211)
shares = tibble(run = 1:1000) |>
  mutate(share = map_dbl(run, \(i) mean(rbinom(nrow(aa), size = 1, prob = aa$p))))

ggplot(shares, aes(x = share)) +
  geom_histogram(binwidth = 0.002, boundary = 0, color = "white") +
  geom_vline(xintercept = mean(aa$p), color = "darkorange3", linewidth = 1.2) +
  labs(x = "Share of AA-named résumés called back, in one run", y = "Runs (out of 1,000)")

# the 1,000 shares average out to the average p_i
mean(shares$share)


# 11. Unbiased for the gap, or for the effect? -------------------------------

# the algebra says the difference in means is centered on the gap between the
# groups, p_AA - p_white. Whether that gap is the EFFECT of the name depends
# on randomization. Same DGP, one change: AA names go to résumés with 0-5
# years of experience, white names to résumés with 5-10
make_resumes_conf = function() {
  tibble(name = sample(c("white", "black"), size = 4870, replace = TRUE),
         age = sample(22:55, size = 4870, replace = TRUE),
         distance = sample(0:30, size = 4870, replace = TRUE)) |>
    mutate(exper = if_else(name == "black", sample(0:5, size = 4870, replace = TRUE),    # 0-5 years
                                            sample(5:10, size = 4870, replace = TRUE)),  # 5-10 years
           p = 0.099 - 0.032 * (name == "black") + 0.004 * exper -
               0.0005 * (age - 22) - 0.001 * distance,
           call = rbinom(4870, size = 1, prob = p))
}
run_resumes_conf = function() {
  make_resumes_conf() |>
    summarize(gap = mean(call[name == "black"]) - mean(call[name == "white"])) |>
    pull(gap)
}

# 1,000 studies without randomization, stacked with section 2's 1,000 with it
set.seed(211)
gaps_design = bind_rows(
  gaps |> mutate(design = "Names assigned by coin flip"),
  tibble(run = 1:1000) |>
    mutate(gap = map_dbl(run, \(i) run_resumes_conf()),
           design = "AA names on less-experienced résumés"))

# where each design's sampling distribution centers (in points)
design_centers = gaps_design |>
  group_by(design) |>
  summarize(center = 100 * mean(gap))
design_centers

# orange: the effect of the name, -3.2 in both worlds. Dashed: where each
# sampling distribution centers, the gap between the groups
ggplot(gaps_design, aes(x = 100 * gap)) +
  geom_histogram(binwidth = 0.25, center = 0, color = "white") +
  geom_vline(data = design_centers, aes(xintercept = center), linetype = "dashed") +
  geom_vline(xintercept = -3.2, color = "darkorange3", linewidth = 1.2) +
  facet_wrap(~design) +
  labs(x = "Callback gap in one study (percentage points)", y = "Studies (out of 1,000)")

# Unbiased is the first of four properties an estimator can have:
#   unbiased                its sampling distribution is centered on the estimand (today)
#   consistent              it lands ever closer to the estimand as n grows (weeks 6-7)
#   asymptotically normal   its sampling distribution is bell-shaped once n is large (week 6)
#   efficient               among unbiased estimators, the one with the smallest spread (week 7)


# 12. A coin flip's variance is p(1 - p) -------------------------------------

# Var(X) = E[(X - E[X])^2]: each possible squared miss, weighted by its
# probability. For a coin flip that works out to p(1 - p). For résumé #1:
p1 * (1 - p1)

# the variance peaks at p = 0.5 and shrinks as the outcome gets nearly
# certain. The orange points are the real study's two callback rates
rates = resume |>
  group_by(race) |>
  summarize(p = mean(call))

tibble(p = seq(0, 1, by = 0.001)) |>
  ggplot(aes(x = p, y = p * (1 - p))) +
  geom_line(linewidth = 1.2) +
  geom_point(data = rates, size = 4, color = "darkorange3") +
  labs(x = "p: the chance of a 1", y = "Var(Y) = p(1 - p)")


# 13. Check it by simulation -------------------------------------------------

# the same 100,000 flips of résumé #1's coin as section 8. var() and sd() are
# week 3's functions, applied to draws instead of data
set.seed(211)
tibble(draw = rbinom(100000, size = 1, prob = 0.0545)) |>
  summarize(average = mean(draw),
            variance = var(draw),
            sd = sd(draw))


# 14. The rules for variances ------------------------------------------------

#   Var(X + k) = Var(X)          adding a constant doesn't change the spread
#   Var(k) = 0                   a constant has no spread
#   Var(kX) = k^2 Var(X)         every miss is k times bigger, then squared
#   Var(X + Y) = Var(X) + Var(Y) + 2 Cov(X, Y)
#   Var(X - Y) = Var(X) + Var(Y) - 2 Cov(X, Y)
#
# Why we need them: a callback share divides a SUM of coin flips by n, and the
# gap is a DIFFERENCE of two shares

# check the k^2 rule with a fair coin (variance 0.5 x 0.5 = 0.25). The rules
# say Var(2X) = 2^2 x 0.25 = 1, and Var(X + 10) = 0.25
set.seed(211)
tibble(x = rbinom(100000, size = 1, prob = 0.5)) |>
  mutate(doubled = 2 * x,          # 0 or 2
         shifted = x + 10) |>      # 10 or 11
  summarize(var_x = var(x), var_doubled = var(doubled), var_shifted = var(shifted))


# 🚨 Your turn: a protest's size as a random variable ----

# A protest tomorrow will be small (100 people), medium (1,000), or large
# (10,000), with probabilities 0.6, 0.3, and 0.1
#
# 1. By hand: E[X] and Var(X). (Make a table like the coin flip's: value,
#    probability, miss, squared miss)
# 2. Simulate 100,000 protests with sample(). Do mean() and var() match your
#    answers?



# 15. Two résumés: separate coins vs. one coin --------------------------------

# Two résumés, each with a 0.1 chance of a call.
#   Scenario A: two different employers, each flipping their own coin
#   Scenario B: one employer who decides about the pair at once (both or neither)
# Either way the pair gets 0.2 callbacks on average. Does the number of
# callbacks vary by the same amount?
set.seed(211)
two = tibble(run = 1:100000) |>
  mutate(y1 = rbinom(n(), size = 1, prob = 0.1),      # résumé 1's call
         y2_own = rbinom(n(), size = 1, prob = 0.1),  # A: résumé 2 gets its own coin
         y2_same = y1)                                # B: résumé 2 gets the same coin

# same expectation, but the linked coins vary twice as much
two |>
  summarize(mean_A = mean(y1 + y2_own), var_A = var(y1 + y2_own),
            mean_B = mean(y1 + y2_same), var_B = var(y1 + y2_same))

# the rule: Var(X + Y) = Var(X) + Var(Y) + 2 Cov(X, Y). Each résumé's variance
# is 0.1 x 0.9 = 0.09.
#   A: Cov = 0 (the coins don't move together):  0.09 + 0.09 + 0       = 0.18
#   B: Cov = 0.09 (one coin moves with itself):   0.09 + 0.09 + 2(0.09) = 0.36
#
# Where the rule comes from, for anyone who wants it: write X* = X - E[X] and
# Y* = Y - E[Y]. Then Var(X + Y) = E[(X* + Y*)^2]
#   = E[X*^2] + E[Y*^2] + 2 E[X* Y*] = Var(X) + Var(Y) + 2 Cov(X, Y)


# 16. Independent: one résumé's call tells you nothing about the other's ------

# conditional probability: résumé 2's chance of a call, among only the runs
# where résumé 1 got one. Filter to those runs, then take the mean
two |>
  filter(y1 == 1) |>
  summarize(A = mean(y2_own),    # still about 0.1: independent
            B = mean(y2_same))   # 1: knowing résumé 1 tells you everything


# 17. With more than two, every pair adds a covariance term -------------------

# Var(Y_1 + ... + Y_n) has n variance terms and one covariance term per pair:
# n(n - 1)/2 of them. For the real study's AA résumés:
n_aa = sum(resume$race == "black")
n_aa                      # variance terms
n_aa * (n_aa - 1) / 2     # covariance terms, all zero if the résumés are independent


# 18. Plug in the one run we have --------------------------------------------

# Payoff 4: SE(gap) = sqrt( p_AA(1 - p_AA)/n_AA + p_white(1 - p_white)/n_white ).
# We don't know the true p's, so we plug in the shares from the data (p-hat)
pieces = resume |>
  group_by(race) |>
  summarize(n = n(), p_hat = mean(call)) |>
  mutate(var = p_hat * (1 - p_hat),   # the Bernoulli variance, p(1 - p)
         piece = var / n)             # one group's piece of the formula
pieces

se_formula = sqrt(sum(pieces$piece))
se_formula


# 19. Three paths, one number ------------------------------------------------

sd(gaps$gap)   # simulation (section 2): the SD of 1,000 reruns of our made-up DGP
se_formula     # analytic (today): the formula, from one run of the real data
se_lm          # week 1's table: lm(), from the same one run
