# POL 211 Week 4: Probability, or what if we could do it all over again? -----
#
# This script repeats the analysis from class, in the order of the slides.
# Save it in your POL 211 project's notes/ folder and run it one chunk at a
# time.
#
# Before you start:
#   - resume.csv (from week 3 on the course schedule) should already be in
#     your project's data/ folder
#
# Today's activity (where does the uncertainty come from?) is a discussion,
# so there is no code for it: the questions are at the end of section 6.


# Packages -------------------------------------------------------------------

library(tidyverse)   # wrangling + plotting
library(here)        # file paths that work on any computer


# 1. Reminder: the standard error from week 1's table ------------------------

# the résumé experiment (Bertrand and Mullainathan 2004): 4,870 résumés,
# each randomly given a white- or Black-sounding name. call = 1 if the
# employer called back
resume = read_csv(here("data", "resume.csv"))

# week 1's regression. The slope is the estimated gap (-0.032: Black-sounding
# names got callbacks 3.2 points less often). The number under it is the
# standard error: "how much the estimate would vary if we ran the study again"
m_resume = lm(call ~ I(race == "black"), data = resume)
summary(m_resume)

# pull the standard error out on its own; we'll compare to it at the end
se_lm = summary(m_resume)$coefficients[2, "Std. Error"]
se_lm


# 2. Step 1, the systematic part: each résumé's chance of a call -------------

# a made-up data-generating process (DGP) shaped like the real study. We write
# the rule ourselves, so we know the systematic part exactly: each résumé's
# chance of a call, p, depends on its name, experience, age, and distance
set.seed(211)
resumes = tibble(name = sample(c("white", "black"), size = 4870, replace = TRUE),  # a coin flip picks each name
                 exper = sample(0:10, size = 4870, replace = TRUE),       # years of experience
                 age = sample(22:55, size = 4870, replace = TRUE),
                 distance = sample(0:30, size = 4870, replace = TRUE)) |> # miles from the job
  mutate(p = 0.099 - 0.032 * (name == "black") +   # Black-sounding name: -3.2 points
               0.004 * exper -                      # each year of experience: +0.4 points
               0.0005 * (age - 22) -                # each 10 years older: -0.5 points
               0.001 * distance)                    # each 10 miles away: -1 point
resumes


# 3. Step 2, the random part: each employer flips a weighted coin ------------

# rbinom() flips one coin per résumé, each weighted by its own p.
# call = 1 (a callback) with probability p, 0 otherwise
resumes = resumes |>
  mutate(call = rbinom(4870, size = 1, prob = p))
resumes


# 4. One run of the study gives one estimate; run it again, get another ------

# the same estimator as week 1: the difference in callback rates between the
# two names. The true gap we built in is -3.2 points
run_1 = resumes |>
  group_by(name) |>
  summarize(callback_rate = mean(call))
run_1

# run 1's gap, as a single number (Black minus white)
run_1$callback_rate[run_1$name == "black"] - run_1$callback_rate[run_1$name == "white"]

# run it again: same résumés, same p's, new coin flips. Nothing about the
# systematic part changed, but the estimate does
run_2 = resumes |>
  mutate(call = rbinom(4870, size = 1, prob = p)) |>   # re-flip every employer's coin
  group_by(name) |>
  summarize(callback_rate = mean(call))
run_2

run_2$callback_rate[run_2$name == "black"] - run_2$callback_rate[run_2$name == "white"]


# 5. Even with the systematic part known, each résumé's outcome is random ----

# 100 made-up résumés of each kind, drawn from two Bernoullis: a 9.65% chance
# of a call with a white-sounding name, 6.45% with a Black-sounding one.
# The name shifts the chance of a call; it doesn't decide any one résumé
set.seed(4)
grid_draws = expand_grid(name = c("White-sounding name", "Black-sounding name"),
                         row = 1:10, col = 1:10) |>
  mutate(name = fct_inorder(name),
         call = rbinom(n(), size = 1, prob = if_else(name == "White-sounding name", 0.0965, 0.0645)),
         call = if_else(call == 1, "Called back", "No call"))

ggplot(grid_draws, aes(x = col, y = row, color = call)) +
  geom_point(size = 4) +
  facet_wrap(~name) +
  coord_fixed() +
  scale_color_manual(values = c("Called back" = "darkorange3", "No call" = "grey80")) +
  labs(x = NULL, y = NULL, color = NULL) +
  theme(axis.text = element_blank(), panel.grid = element_blank())

# count the callbacks in each grid
grid_draws |>
  count(name, call)


# 6. The data give us p-hat, not p ------------------------------------------

# in the made-up study we wrote p down ourselves. With real data, all we ever
# get is p-hat: the callback share in each group
resume |>
  group_by(race) |>
  summarize(resumes = n(),
            callbacks = sum(call),
            p_hat = mean(call))

# 🚨 Your turn: where does the uncertainty come from? ----

# Randomness enters data in three ways (Gailmard 2014, pp. 73-78):
#   sampling (we see only part of the whole), theoretical (our theory leaves
#   things out), fundamental (people are partly random, even with a
#   complete theory)
#
# Which source is most at work in each case?
# 1. A poll of 1,000 adults reports that 54% approve of the president
# 2. A model of protest uses food prices and elections, but not the weather
# 3. The same senator, on the same bill, might have voted the other way if
#    the vote had been held a day later
# 4. A city draws lottery numbers to decide which neighborhoods get new
#    streetlights


# 7. A probability distribution says how likely each value is ----------------

# résumé #1 from Step 1: its chance of a call, set by the rule we wrote.
# Y_1 (whether résumé #1 gets a call) is a random variable with a Bernoulli
# distribution: P(Y_1 = 1) = p_1 and P(Y_1 = 0) = 1 - p_1
r1 = resumes |> slice(1)
r1

p1 = r1$p
p1          # P(Y_1 = 1)
1 - p1      # P(Y_1 = 0)

# one parameter, p, pins down the whole distribution. This is what rbinom()
# draws from in Step 2: 1 with probability p, 0 with probability 1 - p
rbinom(10, size = 1, prob = p1)


# 8. Wrap the two steps in a function, so we can run the study with one line -

# a function takes inputs and returns an output (Problem Set 3's mse()).
# This one takes no inputs and returns a whole made-up dataset: the function
# *is* the data-generating process. Each call draws new résumés and new coin
# flips: one whole run of the study
make_resumes = function() {
  tibble(name = sample(c("white", "black"), size = 4870, replace = TRUE),
         exper = sample(0:10, size = 4870, replace = TRUE),
         age = sample(22:55, size = 4870, replace = TRUE),
         distance = sample(0:30, size = 4870, replace = TRUE)) |>
    mutate(p = 0.099 - 0.032 * (name == "black") + 0.004 * exper -
                 0.0005 * (age - 22) - 0.001 * distance,      # step 1: systematic
           call = rbinom(4870, size = 1, prob = p))            # step 2: random
}

# and one that runs the study and returns just the estimate.
# pull() turns a one-cell tibble into a plain number
run_resumes = function() {
  make_resumes() |>
    summarize(gap = mean(call[name == "black"]) - mean(call[name == "white"])) |>
    pull(gap)
}

# try it a few times: a new gap every run
run_resumes()
run_resumes()
run_resumes()


# 9. Rerun the study 1,000 times ---------------------------------------------

# map_dbl() (Problem Set 3) runs the function once per row. The \(i) means
# "for each run"; we don't use i inside. One whole study per row, so each
# gap is an estimate the study *could* have produced
set.seed(211)
gaps = tibble(run = 1:1000) |>
  mutate(gap = map_dbl(run, \(i) run_resumes()))
gaps


# 10. Watch the pile form: 10, 100, then 1,000 studies -----------------------

# a helper: the first k reruns, as a histogram in quarter-point bins. The
# orange line is the true gap we built in (-3.2 points)
plot_pile = function(k) {
  gaps |>
    slice_head(n = k) |>
    ggplot(aes(x = 100 * gap)) +
    geom_histogram(binwidth = 0.25, center = 0, color = "white") +
    geom_vline(xintercept = -3.2, color = "darkorange3", linewidth = 1.2) +
    coord_cartesian(xlim = c(-6, -0.5)) +
    labs(x = "Callback gap in one study (percentage points)", y = "Studies",
         title = paste("After", k, "studies"))
}

plot_pile(10)     # no shape yet
plot_pile(100)    # a shape appears
plot_pile(1000)   # the sampling distribution: every estimate the DGP could produce, and how often


# 11. The pile centers on the truth, and its spread is a standard deviation --

# week 3's two summaries, applied to 1,000 estimates instead of 1,000
# observations. The average of the estimates is the -3.2 points we built in
# (the estimator is unbiased); the SD says how far a typical run lands from
# the truth
gaps |>
  summarize(average_gap = mean(gap),
            spread = sd(gap))

# compare that spread to week 1's standard error, from section 1. Same number.
# The formula behind it (the analytic path) is where the next weeks go
se_lm
