# POL 211 Week 3: Averages, spread, and relationships ------------------------
#
# This script repeats the analysis from class, in the order of the slides.
# Save it in your POL 211 project's notes/ folder and run it one chunk at a
# time.
#
# Before you start:
#   - download face.csv and resume.csv from week 3 on the course schedule,
#     and save them in your project's data/ folder
#   - the civil war data (fearon) and the Congress data (bonica) come with
#     the juanr package
#   - the county data come with the socviz package. If you skipped Quant
#     Camp, install it first: install.packages("socviz")
#
# The "🚨 Your turn" sections set up the in-class activities. The answers
# aren't here: working them out is the point.


# Packages -------------------------------------------------------------------

library(tidyverse)   # wrangling + plotting
library(juanr)       # fearon (civil wars) and bonica (Congress)
library(socviz)      # county_data: US counties, census + election results
library(here)        # file paths that work on any computer


# 1. Recap: democracy and civil war, within income groups --------------------

# same setup as week 2: one row per country-year with a Polity score and GDP
wars = fearon |>
  filter(!is.na(polity2), !is.na(gdp_cap)) |>
  mutate(democracy = if_else(polity2 > 5, "Democracy", "Not a democracy"),
         civil_war = if_else(onset > 0, 1, 0),      # 1 = a civil war began that year
         income = ntile(gdp_cap, 3))                # 3 equal-sized groups: 1 = poorest

# the civil war rate by regime, within each income third.
# Within each income group, the democracy gap shrinks a lot
wars |>
  group_by(income, democracy) |>
  summarize(war_rate = mean(civil_war), .groups = "drop")


# 2. Why the mean? The median callback is zero -------------------------------

# the résumé experiment from week 1 (Bertrand and Mullainathan 2004)
resume = read_csv(here("data", "resume.csv"))

# call is 1 if the employer called back, 0 if not. Most résumés got no call,
# so the median (and the most common value) is 0 for both groups. Only the
# mean picks up the gap
resume |>
  group_by(race) |>
  summarize(mean = mean(call),
            median = median(call))


# 3. Weighted averages and sampling weights ----------------------------------

# three made-up races with 1, 2, and 6 million votes cast
votes = c(1, 2, 6)
mean(votes)                              # the ordinary mean: 3

# count the third race twice
weighted.mean(votes, w = c(1, 1, 2))     # 3.75

# a made-up poll of 100 people that reached too few young people:
# 20 under 40 (70% support) and 80 over 40 (40% support). The town is 50-50
poll = tibble(
  group = c("Under 40", "Over 40"),
  n = c(20, 80),                         # people in the poll
  support = c(70, 40),                   # % who support
  weight = c(0.5 / 0.2, 0.5 / 0.8)       # town share / poll share: 2.5 and 0.625
)

# unweighted: every respondent counts the same (46%)
weighted.mean(poll$support, w = poll$n)

# weighted: each young respondent counts 2.5 times, each older one 0.625 (55%)
weighted.mean(poll$support, w = poll$n * poll$weight)


# 4. Try every guess: the mean sits at the bottom of the MSE curve -----------

# score a guess g by its mean squared error: the average squared miss
mse_toy = function(g) mean((votes - g)^2)

mse_toy(2)   # the median
mse_toy(3)   # the mean: the smallest MSE
mse_toy(4)

# score every guess from 0 to 7, and plot the curve
toy_curve = tibble(guess = seq(0, 7, by = 0.01)) |>
  mutate(mse = map_dbl(guess, mse_toy))   # run mse_toy() once for each guess

ggplot(toy_curve, aes(x = guess, y = mse)) +
  geom_line(linewidth = 1.2) +
  geom_vline(xintercept = mean(votes), linetype = "dashed") +
  labs(x = "Our guess, g (millions of votes)", y = "MSE(g)")


# 5. Properties of the mean --------------------------------------------------

# Property 2: multiply every value by k, and the mean is multiplied by k.
# Callbacks as a share (0/1) or as a percent (0/100): the same finding
resume |>
  group_by(race) |>
  summarize(as_share = mean(call),               # callback as a 0/1
            as_percent_k_y = mean(100 * call),   # mean of k * Y
            as_percent_k_ybar = mean(call) * 100)  # k * Y_bar

# Property 3: add k to every value, and the mean moves by k, but the gap
# between groups doesn't. Polity runs from -10 to 10; add 10 and it runs 0 to 20
wars |>
  group_by(oil) |>                                   # 1 = oil exporter
  summarize(polity = mean(polity2),
            polity_0_to_20_ky = mean(polity2 + 10),     # mean of Y + k
            polity_0_to_20_ky_bar = mean(polity2) + 10) # Y_bar + k


# 6. Faces and Senate races --------------------------------------------------

# Todorov et al. (2005) showed people pairs of Senate candidates' faces for
# one second and asked who looked more competent. One row per race, 2000-2006
face = read_csv(here("data", "face.csv")) |>
  mutate(d_share = 100 * d.votes / (d.votes + r.votes),  # Democrat's % of the two-party vote
         d_comp = 100 * d.comp,                          # % of raters picking the Democrat's face
         total_votes = (d.votes + r.votes) / 1e6,        # votes cast, in millions
         d_won = if_else(w.party == "D", 1, 0),          # 1 = the Democrat won
         d_looks = if_else(d.comp > 0.5, 1, 0))          # 1 = the Democrat looked more competent

# take a look
face |>
  select(year, state, winner, loser, d_comp, d_share)

# how many people vote in a typical Senate race?
ggplot(face, aes(x = total_votes)) +
  geom_histogram(binwidth = 0.5, boundary = 0, color = "white") +
  labs(x = "Votes cast for the two major candidates (millions)", y = "Senate races")


# 🚨 Your turn: mean vs. median, squared vs. absolute ----

my_guess = 2   # try the mean, the median, and anything else

face |>
  summarize(mse = mean((total_votes - my_guess)^2))   # mean squared error
            # add the mean absolute error here: mae = ...

# 1. Find the mean and the median of total_votes
# 2. Score each one on MSE and on MAE. Which wins on which?
# 3. Convince yourself that other guesses aren't as good


# 7. Squaring pays more attention to "outliers" ------------------------------

# add one new race to our three made-up races (1, 2, 6). If it lands near
# the others, the mean barely moves; if it lands far away, the mean chases it
mean(c(votes, 4))      # 3.25
median(c(votes, 4))    # 3
mean(c(votes, 20))     # 7.25
median(c(votes, 20))   # 4

# why: a miss of 17 costs 17 under absolute error, but 289 under squared error
17^2


# 8. Spread: same typical income, very different spread ----------------------

# median household income (2009-2013) for counties in Iowa and Virginia.
# Virginia's independent cities count as counties
counties = county_data |>
  filter(state %in% c("IA", "VA"), grepl("County|city", name), !is.na(hh_income)) |>
  mutate(state = if_else(state == "IA", "Iowa", "Virginia"),
         income = hh_income / 1000)                     # in $1,000s

ggplot(counties, aes(x = income)) +
  geom_histogram(binwidth = 4, boundary = 0, color = "white") +
  facet_wrap(~state, scales = "free_y") +
  labs(x = "Median household income ($1,000s)", y = "Counties")

# each county's distance from its state's average: the poorest, middle, and
# richest county in Virginia
counties |>
  filter(state == "Virginia") |>
  mutate(miss = income - mean(income)) |>
  arrange(income) |>
  slice(c(1, round(n() / 2), n())) |>                   # first, middle, last row
  select(name, income, miss)

# the variance is the average squared distance from the mean; the SD is its
# square root, back in dollars (here, $1,000s). R's var() and sd() can come
# out a hair different from the slides; we'll see why in class
county_stats = counties |>
  group_by(state) |>
  summarize(mean = mean(income),
            var = var(income),
            sd = sd(income))

county_stats

# shade one SD on each side of the mean
ggplot(counties, aes(x = income)) +
  geom_rect(data = county_stats, aes(xmin = mean - sd, xmax = mean + sd, ymin = -Inf, ymax = Inf),
            inherit.aes = FALSE, fill = "orange", alpha = 0.2) +
  geom_histogram(binwidth = 4, boundary = 0, color = "white") +
  geom_vline(data = county_stats, aes(xintercept = mean), linetype = "dashed") +
  facet_wrap(~state, scales = "free_y") +
  labs(x = "Median household income ($1,000s)", y = "Counties")


# 🚨 Your turn: build the SD by hand ----

# d_share (the Democrat's % of the vote) is already in face, from section 6

# 1. Use mutate() to find each race's miss from the mean, then square it
# 2. Use summarize() to average the squared misses (the variance), then take
#    the square root with sqrt()
# 3. Compare your answer to sd(d_share). It should be the same number


# 9. Spread tells us how uncertain an estimate is ----------------------------

# week 1's standard error, built from each group's variance and size
res_groups = resume |>
  group_by(race) |>
  summarize(n = n(), rate = mean(call), var = var(call))

res_groups

# SE = square root of (Var_AA / n_AA + Var_W / n_W)
sqrt(res_groups$var[1] / res_groups$n[1] + res_groups$var[2] / res_groups$n[2])

# the same number as the standard error in week 1's regression table
# (more on lm() in a bit, and much more in POL 212)
lm(call ~ race, data = resume) |>
  summary()


# 10. Spread as the thing we study: polarization -----------------------------

# DW-NOMINATE ideology scores (-1 = left, 1 = right) for candidates in
# Bonica's DIME data
cong = bonica |>
  filter(!is.na(dwnom1), !is.na(total_receipts), party %in% c("D", "R")) |>
  mutate(party = if_else(party == "D", "Democrat", "Republican"))

ggplot(cong, aes(x = dwnom1, fill = party)) +
  geom_histogram(binwidth = 0.05, boundary = 0, color = "white") +
  labs(x = "Ideology (DW-NOMINATE, left to right)", y = "Candidates", fill = NULL)

# the SD for all of Congress is much bigger than the SD within either party:
# most of the spread is between the parties
sd(cong$dwnom1)
cong |>
  group_by(party) |>
  summarize(sd = sd(dwnom1))


# 11. Correlation: direction and strength ------------------------------------

# every US county in the 2016 presidential election
us_counties = county_data |>
  filter(!is.na(per_dem_2016), !is.na(per_dem_2012)) |>
  mutate(dem_2016 = 100 * per_dem_2016,        # Democrat's % of the vote, 2016
         dem_2012 = 100 * per_dem_2012,        # ... and 2012
         density = pop / land_area)            # people per square mile

# direction: positive (denser counties vote more Democratic)...
ggplot(us_counties, aes(x = density, y = dem_2016)) +
  geom_point(alpha = 0.15) +
  scale_x_log10() +                            # a few cities are thousands of times denser
  labs(x = "People per square mile (log scale)", y = "Democrat's % of the vote, 2016")

# ...or negative (whiter counties vote less Democratic)
ggplot(us_counties, aes(x = white, y = dem_2016)) +
  geom_point(alpha = 0.15) +
  labs(x = "% of residents who are white", y = "Democrat's % of the vote, 2016")

# strength: strong (2012 vote predicts 2016 closely)...
ggplot(us_counties, aes(x = dem_2012, y = dem_2016)) +
  geom_point(alpha = 0.15) +
  labs(x = "Democrat's % of the vote, 2012", y = "Democrat's % of the vote, 2016")

# ...or weak (commute time barely helps)
ggplot(us_counties, aes(x = travel_time, y = dem_2016)) +
  geom_point(alpha = 0.15) +
  labs(x = "Average commute (minutes)", y = "Democrat's % of the vote, 2016")


# 12. With categories, a correlation is a gap between groups -----------------

# a crosstab: whose face looked more competent, and who won
face |>
  count(d_looks, d_won)

# the conditional mean, E[Y | X]: average vote share for each value of X
face |>
  group_by(d_looks) |>
  summarize(avg_share = mean(d_share), races = n())


# 13. From a count to the covariance -----------------------------------------

# competence takes a different value in almost every race
ggplot(face, aes(x = d_comp, y = d_share)) +
  geom_point(size = 3, alpha = 0.7) +
  labs(x = "% of raters who picked the Democrat's face", y = "Democrat's % of the vote")

# a simple question first: how many races sit on the same side of both
# averages (above on both, or below on both)?
face |>
  mutate(same_side = (d_comp > mean(d_comp)) == (d_share > mean(d_share))) |>
  count(same_side)

# the covariance: multiply each race's two misses, then average.
# Same-side races add, opposite-side races subtract. R's cov() does it for us
# (a hair different from the slides, for the same reason as sd())
cov(face$d_comp, face$d_share)

# the correlation: divide the covariance by both SDs, and the units cancel
cov(face$d_comp, face$d_share) / (sd(face$d_comp) * sd(face$d_share))
cor(face$d_comp, face$d_share)   # the same number


# 🚨 Your turn: guess the correlation ----

# Look at the four panels on the slide (no code here: the code that makes
# them would give away the answers)

# 1. Write down a guess for the correlation in each panel


# 14. From averages to a line ------------------------------------------------

# chop competence into k bins with equal numbers of races, and take the
# average vote share in each bin
bin_means = function(d, k) {
  d |>
    mutate(bin = ntile(d_comp, k)) |>
    group_by(bin) |>
    summarize(lo = min(d_comp), hi = max(d_comp), share = mean(d_share))
}

bin_means(face, 5)

# draw each bin's average as a flat segment
ggplot(face, aes(x = d_comp, y = d_share)) +
  geom_point(alpha = 0.3) +
  geom_segment(data = bin_means(face, 5), aes(x = lo, xend = hi, y = share, yend = share),
               color = "orange", linewidth = 2) +
  labs(x = "% of raters who picked the Democrat's face", y = "Democrat's % of the vote")

# try more bins: change the 5s above to 2, 10, or 30

# a running average slides the bin along ("loess")
ggplot(face, aes(x = d_comp, y = d_share)) +
  geom_point(alpha = 0.3) +
  geom_smooth(method = "loess", se = FALSE) +
  labs(x = "% of raters who picked the Democrat's face", y = "Democrat's % of the vote")

# a regression line: a running average forced into a straight line
ggplot(face, aes(x = d_comp, y = d_share)) +
  geom_point(alpha = 0.3) +
  geom_smooth(method = "lm", se = FALSE) +
  labs(x = "% of raters who picked the Democrat's face", y = "Democrat's % of the vote")

# the regression behind the table on the slides. The slope on d_comp: each
# extra point of raters picking the Democrat's face goes with about 0.33
# points more of the vote. How it works is POL 212
fit = lm(d_share ~ d_comp, data = face)
summary(fit)

# a race where 70% of raters picked the Democrat's face, vs. one where 30% did
40 * coef(fit)["d_comp"]
