<jupyter_start><jupyter_text>Projected outcomesThis notebook contains the code we used to project outcomes under the 10x5 and 50x1 districting plans using demographic threshold inference model (see section 5 of the report).<jupyter_code>import pandas
import numpy
from collections import namedtuple
from tqdm import tqdm
import matplotlib.pyplot as plt
import random
import itertools<jupyter_output><empty_output><jupyter_text>Projection model code<jupyter_code>class ElectionSampler:
    cases = None
    threshold = None
    
    def decide_case(self, groups: pandas.Series):
        number_over_threshold = (groups > self.threshold).sum()
        return self.cases[number_over_threshold]

    def sample(self, groups: pandas.Series):
        """
        :param pandas.Series groups: a single row of a DataFrame with the percentages
            of each racial group in a ward.
        """
        outcomes, probabilities = self.decide_case(groups)
        outcome = numpy.random.choice(a=outcomes, p=probabilities)
        groups_from_highest_to_lowest = groups.sort_values(ascending=False).keys()
        full_outcome = itertools.chain(
            outcome,
            itertools.repeat(0, len(groups_from_highest_to_lowest) - len(outcome))
        )
        return pandas.Series(full_outcome, index=groups_from_highest_to_lowest)

class ElectionSamplerWithMajority(ElectionSampler):
    def decide_case(self, groups: pandas.Series):
        number_over_threshold = (groups > self.threshold).sum()
        any_majority = (groups > 0.5).any()
        if any_majority and number_over_threshold in self.cases["majority"]:
            return self.cases["majority"][number_over_threshold]
        else:
            return self.cases["plurality"][number_over_threshold]

Case = namedtuple("Case", ["outcomes", "probabilities"])
class TenByFiveSampler(ElectionSampler):
    cases = {
        1: Case(outcomes=((5,), (4,1)), probabilities=(0.5, 0.5)),
        2: Case(((4, 1), (3, 2), (2, 3), (2, 2, 1), (3, 1, 1)), (0.25, 0.55, 0.1, 0.05, 0.05)),
        3: Case(((3, 1, 1), (3, 2), (2, 3), (2, 2, 1), (2, 1, 2)), (0.25, 0.25, 0.15, 0.2, 0.15)),
        4: Case(((3, 1, 1), (3, 2), (2, 3), (2, 2, 1), (2, 1, 2)), (0.2, 0.2, 0.2, 0.2, 0.2))
    }
    threshold = 1/6

class FiftyByOneSampler(ElectionSamplerWithMajority):
    cases = {
        "majority": {
            2: Case(((1,), (0,1)), (0.9, 0.1)),
            3: Case(((1,), (0,1,), (0,0,1)), (0.8, 0.18, 0.02))
        },
        "plurality": {
            1: Case(outcomes=((1,), (0,1)), probabilities=(1, 0)),
            2: Case(((1,), (0,1)), (0.6, 0.4)),
            3: Case(((1,), (0,1), (0,0,1)),(0.45, 0.3, 0.25))
        }
    }
    threshold = 0.25
    
class TenByThreeSampler(ElectionSamplerWithMajority):
    cases = {
        "majority": {
            1: Case(outcomes=((3,), (2,1)), probabilities=(0.9, 0.1)),
            2: Case(((3,), (2,1)), (0.2, 0.8)),
            3: Case(((3,), (2,1), (1,2), (1,1,1)), (0.1, 0.5, 0.3, 0.1)),
            
        },
        "plurality": {
            1: Case(((3,), (2,1)), (0.5, 0.5)),
            2: Case(((3,), (2,1), (1,2), (1,1,1)), (0.1, 0.5, 0.3, 0.1)),
            3: Case(((3,), (2,1), (1,2), (1,1,1)), (0.05, 0.15, 0.1, 0.7))
        }
    }
    threshold = 0.25
def load_data(filename):
    df = pandas.read_csv(filename)
    data = pandas.DataFrame(
        {
            "step": df["step"],
            "ward": df["ward"],
            "White": df["NH_WHITE"] / df["TOTPOP"],
            "Black": df["NH_BLACK"] / df["TOTPOP"],
            "Hispanic": df["HISP"] / df["TOTPOP"],
            "Asian": df["NH_ASIAN"] / df["TOTPOP"],
        }
    )
    return data<jupyter_output><empty_output><jupyter_text>Generating the projections<jupyter_code>def random_sample(data, size, sampler):
    all_plans = {step: data.loc[indices] for step, indices in data.groupby("step").groups.items()}
    groups = ["White", "Black", "Hispanic", "Asian"]
    plan_indices = data["step"].unique()
    
    for plan_index in tqdm(numpy.random.choice(plan_indices, size=size)):
        wards = all_plans[plan_index]
        yield wards[groups].apply(sampler.sample, axis=1).sum()
groups = ["Asian", "Black", "Hispanic", "White"]
data = load_data("../ensembles/prec50/results.csv")
outcomes_prec50 = pandas.DataFrame.from_records(random_sample(data, 1000, FiftyByOneSampler()))
outcomes_prec50.to_csv("./projection_50x1.csv")
outcomes_prec50.groupby(groups).size().to_csv("./projection_50x1_counts.csv", header=True)
data = load_data("../ensembles/prec10/results.csv")
outcomes_prec10 = pandas.DataFrame.from_records(random_sample(data, 10_000, TenByFiveSampler()))
outcomes_prec10.to_csv("./projection_10x5.csv")
outcomes_prec10.groupby(groups).size().to_csv("./projection_10x5_counts.csv", header=True)
data = load_data("../ensembles/ca10/results.csv")
outcomes_ca10_by_5 = pandas.DataFrame.from_records(random_sample(data, 10_000, TenByFiveSampler()))
outcomes_ca10_by_5.to_csv("./projection_10x5_CA.csv")
outcomes_ca10_by_5.groupby(groups).size().to_csv("./projection_10x5_CA_counts.csv", header=True)
data = load_data("../ensembles/ca10/results.csv")
outcomes_ca10_by_3 = pandas.DataFrame.from_records(random_sample(data, 10_000, TenByThreeSampler()))
outcomes_ca10_by_3.to_csv("./projection_10x3_CA.csv")
outcomes_ca10_by_3.groupby(groups).size().to_csv("./projection_10x3_CA_counts.csv", header=True)<jupyter_output><empty_output><jupyter_text>Plotting the resultsHere's an example plot of the projections we just made using the 50x1 ensemble.<jupyter_code>outcomes_prec50[groups].hist(sharex=True, grid=False, figsize=(12,10), color="#0099cd", bins=list(range(30)))
outcomes_prec10_by_5 = pandas.read_csv("./projection_10x5.csv")
outcomes_prec10_by_5[groups].hist(sharex=True, grid=False, figsize=(12,10), color="#0099cd", bins=list(range(30)))<jupyter_output><empty_output><jupyter_text>StatisticsWe can use `pandas` to compute statistics for the projected outcomes:<jupyter_code>outcomes_prec50.mean()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17199139215361695, 'fraction_numerical': 0.058268498592948186, 'mean_word_length': 3.7425431711145998, 'pattern_counts': {'":': 10, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '590641', 'license': 'MIT', 'n_tokens_mistral': 2382, 'n_tokens_neox': 2198, 'n_words': 412, 'path': 'projection/projection/Projection.ipynb', 'repo_name': 'hangulu/mggg_chicago', 'size': '6041'}