<jupyter_start><jupyter_text>Michigan Data<jupyter_code>df = pd.read_json('../../public/michigan.json') #pd.read_json('cache/data.jl', lines=True)<jupyter_output><empty_output><jupyter_text>County Data<jupyter_code>df['county'].value_counts()<jupyter_output><empty_output><jupyter_text>Local Data<jupyter_code>df.sample(10, random_state=42)<jupyter_output><empty_output><jupyter_text>These missing values appear to be missing from the website<jupyter_code>df[df['emails'].apply(len)<1]
df[df['faxes'].apply(len)<1]<jupyter_output><empty_output><jupyter_text>These are suffixes used for cities<jupyter_code>df['city'].str.split(' ').apply(lambda x: x[-1]).value_counts()
df['address'].sample(10, random_state=42)
def get_michigan_index(parts):
    for i, p in enumerate(parts):
        if p.startswith('Michigan'):
            return i
        
    return None

def get_city(address):
    parts = address.split(', ')
    michigan_idx = get_michigan_index(parts)
    return parts[michigan_idx - 1]

df['addr_city'] = df['address'].apply(get_city)
df['addr_city'].sample(10, random_state=42)
df['within'] = df.apply(lambda x: x['addr_city'] in x['city'], axis=1)
print("Only about {} are within their own townships ".format(df['within'].mean()))
df.loc[~df['within'], ['addr_city', 'city', 'county']].sample(20, random_state=42)
df['maybe_within'] = df['addr_city'].apply(
    lambda x: df['city'].str.startswith(x).any()
)
print("Only about {} are possibly within an existing township ".format(df['maybe_within'].mean()))
df.loc[~df['maybe_within'], ['addr_city', 'city', 'county']].sample(20, random_state=42)<jupyter_output><empty_output><jupyter_text>Is Township vs. Charter Township Identifying?Yes -- there is both a Township and a Charter Township for each of these place names<jupyter_code>df_charter_township = df[df['city'].str.endswith('Charter Township')]
charter_names = df_charter_township['city'].str.replace(' Charter ', ' ')
all_names = set(df['city'])
duplicate_names = [name for name in charter_names if name in all_names]
for name in duplicate_names:
    print(name)<jupyter_output>Benton Township
Caledonia Township
Lake Township
Lincoln Township
Shelby Township
Sheridan Township
{'fraction_non_alphanumeric': 0.1592356687898089, 'fraction_numerical': 0.011373976342129208, 'mean_word_length': 7.114391143911439, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '639491', 'license': 'Apache-2.0', 'n_tokens_mistral': 798, 'n_tokens_neox': 760, 'n_words': 146, 'path': 'analysis/Michigan_Analysis.ipynb', 'repo_name': 'TheReFTW/election-official-data', 'size': '2198'}