<jupyter_start><jupyter_text>FEC Campaign Finance Committee Master<jupyter_code>import pandas as pd
import numpy as np

from datetime import datetime as dt
year = '2018'
# read in & clean
cm = pd.read_csv(
    f'https://www.fec.gov/files/bulk-downloads/{year}/cm{year[2:]}.zip', 
    sep = '|', 
    error_bad_lines = False, 
    header = None, 
)
print('original length:', len(cm))

# add column headers from separate file
headers = pd.read_csv(
    'https://www.fec.gov/files/bulk-downloads/data_dictionaries/cm_header_file.csv', 
)
cm.columns = [x.lower() for x in headers.columns]

# fillna
cm['cmte_pty_affiliation'].fillna('UNK', inplace = True)

# preserve uncleaned tres_nm & cmte_st1
cm['orig_tres_nm'] = list(cm['tres_nm'].values)
cm['orig_cmte_st1'] = list(cm['cmte_st1'].values)

# prep strings for deduping
change_dict = {
    ' STREET' : ' ST',
    'AVENUE' : 'AVE',
    ' DRIVE' : ' DR',
    ' ROAD' : ' RD',
    ' SUITE' : ' STE',
    '.' : '',
    ',' : '',
    'BOULEVARD' : 'BLVD',
    ' PLACE' : ' PL',
    
}
keys = list(set(change_dict.keys()))
for key in keys:
    cm['cmte_st1'] = [str(x).replace(key, change_dict[key]) for x in cm['cmte_st1']]

keep_0 = [
    ' STE ',
    '#',
    ' ST ',
    ' AVE ',
    ' RD ',
    ' PL ', 
    ' BLVD ', 
    ' DR ',
]

cm['cmte_st1'] = [x.split(' STE ')[0].split('#')[0].split(' NUM ')[0] for \
                  x in cm['cmte_st1']]
cm['cmte_st1'] = [' '.join(str(x).split()) for x in cm['cmte_st1']]

cm['cmte_zip'] = [str(x).strip() for x in cm['cmte_zip']]
cm.loc[cm['cmte_zip'].apply(lambda x: len(str(x))) == 9, 'cmte_zip'] = \
cm.loc[cm['cmte_zip'].apply(lambda x: len(str(x))) == 9, 'cmte_zip'].apply(lambda x: x[:5])

cm['cmte_zip'].apply(lambda x: len(x)).value_counts().sort_index()
cands = pd.read_csv('data/02a_cand_clean.csv')

cands = list(cands['cand_id'])

mask = []
for x in cm['cand_id']:
    if type(x) == float:
        mask.append(True)
    else:
        if x in cands:
            mask.append(True)
        else:
            mask.append(False)
        
cm = cm[mask]

print(len(cm))
cm.loc[cm['cmte_zip'].apply(lambda x: len(str(x))) != 5, 
       ['cmte_st1', 'cmte_city', 'cmte_st', 'cmte_zip']].sort_values([
    'cmte_zip', 
    'cmte_st1', 
    'cmte_city',
    'cmte_st',
])
fix_zip = {
    '291501' : '29150',
    '89053145' : '89015',
    '04342443' : '70433',
    '21232813' : '02126',
    '7276' : '72762',
    '2021' : '02021',
    '2026' : '02026',
    '2446' : '02446',
    '3768' : '03768',
    '7201' : '07201',
    '7724' : '07724',
    
}
for key in list(fix_zip.keys()):
    cm.loc[cm['cmte_zip'] == key, 'cmte_zip'] = fix_zip[key]

cm.loc[cm['cmte_st1'] == '1055 CATALPA RD', 'cmte_zip'] = '91007'
cm.loc[cm['cmte_st1'] == '2008 OLDTOWN VALLEY RD SE', 'cmte_zip'] = '44663'
cm.loc[
    (cm['cmte_st1'] == '247 THIRD ST') & \
    (cm['cmte_city'] == 'ASHLAND'), 
    'cmte_zip'] = '97520'
cm.loc[cm['cmte_st1'] == '420 KEWANNA DR', 'cmte_zip'] = '47130'
cm.loc[cm['cmte_st1'] == '868 CHURCH ST -', 'cmte_zip'] = '11716'
cm.loc[
    (cm['cmte_st1'] == 'PO BOX 334') & \
    (cm['cmte_city'] == 'DAVIDSONVILLE'), 
    'cmte_zip'] = '21035'
cm.loc[
    (cm['cmte_st1'] == 'PO BOX 356') & \
    (cm['cmte_city'] == 'LIVINGSTON'), 
    'cmte_zip'] = '07039'
cm.loc[cm['cmte_zip'].apply(lambda x: len(x) == 3), 'cmte_zip'] = np.nan
# create full address column from cleaned parts
cm['address'] = \
cm['cmte_st1'].apply(lambda x: str(x).strip()) + ', ' + \
cm['cmte_city'].apply(lambda x: str(x).strip()) + ', ' + \
cm['cmte_st'].apply(lambda x: str(x).strip()) + ' ' + \
cm['cmte_zip'].apply(lambda x: str(x).strip())

cm.head(2)
# check for duplicate rows
cm[cm.duplicated(keep = False)]
# drop duplicates
cm.drop_duplicates(keep = 'first', inplace = True)
# see which cmte_ids appear more than once
cm['cmte_id'].value_counts()[cm['cmte_id'].value_counts() > 1]
# check for nulls
cm.isnull().sum()[cm.isnull().sum() > 0].sort_values()
# see which have missing cmte_nm
cm[cm['cmte_nm'].isnull()]
# fill in placeholder cmte_nm
cm.loc[cm['cmte_nm'].isnull(), 'cmte_nm'] = '(cmte_nm missing)'
# see which have missing address
# then see if any other rows have matching cmte_nm
cm[cm['address'].isnull()]
# see which street addresses appear most frequently
cm['address'].value_counts()[:20]
# fill in placeholder for missing treasurer name 
print(len(cm[cm['tres_nm'].isnull()]))
cm.loc[cm['tres_nm'].isnull(), 'tres_nm'] = '(no treasurer listed)'
cm.loc[cm['tres_nm'].apply(lambda x: x in ['.', 'NONE', '']), 'tres_nm'] = '(no treasurer listed)'

# convert to <first name> <last name> format from <last name>, <first name>
cm['tres_nm'] = [' '.join([' '.join(x.split(', ')[1:]), x.split(', ')[0]]) if ', ' in \
                 x else x for x in cm['tres_nm']]
# clean up spacing
# looks like this could just be ' '.join(x.strip()) 
# but i feel like i did it this way for a reason... leaving it
cm['tres_nm'] = [' '.join([x for x in list(x.split(' ')) if len(x) > 1]).strip() for \
                 x in cm['tres_nm']]

# prep for deduping

# substitutions
subs = {
    'XAVUER' : 'XAVIER',
    'JSOEPH' : 'JOSEPH',
    'WILLAIM' : 'WILLIAM',
    'V?RONIQUE' : 'VERONIQUE',
    '.' : '',
    ',' : '',
    
}
for key in list(subs.keys()):
    cm['tres_nm'] = [x.replace(key, subs[key]) for x in cm['tres_nm']]

# drops
drops = [
    'MRS',
    'MS',
    'MR',
    'MISS',
    'JR',
    'SR',
    'UNDEFINED',
    'JD',
    'CPA',
    'HONORABLE',
    'FED',
    'II', 
    'III',
    'IV',
    'ESQ',
    'DR',
    'MD',
    'DO',
    'CFO',
    'CEO',
    'CTO',
    'TREAS',
    'US',
    '1970',
    '--SELECT',
    '1973',
    '1979',
    
]
for droplet in drops:
    cm['tres_nm'] = [' '.join([x for x in x.split() if \
                               ((x not in drops) & (len(x) > 1))]) for x in cm['tres_nm']]

# a bunch of entries have last name appearing twice; 
# check for this and keep only first instance
cm['tres_nm'] = [x.split() for x in cm['tres_nm']]
def f7(seq):
    '''https://stackoverflow.com/questions/480214/\
    how-do-you-remove-duplicates-from-a-list-whilst-preserving-order'''
    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x in seen or seen_add(x))]            
cm['tres_nm'] = [' '.join(f7(x)) for x in cm['tres_nm']]

# see tres_nms appearing most frequently
cm['tres_nm'].value_counts()[:20]
# what is street address of committees not listing treasurer
cm.loc[cm['tres_nm'] == '(no treasurer listed)', 'address'].value_counts()[:20]
list(cm.loc[cm['address'] == '1201 CONNECTICUT AVE NW, WASHINGTON, DC 20036', 'cmte_nm'])
list(cm.loc[
    (cm['address'] == '700 13TH ST NW, WASHINGTON, DC 20005') & \
    (cm['tres_nm'] == '(no treasurer listed)'), 'cmte_nm'])
list(cm.loc[
    (cm['address'] == '555 CAPITOL MALL, SACRAMENTO, CA 95814') & \
    (cm['tres_nm'] == '(no treasurer listed)'), 'cmte_nm'])
# mark principal campaign committees

pccs = pd.read_csv('data/02a_principalcampaigncommittees.csv')

pccs.head()
cm['pcc'] = [True if (x in list(pccs['cmte_id'].values)) else False for x in cm['cmte_id']]
cm['pcc'].value_counts()<jupyter_output><empty_output><jupyter_text>Dedupe treasurer namesThis is a journey.<jupyter_code>def findall(name):
    '''find pattern in treasurer names columns'''
    all_tres = list(set(cm['tres_nm']))
    if type(name) == str:
        these = [x for x in all_tres if name.lower() in str(x).lower()]
    else:  
        these = all_tres
        for which in name:
            these = [x for x in these if which in str(x).lower()]
    return sorted(these)
def getall(these, new):
    '''assign deduped treasurer name'''
    print('orig len:', len(cm[cm['tres_nm'] == new]))
    for which in these:
        cm.loc[cm['tres_nm'] == which, 'tres_nm'] = new
    print('new len:', len(cm[cm['tres_nm'] == new]))
    return 
# this list keeps track of the ones we've already addressed
dones = ['(no treasurer listed)']
findall(' kil')
findall(' joyce')
dones.extend(findall(' kil'))
findall(' lis')
dones.extend(findall(' lis'))
findall('lisker')
new = '<NAME>'
dones.append(new)

these = findall('lisker')

getall(these, new)
findall(['c', 'marst'])
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>'].head()
new = '<NAME>'
dones.append(new)

these = findall(['c', 'marst'])

getall(these, new)
findall(['st', ' martin'])
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>'].head()
dones.extend(findall(['st', ' martin']))
findall(' crat')
dones.extend(findall(' crat'))
findall('datwyler')
new = '<NAME>'
dones.append(new)

these = findall('datwyler')

getall(these, new)
findall('zamore')
new = '<NAME>'
dones.append(new)

these = findall('zamore')

getall(these, new)
findall(' pett')
dones.extend(findall(' pett'))
findall(['jen', 'may'])
dones.append('<NAME>')
# first jennifer for driskell
cm[cm['tres_nm'] == '<NAME>']
# second jennifer for driskell.  same?
cm[(cm['tres_nm'] == '<NAME>') & (cm['cmte_st1'] == 'PO BOX 464')]
findall(' williamson')
dones.extend(findall(' williamson'))
findall(' ky')
dones.extend(findall(' ky'))
findall(' watkins')
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
new = '<NAME>'
dones.append(new)

these = ['<NAME>', '<NAME>']

getall(these, new)
dones.extend(findall(' watkins'))
findall('hobbs')
findall('giles')
dones.extend(findall('giles'))
dones.extend(findall('hobbs'))
findall(' satt')
dones.extend(findall(' satt'))
findall(' curtis')
findall('brackett')
findall(['mi', 'dean'])
new = '<NAME>'
dones.append(new)

these = findall(['liz', 'curtis'])

getall(these, new)
dones.extend(findall(' curtis'))
findall(' davis')
cm[cm['tres_nm'].apply(lambda x: x in ['<NAME>', '<NAME>'])]
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>'].head()
new = '<NAME>' 
dones.append(new)

these = ['<NAME>', '<NAME>']

getall(these, new)
cm[cm['tres_nm'].apply(lambda x: x in ['<NAME>', '<NAME>'])]
dones.extend(findall(' davis'))
findall(['phillips', 'ob'])
new = '<NAME>' 
dones.append(new)

these = findall(['phillips', 'ob'])

getall(these, new)
findall(' mele')
cm[cm['tres_nm'] == 'STEVE MELE']
cm[cm['tres_nm'] == 'STEVEN MELE'].head()
new = '<NAME>'
dones.append(new)

these = findall(['steve', ' mele'])

getall(these, new)
findall(['su', 'jackson'])
new = '<NAME>'
dones.append(new)

these = findall(['su', 'jackson'])

getall(these, new)
findall(' otte')
new = '<NAME>'
dones.append(new)

these = findall(' otte')

getall(these, new)
findall(' mcmi')
dones.extend(findall(' mcmi'))
findall(' lew')
findall(['and', ' james'])
findall(['cord', 'burk'])
findall('charbon')
findall(['nat', 'wurtz'])
findall(' alston')
cm[cm['tres_nm'] == 'WENDY LEWIS']
cm[cm['tres_nm'] == 'WENDY LEWIS ARMSTRONG']
new = 'WENDY LEWIS'
dones.append(new)

these = findall('wendy lewis')

getall(these, new)
dones.extend(findall(' lew'))
findall(' lawl')
dones.extend(findall(' lawl'))
findall(' crum')
dones.extend(findall(' crum'))
findall(' slater')
dones.extend(findall(' slater'))
findall(' mccaul')
dones.extend(findall(' mccaul'))
findall(' montg')
dones.extend(findall(' montg'))
findall(' nis')
dones.extend(findall(' nis'))
findall(' matthews')
dones.extend(findall(' matthews'))
findall(' goul')
cm[cm['tres_nm'].apply(lambda x: ('ADAM' in x) & ('GOULD' in x))]
new = '<NAME>'
dones.append(new)

these = findall(['adam', 'gould'])

getall(these, new)
dones.extend(findall(' goul'))
findall(' gan')
findall('borja')
findall(['mary', 'randolph'])
dones.extend(findall(' gan'))
findall(' carli')
dones.extend(findall(' carli'))
findall(' kra')
cm[cm['tres_nm'] == 'THOMAS KRAUS']
cm[cm['tres_nm'] == 'TOM KRAUSE']
dones.extend(findall(' kra'))
findall(' koch')
dones.extend(findall(' koch'))
findall(' win')
dones.extend(findall(' win'))
findall(' rit')
dones.extend(findall(' rit'))
findall(' tat')
cm[cm['tres_nm'].apply(lambda x: x in ['BRIAN TATUM', 'BRYAN TATUM'])]
dones.extend(findall(' tat'))
findall('ralls')
cm[cm['tres_nm'] == 'STEVEN RALLS']
cm[cm['tres_nm'] == 'STEVE RALLS'].head()
new = 'STEVE RALLS'
dones.append(new)

these = findall('ralls')

getall(these, new)
findall(' has')
dones.extend(findall(' has'))
findall(' buch')
dones.extend(findall(' buch'))
findall(' bac')
dones.extend(findall(' bac'))
findall(' brog')
dones.extend(findall(' brog'))
findall(' gia')
findall('longacre')
dones.extend(findall(' gia'))
findall(' deane')
dones.extend(findall(' deane'))
findall(' hag')
dones.extend(findall(' hag'))
findall(' anger')
dones.extend(findall(' anger'))
findall(' lowe')
dones.extend(findall(' lowe'))
findall(' bau')
findall(['ju', 'mart'])
dones.extend(findall(' bau'))
findall(' evans')
findall(' doyle')
dones.extend(findall(' doyle'))
dones.extend(findall(' evans'))
findall(' mason')
cm[cm['tres_nm'] == 'DAVE MASON'].head()
cm[(cm['tres_nm'] == 'DAVID MASON') & (cm['cmte_st'] == 'SC')].head()
findall('cantrell')
cm[cm['tres_nm'] == '<NAME>ASON'].head()
cm[(cm['tres_nm'] == '<NAME>') & (cm['cmte_st'] == 'TX')].head()
new = '<NAME>'
dones.append(new)

these = findall(['michael', 'mason'])

getall(these, new)
findall(' hale')
dones.extend(findall(' mason'))
findall(' lloyd')
findall(' mcneel')
dones.extend(findall(' lloyd'))
findall(' johnson')
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
new = '<NAME>'
dones.append(new)

these = findall(['lor', 'etta', 'johnson'])

getall(these, new)
findall(['mi', 'gibson'])
dones.extend(findall(' johnson'))
findall(' goldstein')
dones.extend(findall(' goldstein'))
findall(' marks')
dones.extend(findall(' marks'))
findall(' grav')
dones.extend(findall(' grav'))
findall(' eich')
dones.extend(findall(' eich'))
findall(' fou')
dones.extend(findall(' fou'))
findall(' martin')
findall(' salazar')
findall('salzburg')
cm[cm['tres_nm'] == '<NAME>']
cm[(cm['tres_nm'] == '<NAME>') & (cm['cmte_st'] == 'NC')]
dones.extend(findall(' martin'))
findall(' davidson')
findall(['paul', 'reynolds'])
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
new = '<NAME>'
dones.append(new)

these = findall(['paul', 'reynolds'])

getall(these, new)
dones.extend(findall(' davidson'))
findall(' bren')
findall(' mantan')
dones.extend(findall(' bren'))
findall(' mont')
dones.extend(findall(' mont'))
findall(' hank')
dones.extend(findall(' hank'))
findall(' lesh')
dones.extend(findall(' lesh'))
findall(' bre')
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
new = '<NAME>'
dones.append(new)

these = findall(['james', 'brewer'])

getall(these, new)
findall(' ringham')
dones.extend(findall(' bre'))
findall(' sel')
findall(['ja', 'peters'])
findall(['sar', 'scot'])
dones.extend(findall(' sel'))
findall(' owen')
findall(' mosier')
dones.extend(findall(' owen'))
findall(' vo')
findall(' luza')
dones.extend(findall(' vo'))
findall(' bol')
dones.extend(findall(' bol'))
findall(' hub')
findall(' hauser')
dones.extend(findall(' hub'))
findall(' oz')
findall(' tunc')
dones.extend(findall(' oz'))
findall(' groe')
dones.extend(findall(' groe'))
findall(' sola')
findall(['ed', 'patterson'])
dones.extend(findall(' sola'))
findall(' rupp')
dones.extend(findall(' rupp'))
findall(' schi')
dones.extend(findall(' schi'))
findall(' guin')
dones.extend(findall(' guin'))
findall(' stone')
dones.extend(findall(' stone'))
findall(' cros')
dones.extend(findall(' cros'))
findall(' gla')
findall('hemmig')
dones.extend(findall(' gla'))
findall(' ragan')
dones.extend(findall(' ragan'))
findall(' mau')
findall('littlejohn')
findall('buchser')
dones.extend(findall(' mau'))
findall(' snyd')
dones.extend(findall(' snyd'))
findall(' pur')
findall(' balc')
dones.extend(findall(' pur'))
findall(' lei')
findall('bronson')
findall('kasz')
findall(['cynth', 'apple'])
findall('fowler')
findall('keller')
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
findall(' hubbard')
dones.extend(findall(' lei'))
findall(' tea')
dones.extend(findall(' tea'))
findall(' rut')
findall('bellar')
findall(' mayo')
dones.extend(findall(' mayo'))
dones.extend(findall(' rut'))
findall(' elli')
findall(' robey')
findall(' dahl')
dones.extend(findall(' elli'))
findall(' kee')
findall(' segars')
dones.extend(findall(' kee'))
findall(' jan')
findall(' janssen')
findall(' thakh')
findall('osgerby')
findall('wharton')
findall(' tanis')
dones.extend(findall(' jan'))
findall(' dupr')
dones.extend(findall(' dupr'))
findall(' schwartz')
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
dones.extend(findall(' schwartz'))
findall(' kau')
cm[cm['tres_nm'] == '<NAME>']
cm[cm['tres_nm'] == '<NAME>']
new = '<NAME>'
dones.append(new)

these = findall(['ron', ' kaufman'])

getall(these, new)
dones.extend(findall(' kau'))
findall(' riesc')
dones.extend(findall(' riesc'))
findall(' cop')
dones.extend(findall(' cop'))
findall(' bal')
dones.extend(findall(' bal'))
findall(' tall')
dones.extend(findall(' tall'))
findall(' juk')
dones.extend(findall(' juk'))
findall(' milln')
dones.extend(findall(' milln'))
findall(' goode')
dones.extend(findall(' goode'))
findall(' kun')
cm[cm['tres_nm'].apply(lambda x: ('KUNITZ' in x) & ('JOHN' in x))]
new = '<NAME>'
dones.append(new)

these = findall(['kunitz', 'john'])

getall(these, new)
dones.extend(findall(' kun'))
findall('lefko')
cm[cm['tres_nm'].apply(lambda x: 'LEFKOWITZ' in x)]
new = '<NAME>'
dones.append(new)

these = findall('lefkowitz')

getall(these, new)
findall(' sif')
dones.extend(findall(' sif'))
findall(' vand')
dones.extend(findall(' vand'))
findall(' changkyu')
new = '<NAME>'
dones.append(new)

these = findall(' changkyu')

getall(these, new)
findall(['ash', 'newman'])
dones.extend(findall(['ash', 'newman']))
findall(' rogers')
dones.extend(findall(' rogers'))
findall(' kennedy')
cm[cm['tres_nm'].apply(lambda x: x in ['<NAME>', '<NAME>'])]
cm[cm['tres_nm'].apply(lambda x: x in ['<NAME>', '<NAME>'])]
new = '<NAME>'
dones.append(new)

these = findall(['kat', 'kennedy'])

getall(these, new)
cm[cm['tres_nm'].apply(lambda x: x in [
    '<NAME>', 
    '<NAME>', 
])]
dones.extend(findall(' kennedy'))
findall(['mel', 'allen'])
findall(' himmel')
dones.extend(findall(' himmel'))
dones.extend(findall(['mel', 'allen']))
findall(' reis')
dones.extend(findall(' reis'))
findall(' doz')
dones.extend(findall(' doz'))
findall(' roz')
dones.extend(findall(' roz'))
findall(' patt')
dones.extend(findall(' patt'))<jupyter_output><empty_output><jupyter_text>The work is never done...<jupyter_code># checks which have already been deduped and does not display those
mask = [False if cm.loc[i, 'tres_nm'] in dones else True for i in list(cm.index)]
cm.loc[mask, 'tres_nm'].value_counts()[cm.loc[mask, 'tres_nm'].value_counts() > 1][:20]
dupename = list(cm['cmte_nm'].value_counts()[cm['cmte_nm'].value_counts() > 2].index)

for dupe in dupename:
    lil_cm = cm.loc[cm['cmte_nm'] == dupe, [
#         'cmte_id',
        'tres_nm',
        'address',
    ]].sort_values(['address'])
    print(dupe)
    print(lil_cm)
    print('\n')
# show all cmte_nm & tres_nm for committees sharing a street address
multiadd = list(cm['address'].value_counts()[cm['address'].value_counts() > 10].index)

for add in multiadd:
    lil_cm = cm.loc[cm['address'] == add, ['tres_nm', 'cmte_nm']]\
    .sort_values(['tres_nm', 'cmte_nm'], ascending = [True, True]).reset_index(drop = True)
    print('-----------------------------------------------------------')
    print(add)
    print(len(lil_cm))
    print('\n')
    treses = list(lil_cm['tres_nm'].value_counts().index)
    for tres in treses:
        lil_tres = lil_cm[lil_cm['tres_nm'] == tres]
        if len(lil_tres) > 1:
            committees = sorted(list(set(lil_tres['cmte_nm'])))
            print(tres, len(committees))
            print('\n')
            for committee in committees:
                print('    ' + committee)
            print('\n')
    print('\n')
    
# "look here"<jupyter_output>-----------------------------------------------------------
228 S WASHINGTON ST, ALEXANDRIA, VA 22314
156


<NAME> 74


    AMERICANS UNITED FOR FREEDOM
    ARIZONA GRASSROOTS ACTION PAC
    BACON VICTORY FUND
    BOLD ACTIVE CONSERVATIVES OF NEBRASKA PAC-BACON PAC
    BRIDGING THE GAP
    BUCKEYE JOINT FUNDRAISING COMMITTEE
    BUILDING OUR BASE - BOB PAC
    BUILDING RENEWAL IN AMERICA NOW PAC
    CITIZENS FOR PROSPERITY IN AMERICA TODAY PAC
    CONCERNED AMERICANS FOR FREEDOM & OPPORTUNITY PAC (CAFO PAC)
    CRAPO VICTORY COMMITTEE
    CREATING OPPORTUNITIES, MARKETS AND ENTHUSIASM IN RURAL KENTUCKY PAC
    FREEDOM AND SECURITY PAC
    FREEDOM PROJECT; THE
    FRIENDS OF WINNING WOMEN 2016
    FRIENDS OF WINNING WOMEN 2018
    GARDNER VICTORY COMMITTEE
    GOP MAJORITY VICTORY FUND
    GOP WINNING WOMEN
    GRAHAM MAJORITY FUND
    GROWING REPUBLICAN ACHIEVEMENTS AND PROMOTING EXCELLENCE-GRAPE JFC
    HONOR AND PRINCIPLES PAC
    HOUSE CONSERVATIVES FUND
    IOWA CONGRESSIONAL [...]<jupyter_text>Other columns<jupyter_code>cmtetp_dict = {
    'C' : 'cmte_CommunicationCost',
    'D' : 'cmte_DelegateCommittee',
    'E' : 'cmte_ElectioneeringComms',
    'H' : 'cmte_House',
    'I' : 'cmte_IndependentExpenditor',
    'S' : 'cmte_Senate',
    'N' : 'cmte_PACnonqual',
    'Q' : 'cmte_PACqual',
    'Y' : 'cmte_Partyqual',
    'O' : 'cmte_SuperPAC',
    'P' : 'cmte_Presidential',
    'U' : 'cmte_Singlecand',
    'V' : 'cmte_PACnoncontrib_nonqual',
    'W' : 'cmte_PACnoncontrib_qual',
    'X' : 'cmte_Partynonqual',
    np.nan : np.nan,
}

cm['cmte_tp'] = [cmtetp_dict[x] for x in cm['cmte_tp']]
cm['cmte_tp'].value_counts()
cmtedsgn_dict = {
    'A' : 'Authorized by a candidate',
    'B' : 'Lobbyist/Registrant PAC',
    'D' : 'Leadership PAC',
    'J' : 'Joint fundraiser',
    'P' : 'Principal campaign committee',
    'U' : 'Unauthorized',
    np.nan : np.nan,
}

cm['cmte_dsgn'] = [cmtedsgn_dict[x] for x in cm['cmte_dsgn']]
cm['cmte_dsgn'].value_counts()
pty_affil = [
    np.nan,
    '.',
    'UKN',
    '  0',
    'UN',
]

cm['cmte_pty_affiliation'] = ['UNK' if (x in pty_affil) else x for x in cm['cmte_pty_affiliation']]
cm['cmte_pty_affiliation'].value_counts()
cm['cmte_filing_freq'].value_counts()
orgtp_dict = {
    'C' : 'org_Corporation',
    'H' : 'org_HnotonFECsite',
    'I' : 'org_InotonFECsite',
    'L' : 'org_Labor organization',
    'M' : 'org_Membership organization',
    'T' : 'org_Trade association',
    'V' : 'org_Cooperative',
    'W' : 'org_Corporation without capital stock',
    np.nan : np.nan,
}

cm['org_tp'] = [orgtp_dict[x] for x in cm['org_tp']]
cm['org_tp'].value_counts()
cm['connected_org_nm'].value_counts()[:10]
cm.loc[cm['connected_org_nm'] == 'NONE', 'connected_org_nm'] = np.nan
cm['connected_org_nm'].value_counts()[:10]
# sorry! these print ugly but it's to display them unabbreviated
treses = list(cm['tres_nm'].value_counts()[cm['tres_nm'].value_counts() > 10].index)

for tres in treses:
    if tres != '(no treasurer listed)':
        lil_cm = cm[cm['tres_nm'] == tres]
        lil_cm = lil_cm[[
            'cmte_id',
            'cand_id',
            'cmte_nm',

        ]].sort_values(['cand_id', 'cmte_nm']).reset_index(drop = True)
        pty = list(set(cm.loc[list(lil_cm.index), 'cmte_pty_affiliation']))
        ptys = sorted([x for x in pty if (type(x) != float)])
        print(''.join([tres, ', treasurer']))
        print(len(lil_cm), 'committees')
        print('parties:', str(ptys))
        print(lil_cm.to_csv(None))
        print('\n\n')
cm.to_csv('data/03a_committees.csv', index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.22020648340910057, 'fraction_numerical': 0.013128266134171729, 'mean_word_length': 4.575082899099953, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 98, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '20292', 'license': 'Apache-2.0', 'n_tokens_mistral': 9776, 'n_tokens_neox': 9496, 'n_words': 1994, 'path': 'projects/FEC/2018/03a - 2018_CommitteeMaster_clean.ipynb', 'repo_name': 'volsweep/volsweep.github.io', 'size': '24110'}