<commit_before>import glob
import web
import scrapenj

db = web.database(dbn='postgres', db='watchdog_dev', user='postgres', pw='')
ALMANAC_DIR = 'almanac/nationaljournal.com/pubs/almanac/2008/people/'

for fn in glob.glob(ALMANAC_DIR + '*/rep*'):
    dist = web.lstrips(web.rstrips(fn.split('/')[-1], '.htm'), 'rep_')
    dist = dist[0:2].upper() + '-' + dist[-2:]
    
    d = scrapenj.scrape1(fn)
    if 'demographics' not in d: continue
    cook_index = d['demographics']['Cook Partisan Voting Index'].replace(' ', '')
    area_sqmi = d['demographics']['Area size']
    area_sqmi = web.rstrips(area_sqmi, ' sq. mi.').replace(',', '').replace(' ', '')
    
    db.insert('district', seqname=False, name=dist, state=dist[:2], district=dist[-2:], cook_index=cook_index, area_sqmi=area_sqmi)<commit_msg>dummy commit<commit_after>import glob
import web
import scrapenj

ALMANAC_DIR = 'almanac/nationaljournal.com/pubs/almanac/2008/people/'

for fn in glob.glob(ALMANAC_DIR + '*/rep*'):
    dist = web.lstrips(web.rstrips(fn.split('/')[-1], '.htm'), 'rep_')
    dist = dist[0:2].upper() + '-' + dist[-2:]
    
    d = scrapenj.scrape1(fn)
    if 'demographics' not in d: continue
    cook_index = d['demographics']['Cook Partisan Voting Index'].replace(' ', '')
    area_sqmi = d['demographics']['Area size']
    area_sqmi = web.rstrips(area_sqmi, ' sq. mi.').replace(',', '').replace(' ', '')
    
    web.insert('district', seqname=False, name=dist, state=dist[:2], district=dist[-2:], cook_index=cook_index, area_sqmi=area_sqmi)
{'fraction_non_alphanumeric': 0.20209287115761937, 'fraction_numerical': 0.014388489208633094, 'mean_word_length': 4.5234657039711195, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '569507', 'n_tokens_mistral': 589, 'n_tokens_neox': 567, 'n_words': 101}