<commit_before><commit_msg>a script for computing summary stats on violations<commit_after>import csv
import json
import os
import itertools

histogram={}
#going to need to do something different with the path
with open('C:\Users\Alexander\Documents\GitHub\injustice_dropper\data\\violations.csv', 'rb') as citations:
    reader = csv.DictReader(citations)
    for row in reader:
        key=row['violation_description']
        if key not in histogram:
            histogram[key]={'fine_amount':{'sum':0,'total':0},'court_cost':{'sum':0,'total':0}}
        if len(row['court_cost'])>0:
            cost=float(row['court_cost'].strip('$'))
            histogram[key]['court_cost']['sum']+=cost
            histogram[key]['court_cost']['total']+=1
        if len(row['fine_amount'])>0:
            cost=float(row['fine_amount'].strip('$'))
            histogram[key]['fine_amount']['sum']+=cost
            histogram[key]['fine_amount']['total']+=1
print histogram
output={}
for key in histogram:
    output[key]={
        'fine_amount':float(histogram[key]['fine_amount']['sum'])/float(histogram[key]['fine_amount']['total']),
        'court_cost':float(histogram[key]['court_cost']['sum'])/float(histogram[key]['court_cost']['total']),
    }
print(output)        
        
{'fraction_non_alphanumeric': 0.1791044776119403, 'fraction_numerical': 0.006284367635506678, 'mean_word_length': 4.157894736842105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4182937', 'n_tokens_mistral': 401, 'n_tokens_neox': 382, 'n_words': 65}