<commit_before>#!/usr/bin/env python

import sys
import os.path
import ColumnMap
import Mapping
import Execution
from collections import defaultdict

import pprint, json, csv

def prettyprint(AllExecutions):
	pprint.pprint(AllExecutions)

def writeJSON(AllExecutions):
	with open("data.json", "w") as outfile:
	    json.dump(AllExecutions, outfile, indent=4)

def writeCSV(AllExecutions):
	spamWriter = csv.writer(open('espy.csv', 'wb'), delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
	spamWriter.writerow(AllExecutions[1].keys())
	for execution in AllExecutions:
		spamWriter.writerow(execution.values())

def writeCSVList(AllExecutions):
	spamWriter = csv.writer(open('data.csv', 'wb'), delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
	#spamWriter.writerow(AllExecutions[1].keys())
	for execution in AllExecutions:
		spamWriter.writerow(execution)


Maps = ColumnMap.Maps
CrimeMappings = Mapping.CrimeMappings
AllExecutions = []

if not os.path.isfile('08451-0001-Data.txt'):
	print "Couldn't find '08451-0001-Data.txt'. You need to download it manually (see README)"
	sys.exit()

#1608-2002
with open('08451-0001-Data.txt') as asciidata:
 for penalty in asciidata:
 	current_execution = Execution.Execution()
 	for column in Maps:
 		value = penalty[column['begin']:column['end']+1]
 		if value.isdigit():
 			if column['expand'] == True:
	 			value = int(value)
 				if value == 0:
 					attribute = 0
	 			else:
		 			attribute = CrimeMappings[column['name']][value]

		 	else:
		 		attribute = value
		else:
			attribute = value.strip()
		setattr(current_execution,column['name'],attribute)
	AllExecutions.append(current_execution.getExecution())

writeCSV(AllExecutions)

print "found %d total executions in the data" % len(AllExecutions)

if "--states" in sys.argv:
	AllExecutions = [ex for ex in AllExecutions if ex["JurisdictionOfExecution"] == "State"]	
	print "reduced to %d executions carried out by states" % len(AllExecutions)

data = defaultdict(lambda: defaultdict(int))
total = defaultdict(int)

for ex in AllExecutions:
	data[ex['DateYear']][ex['StateOfExecution']] += 1
	total[ex['StateOfExecution']] += 1

headers = sorted(total.keys())
output = [["year"] + headers]

for y in range(1700, 2003):
	datum = [y]
	for state in headers:		
		datum.append(0 if state not in data[str(y)] else data[str(y)][state])
	output.append(datum)

writeCSVList(output)<commit_msg>Cleaned up code and added system arguments for output<commit_after>#!/usr/bin/env python

import sys
import os.path
import ColumnMap
import Mapping
import Execution
from collections import defaultdict
import pprint, json, csv

def writeOutput(AllExecutions):
	if "--csv" in sys.argv:
		if "--states" in sys.argv:
			writeCSVList(AllExecutions)
		else:
			writeCSV(AllExecutions)
	elif "--json" in sys.argv:
		writeJSON(AllExecutions)
	else:
		print "File format argument not specified, printing data here"
		prettyprint(AllExecutions)

def prettyprint(AllExecutions):
	pprint.pprint(AllExecutions)

def writeJSON(AllExecutions):
	with open("data.json", "w") as outfile:
	    json.dump(AllExecutions, outfile, indent=4)
	print "Wrote JSON file"

def writeCSV(AllExecutions):
	spamWriter = csv.writer(open('data.csv', 'wb'), delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
	spamWriter.writerow(AllExecutions[1].keys())
	for execution in AllExecutions:
		spamWriter.writerow(execution.values())
	print "Wrote CSV file"


def writeCSVList(AllExecutions):
	spamWriter = csv.writer(open('data.csv', 'wb'), delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
	#spamWriter.writerow(AllExecutions[1].keys())
	for execution in AllExecutions:
		spamWriter.writerow(execution)
	print "Wrote CSV file"


Maps = ColumnMap.Maps
CrimeMappings = Mapping.CrimeMappings
AllExecutions = []

if not os.path.isfile('08451-0001-Data.txt'):
	print "Couldn't find '08451-0001-Data.txt'. You need to download it manually (see README)"
	sys.exit()

#1608-2002
with open('08451-0001-Data.txt') as asciidata:
 for penalty in asciidata:
 	current_execution = Execution.Execution()
 	for column in Maps:
 		value = penalty[column['begin']:column['end']+1]
 		if value.isdigit():
 			if column['expand'] == True:
	 			value = int(value)
 				if value == 0:
 					attribute = 0
	 			else:
		 			attribute = CrimeMappings[column['name']][value]

		 	else:
		 		attribute = value
		else:
			attribute = value.strip()
		setattr(current_execution,column['name'],attribute)
	AllExecutions.append(current_execution.getExecution())

print "found %d total executions in the data" % len(AllExecutions)

if "--states" in sys.argv:
	AllExecutions = [ex for ex in AllExecutions if ex["JurisdictionOfExecution"] == "State"]	
	print "reduced to %d executions carried out by states" % len(AllExecutions)
	data = defaultdict(lambda: defaultdict(int))
	total = defaultdict(int)

	for ex in AllExecutions:
		data[ex['DateYear']][ex['StateOfExecution']] += 1
		total[ex['StateOfExecution']] += 1

	headers = sorted(total.keys())
	output = [["year"] + headers]
	for y in range(1700, 2003):
		datum = [y]
		for state in headers:		
			datum.append(0 if state not in data[str(y)] else data[str(y)][state])
		output.append(datum)
	AllExecutions = output

writeOutput(AllExecutions)
{'fraction_non_alphanumeric': 0.13380416109944646, 'fraction_numerical': 0.019851116625310174, 'mean_word_length': 5.193853427895981, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3117397', 'n_tokens_mistral': 1983, 'n_tokens_neox': 1780, 'n_words': 437}