<commit_before>import sqlite3
import sys

usage = "Usage: python parser.py dataFile"

if len(sys.argv) < 2:
	print usage
	sys.exit(1)

conn = sqlite3.connect('incidents.db')
c = conn.cursor()

data_file = open(sys.argv[1], 'r')

headers = None

data = {}

incidents = []
for line in data_file:
	if not headers:
		headers = True

	fields = line.split("\t")

	print fields

	aggregate_level = fields[0]
	
	# TODO - parse other fields and insert into database
	incident = (
		aggregate_level,
		None,
		None,
		None,
		None,
		None,
		None,
		None,
		None,
		None,
		None,
		None,
		None
		)

	incidents.append(incident)
c.executemany('INSERT INTO incidents VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)', incidents)
c.close()
conn.commit()
conn.close()<commit_msg>Instead of writing to DB, simplify and just output json<commit_after>import sqlite3
import sys

usage = "Usage: python parser.py dataFile"

if len(sys.argv) < 2:
	print usage
	sys.exit(1)

data_file = open(sys.argv[1], 'r')

headers = None

data = []

for line in data_file:
	if not headers:
		headers = True

	fields = line.split("\t")

	incident = {}

	aggregate_level = fields[0]
	incident['aggregate_level'] = aggregate_level

	# TODO - parse other fields and add to map

	data.append(incident)

print data
{'fraction_non_alphanumeric': 0.1379857256145916, 'fraction_numerical': 0.008723235527359239, 'mean_word_length': 3.8725868725868726, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7608595', 'n_tokens_mistral': 494, 'n_tokens_neox': 454, 'n_words': 128}