<commit_before>import fileinput
from dateutil.parser import parse

header = True
for line in fileinput.input():
	if header:
		print line
		header = False
	else:
		ss=line.rstrip().split(",")
		#print ss
		tStart = parse(ss[2])
		if "FAILED" in ss[4]:
			print "{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}".format(
				ss[0],ss[1],ss[2],ss[3],"-",ss[4],"-", "-", "-")
		else:
			tEnd = parse(ss[4])
			duration = tEnd - tStart
			print "{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}".format(
				ss[0],ss[1],ss[2],ss[3],duration,ss[4],ss[5],ss[6],ss[7])
<commit_msg>Added milli seconds column to the output.<commit_after>import fileinput
from dateutil.parser import parse

header = True
for line in fileinput.input():
	if header:
		print line
		header = False
	else:
		ss=line.rstrip().split(",")
		#print ss
		tStart = parse(ss[2])
		if "FAILED" in ss[4]:
			print "{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}".format(
				ss[0],ss[1],ss[2],ss[3],"-",ss[4],"-", "-", "-")
		else:
			tEnd = parse(ss[4])
			duration = tEnd - tStart
			duration_milliseconds = duration.total_seconds()*1000
			print "{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}".format(
				ss[0],ss[1],ss[2],ss[3],duration,duration_milliseconds,ss[4],ss[5],ss[6],ss[7])

{'fraction_non_alphanumeric': 0.2617124394184168, 'fraction_numerical': 0.05896607431340872, 'mean_word_length': 4.1625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5582109', 'n_tokens_mistral': 576, 'n_tokens_neox': 532, 'n_words': 121}