<commit_before>#!/usr/bin/python
import sys
import unicodecsv as csv # using this to get round UTF-8 error
import argparse
import ijson 

csvwriter = csv.writer(sys.stdout)

with open(sys.argv[1], 'rb') as input_file:
	cves = ijson.items(input_file, 'CVE_Items.item')
	for cve in cves:
		#print cve

		id = cve['cve']['CVE_data_meta']['ID']
		desc = cve['cve']['description']['description_data'][0]['value']
		severity = "Unknown"
		try:
			severity = cve['impact']['baseMetricV2']['severity']
		except:
			a = 'a'

		row = [ id, severity, desc ]
		csvwriter = csv.writer(sys.stdout)
		csvwriter.writerows( [row] )
<commit_msg>Now works with python3 better<commit_after>#!/usr/bin/python
import sys
import csv 
import argparse
import ijson 

csvwriter = csv.writer(sys.stdout)

with open(sys.argv[1], 'rb') as input_file:
	cves = ijson.items(input_file, 'CVE_Items.item')
	for cve in cves:
		#print cve

		id = cve['cve']['CVE_data_meta']['ID']
		desc = cve['cve']['description']['description_data'][0]['value']
		severity = "Unknown"
		try:
			severity = cve['impact']['baseMetricV2']['severity']
		except:
			a = 'a'

		row = [ str(id), str(severity), str(desc) ]
		csvwriter = csv.writer(sys.stdout)
		csvwriter.writerow( row )

{'fraction_non_alphanumeric': 0.16734362307067424, 'fraction_numerical': 0.006498781478472786, 'mean_word_length': 4.730232558139535, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2036060', 'n_tokens_mistral': 508, 'n_tokens_neox': 457, 'n_words': 103}