<commit_before>import json, sys, argparse
from collections import defaultdict

parser = argparse.ArgumentParser(description='Transform ADAGE formatted raw JSON data.')
parser.add_argument('-f', metavar='FILE.json', type=str, help='ADAGE API formatted JSON file', required=True)
parser.add_argument('--elem', metavar='FIELD', type=str, help='Field in data (e.g., user_id)', nargs='+')
parser.add_argument('--usersum', action="store_true", help='numeric field(s) summed over a user_id')
parser.add_argument('--totalsum', action="store_true", help='numeric field(s) summed over the entire file')
args = parser.parse_args()

print args

jsonfile = open(args.f,'rb')
data = json.loads(jsonfile.read())

if args.fields:
    searchfields = args.fields

    for elem in data:
        print [elem[x] for x in elem if x in searchfields] 
<commit_msg>added csvfy<commit_after>import json, sys, argparse
from collections import defaultdict

parser = argparse.ArgumentParser(description='Transform ADAGE formatted raw JSON data.')
parser.add_argument('-f', metavar='FILE.json', type=str, help='ADAGE API formatted JSON file', required=True)
parser.add_argument('--elem', metavar='FIELD', type=str, help='Field in data (e.g., user_id)', nargs='+')
parser.add_argument('--usersum', action="store_true", help='numeric field(s) summed over a user_id')
parser.add_argument('--totalsum', action="store_true", help='numeric field(s) summed over the entire file')
parser.add_argument('--csvfy', action="store_true", help='turns entire file into CSV (note: 2n for n lines to get all fields)')
args = parser.parse_args()

jsonfile = open(args.f,'rb')
data = json.loads(jsonfile.read())

if args.csvfy:
    columns = defaultdict(bool)
    for line in data:
        for col in line:
            columns[col] = True
    for col in sorted(columns):
        print col,',',
    print ''
    for line in data:
        lineout = defaultdict(str)
        outputlist = []
        for elem in line:
            lineout[elem] = line[elem]
        for col in sorted(columns):
            outputlist.append(str(lineout[col]))
        print "\"",'","'.join(outputlist),"\""

elif args.fields:
    searchfields = args.fields

    for elem in data:
        print [elem[x] for x in elem if x in searchfields] 

{'fraction_non_alphanumeric': 0.14059056853239313, 'fraction_numerical': 0.00044072278536800354, 'mean_word_length': 4.2304147465437785, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5673765', 'n_tokens_mistral': 707, 'n_tokens_neox': 694, 'n_words': 208}