<commit_before>#!/usr/bin/python
import sys
import json

def doMain(fieldlist,constantValues):
    template = {}
    for x in constantValues:
        (k,d,v) = x.partition("=")
        template[k] = v
    genId='id' in template
    fields = fieldlist.split(",")
    N = len(fields)
    i=0
    print "["
    delim=False
    for line in sys.stdin:
        i+=1
        vlist = line.strip().split("\t")
        for vi in range(len(vlist)):
            if vlist[vi].startswith("@python"):
                vlist[vi] = eval(vlist[vi][7:])
            elif len(vlist[vi]) == 0:
                vlist[vi] = None
        if len(vlist) != N:
            assert "Expected %d fields on line %d (saw %d)" % (N,i,len(vlist))
        x = dict(template)
        x.update(zip(fields,vlist))
        if genId: x['id'] = "%s%d" % (template['id'],i)
        if delim: print ","
        else: delim=True
        print json.dumps(x),
    print "\n]"

if __name__=="__main__":
    if len(sys.argv) < 3:
        print "Usage: python %s field1,field2,field3 fieldX=value fieldY=value ..."
        exit(1)
    doMain(sys.argv[1],sys.argv[2:])<commit_msg>logic errors<commit_after>#!/usr/bin/python
import sys
import json

def doMain(fieldlist,constantValues):
    template = {}
    for x in constantValues:
        (k,d,v) = x.partition("=")
        template[k] = v
    genId='id' in template
    fields = fieldlist.split(",")
    N = len(fields)
    i=0
    print "["
    delim=False
    for line in sys.stdin:
        i+=1
        vlist = line.strip().split("\t")
        for vi in range(len(vlist)):
            if vlist[vi].startswith("@python"):
                vlist[vi] = eval(vlist[vi][7:])
            if len(vlist[vi]) == 0:
                vlist[vi] = None
        if len(vlist) != N:
            assert "Expected %d fields on line %d (saw %d)" % (N,i,len(vlist))
        x = dict(template)
        x.update(zip(fields,vlist))
        for k in x.keys():
            if x[k] == None: del x[k]
        if genId: x['id'] = "%s%d" % (template['id'],i)
        if delim: print ","
        else: delim=True
        print json.dumps(x),
    print "\n]"

if __name__=="__main__":
    if len(sys.argv) < 3:
        print "Usage: python %s field1,field2,field3 fieldX=value fieldY=value ..."
        exit(1)
    doMain(sys.argv[1],sys.argv[2:])
{'fraction_non_alphanumeric': 0.16767241379310344, 'fraction_numerical': 0.009482758620689655, 'mean_word_length': 2.107095046854083, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6924777', 'n_tokens_mistral': 839, 'n_tokens_neox': 819, 'n_words': 189}