<commit_before>#!/usr/bin/env python3
#
# Change json output to be human-readable
import argparse
import csv
import json

def convert_dict(d):
    # for addresses, options, records, etc, just dereference
    if "type" in d and d["type"] in ["address", "option_or_ind", "record", "Collection", "Map", "Seq"]:
        return convert_any(d["value"])
    else:
        res = {}
        for key in d:
            res[key] = convert_any(d[key])
        return res;

def convert_list(xs):
    res = []
    for x in xs:
        res.append([convert_any(x)])
    return res

def convert_any(x):
    if type(x) is dict:
        return convert_dict(x)
    elif type(x) is list:
        return convert_list(x)
    else:
        return x

def convert_file(file_nm):
    out_file = file_nm + "_out"
    with open(file_nm, 'r', newline='') as csvfile:
        with open(out_file, 'w', newline='') as out_csvfile:
            reader = csv.reader(csvfile, delimiter='|', quotechar="'")
            writer = csv.writer(out_csvfile, delimiter='|', quotechar="'")
            for row in reader:
                res = []
                for x in row:
                    # print(x)
                    obj = json.loads(x)
                    # print("obj: " + str(obj))
                    new_obj = convert_any(obj)
                    print(new_obj)
                    res += [json.dumps(new_obj)]
                writer.writerow(res)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("json_files", type=str, nargs='+', help="Specify path")
    args = parser.parse_args()
    for j in args.json_files:
        convert_file(j)

if __name__=='__main__':
    main ()<commit_msg>mosaic: remove debug stuff from clean_json.py<commit_after>#!/usr/bin/env python3
#
# Change json output to be human-readable
import argparse
import csv
import json

def convert_dict(d):
    # for addresses, options, records, etc, just dereference
    if "type" in d and d["type"] in ["address", "option_or_ind", "record", "Collection", "Map", "Seq"]:
        return convert_any(d["value"])
    else:
        res = {}
        for key in d:
            res[key] = convert_any(d[key])
        return res;

def convert_list(xs):
    res = []
    for x in xs:
        res.append([convert_any(x)])
    return res

def convert_any(x):
    if type(x) is dict:
        return convert_dict(x)
    elif type(x) is list:
        return convert_list(x)
    else:
        return x

def convert_file(file_nm):
    out_file = file_nm + "_out"
    with open(file_nm, 'r', newline='') as csvfile:
        with open(out_file, 'w', newline='') as out_csvfile:
            reader = csv.reader(csvfile, delimiter='|', quotechar="'")
            writer = csv.writer(out_csvfile, delimiter='|', quotechar="'")
            for row in reader:
                res = []
                for x in row:
                    #print(x)
                    obj = json.loads(x)
                    # print("obj: " + str(obj))
                    new_obj = convert_any(obj)
                    #print(new_obj)
                    res += [json.dumps(new_obj)]
                writer.writerow(res)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("json_files", type=str, nargs='+', help="Specify path")
    args = parser.parse_args()
    for j in args.json_files:
        convert_file(j)

if __name__=='__main__':
    main ()
{'fraction_non_alphanumeric': 0.13103651354534745, 'fraction_numerical': 0.0005889281507656066, 'mean_word_length': 2.11651376146789, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5089592', 'n_tokens_mistral': 1104, 'n_tokens_neox': 1066, 'n_words': 283}