<commit_before>#!/usr/bin/env python

import sys
import json

# python json_csv.py instances.json Reservations.Instances.PublicDnsName,Reservations.Instances.PrivateDnsName

# {'Reservations': {'Instances': {'PrivateDnsName': {}, 'PublicDnsName': {}}}}

def parse_filter_opts(filter_ops):
    pieces = filter_opts.split(',')
    tree = {}
    for piece in pieces:
        curr_node = tree
        for idx, selector in enumerate(piece.split('.')):
            if selector not in curr_node:
                curr_node[selector] = {}
            if idx == len(piece.split('.')) - 1:
                curr_node[selector]['terminal'] = True
            curr_node = curr_node[selector]
    return tree

# Recurse along both options and trees in parallel
# In case of a list we descend through each child as well
def json_to_csv(json_tree, opts_tree, so_far):
    keys = opts_tree.keys()
    if len(keys) == 1:
        key = keys[0]
        json_tree = json_tree[key]
        if isinstance(json_tree, list):
            for item in json_tree:
                json_to_csv(item, opts_tree[key], so_far)
        else:
            if 'terminal' in opts_tree[key]:
                print json_tree[key]
            else:
                json_to_csv(json_tree, opts_tree[key], so_far)
    else:
        for key in keys:
            if 'terminal' in opts_tree[key]:
                print json_tree[key],
            else:
                # Figure this out
                pass
        print

if __name__ == '__main__':
    filename = sys.argv[1]
    filter_opts = sys.argv[2]
<commit_msg>Clean up code a little bit<commit_after>#!/usr/bin/env python

import sys
import json

# python json_csv.py instances.json Reservations.Instances.PublicDnsName,Reservations.Instances.PrivateDnsName

# {'Reservations': {'Instances': {'PrivateDnsName': {}, 'PublicDnsName': {}}}}

def parse_filter_opts(filter_ops):
    pieces = filter_opts.split(',')
    tree = {}
    for piece in pieces:
        curr_node = tree
        for idx, selector in enumerate(piece.split('.')):
            if selector not in curr_node:
                curr_node[selector] = {}
            if idx == len(piece.split('.')) - 1:
                curr_node[selector]['terminal'] = True
            curr_node = curr_node[selector]
    return tree

def json_to_csv(json_tree, opts_tree, so_far):
    keys = opts_tree.keys()
    for key in keys:
        if 'terminal' in opts_tree[key]:
            print json_tree[key],
        else:
            json_tree = json_tree[key]
            if isinstance(json_tree, list):
                for item in json_tree:
                    json_to_csv(item, opts_tree[key], so_far)
            else:
                json_to_csv(json_tree, opts_tree[key], so_far)
    print

if __name__ == '__main__':
    filename = sys.argv[1]
    filter_opts = sys.argv[2]

{'fraction_non_alphanumeric': 0.11001410437235543, 'fraction_numerical': 0.0028208744710860366, 'mean_word_length': 2.2684331797235022, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2697715', 'n_tokens_mistral': 892, 'n_tokens_neox': 867, 'n_words': 216}