<commit_before>#!/usr/bin/env python

import sys
import json

import jsonselectjs

import gflags
FLAGS = gflags.FLAGS


def selector_to_ids(selector, obj):
    return [id(node) for node in jsonselectjs.match(selector, obj)]


UNSPECIFIED = 0
KEEP = 1
DELETE = 2

def filter_object(obj, marks, presumption=DELETE):
    '''Filter down obj based on marks, presuming keys should be kept/deleted.

    Args:
        obj: The object to be filtered. Filtering is done in-place.
        marks: An object mapping id(obj) --> {DELETE,KEEP}
               These values apply to the entire subtree, unless inverted.
        presumption: The default action to take on all keys.
    '''
    if isinstance(obj, list):
        keys = reversed(range(0, len(obj)))
    else:
        keys = obj.keys()

    for k in keys:
        v = obj[k]
        m = marks.get(id(v), UNSPECIFIED)
        if m == DELETE:
            del obj[k]  # an explicit deletion is irreversible.
        elif m == KEEP or presumption==KEEP:
            # keep descending, in case there are nodes we should delete.
            if isinstance(v, list) or isinstance(v, dict):
                filter_object(v, marks, presumption=KEEP)
        elif m == UNSPECIFIED:
            # ... and presumption == DELETE
            if isinstance(v, list) or isinstance(v, dict):
                filter_object(v, marks, presumption=DELETE)
                if len(v) == 0:
                    del obj[k]
            else:
                del obj[k]


def usage():
    print '''%s [selector] [-v exclude_selector]

...
'''


def run(args):
    path = args.pop()  # TODO: allow stdin
    actions = args

    obj = json.load(open(path))
    while actions:
        action = actions[0]
        del actions[0]
        mode = KEEP
        presumption = DELETE
        if action == '-v':
            mode = DELETE
            action = actions[0]
            presumption = KEEP
            del actions[0]

        marks = {k: mode for k in selector_to_ids(action, obj)}
        filter_object(obj, marks, presumption=presumption)

    return obj


if __name__ == '__main__':
    obj = run(sys.argv[1:])
    # TODO: preserve the input ordering of keys
    print json.dumps(obj, indent=2, sort_keys=True)<commit_msg>Ordered dict in cli.py; log status -- it's slow!<commit_after>#!/usr/bin/env python

import sys
import json
from collections import OrderedDict

import jsonselectjs

import gflags
FLAGS = gflags.FLAGS


def selector_to_ids(selector, obj):
    return [id(node) for node in jsonselectjs.match(selector, obj)]


UNSPECIFIED = 0
KEEP = 1
DELETE = 2

def filter_object(obj, marks, presumption=DELETE):
    '''Filter down obj based on marks, presuming keys should be kept/deleted.

    Args:
        obj: The object to be filtered. Filtering is done in-place.
        marks: An object mapping id(obj) --> {DELETE,KEEP}
               These values apply to the entire subtree, unless inverted.
        presumption: The default action to take on all keys.
    '''
    if isinstance(obj, list):
        keys = reversed(range(0, len(obj)))
    else:
        keys = obj.keys()

    for k in keys:
        v = obj[k]
        m = marks.get(id(v), UNSPECIFIED)
        if m == DELETE:
            del obj[k]  # an explicit deletion is irreversible.
        elif m == KEEP or presumption==KEEP:
            # keep descending, in case there are nodes we should delete.
            if isinstance(v, list) or isinstance(v, dict):
                filter_object(v, marks, presumption=KEEP)
        elif m == UNSPECIFIED:
            # ... and presumption == DELETE
            if isinstance(v, list) or isinstance(v, dict):
                filter_object(v, marks, presumption=DELETE)
                if len(v) == 0:
                    del obj[k]
            else:
                del obj[k]


def usage():
    print '''%s [selector] [-v exclude_selector]

...
'''


def run(args):
    path = args.pop()  # TODO: allow stdin
    actions = args

    sys.stderr.write('Loading JSON...\n')
    obj = json.load(open(path), object_pairs_hook=OrderedDict)
    sys.stderr.write('done\n')
    while actions:
        action = actions[0]
        del actions[0]
        mode = KEEP
        presumption = DELETE
        if action == '-v':
            mode = DELETE
            action = actions[0]
            presumption = KEEP
            del actions[0]

        sys.stderr.write('Gathering marks...\n')
        marks = {k: mode for k in selector_to_ids(action, obj)}
        sys.stderr.write('done\n')
        sys.stderr.write('filtering object...\n')
        filter_object(obj, marks, presumption=presumption)
        sys.stderr.write('done\n')

    return obj


if __name__ == '__main__':
    obj = run(sys.argv[1:])
    print json.dumps(obj, indent=2)
{'fraction_non_alphanumeric': 0.1088506711409396, 'fraction_numerical': 0.004614093959731544, 'mean_word_length': 2.1088657105606257, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4896006', 'n_tokens_mistral': 1485, 'n_tokens_neox': 1402, 'n_words': 454}