<commit_before>try:
    import json
except ImportError:
    import simplejson as json

import sys
import optparse
from copy import deepcopy

class InvalidFormat(ValueError): pass

# http://www.xormedia.com/recursively-merge-dictionaries-in-python/
def dict_recursive_merge(a, b):
    '''recursively merges dict's. not just simple a['key'] = b['key'], if
    both a and bhave a key who's value is a dict then dict_merge is called
    on both values and the result stored in the returned dictionary.'''
    if not isinstance(b, dict):
        return b
    result = deepcopy(a)
    for k, v in b.iteritems():
        if k in result and isinstance(result[k], dict):
                result[k] = dict_merge(result[k], v)
        else:
            result[k] = deepcopy(v)
    return result

def get_cli():
    cli = optparse.OptionParser()
    return cli

class Parser(object):
    def parse(self, parts):
        document = {}
        for part in parts:
            document = dict_recursive_merge(document, self.parse_part(part))
        return document

    def split(self, part):
        return part.split('=', 1)

    def create_nested_hash(self, key, value):
        parts = key.split('.')
        document = {}<commit_msg>Fix incorrect func call on recursion<commit_after>try:
    import json
except ImportError:
    import simplejson as json

import sys
import optparse
from copy import deepcopy

class InvalidFormat(ValueError): pass

# http://www.xormedia.com/recursively-merge-dictionaries-in-python/
def dict_recursive_merge(a, b):
    '''recursively merges dict's. not just simple a['key'] = b['key'], if
    both a and bhave a key who's value is a dict then dict_merge is called
    on both values and the result stored in the returned dictionary.'''
    if not isinstance(b, dict):
        return b
    result = deepcopy(a)
    for k, v in b.iteritems():
        if k in result and isinstance(result[k], dict):
            result[k] = dict_recursive_merge(result[k], v)
        else:
            result[k] = deepcopy(v)
    return result

def get_cli():
    cli = optparse.OptionParser()
    return cli

class Parser(object):
    def parse(self, parts):
        document = {}
        for part in parts:
            document = dict_recursive_merge(document, self.parse_part(part))
        return document

    def split(self, part):
        return part.split('=', 1)

    def create_nested_hash(self, key, value):
        parts = key.split('.')
        document = {}
{'fraction_non_alphanumeric': 0.10270926000808735, 'fraction_numerical': 0.0008087343307723412, 'mean_word_length': 2.794478527607362, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1253756', 'n_tokens_mistral': 750, 'n_tokens_neox': 728, 'n_words': 247}