<commit_before>#!/usr/bin/env python
__author__ = "<NAME>"
__copyright__ = "Copyright 2016, Stephens lab"
__email__ = "<EMAIL>"
__license__ = "MIT"

import sys, os, re, yaml, itertools, collections, sympy
from itertools import cycle, chain, islice
from fnmatch import fnmatch
from difflib import SequenceMatcher
from xxhash import xxh32 as xxh
from .constant import HTML_CSS, HTML_JS

def no_duplicates_constructor(loader, node, deep=False):
    """YAML check for duplicate keys."""
    mapping = {}
    for key_node, value_node in node.value:
        key = loader.construct_object(key_node, deep=deep)
        value = loader.construct_object(value_node, deep=deep)
        if key in mapping:
            raise yaml.constructor.ConstructorError("while constructing a mapping", node.start_mark,
                                   "found duplicate key (%s)" % key, key_node.start_mark)
        mapping[key] = value
    return collections.OrderedDict(loader.construct_pairs(node, deep))

def dict_representer(dumper, data):
    return dumper.represent_dict(data.iteritems())

yaml.add_representer(collections.OrderedDict, dict_representer)
yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, no_duplicates_constructor)
yaml.Dumper.ignore_aliases = lambda self, value: True

class Logger:
    def __init__(self):
        self.__width_cache = 1
        self.verbosity = 2

    def error(self, msg = None, q = True):
        if msg is None:
            sys.stderr.write('\n')
            return
        if isinstance(msg, list):
            msg = ' '.join(map(str, msg))
        else:
            msg = str(msg)
        start = '\n' if msg.startswith('\n') else ''
        end = '\n' if msg.endswith('\n') else ''
        msg = msg.strip()
        if q:
            sys.stderr.write(start + f"\033[1;33mERROR: {self.emphasize(msg, 33)}\033[0m\n" + end)<commit_msg>Fix an iteritems bug for dictionary<commit_after>#!/usr/bin/env python
__author__ = "<NAME>"
__copyright__ = "Copyright 2016, Stephens lab"
__email__ = "<EMAIL>"
__license__ = "MIT"

import sys, os, re, yaml, itertools, collections, sympy
from itertools import cycle, chain, islice
from fnmatch import fnmatch
from difflib import SequenceMatcher
from xxhash import xxh32 as xxh
from .constant import HTML_CSS, HTML_JS

def no_duplicates_constructor(loader, node, deep=False):
    """YAML check for duplicate keys."""
    mapping = {}
    for key_node, value_node in node.value:
        key = loader.construct_object(key_node, deep=deep)
        value = loader.construct_object(value_node, deep=deep)
        if key in mapping:
            raise yaml.constructor.ConstructorError("while constructing a mapping", node.start_mark,
                                   "found duplicate key (%s)" % key, key_node.start_mark)
        mapping[key] = value
    return collections.OrderedDict(loader.construct_pairs(node, deep))

def dict_representer(dumper, data):
    return dumper.represent_dict(data.items())

yaml.add_representer(collections.OrderedDict, dict_representer)
yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, no_duplicates_constructor)
yaml.Dumper.ignore_aliases = lambda self, value: True

class Logger:
    def __init__(self):
        self.__width_cache = 1
        self.verbosity = 2

    def error(self, msg = None, q = True):
        if msg is None:
            sys.stderr.write('\n')
            return
        if isinstance(msg, list):
            msg = ' '.join(map(str, msg))
        else:
            msg = str(msg)
        start = '\n' if msg.startswith('\n') else ''
        end = '\n' if msg.endswith('\n') else ''
        msg = msg.strip()
        if q:
            sys.stderr.write(start + f"\033[1;33mERROR: {self.emphasize(msg, 33)}\033[0m\n" + end)
{'fraction_non_alphanumeric': 0.11199575371549894, 'fraction_numerical': 0.010615711252653927, 'mean_word_length': 3.052688172043011, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6364176', 'n_tokens_mistral': 1206, 'n_tokens_neox': 1150, 'n_words': 304}