<commit_before>import yaml
import os
from collections import OrderedDict

# codes form stackoverflow.com
# http://stackoverflow.com/questions/6432605/any-yaml-libraries-in-python-that-support-dumping-of-long-strings-as-block-liter
# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data

class folded_unicode(unicode): pass
class literal_unicode(unicode): pass
class quoted(str): pass
class literal(str): pass


def folded_unicode_representer(dumper, data):
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='>')
def literal_unicode_representer(dumper, data):
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|')
def quoted_presenter(dumper, data):
    return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
def literal_presenter(dumper, data):
    return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
def ordered_dict_presenter(dumper, data):
    return dumper.represent_dict(data.items())


yaml.add_representer(folded_unicode, folded_unicode_representer)
yaml.add_representer(literal_unicode, literal_unicode_representer)
yaml.add_representer(quoted, quoted_presenter)
yaml.add_representer(literal, literal_presenter)
yaml.add_representer(OrderedDict, ordered_dict_presenter)


def include_constructor(loader, node):
    filename = loader.construct_scalar(node)
    if not os.path.exists(filename):
        stream_filename = os.path.abspath(loader.stream.name)
        stream_dirname = os.path.dirname(stream_filename)
        filename = os.path.join(stream_dirname, filename)

    with open(filename) as f:
        data = yaml.load(f)
    return data

yaml.add_constructor(u'!include', include_constructor)<commit_msg>add range constructor<commit_after>import yaml
import os
import re
from collections import OrderedDict

# codes form stackoverflow.com
# http://stackoverflow.com/questions/6432605/any-yaml-libraries-in-python-that-support-dumping-of-long-strings-as-block-liter
# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data

class folded_unicode(unicode): pass
class literal_unicode(unicode): pass
class quoted(str): pass
class literal(str): pass


def folded_unicode_representer(dumper, data):
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='>')
def literal_unicode_representer(dumper, data):
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|')
def quoted_presenter(dumper, data):
    return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
def literal_presenter(dumper, data):
    return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
def ordered_dict_presenter(dumper, data):
    return dumper.represent_dict(data.items())


yaml.add_representer(folded_unicode, folded_unicode_representer)
yaml.add_representer(literal_unicode, literal_unicode_representer)
yaml.add_representer(quoted, quoted_presenter)
yaml.add_representer(literal, literal_presenter)
yaml.add_representer(OrderedDict, ordered_dict_presenter)


def include_constructor(loader, node):
    filename = loader.construct_scalar(node)
    if not os.path.exists(filename):
        stream_filename = os.path.abspath(loader.stream.name)
        stream_dirname = os.path.dirname(stream_filename)
        filename = os.path.join(stream_dirname, filename)

    with open(filename) as f:
        data = yaml.load(f)
    return data

yaml.add_constructor(u'!include', include_constructor)


def range_constructor(loader, node):
    data = loader.construct_scalar(node)
    match = range_regex.match(data)
    if match:
        start, stop = map(int, match.groups())
        return range(start, stop + 1)
    else:
        raise ValueError('Invalid range')

range_regex = re.compile(r'^\s*(\d+)\s*\.\.\s*(\d+)\s*$')
yaml.add_constructor(u'!range', range_constructor)
yaml.add_implicit_resolver(u'!range', range_regex)
{'fraction_non_alphanumeric': 0.12329459322890349, 'fraction_numerical': 0.015411824153612936, 'mean_word_length': 5.802405498281787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3587646', 'n_tokens_mistral': 1362, 'n_tokens_neox': 1255, 'n_words': 216}