<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

__version__ = '0.2.1'

import re
import sys

# the helpers

escape_x_re = re.compile(r'(?:\\x[0-9a-f]{2})+')
escape_u_re = re.compile(r'(?:\\u[0-9a-f]{4}|\\U[0-9a-f]{8})+')
encoding = sys.getfilesystemencoding()

def dexuescape(s):
    r'''decode the \x, \u and \U in a escaped string -> encoded string'''
    s = escape_x_re.sub(lambda m: m.group().decode('string-escape'), s)
    s = escape_u_re.sub(lambda m: m.group().decode('unicode-escape').encode(encoding), s)
    return s

# make uniout
uniout = lambda: 'middleware of stdout' # any instance

# make uniout look like stdout
for attrname in dir(sys.stdout):
    if not attrname.startswith('__'):
        setattr(uniout, attrname, getattr(sys.stdout, attrname))

# modify the write method to de-escape
uniout.write = lambda s: sys.__stdout__.write(dexuescape(s))

# install the uniout
sys.stdout = uniout
<commit_msg>fixed a bug when Python < 2.7<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

__version__ = '0.2.1'

import re
import sys

# the helpers

escape_x_re = re.compile(r'(?:\\x[0-9a-f]{2})+')
escape_u_re = re.compile(r'(?:\\u[0-9a-f]{4}|\\U[0-9a-f]{8})+')
encoding = sys.getfilesystemencoding()

def dexuescape(s):
    r'''decode the \x, \u and \U in a escaped string -> encoded string'''
    s = escape_x_re.sub(lambda m: m.group().decode('string-escape'), s)
    s = escape_u_re.sub(lambda m: m.group().decode('unicode-escape').encode(encoding), s)

    # for Python < 2.7
    if isinstance(s, unicode):
        s = s.encode(encoding)

    return s

# make uniout
uniout = lambda: 'middleware of stdout' # any instance

# make uniout look like stdout
for attrname in dir(sys.stdout):
    if not attrname.startswith('__'):
        setattr(uniout, attrname, getattr(sys.stdout, attrname))

# modify the write method to de-escape
uniout.write = lambda s: sys.__stdout__.write(dexuescape(s))

# install the uniout
sys.stdout = uniout

{'fraction_non_alphanumeric': 0.16927741283476502, 'fraction_numerical': 0.015159171298635674, 'mean_word_length': 3.8175182481751824, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1289188', 'n_tokens_mistral': 754, 'n_tokens_neox': 730, 'n_words': 181}