<commit_before>from __future__ import absolute_import

import sys
import re

from six import StringIO


EXTENSIONS = ('txt', 'text')
BUFFER_MAX = 8192
ENCODING = sys.getfilesystemencoding()
NON_ASCII_SUB = re.compile('[\x00-\x1F\x7F-\xFF]')


def _get_file(f, **kwargs):
    buffer = StringIO()
    enc = kwargs.get('enc', ENCODING)

    while True:
        text = f.read(BUFFER_MAX)

        if not text:
            break

        try:
            text = text.decode(enc, 'replace')
        except AttributeError:
            pass

        text = NON_ASCII_SUB.sub(' ', text)
        buffer.write(text)

    return buffer.getvalue()<commit_msg>Handle text better.<commit_after>from __future__ import absolute_import

import sys
import re
import string

from six import StringIO

try:
    from string import maketrans

    def bytes(chars):
        return ''.join(map(chr, chars))

except ImportError:
    maketrans = bytes.maketrans


EXTENSIONS = ('txt', 'text')
ENCODING = sys.getfilesystemencoding()
BUFFER_MAX = 1024 * 1024

DELETE = bytes([
    i for i in range(256) if chr(i) not in string.printable
])
TRANSLATE = maketrans(b'\r\n', b'  ')


def _get_file(f, **kwargs):
    buffer = StringIO()
    enc = kwargs.get('encoding', ENCODING)

    while True:
        text = f.read(BUFFER_MAX)

        if not text:
            break

        text = text.translate(TRANSLATE, DELETE)
        text = text.decode('ascii', 'ignore')

        # Emulate the `strings` CLI tool.
        buffer.write(text)
    
    return buffer.getvalue()
{'fraction_non_alphanumeric': 0.11263020833333333, 'fraction_numerical': 0.012369791666666666, 'mean_word_length': 2.6947115384615383, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '895409', 'n_tokens_mistral': 531, 'n_tokens_neox': 488, 'n_words': 132}