<commit_before>import os
import sys

class FileType:
    BINARY_IMAGES = ('png', 'jpeg', 'jpg', 'gif', 'ps', 'eps', 'eps')
    TEXT_IMAGES = ('svg', 'ps', 'eps', 'pdf')  # pdf isn't, strictly, but...
    TEXT_FLAT_FILES = ('csv', 'tsv', 'psv')
    OTHER_TEXTS = ('txt',  'tex',
                   'md', 'markdown', 'rst', 'tex'
                   'html', 'htm',
                   'json', 'xml', 'yaml',
                   'sh', 'py', 'R', 'sql')
    TEXT_FILES = TEXT_IMAGES + TEXT_FLAT_FILES + OTHER_TEXTS
    IMAGE_FILES = BINARY_IMAGES + TEXT_IMAGES

    def __init__(self, path):
        self.orig_path = path
        self.ext = get_short_ext(path)
        name = os.path.basename(path)

        self.binary = self.ext in self.BINARY_IMAGES
        self.text   = self.ext in self.TEXT_FILES or name == 'Makefile'
        self.image  = self.ext in self.IMAGE_FILES
        self.encoding = 'iso-8859-1' if self.ext == 'pdf' else None
        self.orig_encoding = self.encoding  # encoding might be modified

    def is_unknown(self):
        return not self.binary and not self.text


def guess_encoding(path):
    ext = get_short_ext(path)
    if ext == 'pdf':
        return 'iso-8859-1'
    return 'utf-8'


def normalize_encoding(encoding):
    lc = encoding.lower()
    return 'utf-8' if lc == 'utf8' else lc


def get_encoding(path, encoding=None):
    if encoding is None:
        return guess_encoding(path)
    else:
        return normalize_encoding(encoding)


def get_short_ext(path):
    """Returns path extension, with dot removed"""
    return os.path.splitext(path)[1].lower()[1:] if path else ''


def protected_readlines(path, filetype):
    """
    Attempts to read path base on information in filetype.

    If the file is binary, return None

    If the file can't be read using the filetype
    """
    filetype = filetype or FileType(path)
    if filetype.binary:
        return
    enc = filetype.encoding if filetype else None
    try:
        with open(path, encoding=enc) as f:
            return f.readlines()
    except UnicodeDecodeError:
        if filetype.text:  # really was expecting text
            try:
                with open(path, encoding='iso-8859-1') as f:
                    lines = f.readlines()
                    filetype.encoding = 'iso-8859-1'
                    return lines
            except UnicodeDecodeError:
                filetype.binary = True
                filetype.text = False
                filetype.encoding = None
                print('Could not read %s as text file; treating as binary'
                      % path, file=sys.stderr)
<commit_msg>Update file list in utils.<commit_after>import os
import sys

class FileType:
    BINARY_IMAGES = ('png', 'jpeg', 'jpg', 'gif', 'ps', 'eps', 'eps')
    TEXT_IMAGES = ('svg', 'ps', 'eps', 'pdf')  # pdf isn't, strictly, but...
    TEXT_FLAT_FILES = ('csv', 'tsv', 'psv')
    OTHER_TEXTS = ('txt',  'tex',
                   'md', 'markdown', 'rst', 'tex'
                   'html', 'htm', 'css', 'js',
                   'json', 'xml', 'yaml',
                   'sh', 'py', 'R', 'sql')
    TEXT_FILES = TEXT_IMAGES + TEXT_FLAT_FILES + OTHER_TEXTS
    IMAGE_FILES = BINARY_IMAGES + TEXT_IMAGES

    def __init__(self, path):
        self.orig_path = path
        self.ext = get_short_ext(path)
        name = os.path.basename(path)

        self.binary = self.ext in self.BINARY_IMAGES
        self.text   = self.ext in self.TEXT_FILES or name == 'Makefile'
        self.image  = self.ext in self.IMAGE_FILES
        self.encoding = 'iso-8859-1' if self.ext == 'pdf' else None
        self.orig_encoding = self.encoding  # encoding might be modified

    def is_unknown(self):
        return not self.binary and not self.text


def guess_encoding(path):
    ext = get_short_ext(path)
    if ext == 'pdf':
        return 'iso-8859-1'
    return 'utf-8'


def normalize_encoding(encoding):
    lc = encoding.lower()
    return 'utf-8' if lc == 'utf8' else lc


def get_encoding(path, encoding=None):
    if encoding is None:
        return guess_encoding(path)
    else:
        return normalize_encoding(encoding)


def get_short_ext(path):
    """Returns path extension, with dot removed"""
    return os.path.splitext(path)[1].lower()[1:] if path else ''


def protected_readlines(path, filetype):
    """
    Attempts to read path base on information in filetype.

    If the file is binary, return None

    If the file can't be read using the filetype
    """
    filetype = filetype or FileType(path)
    if filetype.binary:
        return
    enc = filetype.encoding if filetype else None
    try:
        with open(path, encoding=enc) as f:
            return f.readlines()
    except UnicodeDecodeError:
        if filetype.text:  # really was expecting text
            try:
                with open(path, encoding='iso-8859-1') as f:
                    lines = f.readlines()
                    filetype.encoding = 'iso-8859-1'
                    return lines
            except UnicodeDecodeError:
                filetype.binary = True
                filetype.text = False
                filetype.encoding = None
                print('Could not read %s as text file; treating as binary'
                      % path, file=sys.stderr)

{'fraction_non_alphanumeric': 0.11433447098976109, 'fraction_numerical': 0.009480470231323474, 'mean_word_length': 2.125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '93772', 'n_tokens_mistral': 1681, 'n_tokens_neox': 1573, 'n_words': 468}