<commit_before>""" fileloader.py

File-based metaclass to reduce duplicate code.
"""

import os.path
import requests

try:
    e = FileNotFoundError
except NameError:
    e = IOError


class FileLoader(object):
    def __init__(self, lfile):
        self.bytes = bytes()
        self.path = None

        # None means we're creating a new file, probably in-memory
        if lfile is None:
            pass

        # we have been passed a file-like object
        elif hasattr(lfile, "read"):
            self.bytes = bytes(lfile.read())

            # try to extract the path, too
            if hasattr(lfile, "name") and os.path.exists(os.path.realpath(lfile.name)):
                self.path = lfile.name

        # str without NUL bytes means this is likely a file path or URL
        # because in 2.x, bytes is just an alias of str
        elif type(lfile) is str and '\x00' not in lfile:
            # is this a URL?
            if "://" in lfile:
                r = requests.get(lfile, verify=True)

                if not r.ok:
                    raise e(lfile)

                self.bytes = r.content

            # this may be a file path, then
            # does the path already exist?
            elif os.path.exists(lfile):
                self.path = os.path.realpath(lfile)

                with open(lfile, 'rb') as lf:
                    self.bytes = bytes(lf.read())<commit_msg>made file path detection more robust - this can probably be pared down a bit later<commit_after>""" fileloader.py

File-based metaclass to reduce duplicate code.
"""
import os
import os.path
import requests

try:
    e = FileNotFoundError
except NameError:
    e = IOError


class FileLoader(object):
    @staticmethod
    def is_path(ppath):
        if bytes is not str and type(ppath) is bytes:
            return False

        # if we get to this point, testing will need to be a little more involved
        # the POSIX specification (http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_276)
        # for what constitutes a valid filename includes the following characters:
        # A-Z, a-z, 0-9, ., _, -
        #
        # path separators are / on POSIX and \ on Windows
        #
        # lots of other characters that are not present here can indeed be part of a filename

        # this should be adequate most of the time
        # we'll detect all unprintable and extended ASCII characters as 'bad' - their presence will denote 'not a path'
        badchars = [ chr(c) for c in range(0, 32) ]
        badchars += [ chr(c) for c in range(128, 256) ]

        # Windows also specifies some reserved characters
        if os.name == "nt":
            badchars += ['<', '>', ':', '"', '/', '\\', '|', '?', '*']

        if any(c in ppath for c in badchars):
            return False

        return True

    def __init__(self, lfile):
        self.bytes = bytes()
        self.path = None

        # None means we're creating a new file, probably in-memory
        if lfile is None:
            pass

        # we have been passed a file-like object
        elif hasattr(lfile, "read"):
            self.bytes = bytes(lfile.read())

            # try to extract the path, too
            if hasattr(lfile, "name") and os.path.exists(os.path.realpath(lfile.name)):
                self.path = lfile.name

        # str without NUL bytes means this is likely a file path or URL
        # because in 2.x, bytes is just an alias of str
        elif FileLoader.is_path(lfile):
            # is this a URL?
            if "://" in lfile and '\n' not in lfile:
                r = requests.get(lfile, verify=True)

                if not r.ok:
                    raise e(lfile)

                self.bytes = r.content

            # this may be a file path, then
            # does the path already exist?
            elif os.path.exists(lfile):
                self.path = os.path.realpath(lfile)

                with open(lfile, 'rb') as lf:
                    self.bytes = bytes(lf.read())
{'fraction_non_alphanumeric': 0.0935483870967742, 'fraction_numerical': 0.008188585607940446, 'mean_word_length': 1.7780840799448656, 'pattern_counts': {'":': 3, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4695249', 'n_tokens_mistral': 1204, 'n_tokens_neox': 1129, 'n_words': 422}