<commit_before>import os
import fnmatch

try:
    from . import msg, shared as G, utils
    assert G and msg and utils
except ImportError:
    import msg
    import shared as G


IGNORE_FILES = ['.gitignore', '.hgignore', '.flignore', '.flooignore']
# TODO: make this configurable
HIDDEN_WHITELIST = ['.floo'] + IGNORE_FILES
# TODO: grab global git ignores:
# gitconfig_file = popen("git config -z --get core.excludesfile", "r");
DEFAULT_IGNORES = ['extern', 'node_modules', 'tmp', 'vendor']
MAX_FILE_SIZE = 1024 * 1024 * 5


def create_flooignore(path):
    flooignore = os.path.join(path, '.flooignore')
    # A very short race condition, but whatever.
    if os.path.exists(flooignore):
        return
    try:
        with open(flooignore, 'wb') as fd:
            fd.write('\n'.join(DEFAULT_IGNORES).encode('utf-8'))
    except Exception as e:
        msg.error('Error creating default .flooignore: %s' % str(e))


class Ignore(object):
    def __init__(self, parent, path):
        self.parent = parent
        self.size = 0
        self.children = []
        self.files = []
        self.ignores = {
            '/TOO_BIG/': []
        }
        self.path = utils.unfuck_path(path)

        msg.debug('Initializing ignores for %s' % path)
        for ignore_file in IGNORE_FILES:
            try:
                self.load(ignore_file)
            except:
                pass

        try:
            paths = os.listdir(self.path)
        except Exception as e:
            msg.error('Error listing path %s: %s' % (path, unicode(e)))
            return
        for p in paths:
            p_path = os.path.join(path, p)
            if p[0] == '.' and p not in HIDDEN_WHITELIST:
                msg.log('Ignoring hidden path %s' % p_path)
                continue
            is_ignored = self.is_ignored(p_path)
            if is_ignored:
                msg.debug(is_ignored)
                continue
            try:
                s = os.stat(p_path)
            except Exception as e:
                msg.error('Error lstat()ing path %s: %s' % (p_path, unicode(e)))
                continue
            if s.S_ISDIR(s.st_mode):
                ig = Ignore(self, p_path)
                self.children.append(ig)
                self.size += ig.size
                continue
            elif s.S_ISREG(s.st_mode):<commit_msg>Show room info message after we're done uploading.<commit_after>import os
import fnmatch
import stat

try:
    from . import msg, shared as G, utils
    assert G and msg and utils
except ImportError:
    import msg
    import shared as G


IGNORE_FILES = ['.gitignore', '.hgignore', '.flignore', '.flooignore']
# TODO: make this configurable
HIDDEN_WHITELIST = ['.floo'] + IGNORE_FILES
# TODO: grab global git ignores:
# gitconfig_file = popen("git config -z --get core.excludesfile", "r");
DEFAULT_IGNORES = ['extern', 'node_modules', 'tmp', 'vendor']
MAX_FILE_SIZE = 1024 * 1024 * 5


def create_flooignore(path):
    flooignore = os.path.join(path, '.flooignore')
    # A very short race condition, but whatever.
    if os.path.exists(flooignore):
        return
    try:
        with open(flooignore, 'wb') as fd:
            fd.write('\n'.join(DEFAULT_IGNORES).encode('utf-8'))
    except Exception as e:
        msg.error('Error creating default .flooignore: %s' % str(e))


class Ignore(object):
    def __init__(self, parent, path):
        self.parent = parent
        self.size = 0
        self.children = []
        self.files = []
        self.ignores = {
            '/TOO_BIG/': []
        }
        self.path = utils.unfuck_path(path)

        msg.debug('Initializing ignores for %s' % path)
        for ignore_file in IGNORE_FILES:
            try:
                self.load(ignore_file)
            except:
                pass

        try:
            paths = os.listdir(self.path)
        except Exception as e:
            msg.error('Error listing path %s: %s' % (path, unicode(e)))
            return
        for p in paths:
            p_path = os.path.join(path, p)
            if p[0] == '.' and p not in HIDDEN_WHITELIST:
                msg.log('Ignoring hidden path %s' % p_path)
                continue
            is_ignored = self.is_ignored(p_path)
            if is_ignored:
                msg.debug(is_ignored)
                continue
            try:
                s = os.stat(p_path)
            except Exception as e:
                msg.error('Error lstat()ing path %s: %s' % (p_path, unicode(e)))
                continue
            if stat.S_ISDIR(s.st_mode):
                ig = Ignore(self, p_path)
                self.children.append(ig)
                self.size += ig.size
                continue
            elif stat.S_ISREG(s.st_mode):
{'fraction_non_alphanumeric': 0.11334320152252061, 'fraction_numerical': 0.0050750687248889825, 'mean_word_length': 1.8171530673019654, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1133624', 'n_tokens_mistral': 1546, 'n_tokens_neox': 1451, 'n_words': 382}