<commit_before># -*- coding: utf-8 -*-
import functools
import itertools
import os
import re
from . import utils
try:
    # scandir-powered walk is in the stdlib in python 3.5+
    from os import scandir  # NOQA
    from os import walk as _walk
except ImportError:
    # otherwise, grab it from scandir for the performance boost
    from scandir import walk as _walk
try:
    # pathlib is in python stdlib in python 3.5+
    from pathlib import Path, PurePath
except ImportError:
    from pathlib2 import Path, PurePath


def strict_subpath(parent_path, child_path):
    if os.path.join(child_path, '') == os.path.join(parent_path, ''):
        return ''
    else:
        child = PurePath(os.path.abspath(child_path))
        parent = PurePath(os.path.abspath(parent_path))
        if parent not in child.parents:
            raise ValueError('{c} is not a child of {p}'.format(
                c=child_path, p=parent_path
            ))
        return str(child.relative_to(parent))


def walk(directory, onerror=None, filename='.gitignore',
         ignore_completely=None):
    """
    Generate the file names in a directory tree by walking the tree
    top-down, while obeying the rules of .gitignore. Links will not
    be followed.
    """
    if ignore_completely is None:
        ignore_completely = ['.git']

    starting_directory = os.path.abspath(directory)

    # Rule list will be a dict, keyed by directory with each value a
    # possibly-empty list of IgnoreRules
    rule_list = {}
    while True:
        for root, dirs, files in _walk(directory, onerror=onerror):
            rules = []
            if filename in files:
                with open(os.path.join(root, filename)) as ignore_file:
                    for line in ignore_file:
                        line = line.rstrip('\n')
                        rule = rule_from_pattern(line, os.path.abspath(root))
                        if rule:
                            rules.append(rule)
            current_dir = os.path.abspath(root)
            # SBC fix this up
            # We know we are either within the base_dir...
            rel_path = strict_subpath(starting_directory, current_dir)
            rule_list[rel_path] = rules
            # Now, make a list of rules, working our way back to the
            # base directory.
            applicable_rules = [rule_list[rel_path]]
            if root != directory:
                for p in Path(root).parents:
                    rel_parent = strict_subpath(starting_directory, str(p))
                    applicable_rules.append(rule_list[rel_parent])
                    if p not in Path(starting_directory).parents:
                        break
            # Our rules are actually ordered from the base down
            applicable_rules = applicable_rules[::-1]
            flat_list = list(
                itertools.chain.from_iterable(applicable_rules)<commit_msg>Forcing root-level rules into the applicable ruleset<commit_after># -*- coding: utf-8 -*-
import functools
import itertools
import os
import re
from . import utils
try:
    # scandir-powered walk is in the stdlib in python 3.5+
    from os import scandir  # NOQA
    from os import walk as _walk
except ImportError:
    # otherwise, grab it from scandir for the performance boost
    from scandir import walk as _walk
try:
    # pathlib is in python stdlib in python 3.5+
    from pathlib import Path, PurePath
except ImportError:
    from pathlib2 import Path, PurePath


def strict_subpath(parent_path, child_path):
    if os.path.join(child_path, '') == os.path.join(parent_path, ''):
        return ''
    else:
        child = PurePath(os.path.abspath(child_path))
        parent = PurePath(os.path.abspath(parent_path))
        if parent not in child.parents:
            raise ValueError('{c} is not a child of {p}'.format(
                c=child_path, p=parent_path
            ))
        return str(child.relative_to(parent))


def walk(directory, onerror=None, filename='.gitignore',
         ignore_completely=None):
    """
    Generate the file names in a directory tree by walking the tree
    top-down, while obeying the rules of .gitignore. Links will not
    be followed.
    """
    if ignore_completely is None:
        ignore_completely = ['.git']

    starting_directory = os.path.abspath(directory)

    # Rule list will be a dict, keyed by directory with each value a
    # possibly-empty list of IgnoreRules
    rule_list = {}
    while True:
        for root, dirs, files in _walk(directory, onerror=onerror):
            rules = []
            if filename in files:
                with open(os.path.join(root, filename)) as ignore_file:
                    for line in ignore_file:
                        line = line.rstrip('\n')
                        rule = rule_from_pattern(line, os.path.abspath(root))
                        if rule:
                            rules.append(rule)
            current_dir = os.path.abspath(root)
            # SBC fix this up
            # We know we are either within the base_dir...
            rel_path = strict_subpath(starting_directory, current_dir)
            rule_list[rel_path] = rules
            # Now, make a list of rules, working our way back to the
            # base directory.
            applicable_rules = [rule_list[rel_path]]
            if root != directory:
                for p in Path(root).parents:
                    rel_parent = strict_subpath(starting_directory, str(p))
                    applicable_rules.append(rule_list[rel_parent])
                    if p not in Path(starting_directory).parents:
                        break
            applicable_rules.append(rule_list[''])
            # Our rules are actually ordered from the base down
            applicable_rules = applicable_rules[::-1]
            flat_list = list(
                itertools.chain.from_iterable(applicable_rules)
{'fraction_non_alphanumeric': 0.07994579945799458, 'fraction_numerical': 0.0023712737127371273, 'mean_word_length': 2.0532574974146844, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2046730', 'n_tokens_mistral': 1664, 'n_tokens_neox': 1602, 'n_words': 514}