<commit_before># coding: utf-8

from __future__ import division

import io
import sys
import posixpath
import zipfile
import functools
import itertools

import more_itertools

__metaclass__ = type


def _parents(path):
    """
    Given a path with elements separated by
    posixpath.sep, generate all parents of that path.

    >>> list(_parents('b/d'))
    ['b']
    >>> list(_parents('/b/d/'))
    ['/b']
    >>> list(_parents('b/d/f/'))
    ['b/d', 'b']
    >>> list(_parents('b'))
    []
    >>> list(_parents(''))
    []
    """
    return itertools.islice(_ancestry(path), 1, None)


def _ancestry(path):
    """
    Given a path with elements separated by
    posixpath.sep, generate all elements of that path

    >>> list(_ancestry('b/d'))
    ['b/d', 'b']
    >>> list(_ancestry('/b/d/'))
    ['/b/d', '/b']
    >>> list(_ancestry('b/d/f/'))
    ['b/d/f', 'b/d', 'b']
    >>> list(_ancestry('b'))
    ['b']
    >>> list(_ancestry(''))
    []
    """
    path = path.rstrip(posixpath.sep)
    while path and path != posixpath.sep:
        yield path
        path, tail = posixpath.split(path)


class FastZip(zipfile.ZipFile):
    """
    ZipFile subclass to ensure implicit
    dirs exist and are resolved rapidly.
    """
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.__setup_caches()

    def __setup_caches(self):
        self.__names = super().namelist()
        self.__names += self._implied_dirs(self.__names)
        self.__lookup = set(self.__names)

    def find(self, name):
        if name not in self.__lookup and name + '/' in self.__lookup:
            return name + '/'
        return name

    @staticmethod
    def _implied_dirs(names):
        parents = itertools.chain.from_iterable(map(_parents, names))
        # Deduplicate entries in original order
        implied_dirs = OrderedDict.fromkeys(
            p + posixpath.sep for p in parents
            # Cast names to a set for O(1) lookups
            if p + posixpath.sep not in set(names)
        )
        return implied_dirs

    def namelist(self):
        return self.__names

    @classmethod
    def make(cls, source):
        """
        Given a source (filename or zipfile), return an
        appropriate subclass.
        """
        if isinstance(source, cls):
            return source

        if not isinstance(source, zipfile.ZipFile):
            return cls(_pathlib_compat(source))

        res = cls.__new__(cls)
        vars(res).update(vars(source))
        res.__setup_caches()
        return res


def _pathlib_compat(path):
    """
    For path-like objects, convert to a filename for compatibility
    on Python 3.6.1 and earlier.
    """
    try:
        return path.__fspath__()
    except AttributeError:
        return str(path)


class Path:<commit_msg>Extract CompleteDirs subclass, for when caching is undesirable.<commit_after># coding: utf-8

from __future__ import division

import io
import sys
import posixpath
import zipfile
import functools
import itertools

import more_itertools

__metaclass__ = type


def _parents(path):
    """
    Given a path with elements separated by
    posixpath.sep, generate all parents of that path.

    >>> list(_parents('b/d'))
    ['b']
    >>> list(_parents('/b/d/'))
    ['/b']
    >>> list(_parents('b/d/f/'))
    ['b/d', 'b']
    >>> list(_parents('b'))
    []
    >>> list(_parents(''))
    []
    """
    return itertools.islice(_ancestry(path), 1, None)


def _ancestry(path):
    """
    Given a path with elements separated by
    posixpath.sep, generate all elements of that path

    >>> list(_ancestry('b/d'))
    ['b/d', 'b']
    >>> list(_ancestry('/b/d/'))
    ['/b/d', '/b']
    >>> list(_ancestry('b/d/f/'))
    ['b/d/f', 'b/d', 'b']
    >>> list(_ancestry('b'))
    ['b']
    >>> list(_ancestry(''))
    []
    """
    path = path.rstrip(posixpath.sep)
    while path and path != posixpath.sep:
        yield path
        path, tail = posixpath.split(path)


class CompleteDirs(zipfile.ZipFile):
    @staticmethod
    def _implied_dirs(names):
        parents = itertools.chain.from_iterable(map(_parents, names))
        # Deduplicate entries in original order
        implied_dirs = OrderedDict.fromkeys(
            p + posixpath.sep for p in parents
            # Cast names to a set for O(1) lookups
            if p + posixpath.sep not in set(names)
        )
        return implied_dirs

    def namelist(self):
        names = super().namelist()
        return names + list(self._implied_dirs(names))

    def _name_set(self):
        return set(self.namelist())

    def find(self, name):
        names = self._name_set()
        if name not in names and name + '/' in names:
            return name + '/'
        return name


class FastZip(CompleteDirs):
    """
    ZipFile subclass to ensure implicit
    dirs exist and are resolved rapidly.
    """
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.__setup_caches()

    def __setup_caches(self):
        self.__names = super().namelist()
        self.__lookup = super()._name_set()

    def namelist(self):
        return self.__names

    def _name_set(self):
        return self.__lookup

    @classmethod
    def make(cls, source):
        """
        Given a source (filename or zipfile), return an
        appropriate subclass.
        """
        if isinstance(source, cls):
            return source

        if not isinstance(source, zipfile.ZipFile):
            return cls(_pathlib_compat(source))

        res = cls.__new__(cls)
        vars(res).update(vars(source))
        res.__setup_caches()
        return res


def _pathlib_compat(path):
    """
    For path-like objects, convert to a filename for compatibility
    on Python 3.6.1 and earlier.
    """
    try:
        return path.__fspath__()
    except AttributeError:
        return str(path)


class Path:
{'fraction_non_alphanumeric': 0.1316100692684575, 'fraction_numerical': 0.0020273694880892043, 'mean_word_length': 2.3598183881952326, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '27006', 'n_tokens_mistral': 1889, 'n_tokens_neox': 1833, 'n_words': 487}