<commit_before>from __future__ import absolute_import

import itertools
import operator
import os
from plumbum.lib import six
from abc import abstractmethod, abstractproperty
import warnings

from functools import reduce


class FSUser(int):
    """A special object that represents a file-system user. It derives from ``int``, so it behaves
    just like a number (``uid``/``gid``), but also have a ``.name`` attribute that holds the
    string-name of the user, if given (otherwise ``None``)
    """
    def __new__(cls, val, name = None):
        self = int.__new__(cls, val)
        self.name = name
        return self

class Path(str, six.ABC):
    """An abstraction over file system paths. This class is abstract, and the two implementations
    are :class:`LocalPath <plumbum.machines.local.LocalPath>` and
    :class:`RemotePath <plumbum.path.remote.RemotePath>`.
    """

    CASE_SENSITIVE = True

    def __repr__(self):
        return "<%s %s>" % (self.__class__.__name__, str(self))
    def __div__(self, other):
        """Joins two paths"""
        return self.join(other)
    __truediv__ = __div__
    def __floordiv__(self, expr):
        """Returns a (possibly empty) list of paths that matched the glob-pattern under this path"""
        return self.glob(expr)
    def __iter__(self):
        """Iterate over the files in this directory"""
        return iter(self.list())
    def __eq__(self, other):
        if isinstance(other, Path):
            return self._get_info() == other._get_info()
        elif isinstance(other, str):
            if self.CASE_SENSITIVE:
                return str(self) == other
            else:
                return str(self).lower() == other.lower()
        else:
            return NotImplemented
    def __ne__(self, other):
        return not (self == other)
    def __gt__(self, other):
        return str(self) > str(other)
    def __ge__(self, other):
        return str(self) >= str(other)
    def __lt__(self, other):
        return str(self) < str(other)
    def __le__(self, other):
        return str(self) <= str(other)
    def __hash__(self):
        if self.CASE_SENSITIVE:
            return hash(str(self))
        else:
            return hash(str(self).lower())
    def __nonzero__(self):
        return bool(str(self))
    __bool__ = __nonzero__

    def __fpath__(self):
        """Added for Python 3.6 support"""
        return str(self)

    def __contains__(self, item):
        """Paths should support checking to see if an file or folder is in them."""
        try:
            return (self.path / item.name).exists()
        except AttributeError:
            return (self.path / item).exists()

    @abstractmethod
    def _form(self, *parts):
        pass

    def up(self, count = 1):
        """Go up in ``count`` directories (the default is 1)"""
        return self.join("../" * count)
    def walk(self, filter = lambda p: True, dir_filter = lambda p: True):  # @ReservedAssignment
        """traverse all (recursive) sub-elements under this directory, that match the given filter.
        By default, the filter accepts everything; you can provide a custom filter function that
        takes a path as an argument and returns a boolean

        :param filter: the filter (predicate function) for matching results. Only paths matching
                       this predicate are returned. Defaults to everything.
        :param dir_filter: the filter (predicate function) for matching directories. Only directories
                           matching this predicate are recursed into. Defaults to everything.
        """<commit_msg>Small bugfix for in not working<commit_after>from __future__ import absolute_import

import itertools
import operator
import os
from plumbum.lib import six
from abc import abstractmethod, abstractproperty
import warnings

from functools import reduce


class FSUser(int):
    """A special object that represents a file-system user. It derives from ``int``, so it behaves
    just like a number (``uid``/``gid``), but also have a ``.name`` attribute that holds the
    string-name of the user, if given (otherwise ``None``)
    """
    def __new__(cls, val, name = None):
        self = int.__new__(cls, val)
        self.name = name
        return self

class Path(str, six.ABC):
    """An abstraction over file system paths. This class is abstract, and the two implementations
    are :class:`LocalPath <plumbum.machines.local.LocalPath>` and
    :class:`RemotePath <plumbum.path.remote.RemotePath>`.
    """

    CASE_SENSITIVE = True

    def __repr__(self):
        return "<%s %s>" % (self.__class__.__name__, str(self))
    def __div__(self, other):
        """Joins two paths"""
        return self.join(other)
    __truediv__ = __div__
    def __floordiv__(self, expr):
        """Returns a (possibly empty) list of paths that matched the glob-pattern under this path"""
        return self.glob(expr)
    def __iter__(self):
        """Iterate over the files in this directory"""
        return iter(self.list())
    def __eq__(self, other):
        if isinstance(other, Path):
            return self._get_info() == other._get_info()
        elif isinstance(other, str):
            if self.CASE_SENSITIVE:
                return str(self) == other
            else:
                return str(self).lower() == other.lower()
        else:
            return NotImplemented
    def __ne__(self, other):
        return not (self == other)
    def __gt__(self, other):
        return str(self) > str(other)
    def __ge__(self, other):
        return str(self) >= str(other)
    def __lt__(self, other):
        return str(self) < str(other)
    def __le__(self, other):
        return str(self) <= str(other)
    def __hash__(self):
        if self.CASE_SENSITIVE:
            return hash(str(self))
        else:
            return hash(str(self).lower())
    def __nonzero__(self):
        return bool(str(self))
    __bool__ = __nonzero__

    def __fpath__(self):
        """Added for Python 3.6 support"""
        return str(self)

    def __contains__(self, item):
        """Paths should support checking to see if an file or folder is in them."""
        try:
            return (self / item.name).exists()
        except AttributeError:
            return (self / item).exists()

    @abstractmethod
    def _form(self, *parts):
        pass

    def up(self, count = 1):
        """Go up in ``count`` directories (the default is 1)"""
        return self.join("../" * count)
    def walk(self, filter = lambda p: True, dir_filter = lambda p: True):  # @ReservedAssignment
        """traverse all (recursive) sub-elements under this directory, that match the given filter.
        By default, the filter accepts everything; you can provide a custom filter function that
        takes a path as an argument and returns a boolean

        :param filter: the filter (predicate function) for matching results. Only paths matching
                       this predicate are returned. Defaults to everything.
        :param dir_filter: the filter (predicate function) for matching directories. Only directories
                           matching this predicate are recursed into. Defaults to everything.
        """
{'fraction_non_alphanumeric': 0.10242959690778575, 'fraction_numerical': 0.0011043622308117063, 'mean_word_length': 2.5238326848249026, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '73589', 'n_tokens_mistral': 2019, 'n_tokens_neox': 1962, 'n_words': 725}