<commit_before>"""

"""
import posixpath
import mimetypes

from vcs.utils.lazy import LazyProperty
from vcs.exceptions import VCSError

from pygments import lexers

class NodeError(VCSError):
    pass

class RemovedFileNodeError(NodeError):
    pass

class NodeKind:
    DIR = 1
    FILE = 2

class NodeState:
    ADDED = 'added'
    CHANGED = 'changed'
    NOT_CHANGED = 'not changed'
    REMOVED = 'removed'

class Node(object):
    """
    Simplest class representing file or directory on repository.  SCM backends
    should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node``
    directly.

    Node's ``path`` cannot start with slash as we oparete on *relative* paths
    only. Moreover, every single node is identified by the ``path`` attribute,
    so it cannot end with slash, too. Otherwise, path could lead to mistakes.
    """

    def __init__(self, path, kind):
        if path.startswith('/'):
            raise NodeError("Cannot initialize Node objects with slash at "
                "the beginning as only relative paths are supported")
        self.path = path.rstrip('/')
        if path == '' and kind != NodeKind.DIR:
            raise NodeError("Only DirNode and its subclasses may be initialized"
                " with empty path")
        self.kind = kind
        #self.dirs, self.files = [], []
        if self.is_root() and not self.is_dir():
            raise NodeError, "Root node cannot be FILE kind"

    @LazyProperty
    def parent(self):
        parent_path = self.get_parent_path()
        if parent_path:
            if self.changeset:
                return self.changeset.get_node(parent_path)
            return DirNode(parent_path)
        return None

    @LazyProperty
    def name(self):
        """
        Returns name of the node so if its path
        then only last part is returned.
        """
        return self.path.rstrip('/').split('/')[-1]

    def _get_kind(self):
        return self._kind

    def _set_kind(self, kind):
        if hasattr(self, '_kind'):
            raise NodeError, "Cannot change node's kind"
        else:
            self._kind = kind
            # Post setter check (path's trailing slash)
            if self.path.endswith('/'):
                raise NodeError, "Node's path cannot end with slash"

    kind = property(_get_kind, _set_kind)

    def __cmp__(self, other):
        """
        Comparator using name of the node, needed for quick list sorting.
        """
        kind_cmp = cmp(self.kind, other.kind)
        if kind_cmp:
            return kind_cmp
        return cmp(self.name, other.name)

    def __eq__(self, other):
        for attr in self.__dict__:
            if self.__dict__[attr] != other.__dict__[attr]:
                return False
        return True

    def __nq__(self, other):
        return not self == other

    def __repr__(self):
        return '<%s %r>' % (self.__class__.__name__, self.path)

    def __str__(self):
        return self.__repr__()

    def __unicode__(self):
        return unicode(self.name)

    def get_parent_path(self):
        """
        Returns node's parent path or empty string if node is root.
        """
        if self.is_root():
            return ''
        return posixpath.dirname(self.path.rstrip('/')) + '/'

    def is_file(self):
        """
        Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False``
        otherwise.
        """
        return self.kind == NodeKind.FILE

    def is_dir(self):
        """
        Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False``
        otherwise.
        """
        return self.kind == NodeKind.DIR

    def is_root(self):
        """
        Returns ``True`` if node is a root node and ``False`` otherwise.
        """
        return self.kind == NodeKind.DIR and self.path == ''

    @LazyProperty
    def added(self):
        return self.state is NodeState.ADDED

    @LazyProperty
    def changed(self):
        return self.state is NodeState.CHANGED

    @LazyProperty
    def not_changed(self):
        return self.state is NodeState.NOT_CHANGED

    @LazyProperty
    def removed(self):
        return self.state is NodeState.REMOVED


class FileNode(Node):
    """
    Class representing file nodes.

    :attribute: path: path to the node, relative to repostiory's root
    :attribute: content: if given arbitrary sets content of the file
    :attribute: changeset: if given, first time content is accessed, callback
    """

    def __init__(self, path, content=None, changeset=None):
        """
        Only one of ``content`` and ``changeset`` may be given. Passing both
        would raise ``NodeError`` exception.

        :param path: relative path to the node
        :param content: content may be passed to constructor
        :param changeset: if given, will use it to lazily fetch content
        """

        if content and changeset:
            raise NodeError("Cannot use both content and changeset")
        super(FileNode, self).__init__(path, kind=NodeKind.FILE)
        self.changeset = changeset
        self._content = content

    @LazyProperty
    def content(self):
        """
        Returns lazily content of the FileNode. If possible, would try to
        decode content from UTF-8.
        """
        if self.changeset:
            content = self.changeset.get_file_content(self.path)
        else:
            content = self._content
        try:
            content = content.decode('utf-8')
        except UnicodeDecodeError:
            pass
        return content

    @LazyProperty
    def nodes(self):
        raise NodeError("%s represents a file and has no ``nodes`` attribute"
            % self)

    @LazyProperty
    def size(self):
        if self.changeset:
            return self.changeset.get_file_size(self.path)
        raise NodeError("Cannot retrieve size of the file without related "
            "changeset attribute")<commit_msg>Now before getting content of filenode vcs tries to encode to utf-8 first<commit_after>"""

"""
import posixpath
import mimetypes

from vcs.utils.lazy import LazyProperty
from vcs.exceptions import VCSError

from pygments import lexers

class NodeError(VCSError):
    pass

class RemovedFileNodeError(NodeError):
    pass

class NodeKind:
    DIR = 1
    FILE = 2

class NodeState:
    ADDED = 'added'
    CHANGED = 'changed'
    NOT_CHANGED = 'not changed'
    REMOVED = 'removed'

class Node(object):
    """
    Simplest class representing file or directory on repository.  SCM backends
    should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node``
    directly.

    Node's ``path`` cannot start with slash as we oparete on *relative* paths
    only. Moreover, every single node is identified by the ``path`` attribute,
    so it cannot end with slash, too. Otherwise, path could lead to mistakes.
    """

    def __init__(self, path, kind):
        if path.startswith('/'):
            raise NodeError("Cannot initialize Node objects with slash at "
                "the beginning as only relative paths are supported")
        self.path = path.rstrip('/')
        if path == '' and kind != NodeKind.DIR:
            raise NodeError("Only DirNode and its subclasses may be initialized"
                " with empty path")
        self.kind = kind
        #self.dirs, self.files = [], []
        if self.is_root() and not self.is_dir():
            raise NodeError, "Root node cannot be FILE kind"

    @LazyProperty
    def parent(self):
        parent_path = self.get_parent_path()
        if parent_path:
            if self.changeset:
                return self.changeset.get_node(parent_path)
            return DirNode(parent_path)
        return None

    @LazyProperty
    def name(self):
        """
        Returns name of the node so if its path
        then only last part is returned.
        """
        return self.path.rstrip('/').split('/')[-1]

    def _get_kind(self):
        return self._kind

    def _set_kind(self, kind):
        if hasattr(self, '_kind'):
            raise NodeError, "Cannot change node's kind"
        else:
            self._kind = kind
            # Post setter check (path's trailing slash)
            if self.path.endswith('/'):
                raise NodeError, "Node's path cannot end with slash"

    kind = property(_get_kind, _set_kind)

    def __cmp__(self, other):
        """
        Comparator using name of the node, needed for quick list sorting.
        """
        kind_cmp = cmp(self.kind, other.kind)
        if kind_cmp:
            return kind_cmp
        return cmp(self.name, other.name)

    def __eq__(self, other):
        for attr in self.__dict__:
            if self.__dict__[attr] != other.__dict__[attr]:
                return False
        return True

    def __nq__(self, other):
        return not self == other

    def __repr__(self):
        return '<%s %r>' % (self.__class__.__name__, self.path)

    def __str__(self):
        return self.__repr__()

    def __unicode__(self):
        return unicode(self.name)

    def get_parent_path(self):
        """
        Returns node's parent path or empty string if node is root.
        """
        if self.is_root():
            return ''
        return posixpath.dirname(self.path.rstrip('/')) + '/'

    def is_file(self):
        """
        Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False``
        otherwise.
        """
        return self.kind == NodeKind.FILE

    def is_dir(self):
        """
        Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False``
        otherwise.
        """
        return self.kind == NodeKind.DIR

    def is_root(self):
        """
        Returns ``True`` if node is a root node and ``False`` otherwise.
        """
        return self.kind == NodeKind.DIR and self.path == ''

    @LazyProperty
    def added(self):
        return self.state is NodeState.ADDED

    @LazyProperty
    def changed(self):
        return self.state is NodeState.CHANGED

    @LazyProperty
    def not_changed(self):
        return self.state is NodeState.NOT_CHANGED

    @LazyProperty
    def removed(self):
        return self.state is NodeState.REMOVED


class FileNode(Node):
    """
    Class representing file nodes.

    :attribute: path: path to the node, relative to repostiory's root
    :attribute: content: if given arbitrary sets content of the file
    :attribute: changeset: if given, first time content is accessed, callback
    """

    def __init__(self, path, content=None, changeset=None):
        """
        Only one of ``content`` and ``changeset`` may be given. Passing both
        would raise ``NodeError`` exception.

        :param path: relative path to the node
        :param content: content may be passed to constructor
        :param changeset: if given, will use it to lazily fetch content
        """

        if content and changeset:
            raise NodeError("Cannot use both content and changeset")
        super(FileNode, self).__init__(path, kind=NodeKind.FILE)
        self.changeset = changeset
        self._content = content

    @LazyProperty
    def content(self):
        """
        Returns lazily content of the FileNode. If possible, would try to
        decode content from UTF-8.
        """
        if self.changeset:
            content = self.changeset.get_file_content(self.path)
        else:
            content = self._content
        try:
            content = content.encode('utf-8').decode('utf-8')
        except (UnicodeDecodeError, UnicodeEncodeError):
            pass
        return content

    @LazyProperty
    def nodes(self):
        raise NodeError("%s represents a file and has no ``nodes`` attribute"
            % self)

    @LazyProperty
    def size(self):
        if self.changeset:
            return self.changeset.get_file_size(self.path)
        raise NodeError("Cannot retrieve size of the file without related "
            "changeset attribute")
{'fraction_non_alphanumeric': 0.09707602339181287, 'fraction_numerical': 0.0010025062656641604, 'mean_word_length': 2.169446650781043, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5765266', 'n_tokens_mistral': 3391, 'n_tokens_neox': 3273, 'n_words': 1114}