<commit_before># -*- coding: utf-8 -*-

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#    Copyright (C) 2013 Rackspace Hosting All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import contextlib
import copy
import posixpath as pp

from taskflow import exceptions as exc
from taskflow.persistence import path_based
from taskflow.types import tree
from taskflow.utils import lock_utils


class FakeFilesystem(object):
    """An in-memory filesystem-like structure."""

    #: Root path of the in-memory filesystem.
    root_path = pp.sep

    @classmethod
    def _normpath(cls, path):
        if not path.startswith(cls.root_path):
            raise ValueError("This filesystem can only normalize absolute"
                             " paths: '%s' is not valid" % path)
        return pp.normpath(path)

    def __init__(self, deep_copy=True):
        self._root = tree.Node(self.root_path, value=None)
        if deep_copy:
            self._copier = copy.deepcopy
        else:
            self._copier = copy.copy

    def ensure_path(self, path):
        path = self._normpath(path)
        # Ignore the root path as we already checked for that; and it
        # will always exist/can't be removed anyway...
        if path == self._root.item:
            return
        node = self._root
        for piece in self._iter_pieces(path):
            child_node = node.find(piece, only_direct=True,
                                   include_self=False)
            if child_node is None:
                child_node = tree.Node(piece, value=None)
                node.add(child_node)
            node = child_node

    def _fetch_node(self, path):
        node = self._root
        path = self._normpath(path)
        if path == self._root.item:
            return node
        for piece in self._iter_pieces(path):
            node = node.find(piece, only_direct=True,
                             include_self=False)
            if node is None:
                raise exc.NotFound("Path '%s' not found" % path)
        return node

    def _get_item(self, path, links=None):
        node = self._fetch_node(path)
        if 'target' in node.metadata:
            # Follow the link (and watch out for loops)...
            path = node.metadata['target']
            if links is None:
                links = []
            if path in links:
                raise ValueError("Recursive link following not"
                                 " allowed (loop %s detected)"
                                 % (links + [path]))
            else:
                links.append(path)
            return self._get_item(path, links=links)
        else:
            return self._copier(node.metadata['value'])

    def ls(self, path):
        return [node.item for node in self._fetch_node(path)]

    def _iter_pieces(self, path, include_root=False):
        if path == self._root.item:
            # Check for this directly as the following doesn't work with
            # split correctly:
            #
            # >>> path = "/"
            # path.split(pp.sep)
            # ['', '']
            parts = []
        else:
            parts = path.split(pp.sep)[1:]
        if include_root:
            parts.insert(0, self._root.item)
        for piece in parts:
            yield piece

    def __delitem__(self, path):
        node = self._fetch_node(path)
        if node is self._root:
            raise ValueError("Can not delete '%s'" % self._root.item)
        node.disassociate()

    def pformat(self):
        return self._root.pformat()

    def symlink(self, src_path, dest_path):
        dest_path = self._normpath(dest_path)
        src_path = self._normpath(src_path)
        dirname, basename = pp.split(dest_path)
        parent_node = self._fetch_node(dirname)
        child_node = parent_node.find(basename,
                                      only_direct=True,
                                      include_self=False)
        if child_node is None:
            child_node = tree.Node(basename, value=None)<commit_msg>Add more comments to fake in-memory filesystem<commit_after># -*- coding: utf-8 -*-

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#    Copyright (C) 2013 Rackspace Hosting All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import contextlib
import copy
import posixpath as pp

from taskflow import exceptions as exc
from taskflow.persistence import path_based
from taskflow.types import tree
from taskflow.utils import lock_utils


class FakeFilesystem(object):
    """An in-memory filesystem-like structure.

    This filesystem uses posix style paths **only** so users must be careful
    to use the ``posixpath`` module instead of the ``os.path`` one which will
    vary depending on the operating system which the active python is running
    in (the decision to use ``posixpath`` was to avoid the path variations
    which are not relevant in an implementation of a in-memory fake
    filesystem).

    Example usage:

    >>> from taskflow.persistence.backends import impl_memory
    >>> fs = impl_memory.FakeFilesystem()
    >>> fs.ensure_path('/a/b/c')
    >>> fs['/a/b/c'] = 'd'
    >>> print(fs['/a/b/c'])
    d
    >>> del fs['/a/b/c']
    >>> fs.ls("/a/b")
    []
    """

    #: Root path of the in-memory filesystem.
    root_path = pp.sep

    @classmethod
    def _normpath(cls, path):
        if not path.startswith(cls.root_path):
            raise ValueError("This filesystem can only normalize absolute"
                             " paths: '%s' is not valid" % path)
        return pp.normpath(path)

    def __init__(self, deep_copy=True):
        self._root = tree.Node(self.root_path, value=None)
        if deep_copy:
            self._copier = copy.deepcopy
        else:
            self._copier = copy.copy

    def ensure_path(self, path):
        """Ensure the path (and parents) exists."""
        path = self._normpath(path)
        # Ignore the root path as we already checked for that; and it
        # will always exist/can't be removed anyway...
        if path == self._root.item:
            return
        node = self._root
        for piece in self._iter_pieces(path):
            child_node = node.find(piece, only_direct=True,
                                   include_self=False)
            if child_node is None:
                child_node = tree.Node(piece, value=None)
                node.add(child_node)
            node = child_node

    def _fetch_node(self, path):
        node = self._root
        path = self._normpath(path)
        if path == self._root.item:
            return node
        for piece in self._iter_pieces(path):
            node = node.find(piece, only_direct=True,
                             include_self=False)
            if node is None:
                raise exc.NotFound("Path '%s' not found" % path)
        return node

    def _get_item(self, path, links=None):
        node = self._fetch_node(path)
        if 'target' in node.metadata:
            # Follow the link (and watch out for loops)...
            path = node.metadata['target']
            if links is None:
                links = []
            if path in links:
                raise ValueError("Recursive link following not"
                                 " allowed (loop %s detected)"
                                 % (links + [path]))
            else:
                links.append(path)
            return self._get_item(path, links=links)
        else:
            return self._copier(node.metadata['value'])

    def ls(self, path):
        """Return list of all children of the given path."""
        return [node.item for node in self._fetch_node(path)]

    def _iter_pieces(self, path, include_root=False):
        if path == self._root.item:
            # Check for this directly as the following doesn't work with
            # split correctly:
            #
            # >>> path = "/"
            # path.split(pp.sep)
            # ['', '']
            parts = []
        else:
            parts = path.split(pp.sep)[1:]
        if include_root:
            parts.insert(0, self._root.item)
        for piece in parts:
            yield piece

    def __delitem__(self, path):
        node = self._fetch_node(path)
        if node is self._root:
            raise ValueError("Can not delete '%s'" % self._root.item)
        node.disassociate()

    def pformat(self):
        """Pretty format this in-memory filesystem."""
        return self._root.pformat()

    def symlink(self, src_path, dest_path):
        """Link the destionation path to the source path."""
        dest_path = self._normpath(dest_path)
        src_path = self._normpath(src_path)
        dirname, basename = pp.split(dest_path)
        parent_node = self._fetch_node(dirname)
        child_node = parent_node.find(basename,
                                      only_direct=True,
                                      include_self=False)
        if child_node is None:
            child_node = tree.Node(basename, value=None)
{'fraction_non_alphanumeric': 0.09229249011857707, 'fraction_numerical': 0.0029644268774703555, 'mean_word_length': 1.9192385347562735, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5890892', 'n_tokens_mistral': 2790, 'n_tokens_neox': 2709, 'n_words': 883}