<commit_before>import collections
import errno
from io import BytesIO, StringIO
import os


# FIXME does not support binary files
class MemoryFile(object):
    def __init__(self):
        self.content = b''
        self.delegate = None
        self.mode = None

    def get_node(self, split_path):
        # FIXME this is neither tested nor nice
        try:
            next(iter(split_path))
        except StopIteration:
            return self
        else:
            raise OSError(errno.ENOENT, 'No such file or directory.')

    def open(self, mode='r'):
        self.mode = mode
        if 'b' in mode:
            self.delegate = BytesIO(self.content)
        else:
            self.delegate = StringIO(self.content.decode())
        return self

    def flush(self):
        self.delegate.flush()
        if 'b' in self.mode:
            self.content = self.delegate.getvalue()
        else:
            self.content = self.delegate.getvalue().encode()

    def close(self):
        self.flush()
        self.delegate.close()

    def __getattr__(self, name):
        return getattr(self.delegate, name)

    def __enter__(self):
        return self

    def __exit__(self, err_type, value, traceback):
        self.close()


class MemoryFS(object):
    def __init__(self, parent=None):
        self.children = {}
        if parent is None:
            parent = self
        self.parent = parent

    def _split_whole_path(self, path):
        split = collections.deque()
        while path != '':
            path, tail = os.path.split(path)
            split.appendleft(tail)
        return split

    def get_node(self, split_path):
        it = iter(split_path)
        try:
            name = next(it)
        except StopIteration:
            return self

        if name == os.path.curdir:
            node = self
        elif name == os.path.pardir:
            node = self.parent
        else:
            node = self.children[name]
        return node.get_node(it)

    def mkdir(self, path):
        split_path = self._split_whole_path(path)
        dirname = split_path.pop()
        node = self.get_node(split_path)
<commit_msg>Add common node base class for MemoryFS.<commit_after>import collections
import errno
from io import BytesIO, StringIO
import os


class MemoryFSNode(object):
    def __init__(self, parent=None):
        if parent is None:
            parent = self
        self.parent = parent
        self.children = {}

    def get_node(self, split_path):
        it = iter(split_path)
        try:
            name = next(it)
        except StopIteration:
            return self

        if name == os.path.curdir:
            node = self
        elif name == os.path.pardir:
            node = self.parent
        else:
            node = self.children[name]
        return node.get_node(it)


# FIXME does not support binary files
class MemoryFile(MemoryFSNode):
    def __init__(self, parent=None):
        super(MemoryFile, self).__init__(parent)
        self.content = b''
        self.delegate = None
        self.mode = None

    def open(self, mode='r'):
        self.mode = mode
        if 'b' in mode:
            self.delegate = BytesIO(self.content)
        else:
            self.delegate = StringIO(self.content.decode())
        return self

    def flush(self):
        self.delegate.flush()
        if 'b' in self.mode:
            self.content = self.delegate.getvalue()
        else:
            self.content = self.delegate.getvalue().encode()

    def close(self):
        self.flush()
        self.delegate.close()

    def __getattr__(self, name):
        return getattr(self.delegate, name)

    def __enter__(self):
        return self

    def __exit__(self, err_type, value, traceback):
        self.close()


class MemoryFS(MemoryFSNode):
    def _split_whole_path(self, path):
        split = collections.deque()
        while path != '':
            path, tail = os.path.split(path)
            split.appendleft(tail)
        return split

    def mkdir(self, path):
        split_path = self._split_whole_path(path)
        dirname = split_path.pop()
        node = self.get_node(split_path)

{'fraction_non_alphanumeric': 0.09824645688205622, 'fraction_numerical': 0.0, 'mean_word_length': 1.9057920446615493, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3781851', 'n_tokens_mistral': 1215, 'n_tokens_neox': 1181, 'n_words': 311}