<commit_before>from copy import deepcopy


__MIXINS__ = {}


def mix_classes(cls, mixins):

    if not isinstance(mixins, tuple):
        mixins = (mixins,)

    classes = (cls,) + mixins
    cls_names = [cls.__name__] + [m.__name__ for m in mixins]
    mixed_name = '_'.join(cls_names)
    inheritance = ', '.join(cls_names)
    inits = '%s.__init__(self, *args, **kwargs)\n' % cls.__name__
    inits += '\n'.join(['        %s.__init__(self)' % \
                        m.__name__ for m in mixins])
    cls_def = '''class %(mixed_name)s(%(inheritance)s):
    def __init__(self, *args, **kwargs):
        %(inits)s''' % locals()
    namespace = dict([(c.__name__, c) for c in classes])
    exec cls_def in namespace
    return namespace[mixed_name]


class TreeObject(object):

    __slots__ = 'tree', 'name', 'prefix'

    def __init__(self, tree, name, prefix):

        self.tree = tree
        self.name = name
        self.prefix = prefix

    def __eq__(self, other):

        return self.name == other.name and \
               self.prefix == other.prefix

    def __getitem__(self, attr):

        return getattr(self, attr)

    def __setitem__(self, attr, value):

        setattr(self.tree, self.prefix + attr, value)

    def __getattr__(self, attr):

        return getattr(self.tree, self.prefix + attr)


class TreeCollectionObject(TreeObject):

    __slots__ = 'index'

    def __init__(self, tree, name, prefix, index):

        self.index = index
        super(TreeCollectionObject, self).__init__(tree, name, prefix)

    def __eq__(self, other):

        return self.index == other.index and \
               TreeObject.__eq__(self, other)

    def __getattr__(self, attr):

        try:
            return getattr(self.tree, self.prefix + attr)[self.index]
        except IndexError:
            raise IndexError("index %i out of range for "
                             "attribute %s of collection %s of size %i" % \
                             (self.index, attr, self.prefix,
                             len(getattr(self.tree, self.prefix + attr))))


class TreeCollection(object):

    __slots__ = 'tree', 'name', \
                'prefix', 'size', \
                'selection', 'tree_object_cls', \
                '__cache_objects', '__cache'

    def __init__(self, tree, name, prefix, size, mix=None, cache=True):

        # TODO support tuple of mixins

        super(TreeCollection, self).__init__()
        self.tree = tree
        self.name = name
        self.prefix = prefix
        self.size = size
        self.selection = None

        self.__cache_objects = cache
        self.__cache = {}

        self.tree_object_cls = TreeCollectionObject
        if mix is not None:
            if mix in __MIXINS__:
                self.tree_object_cls = __MIXINS__[mix]
            else:
                self.tree_object_cls = mix_classes(TreeCollectionObject, mix)
                __MIXINS__[mix] = self.tree_object_cls

    def __nonzero__(self):

        return len(self) > 0

    def reset(self):

        self.selection = None
        self.__cache = {}

    def select(self, func):

        if self.selection is None:
            self.selection = range(len(self))<commit_msg>add remove/pop to treeobjectcollection<commit_after>from copy import deepcopy


__MIXINS__ = {}


def mix_classes(cls, mixins):

    if not isinstance(mixins, tuple):
        mixins = (mixins,)

    classes = (cls,) + mixins
    cls_names = [cls.__name__] + [m.__name__ for m in mixins]
    mixed_name = '_'.join(cls_names)
    inheritance = ', '.join(cls_names)
    inits = '%s.__init__(self, *args, **kwargs)\n' % cls.__name__
    inits += '\n'.join(['        %s.__init__(self)' % \
                        m.__name__ for m in mixins])
    cls_def = '''class %(mixed_name)s(%(inheritance)s):
    def __init__(self, *args, **kwargs):
        %(inits)s''' % locals()
    namespace = dict([(c.__name__, c) for c in classes])
    exec cls_def in namespace
    return namespace[mixed_name]


class TreeObject(object):

    __slots__ = 'tree', 'name', 'prefix'

    def __init__(self, tree, name, prefix):

        self.tree = tree
        self.name = name
        self.prefix = prefix

    def __eq__(self, other):

        return self.name == other.name and \
               self.prefix == other.prefix

    def __getitem__(self, attr):

        return getattr(self, attr)

    def __setitem__(self, attr, value):

        setattr(self.tree, self.prefix + attr, value)

    def __getattr__(self, attr):

        return getattr(self.tree, self.prefix + attr)


class TreeCollectionObject(TreeObject):

    __slots__ = 'index'

    def __init__(self, tree, name, prefix, index):

        self.index = index
        super(TreeCollectionObject, self).__init__(tree, name, prefix)

    def __eq__(self, other):

        return self.index == other.index and \
               TreeObject.__eq__(self, other)

    def __getattr__(self, attr):

        try:
            return getattr(self.tree, self.prefix + attr)[self.index]
        except IndexError:
            raise IndexError("index %i out of range for "
                             "attribute %s of collection %s of size %i" % \
                             (self.index, attr, self.prefix,
                             len(getattr(self.tree, self.prefix + attr))))


class TreeCollection(object):

    __slots__ = 'tree', 'name', \
                'prefix', 'size', \
                'selection', 'tree_object_cls', \
                '__cache_objects', '__cache'

    def __init__(self, tree, name, prefix, size, mix=None, cache=True):

        # TODO support tuple of mixins

        super(TreeCollection, self).__init__()
        self.tree = tree
        self.name = name
        self.prefix = prefix
        self.size = size
        self.selection = None

        self.__cache_objects = cache
        self.__cache = {}

        self.tree_object_cls = TreeCollectionObject
        if mix is not None:
            if mix in __MIXINS__:
                self.tree_object_cls = __MIXINS__[mix]
            else:
                self.tree_object_cls = mix_classes(TreeCollectionObject, mix)
                __MIXINS__[mix] = self.tree_object_cls

    def __nonzero__(self):

        return len(self) > 0

    def reset(self):

        self.selection = None
        self.__cache = {}

    def remove(self, thing):

        if self.selection is None:
            self.selection = range(len(self))
        for i, other in enumerate(self):
            if thing == other:
                self.selection.pop(i)
                break

    def pop(self, index):

        if self.selection is None:
            self.selection = range(len(self))
        thing = self[index]
        self.selection.pop(index)
        return thing

    def select(self, func):

        if self.selection is None:
            self.selection = range(len(self))
{'fraction_non_alphanumeric': 0.11427326847684043, 'fraction_numerical': 0.0002904022070567736, 'mean_word_length': 2.0397175639894085, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3483667', 'n_tokens_mistral': 2100, 'n_tokens_neox': 1988, 'n_words': 521}