<commit_before>import xml.etree.ElementPath

import attr


@attr.s
class Element:
    tag = attr.ib()
    attrib = attr.ib(default=())
    children = attr.ib(default=attr.Factory(list))

    def append(self, element):
        self.children.append(element)

    def extend(self, elements):
        self.children.extend(elements)

    def insert(self, index, element):
        self.children.insert(index, element)

    def remove(self, element):
        self.children.remove(element)

    def findall(self, path):
        return list(self.iterfind(path))

    def find(self, path):
        return next(self.iterfind(path), None)

    def iterfind(self, path):
        queryable_element = QueryableElement(self)
        found = xml.etree.ElementPath.iterfind(queryable_element, path)
        for item in found:
            if isinstance(item, ListBackedElement):
                yield item.list
            elif isinstance(item, QueryableElement):
                yield item.element

    def iter(self, tag=None):
        return iterate_element(self, tag)

    def __iter__(self):
        return iter(self.children)

    def __getitem__(self, index):
        return self.children[index]

    def __setitem__(self, index, element):
        self.children[index] = element

    def __len__(self):
        return len(self.children)


@attr.s
class QueryableElement:
    element = attr.ib()

    @property
    def tag(self):
        return self.element.tag

    def iter(self, tag=None):
        return iterate_element(self, tag)

    def __iter__(self):
        for item in self.element:
            if isinstance(item, Element):
                yield QueryableElement(item)
            elif isinstance(item, list):
                yield ListBackedElement(item)

    def __len__(self):
        return len(self.element)


@attr.s
class ListBackedElement:
    list = attr.ib()

    @property
    def tag(self):
        return self.list[0]

    def iter(self, tag=None):
        if tag is None or self.tag == tag:
            yield self

    def __iter__(self):
        return iter(())

    def __len__(self):
        return 0


def iterate_element(element, tag):
    if tag is None or element.tag == tag:
        yield element
    if not len(element):
        return
    for item in element:
        if hasattr(item, 'iter'):
            for subitem in item.iter(tag):
                yield subitem
        elif tag is None:
            yield item<commit_msg>Remove length method from QueryableElement<commit_after>import xml.etree.ElementPath

import attr


@attr.s
class Element:
    tag = attr.ib()
    attrib = attr.ib(default=())
    children = attr.ib(default=attr.Factory(list))

    def append(self, element):
        self.children.append(element)

    def extend(self, elements):
        self.children.extend(elements)

    def insert(self, index, element):
        self.children.insert(index, element)

    def remove(self, element):
        self.children.remove(element)

    def findall(self, path):
        return list(self.iterfind(path))

    def find(self, path):
        return next(self.iterfind(path), None)

    def iterfind(self, path):
        queryable_element = QueryableElement(self)
        found = xml.etree.ElementPath.iterfind(queryable_element, path)
        for item in found:
            if isinstance(item, ListBackedElement):
                yield item.list
            elif isinstance(item, QueryableElement):
                yield item.element

    def iter(self, tag=None):
        return iterate_element(self, tag)

    def __iter__(self):
        return iter(self.children)

    def __getitem__(self, index):
        return self.children[index]

    def __setitem__(self, index, element):
        self.children[index] = element

    def __len__(self):
        return len(self.children)


@attr.s
class QueryableElement:
    element = attr.ib()

    @property
    def tag(self):
        return self.element.tag

    def iter(self, tag=None):
        return iterate_element(self, tag)

    def __iter__(self):
        for item in self.element:
            if isinstance(item, Element):
                yield QueryableElement(item)
            elif isinstance(item, list):
                yield ListBackedElement(item)


@attr.s
class ListBackedElement:
    list = attr.ib()

    @property
    def tag(self):
        return self.list[0]

    def iter(self, tag=None):
        if tag is None or self.tag == tag:
            yield self

    def __iter__(self):
        return iter(())


def iterate_element(element, tag):
    if tag is None or element.tag == tag:
        yield element
    for item in element:
        if hasattr(item, 'iter'):
            for subitem in item.iter(tag):
                yield subitem
        elif tag is None:
            yield item
{'fraction_non_alphanumeric': 0.09810060530160718, 'fraction_numerical': 0.0006261740763932373, 'mean_word_length': 2.2598639455782314, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6223202', 'n_tokens_mistral': 1408, 'n_tokens_neox': 1344, 'n_words': 371}