<commit_before># encoding=utf8
from lxml import html


def _recursive_convert(element):
    fragment_root_element = {
        '_': element.tag
    }

    content = []
    if element.text:
        content.append({'t': element.text})

    if element.attrib:
        fragment_root_element.update({
            'a': dict(element.attrib)
        })

    for el in element:
        content.append(_recursive_convert(el))
        # Append Text node after element, if exists
        if el.tail:
            content.append({'t': el.tail})

    if len(content):
        fragment_root_element.update({
            'c': content
        })

    return fragment_root_element


def convert_html_to_telegraph_format(html_string):
    return [
        _recursive_convert(fragment) for fragment in html.fragments_fromstring(html_string)
    ]<commit_msg>All strings outside tags should be ignored<commit_after># encoding=utf8
from lxml import html


def _recursive_convert(element):
    # All strings outside tags should be ignored
    if not isinstance(element, html.HtmlElement):
        return

    fragment_root_element = {
        '_': element.tag
    }

    content = []
    if element.text:
        content.append({'t': element.text})

    if element.attrib:
        fragment_root_element.update({
            'a': dict(element.attrib)
        })

    for child in element:
        content.append(_recursive_convert(child))
        # Append Text node after element, if exists
        if child.tail:
            content.append({'t': child.tail})

    if len(content):
        fragment_root_element.update({
            'c': content
        })

    return fragment_root_element


def convert_html_to_telegraph_format(html_string):
    return [
        _recursive_convert(fragment) for fragment in html.fragments_fromstring(html_string)
    ]
{'fraction_non_alphanumeric': 0.09299781181619256, 'fraction_numerical': 0.0010940919037199124, 'mean_word_length': 2.544573643410853, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3701120', 'n_tokens_mistral': 538, 'n_tokens_neox': 517, 'n_words': 120}