<commit_before>from StringIO import StringIO
from xml.sax.saxutils import escape
import ElementSoup as ES
from xml.etree.ElementTree import Element
from xml.etree import ElementTree as ET

VALID_TAGS = set([
    'a', 'abbr', 'address', 'bdo', 'big', 'blockquote', 'br', 'caption', 'cite',
    'code', 'col', 'colgroup', 'dd', 'del', 'div', 'dfn', 'dl', 'dt', 'em',
    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'img', 'ins', 'kbd', 'li', 'ol', 'p', 'pre',
    'q', 'samp', 'small', 'span', 'strong', 'sub', 'sup', 'table', 'tbody',
    'td', 'tfoot', 'th', 'thead', 'tr', 'tt', 'ul', 'var',
])

REMOVE_IF_EMPTY = set([
    'a', 'abbr', 'address', 'bdo', 'big', 'blockquote', 'caption', 'cite',
    'code', 'dd,' 'del', 'div', 'dfn', 'dl', 'dt', 'em', 'h1', 'h2', 'h3', 'h4',
    'h5', 'h6', 'ins', 'kbd', 'li', 'ol', 'p', 'pre', 'q', 'samp', 'small',
    'span', 'strong', 'sub', 'sup', 'tt', 'ul', 'var',
])

BLOCK_TAGS = set([
    'blockquote', 'div', 'dl', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'ol', 'p',
    'pre', 'table', 'ul',
])

DROP_TAGS = set(['b', 'u', 'i', 'html'])

VALID_ATTRIBS = set([
    'dir', 'class', 'href', 'id', 'lang', 'xml:lang', 'title', 'colspan', 'rowspan',
    'src', 'width', 'height',
])

VALID_URI_SCHEMES = set([
    'http', 'https', 'tel', 'mailto', 'ftp', 'sip', 'telnet', 'xmpp'
])

class HtmlSanitiser(object):
    def __init__(self, html, id_prefix='s', header_offset=0):
        self.html, self.id_prefix = html, id_prefix
        self.header_offset = header_offset
        self.seen_ids = set()

    def get_sanitised(self):
        return self.sanitise_html(self.html)

    def sanitise_html(self, html):
        html = (u"<html>%s</html>" % html).encode('utf8')
        html = ES.parse(StringIO(html), encoding='utf8')

        self.sanitise_node(html)
        self.encapsulate_text_nodes(html)

        return ET.tostring(html)[6:-7]

    def encapsulate_text_nodes(self, html):
    
        html[0:0] = [Element('p')]
        html[0].text, html.text = html.text, None
        <commit_msg>removed a bit of element soup<commit_after>from StringIO import StringIO
from xml.sax.saxutils import escape
from xml.etree.ElementTree import Element
from xml.etree import ElementTree as ET
from lxml import etree

VALID_TAGS = set([
    'a', 'abbr', 'address', 'bdo', 'big', 'blockquote', 'br', 'caption', 'cite',
    'code', 'col', 'colgroup', 'dd', 'del', 'div', 'dfn', 'dl', 'dt', 'em',
    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'img', 'ins', 'kbd', 'li', 'ol', 'p', 'pre',
    'q', 'samp', 'small', 'span', 'strong', 'sub', 'sup', 'table', 'tbody',
    'td', 'tfoot', 'th', 'thead', 'tr', 'tt', 'ul', 'var',
])

REMOVE_IF_EMPTY = set([
    'a', 'abbr', 'address', 'bdo', 'big', 'blockquote', 'caption', 'cite',
    'code', 'dd,' 'del', 'div', 'dfn', 'dl', 'dt', 'em', 'h1', 'h2', 'h3', 'h4',
    'h5', 'h6', 'ins', 'kbd', 'li', 'ol', 'p', 'pre', 'q', 'samp', 'small',
    'span', 'strong', 'sub', 'sup', 'tt', 'ul', 'var',
])

BLOCK_TAGS = set([
    'blockquote', 'div', 'dl', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'ol', 'p',
    'pre', 'table', 'ul',
])

DROP_TAGS = set(['b', 'u', 'i', 'html'])

VALID_ATTRIBS = set([
    'dir', 'class', 'href', 'id', 'lang', 'xml:lang', 'title', 'colspan', 'rowspan',
    'src', 'width', 'height',
])

VALID_URI_SCHEMES = set([
    'http', 'https', 'tel', 'mailto', 'ftp', 'sip', 'telnet', 'xmpp'
])

class HtmlSanitiser(object):
    def __init__(self, html, id_prefix='s', header_offset=0):
        self.html, self.id_prefix = html, id_prefix
        self.header_offset = header_offset
        self.seen_ids = set()

    def get_sanitised(self):
        return self.sanitise_html(self.html)

    def sanitise_html(self, html):
        html = (u"<html>%s</html>" % html).encode('utf8')
        html = etree.parse(StringIO(html), parser=etree.HTMLParser())

        self.sanitise_node(html)
        self.encapsulate_text_nodes(html)

        return ET.tostring(html)[6:-7]

    def encapsulate_text_nodes(self, html):
    
        html[0:0] = [Element('p')]
        html[0].text, html.text = html.text, None
        
{'fraction_non_alphanumeric': 0.2512795515476481, 'fraction_numerical': 0.012429929320009749, 'mean_word_length': 3.777648428405122, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 9}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '222217', 'n_tokens_mistral': 1664, 'n_tokens_neox': 1648, 'n_words': 407}