<commit_before>from HTMLParser import HTMLParser
from django.template import Node, Library, Variable, FilterExpression, TemplateSyntaxError
import math
import re
import logging

register = Library()

CGI_BIN = 'http://www.forkosh.com/mathtex.cgi'

template = r'''
<span>
    <img src="%s?\1"/>
    <span style="display:none">~\1~</span>
</span>
''' % CGI_BIN


@register.filter(name='smart_math', is_safe=True)
def math(page):
    class MathHTMLParser(HTMLParser):
        def __init__(self):
            HTMLParser.__init__(self)
            self.new_page = ''

        def handle_starttag(self, tag, attrs):
            self.new_page += '<' + tag + ' '.join('%s="%s"' % p for p in attrs.iteritems()) + '>'

        def handle_endtag(self, tag):
            self.new_page += '<' + tag + '>'

        def handle_data(self, data):
            self.new_page += re.sub('~(.*?)~', template, data)

    parser = MathHTMLParser()
    parser.feed(page)
    return parser.new_page
<commit_msg>Apparently we're using a SAX-style parser<commit_after>from HTMLParser import HTMLParser
from django.template import Node, Library, Variable, FilterExpression, TemplateSyntaxError
import math
import re
import logging

register = Library()

CGI_BIN = 'http://www.forkosh.com/mathtex.cgi'

template = r'''
<span>
    <img src="%s?\1"/>
    <span style="display:none">~\1~</span>
</span>
''' % CGI_BIN


@register.filter(name='smart_math', is_safe=True)
def math(page):
    class MathHTMLParser(HTMLParser):
        def __init__(self):
            HTMLParser.__init__(self)
            self.new_page = ''
            self.data_buffer = ''

        def handle_starttag(self, tag, attrs):
            self.new_page += '<' + tag + ' '.join('%s="%s"' % p for p in attrs.iteritems()) + '>'

        def handle_endtag(self, tag):
            self.new_page += re.sub('~(.*?)~', template, self.data_buffer)
            self.data_buffer = ''
            self.new_page += '<' + tag + '>'

        def handle_data(self, data):
            self.data_buffer += data

    parser = MathHTMLParser()
    parser.feed(page)
    return parser.new_page

{'fraction_non_alphanumeric': 0.1541371158392435, 'fraction_numerical': 0.0018912529550827422, 'mean_word_length': 2.933085501858736, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6119171', 'n_tokens_mistral': 691, 'n_tokens_neox': 672, 'n_words': 149}