<commit_before># -*- coding: utf-8 -*-
# See LICENSE.txt for licensing terms

from docutils.parsers import rst
from docutils.parsers.rst import directives
from docutils.nodes import General, Inline, Element
from docutils.parsers.rst import roles

import basenodehandler, math_flowable

if 'Directive' in rst.__dict__:

    class Math(rst.Directive):
        has_content = True

        def run(self):
            return [math_node(data=''.join(self.content),
                              rawsource=''.join(self.content))]

        def __repr__(self):
            return u''.join(self.content)

else:

    def Math(name, arguments, options, content, lineno,
            content_offset, block_text, state, state_machine):
        return [math_node(data=''.join(content), rawsource=''.join(content))]

    Math.content = True

directives.register_directive('math', Math)


class math_node(General, Inline, Element):
    children = ()

    def __init__(self, rawsource='', *children, **attributes):
        self.rawsource = rawsource
        self.math_data = attributes['data']
        Element.__init__(self, rawsource, *children, **attributes)


def math_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
    return [math_node(rawtext[7:-1], data=rawtext[7:-1])], []


roles.register_local_role('math', math_role)

class HandleMath(basenodehandler.NodeHandler, math_node):
    def gather_elements(self, client, node, style):
        return [math_flowable.Math(node.math_data)]

    def get_text(self, client, node, replaceEnt):
        mf = math_flowable.Math(node.math_data)
        w, h = mf.wrap(0, 0)
        descent = mf.descent()
        img = mf.genImage()
        client.to_unlink.append(img)
        return '<img src="%s" width=%f height=%f valign=%f/>' % (
            img, w, h, -descent)
<commit_msg>Make this a bit more like sphinx's<commit_after># -*- coding: utf-8 -*-
# See LICENSE.txt for licensing terms

from docutils.parsers import rst
from docutils.parsers.rst import directives
from docutils.nodes import General, Inline, Element
from docutils import utils
from docutils.parsers.rst import roles

import basenodehandler, math_flowable

if 'Directive' in rst.__dict__:

    class Math(rst.Directive):
        has_content = True
        required_arguments = 0
        optional_arguments = 1
        final_argument_whitespace = True
        option_spec = {
            'label': directives.unchanged,
            'nowrap': directives.flag,
        }

        def run(self):
            latex = '\n'.join(self.content)
            if self.arguments and self.arguments[0]:
                latex = self.arguments[0] + '\n\n' + latex
            return [math_node(latex=latex,
                              rawsource=''.join(self.content))]

        def __repr__(self):
            return u''.join(self.content)

else:

    def Math(name, arguments, options, content, lineno,
            content_offset, block_text, state, state_machine):
        return [math_node(latex=''.join(content), rawsource=''.join(content))]

    Math.content = True

directives.register_directive('math', Math)


class math_node(General, Inline, Element):
    children = ()

    def __init__(self, rawsource='', *children, **attributes):
        self.rawsource = rawsource
        self.math_data = attributes['latex']
        Element.__init__(self, rawsource, *children, **attributes)


def math_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
    latex = utils.unescape(text, restore_backslashes=True)
    return [math_node(latex, latex=latex)], []


roles.register_local_role('math', math_role)

class HandleMath(basenodehandler.NodeHandler, math_node):
    def gather_elements(self, client, node, style):
        return [math_flowable.Math(node.math_data)]

    def get_text(self, client, node, replaceEnt):
        mf = math_flowable.Math(node.math_data)
        w, h = mf.wrap(0, 0)
        descent = mf.descent()
        img = mf.genImage()
        client.to_unlink.append(img)
        return '<img src="%s" width=%f height=%f valign=%f/>' % (
            img, w, h, -descent)

{'fraction_non_alphanumeric': 0.12603406326034064, 'fraction_numerical': 0.0034063260340632603, 'mean_word_length': 3.0264446620959844, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3796586', 'n_tokens_mistral': 1310, 'n_tokens_neox': 1262, 'n_words': 296}