<commit_before>from docutils import nodes
from sphinx.writers.html import SmartyPantsHTMLTranslator
from sphinx.builders.html import StandaloneHTMLBuilder

def setup(app):
    app.add_builder(PxBuilder)


BaseHtmlXlator = SmartyPantsHTMLTranslator
class PxTranslator(BaseHtmlXlator):
    """Adjust the HTML translator into a .px translator.
    
    """

    def __init__(self, *args, **kwargs):
        BaseHtmlXlator.__init__(self, *args, **kwargs)
        #self.document.reporter.debug_flag = 1
        # To make the doc title be h0 (skipped), and the next h1.
        self.initial_header_level = 0

    def visit_section(self, node):
        self.section_level += 1

    def depart_section(self, node):
        self.section_level -= 1

    def visit_title(self, node):
        if self.section_level == 1:
            raise nodes.SkipNode
        else:
            BaseHtmlXlator.visit_title(self, node)

    # TODO: get history from here?
    def visit_field_list(self, node):
        raise nodes.SkipNode

    def depart_field_list(self, node):      pass
    def visit_field(self, node):            pass
    def depart_field(self, node):           pass
    def visit_field_name(self, node):       pass
    def depart_field_name(self, node):      pass
    def visit_field_body(self, node):       pass
    def depart_field_body(self, node):      pass

    def visit_literal_block(self, node):
        if node.rawsource != node.astext():
            # most probably a parsed-literal block -- don't highlight
            return BaseHtmlXlator.visit_literal_block(self, node)
        lang = self.highlightlang
        if node.has_key('language'):<commit_msg>Parse document fields to create revision history metadata.<commit_after>from docutils import nodes
from sphinx.writers.html import SmartyPantsHTMLTranslator
from sphinx.builders.html import StandaloneHTMLBuilder

def setup(app):
    app.add_builder(PxBuilder)


BaseHtmlXlator = SmartyPantsHTMLTranslator
class PxTranslator(BaseHtmlXlator):
    """Adjust the HTML translator into a .px translator.
    
    """

    def __init__(self, *args, **kwargs):
        BaseHtmlXlator.__init__(self, *args, **kwargs)
        #self.document.reporter.debug_flag = 1
        # To make the doc title be h0 (skipped), and the next h1.
        self.initial_header_level = 0

    def visit_section(self, node):
        self.section_level += 1

    def depart_section(self, node):
        self.section_level -= 1

    def visit_title(self, node):
        if self.section_level == 1:
            raise nodes.SkipNode
        else:
            BaseHtmlXlator.visit_title(self, node)

    def visit_field_list(self, node):
        self.history = []

    def depart_field_list(self, node):
        if self.history:
            self.body.append("<history>\n")
            for hist in self.history:
                when, what = hist.split(',')
                self.body.append("<what when='%s'>%s</what>\n" % (when, self.encode(what.strip())))
            self.body.append("</history>\n")
            
    def visit_field(self, node):
        if node.children[0].astext() == 'history':
            self.history.append(node.children[1].astext())
        raise nodes.SkipChildren

    def depart_field(self, node):
        pass

    def visit_literal_block(self, node):
        if node.rawsource != node.astext():
            # most probably a parsed-literal block -- don't highlight
            return BaseHtmlXlator.visit_literal_block(self, node)
        lang = self.highlightlang
        if node.has_key('language'):
{'fraction_non_alphanumeric': 0.09991556431184914, 'fraction_numerical': 0.004503236701379116, 'mean_word_length': 2.63023493360572, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7352045', 'n_tokens_mistral': 1083, 'n_tokens_neox': 1039, 'n_words': 261}