<commit_before># -------------------------------------------------------------------------
# Functions for registering and processing generic block tags.
# -------------------------------------------------------------------------

import html
import re

from . import nodes
from . import parsers
from . import escapes


# Use the Pygments module for syntax highlighting, if it's available.
try:
    import pygments
    import pygments.lexers
    import pygments.formatters
except ImportError:
    pygments = None


# Maps tags to registered handler functions.
tagmap = {}


# Decorator function for registering tag handlers.
def register(*tags):

    def register_tag_handler(func):
        for tag in tags:
            tagmap[tag] = func
        return func

    return register_tag_handler


# Entry point.
def process(tag, pargs, kwargs, content, meta):
    if tag in tagmap:
        return tagmap[tag](tag, pargs, kwargs, content, meta)
    else:
        return container_block_handler(tag, pargs, kwargs, content, meta)


# Handler for block-level elements that allow nested blocks. This is the
# default handler for unregistered tags.
def container_block_handler(tag, pargs, kwargs, content, meta):
    node = nodes.Container(tag, kwargs)
    node.children = parsers.BlockParser().parse(content, meta)
    return node


# Handler for block-level elements that do not allow nested block.
@register('p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6')
def leaf_block_handler(tag, pargs, kwargs, content, meta):
    node = nodes.Leaf(tag, kwargs)
    node.append(nodes.Text(str(content)))
    return node


# Hander for block-level elements with raw text content.
@register('script', 'style')
def raw_block_handler(tag, pargs, kwargs, content, meta):
    node = nodes.Raw(tag, kwargs)
    node.append(nodes.Text(str(content)))
    return node


# Handler for the 'esc' tag. Includes content in its raw state.
@register('esc', escapes.char2esc['\\'])
def esc_tag_handler(tag, pargs, kwargs, content, meta):
    node = nodes.Raw()
    node.append(nodes.Text(str(content)))
    return node


# Handler for the ':>>' blockquote sigil.
@register('>>')
def blockquote_tag_handler(tag, pargs, kwargs, content, meta):
    return container_block_handler('blockquote', pargs, kwargs, content, meta)


# Handler for the 'alert' tag and its associated sigil ':!!'
@register('alert', '!!')
def alert_tag_handler(tag, pargs, kwargs, content, meta):
    node = nodes.Container('div', kwargs)
    node.add_class('stx-alert')
    if pargs:
        for arg in pargs:
            node.add_class('stx-%s' % arg)
    node.children = parsers.BlockParser().parse(content, meta)
    return node<commit_msg>Housekeeping<commit_after># -------------------------------------------------------------------------
# Functions for registering and processing generic block tags.
# -------------------------------------------------------------------------

import html
import re

from . import nodes
from . import parsers
from . import escapes


# Use the Pygments module for syntax highlighting, if it's available.
try:
    import pygments
    import pygments.lexers
    import pygments.formatters
except ImportError:
    pygments = None


# Maps tags to registered handler functions.
tagmap = {}


# Decorator function for registering tag handlers.
def register(*tags):

    def register_tag_handler(func):
        for tag in tags:
            tagmap[tag] = func
        return func

    return register_tag_handler


# Entry point.
def process(tag, pargs, kwargs, content, meta):
    if tag in tagmap:
        return tagmap[tag](tag, pargs, kwargs, content, meta)
    else:
        return container_block_handler(tag, pargs, kwargs, content, meta)


# Hanler for 'container' blocks, i.e. blocks that allow nested block-level
# content. This is the default handler for unregistered tags.
def container_block_handler(tag, pargs, kwargs, content, meta):
    node = nodes.Container(tag, kwargs)
    node.children = parsers.BlockParser().parse(content, meta)
    return node


# Handler for 'leaf' blocks, i.e. blocks that do not allow nested block-level
# content.
@register('p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6')
def leaf_block_handler(tag, pargs, kwargs, content, meta):
    node = nodes.Leaf(tag, kwargs)
    node.append(nodes.Text(str(content)))
    return node


# Handler for 'raw' blocks, i.e. blocks whose content should not be processed
# any further but should be included in the output as-is.
@register('script', 'style')
def raw_block_handler(tag, pargs, kwargs, content, meta):
    node = nodes.Raw(tag, kwargs)
    node.append(nodes.Text(str(content)))
    return node


# Handler for the 'esc' tag. Includes content in its raw state.
@register('esc', escapes.char2esc['\\'])
def esc_tag_handler(tag, pargs, kwargs, content, meta):
    node = nodes.Raw()
    node.append(nodes.Text(str(content)))
    return node


# Handler for the ':>>' blockquote sigil.
@register('>>')
def blockquote_tag_handler(tag, pargs, kwargs, content, meta):
    return container_block_handler('blockquote', pargs, kwargs, content, meta)


# Handler for the 'alert' tag and its associated sigil ':!!'
@register('alert', '!!')
def alert_tag_handler(tag, pargs, kwargs, content, meta):
    node = nodes.Container('div', kwargs)
    node.add_class('stx-alert')
    if pargs:
        for arg in pargs:
            node.add_class('stx-%s' % arg)
    node.children = parsers.BlockParser().parse(content, meta)
    return node
{'fraction_non_alphanumeric': 0.16182117388919365, 'fraction_numerical': 0.0025598829767782046, 'mean_word_length': 3.025018395879323, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3657462', 'n_tokens_mistral': 1627, 'n_tokens_neox': 1579, 'n_words': 521}