<commit_before># -*- coding: utf-8 -*-
"""
    textpress.htmlprocessor
    ~~~~~~~~~~~~~~~~~~~~~~~

    We use plain old HTML as blog format so we use BeautifulSoup for
    parsing that. The output from beautiful soup is converted into a
    simplified doctree that is load and dumpable, and can emit events
    on rendering. Plugins can hook in at various places:

    `process-doc-tree`
        Called after the doc tree was created, plugins may want to
        traverse nodes, attach callbacks or replace child nodes.

    `setup-markup-parser`
        Called with the markup parser, that's the last and only chance
        for plugins to modify parsing rules.

    `process-node-callback`
        Called during rendering if there was a callback for a node.

    Plugins can either modify the tree (preferred because cached) or
    replace contents of a node at render time (just for dynamic content
    because slower).

    Here a small example plugin that displays the current time in all
    nodes that have a tp:contents="clock" attribute::

        from datetime import datetime

        def process_tree(event):
            doctree = event.data['doctree']
            for node in doctree.query('*[tp:contents=clock]'):
                del node.attributes['contents']
                node.add_render_callback('testplugin/show_clock')

        def node_callback(event):
            if event.data['identifier'] == 'testplugin/show_clock':
                time = datetime.now().strftime('%H:%M')
                if event.data['text_only']:
                    return time
                return event.data['node'].render(inject=time)

        def setup(app, plugin):
            app.connect_event('process-doc-tree', process_tree)
            app.connect_event('process-node-callback', node_callback)

    In the editor you can then have this snippet to trigger the
    execution::

        The current time is <span tp:contents="clock">time goes here</span>.

    Plugins have to make sure that they delete non HTML compatible
    attributes from the node they control to make sure the output isn't
    that bad. The preferred prefix for plugins is "tp:".

    Alternative Parsers
    ~~~~~~~~~~~~~~~~~~~

    Plugins may change the parser for some input data or reasons. Parsers
    should be subclasses of `textpress.htmlprocessor.BaseParser`

    :copyright: 2007 by <NAME>.
    :license: GNU GPL.
"""
import pickle
from itertools import izip
from weakref import WeakKeyDictionary
from xml.sax.saxutils import quoteattr, escape

from textpress._ext import beautifulsoup as bt

from textpress.application import emit_event, get_request, get_application


#: list of self closing html tags for *rendering*
SELF_CLOSING_TAGS = ['br', 'img', 'area', 'hr', 'param', 'meta',
                     'link', 'base', 'input', 'embed', 'col']<commit_msg>try to use the global installed BeautifulSoup first before use the textpress._ext ones (which can outdated)<commit_after># -*- coding: utf-8 -*-
"""
    textpress.htmlprocessor
    ~~~~~~~~~~~~~~~~~~~~~~~

    We use plain old HTML as blog format so we use BeautifulSoup for
    parsing that. The output from beautiful soup is converted into a
    simplified doctree that is load and dumpable, and can emit events
    on rendering. Plugins can hook in at various places:

    `process-doc-tree`
        Called after the doc tree was created, plugins may want to
        traverse nodes, attach callbacks or replace child nodes.

    `setup-markup-parser`
        Called with the markup parser, that's the last and only chance
        for plugins to modify parsing rules.

    `process-node-callback`
        Called during rendering if there was a callback for a node.

    Plugins can either modify the tree (preferred because cached) or
    replace contents of a node at render time (just for dynamic content
    because slower).

    Here a small example plugin that displays the current time in all
    nodes that have a tp:contents="clock" attribute::

        from datetime import datetime

        def process_tree(event):
            doctree = event.data['doctree']
            for node in doctree.query('*[tp:contents=clock]'):
                del node.attributes['contents']
                node.add_render_callback('testplugin/show_clock')

        def node_callback(event):
            if event.data['identifier'] == 'testplugin/show_clock':
                time = datetime.now().strftime('%H:%M')
                if event.data['text_only']:
                    return time
                return event.data['node'].render(inject=time)

        def setup(app, plugin):
            app.connect_event('process-doc-tree', process_tree)
            app.connect_event('process-node-callback', node_callback)

    In the editor you can then have this snippet to trigger the
    execution::

        The current time is <span tp:contents="clock">time goes here</span>.

    Plugins have to make sure that they delete non HTML compatible
    attributes from the node they control to make sure the output isn't
    that bad. The preferred prefix for plugins is "tp:".

    Alternative Parsers
    ~~~~~~~~~~~~~~~~~~~

    Plugins may change the parser for some input data or reasons. Parsers
    should be subclasses of `textpress.htmlprocessor.BaseParser`

    :copyright: 2007 by <NAME>.
    :license: GNU GPL.
"""
import pickle
from itertools import izip
from weakref import WeakKeyDictionary
from xml.sax.saxutils import quoteattr, escape

try:
    import BautifulSoup as bt
except ImportError:
    from textpress._ext import beautifulsoup as bt

from textpress.application import emit_event, get_request, get_application


#: list of self closing html tags for *rendering*
SELF_CLOSING_TAGS = ['br', 'img', 'area', 'hr', 'param', 'meta',
                     'link', 'base', 'input', 'embed', 'col']
{'fraction_non_alphanumeric': 0.09100221956633088, 'fraction_numerical': 0.0017073587160662455, 'mean_word_length': 2.869220607661823, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3714219', 'n_tokens_mistral': 1565, 'n_tokens_neox': 1483, 'n_words': 623}