<commit_before># -*- coding: utf-8 -*-
"""
    solace.utils.formatting
    ~~~~~~~~~~~~~~~~~~~~~~~

    Implements the formatting.  Uses creoleparser internally.

    :copyright: (c) 2010 by the Solace Team, see AUTHORS for more details.
    :license: BSD, see LICENSE for more details.
"""
from __future__ import with_statement
import re
import creoleparser
from difflib import SequenceMatcher
from operator import itemgetter
from itertools import chain
from genshi.core import Stream, QName, Attrs, START, END, TEXT
from contextlib import contextmanager


_leading_space_re = re.compile(r'^(\s+)(?u)')
_diff_split_re = re.compile(r'(\s+)(?u)')


_parser = creoleparser.Parser(
    dialect=creoleparser.create_dialect(creoleparser.creole10_base),
    method='html'
)


def format_creole(text, inline=False):
    """Format creole markup."""
    kwargs = {}
    if inline:
        kwargs['context'] = 'inline'
    return _parser.render(text, encoding=None, **kwargs)


def format_creole_diff(old, new):
    """Renders a creole diff for two texts."""
    differ = StreamDiffer(_parser.generate(old),
                          _parser.generate(new))
    return differ.get_diff_stream().render('html', encoding=None)


def longzip(a, b):
    """Like `izip` but yields `None` for missing items."""
    aiter = iter(a)
    biter = iter(b)
    try:
        for item1 in aiter:
            yield item1, biter.next()
    except StopIteration:
        for item1 in aiter:
            yield item1, None
    else:
        for item2 in biter:
            yield None, item2


class StreamDiffer(object):
    """A class that can diff a stream of Genshi events.  It will inject
    ``<ins>`` and ``<del>`` tags into the stream.  It probably breaks
    in very ugly ways if you pass a random Genshi stream to it.  I'm
    not exactly sure if it's correct what creoleparser is doing here,<commit_msg>Let format_creole and format_creole_diff return Markup<commit_after># -*- coding: utf-8 -*-
"""
    solace.utils.formatting
    ~~~~~~~~~~~~~~~~~~~~~~~

    Implements the formatting.  Uses creoleparser internally.

    :copyright: (c) 2010 by the Solace Team, see AUTHORS for more details.
    :license: BSD, see LICENSE for more details.
"""
from __future__ import with_statement
import re
import creoleparser
from difflib import SequenceMatcher
from operator import itemgetter
from itertools import chain
from genshi.core import Stream, QName, Attrs, START, END, TEXT
from contextlib import contextmanager

from jinja2 import Markup


_leading_space_re = re.compile(r'^(\s+)(?u)')
_diff_split_re = re.compile(r'(\s+)(?u)')


_parser = creoleparser.Parser(
    dialect=creoleparser.create_dialect(creoleparser.creole10_base),
    method='html'
)


def format_creole(text, inline=False):
    """Format creole markup."""
    kwargs = {}
    if inline:
        kwargs['context'] = 'inline'
    return Markup(_parser.render(text, encoding=None, **kwargs))


def format_creole_diff(old, new):
    """Renders a creole diff for two texts."""
    differ = StreamDiffer(_parser.generate(old),
                          _parser.generate(new))
    return Markup(differ.get_diff_stream().render('html', encoding=None))


def longzip(a, b):
    """Like `izip` but yields `None` for missing items."""
    aiter = iter(a)
    biter = iter(b)
    try:
        for item1 in aiter:
            yield item1, biter.next()
    except StopIteration:
        for item1 in aiter:
            yield item1, None
    else:
        for item2 in biter:
            yield None, item2


class StreamDiffer(object):
    """A class that can diff a stream of Genshi events.  It will inject
    ``<ins>`` and ``<del>`` tags into the stream.  It probably breaks
    in very ugly ways if you pass a random Genshi stream to it.  I'm
    not exactly sure if it's correct what creoleparser is doing here,
{'fraction_non_alphanumeric': 0.11538461538461539, 'fraction_numerical': 0.007016632016632017, 'mean_word_length': 3.2814238042269186, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2197946', 'n_tokens_mistral': 1215, 'n_tokens_neox': 1169, 'n_words': 390}