<commit_before># -*- coding: utf-8  -*-

from re import sub, UNICODE

from markupsafe import escape

def highlight_delta(context, chain, delta):
    processed = []
    dchain = delta.chain
    prev_prev = prev = chain.START
    i = 0
    all_words = chain.text.split()
    paragraphs = chain.text.split("\n")
    for paragraph in paragraphs:
        processed_words = []
        words = paragraph.split(" ")
        for i, word in enumerate(words, i):
            try:
                next = _strip_word(all_words[i + 1])
                try:
                    next_next = _strip_word(all_words[i + 2])
                except IndexError:
                    next_next = chain.END
            except IndexError:
                next = next_next = chain.END
            sword = _strip_word(word)
            middle = (prev, sword) in dchain and next in dchain[(prev, sword)]
            if middle:
                before = after = True
            else:
                b_block = (prev_prev, prev)
                a_block = (sword, next)
                before = b_block in dchain and sword in dchain[b_block]
                after = a_block in dchain and next_next in dchain[a_block]
            is_first = i == 0
            is_last = i + 1 == len(all_words)
            res = _highlight_word(word, before, after, is_first, is_last)
            processed_words.append(res)
            prev_prev = prev
            prev = sword
        processed.append(u" ".join(processed_words))
        i += 1
    return u"<p>" + u"</p>\n<p>".join(processed) + u"</p>"

def _highlight_word(word, before, after, is_first, is_last):
    if before and after:
        # Word is in the middle of a highlighted block, so don't change
        # anything unless this is the first word (force block to start) or the
        # last word (force block to end):
        res = unicode(escape(word))
        if is_first:
            res = u'<span class="cv-hl">' + res
        if is_last:
            res += u'</span>'
    elif before:
        # Word is the last in a highlighted block, so fade it out and then end
        # the block; force open a block before the word if this is the first
        # word:
        res = _fade_word(word, u"out") + u"</span>"
        if is_first:
            res = u'<span class="cv-hl">' + res
    elif after:
        # Word is the first in a highlighted block, so start the block and then
        # fade it in; force close the block after the word if this is the last
        # word:
        res = u'<span class="cv-hl">' + _fade_word(word, u"in")
        if is_last:
            res += u"</span>"
    else:
        # Word is completely outside of a highlighted block, so do nothing:
        res = unicode(escape(word))
    return res

def _fade_word(word, dir):
    if len(word) <= 4:
        word = unicode(escape(word))
        return u'<span class="cv-hl-{0}">{1}</span>'.format(dir, word)
    if dir == u"out":
        before, after = unicode(escape(word[:-4])), unicode(escape(word[-4:]))
        base = u'{0}<span class="cv-hl-out">{1}</span>'
        return base.format(before, after)
    else:
        before, after = unicode(escape(word[:4])), unicode(escape(word[4:]))
        base = u'<span class="cv-hl-in">{0}</span>{1}'
        return base.format(before, after)

def _strip_word(word):
    return sub("[^\w\s-]", "", word.lower(), flags=UNICODE)<commit_msg>Try a new highlighting engine.<commit_after># -*- coding: utf-8  -*-

from re import sub, UNICODE

from markupsafe import escape

def highlight_delta(context, chain, delta):
    degree = chain.degree - 1
    highlights = [False] * degree
    block = [chain.START] * degree
    for word in chain.text.split() + ([chain.END] * degree):
        word = _strip_word(chain, word)
        tblock = tuple(block)
        if tblock in delta.chain and word in delta.chain[tblock]:
            highlights[-1 * degree:] = [True] * degree
            highlights.append(True)
        else:
            highlights.append(False)
        block.pop(0)
        block.append(word)

    i = degree
    numwords = len(chain.text.split())
    processed = []
    for paragraph in chain.text.split("\n"):
        words = []
        for i, word in enumerate(paragraph.split(), i):
            before = highlights[i - 1]
            after = highlights[i + 1]
            first = i == degree
            last = i - degree + 1 == numwords
            words.append(_highlight_word(word, before, after, first, last))
        processed.append(u" ".join(words))
        i += 1

    return u"<p>" + u"</p>\n<p>".join(processed) + u"</p>"

def _highlight_word(word, before, after, first, last):
    if before and after:
        # Word is in the middle of a highlighted block, so don't change
        # anything unless this is the first word (force block to start) or the
        # last word (force block to end):
        res = unicode(escape(word))
        if first:
            res = u'<span class="cv-hl">' + res
        if last:
            res += u'</span>'
    elif before:
        # Word is the last in a highlighted block, so fade it out and then end
        # the block; force open a block before the word if this is the first
        # word:
        res = _fade_word(word, u"out") + u"</span>"
        if first:
            res = u'<span class="cv-hl">' + res
    elif after:
        # Word is the first in a highlighted block, so start the block and then
        # fade it in; force close the block after the word if this is the last
        # word:
        res = u'<span class="cv-hl">' + _fade_word(word, u"in")
        if last:
            res += u"</span>"
    else:
        # Word is completely outside of a highlighted block, so do nothing:
        res = unicode(escape(word))
    return res

def _fade_word(word, dir):
    if len(word) <= 4:
        word = unicode(escape(word))
        return u'<span class="cv-hl-{0}">{1}</span>'.format(dir, word)
    if dir == u"out":
        before, after = unicode(escape(word[:-4])), unicode(escape(word[-4:]))
        base = u'{0}<span class="cv-hl-out">{1}</span>'
        return base.format(before, after)
    else:
        before, after = unicode(escape(word[:4])), unicode(escape(word[4:]))
        base = u'<span class="cv-hl-in">{0}</span>{1}'
        return base.format(before, after)

def _strip_word(chain, word):
    if word == chain.START or word == chain.END:
        return word
    return sub("[^\w\s-]", "", word.lower(), flags=UNICODE)
{'fraction_non_alphanumeric': 0.12908724624205795, 'fraction_numerical': 0.00573376724004339, 'mean_word_length': 2.0939597315436242, 'pattern_counts': {'":': 2, '<': 37, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4022106', 'n_tokens_mistral': 2006, 'n_tokens_neox': 1937, 'n_words': 620}