<commit_before># -*- coding: utf-8 -*-
from django import template
from django.conf import settings
from django.utils.translation import ugettext as _
from django.forms.forms import pretty_name
from yats.diff import generate_patch_html
from yats.shortcuts import has_public_fields, non_previewable_contenttypes
from markdownx.utils import markdownify

import os
import re
try:
    import json
except ImportError:
    from django.utils import simplejson as json

register = template.Library()

@register.filter
def markdown2html(md):
    return markdownify(md)

@register.filter
def hasPreview(mime):
    if not mime:
        return False

    for non in non_previewable_contenttypes:
        if non in mime:
            return False
    return True

@register.filter
def hasPreviewFile(fileid):
    return os.path.isfile('%s%s.preview' % (settings.FILE_UPLOAD_PATH, fileid))

@register.filter
def prettify(value):
    return pretty_name(value)

@register.filter
def contains(value, search):
    if not value or not search:
        return False
    return search in value

@register.filter
def numberToTicketURL(value):
    value = re.sub(u'§([0-9]+)', ur'<a href="/docs/view/\1/">§\1</a>', value)
    return re.sub(u'#([0-9]+)', ur'<a href="/tickets/view/\1/">#\1</a>', value)

@register.filter
def buildToDoList(value):
    class local:
        counter = 0

    def render_item(match):
        local.counter += 1
        state = match.groups()
        checked = ' checked' if state[0].strip() else ''
        return '<input type="checkbox" value="%s" %s/>' % (local.counter, checked)

    return re.sub('\[([ Xx])\]', render_item, value)
<commit_msg>= do not match &#128;<commit_after># -*- coding: utf-8 -*-
from django import template
from django.conf import settings
from django.utils.translation import ugettext as _
from django.forms.forms import pretty_name
from yats.diff import generate_patch_html
from yats.shortcuts import has_public_fields, non_previewable_contenttypes
from markdownx.utils import markdownify

import os
import re
try:
    import json
except ImportError:
    from django.utils import simplejson as json

register = template.Library()

@register.filter
def markdown2html(md):
    return markdownify(md)

@register.filter
def hasPreview(mime):
    if not mime:
        return False

    for non in non_previewable_contenttypes:
        if non in mime:
            return False
    return True

@register.filter
def hasPreviewFile(fileid):
    return os.path.isfile('%s%s.preview' % (settings.FILE_UPLOAD_PATH, fileid))

@register.filter
def prettify(value):
    return pretty_name(value)

@register.filter
def contains(value, search):
    if not value or not search:
        return False
    return search in value

@register.filter
def numberToTicketURL(value):
    # https://regex101.com/r/fC1wX4/10
    value = re.sub(u'(?<!&)§([0-9]+)', ur'<a href="/docs/view/\1/">§\1</a>', value)
    return re.sub(u'(?<!&)#([0-9]+)', ur'<a href="/tickets/view/\1/">#\1</a>', value)

@register.filter
def buildToDoList(value):
    class local:
        counter = 0

    def render_item(match):
        local.counter += 1
        state = match.groups()
        checked = ' checked' if state[0].strip() else ''
        return '<input type="checkbox" value="%s" %s/>' % (local.counter, checked)

    return re.sub('\[([ Xx])\]', render_item, value)

{'fraction_non_alphanumeric': 0.12853317465040168, 'fraction_numerical': 0.010711097887533471, 'mean_word_length': 3.5741496598639455, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6702396', 'n_tokens_mistral': 1091, 'n_tokens_neox': 1074, 'n_words': 297}