<commit_before>import os

import html2text as python_html2text
from django import template
from django.conf import settings
from django.contrib.sites import models as sites_models

from . import Link

register = template.Library()


@register.filter
def filename(value, include_ext=True):
    """Split the filename off the given path."""
    base = os.path.basename(getattr(value, 'name', value))
    return base if include_ext else os.path.splitext(base)[0]


@register.filter
def endswith(value: str, search):
    return value.endswith(search)


@register.filter
def startswith(value: str, search):
    return value.startswith(search)


@register.filter
def cuttrailing(s1, s2):
    """Remove a string at the end of a string."""
    if s1.endswith(s2):
        return s1[:-len(s2)]
    return s1


@register.filter
def nolinebreaks(value):
    """Replace line breaks by spaces."""
    return ' '.join(str(value).split('\n'))


@register.filter
def link_to(text, url):
    """Combine text and url to a link tuple."""
    return Link(text=text, url=url)


@register.filter
def url_for_user(model, user):
    "Return the result of get_absolute_url_for_user()."""
    return model.get_absolute_url_for_user(user)


@register.filter
def html2text(html, preset='mail'):
    text_maker = python_html2text.HTML2Text()
    text_maker.body_width = 0
    if preset == 'mail':
        text_maker.inline_links = False
        text_maker.links_each_paragraph = True
        text_maker.use_automatic_links = True
        text_maker.ignore_images = True
    elif preset == 'import':
        text_maker.escape_snob = True
    return text_maker.handle(html).rstrip()


@register.filter
def full_url(path):
    return '{proto}://{domain}{path}'.format(
            proto=settings.ACCOUNT_DEFAULT_HTTP_PROTOCOL,
            domain=sites_models.Site.objects.get_current().domain,
            path=path)<commit_msg>Unescape html entities, refs #621<commit_after>import html
import os

import html2text as python_html2text
from django import template
from django.conf import settings
from django.contrib.sites import models as sites_models

from . import Link

register = template.Library()


@register.filter
def filename(value, include_ext=True):
    """Split the filename off the given path."""
    base = os.path.basename(getattr(value, 'name', value))
    return base if include_ext else os.path.splitext(base)[0]


@register.filter
def endswith(value: str, search):
    return value.endswith(search)


@register.filter
def startswith(value: str, search):
    return value.startswith(search)


@register.filter
def cuttrailing(s1, s2):
    """Remove a string at the end of a string."""
    if s1.endswith(s2):
        return s1[:-len(s2)]
    return s1


@register.filter
def nolinebreaks(value):
    """Replace line breaks by spaces."""
    return ' '.join(str(value).split('\n'))


@register.filter
def link_to(text, url):
    """Combine text and url to a link tuple."""
    return Link(text=text, url=url)


@register.filter
def url_for_user(model, user):
    "Return the result of get_absolute_url_for_user()."""
    return model.get_absolute_url_for_user(user)


@register.filter
def html2text(html_text, preset='mail'):
    text_maker = python_html2text.HTML2Text()
    text_maker.body_width = 0
    if preset == 'mail':
        text_maker.inline_links = False
        text_maker.links_each_paragraph = True
        text_maker.use_automatic_links = True
        text_maker.ignore_images = True
    elif preset == 'import':
        text_maker.escape_snob = True
    return html.unescape(text_maker.handle(html_text)).rstrip()


@register.filter
def full_url(path):
    return '{proto}://{domain}{path}'.format(
            proto=settings.ACCOUNT_DEFAULT_HTTP_PROTOCOL,
            domain=sites_models.Site.objects.get_current().domain,
            path=path)
{'fraction_non_alphanumeric': 0.11359500909799844, 'fraction_numerical': 0.008058227190018197, 'mean_word_length': 3.8341708542713566, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4759871', 'n_tokens_mistral': 1233, 'n_tokens_neox': 1205, 'n_words': 303}