<commit_before>from datetime import timedelta
from collections import defaultdict

from django import template
from django.utils.safestring import mark_safe
from django.template.defaultfilters import urlizetrunc
from django.utils.translation import ugettext_lazy as _
from django.utils.html import format_html
from django.utils import formats
from django.utils import timezone
from django.contrib.contenttypes.models import ContentType

from django_comments import get_model

from froide.helper.text_utils import (
    split_text_by_separator, redact_plaintext
)
from froide.helper.text_diff import mark_differences

from ..models import FoiRequest, FoiMessage, DeliveryStatus
from ..foi_mail import get_alternative_mail
from ..auth import (
    can_read_foirequest, can_write_foirequest, can_read_foirequest_anonymous
)

Comment = get_model()

register = template.Library()


def unify(text):
    text = text.replace("\r\n", "\n")
    return text


def is_authenticated_read(message, request):
    foirequest = message.request
    return (
        can_write_foirequest(foirequest, request) or
        can_read_foirequest_anonymous(foirequest, request)
    )


def highlight_request(message, request):
    auth_read = is_authenticated_read(message, request)

    real_content = unify(message.get_real_content())
    redacted_content = unify(message.get_content())

    real_description = unify(message.request.description)
    redacted_description = redact_plaintext(<commit_msg>Harden unify function in foirequest_tag<commit_after>from datetime import timedelta
from collections import defaultdict

from django import template
from django.utils.safestring import mark_safe
from django.template.defaultfilters import urlizetrunc
from django.utils.translation import ugettext_lazy as _
from django.utils.html import format_html
from django.utils import formats
from django.utils import timezone
from django.contrib.contenttypes.models import ContentType

from django_comments import get_model

from froide.helper.text_utils import (
    split_text_by_separator, redact_plaintext
)
from froide.helper.text_diff import mark_differences

from ..models import FoiRequest, FoiMessage, DeliveryStatus
from ..foi_mail import get_alternative_mail
from ..auth import (
    can_read_foirequest, can_write_foirequest, can_read_foirequest_anonymous
)

Comment = get_model()

register = template.Library()


def unify(text):
    text = text or ''
    text = text.replace("\r\n", "\n")
    return text


def is_authenticated_read(message, request):
    foirequest = message.request
    return (
        can_write_foirequest(foirequest, request) or
        can_read_foirequest_anonymous(foirequest, request)
    )


def highlight_request(message, request):
    auth_read = is_authenticated_read(message, request)

    real_content = unify(message.get_real_content())
    redacted_content = unify(message.get_content())

    real_description = unify(message.request.description)
    redacted_description = redact_plaintext(
{'fraction_non_alphanumeric': 0.06620093147039255, 'fraction_numerical': 0.0, 'mean_word_length': 5.2907949790794975, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6338898', 'n_tokens_mistral': 898, 'n_tokens_neox': 870, 'n_words': 218}