<commit_before>import string
import re
import urllib

import mistune
import bleach
from html import escape, unescape
from itertools import count
from flask import Markup
from orderedset import OrderedSet
from . import email_with_smart_quotes_regex
from notifications_utils.sanitise_text import SanitiseSMS
import smartypants


LINK_STYLE = 'word-wrap: break-word; color: #1D70B8;'

OBSCURE_WHITESPACE = (
    '\u180E'  # Mongolian vowel separator
    '\u200B'  # zero width space
    '\u200C'  # zero width non-joiner
    '\u200D'  # zero width joiner
    '\u2060'  # word joiner
    '\u00A0'  # non breaking space
    '\uFEFF'  # zero width non-breaking space
)


mistune._block_quote_leading_pattern = re.compile(r'^ *\^ ?', flags=re.M)
mistune.BlockGrammar.block_quote = re.compile(r'^( *\^[^\n]+(\n[^\n]+)*\n*)+')
mistune.BlockGrammar.list_block = re.compile(
    r'^( *)([•*-]|\d+\.)[\s\S]+?'
    r'(?:'
    r'\n+(?=\1?(?:[-*_] *){3,}(?:\n+|$))'  # hrule
    r'|\n+(?=%s)'  # def links
    r'|\n+(?=%s)'  # def footnotes
    r'|\n{2,}'
    r'(?! )'
    r'(?!\1(?:[•*-]|\d+\.) )\n*'
    r'|'
    r'\s*$)' % (
        mistune._pure_pattern(mistune.BlockGrammar.def_links),
        mistune._pure_pattern(mistune.BlockGrammar.def_footnotes),
    )
)
mistune.BlockGrammar.list_item = re.compile(
    r'^(( *)(?:[•*-]|\d+\.)[^\n]*'
    r'(?:\n(?!\2(?:[•*-]|\d+\.))[^\n]*)*)',
    flags=re.M
)
mistune.BlockGrammar.list_bullet = re.compile(r'^ *(?:[•*-]|\d+\.)')
mistune.InlineGrammar.url = re.compile(r'''^(https?:\/\/[^\s<]+[^<.,:"')\]\s])''')

mistune.InlineLexer.default_rules = list(
    OrderedSet(mistune.InlineLexer.default_rules) - set((
        'emphasis',
        'double_emphasis',
    ))
)
mistune.InlineLexer.inline_html_rules = list(
    set(mistune.InlineLexer.inline_html_rules) - set((
        'emphasis',
        'double_emphasis',
    ))
)

govuk_not_a_link = re.compile(
    r'(^|\s)(#|\*|\^)?(GOV)\.(UK)(?!\/|\?|#)',
    re.IGNORECASE
)

smartypants.tags_to_skip = smartypants.tags_to_skip + ['a']

whitespace_before_punctuation = re.compile(r'[ \t]+([,\.])')

hyphens_surrounded_by_spaces = re.compile(r'\s+[-–—]{1,3}\s+')  # check three different unicode hyphens

multiple_newlines = re.compile(r'((\n)\2{2,})')

MAGIC_SEQUENCE = "🇬🇧🐦✉️"

magic_sequence_regex = re.compile(MAGIC_SEQUENCE)

# The Mistune URL regex only matches URLs at the start of a string,
# using `^`, so we slice that off and recompile
url = re.compile(mistune.InlineGrammar.url.pattern[1:])

multiple_spaces_in_a_row = re.compile(r'\s{2,}')

more_than_two_newlines_in_a_row = re.compile(r'\n{3,}')


def unlink_govuk_escaped(message):
    return re.sub(
        govuk_not_a_link,
        r'\1\2\3' + '.\u200B' + r'\4',  # Unicode zero-width space
        message
    )


def nl2br(value):
    return re.sub(r'\n|\r', '<br>', value.strip())


def add_prefix(body, prefix=None):
    if prefix:
        return "{}: {}".format(prefix.strip(), body)
    return body


def create_sanitised_html_for_url(link):
    """
    takes a link and returns an a tag to that link.  does the quote/unquote dance to ensure that " quotes are escaped
    correctly to prevent xss

    input: `http://foo.com/"bar"?x=1#2`
    output: `<a style=... href="http://foo.com/%22bar%22?x=1#2">http://foo.com/"bar"?x=1#2</a>`
    """
    return '<a style="{}" href="{}">{}</a>'.format(
        LINK_STYLE,
        urllib.parse.quote(
            urllib.parse.unquote(link),
            safe=':/?#=&;'
        ),
        link
    )


def autolink_sms(body):
    return url.sub(
        lambda match: create_sanitised_html_for_url(match.group(1)),
        body,
    )


def prepend_subject(body, subject):
    return '# {}\n\n{}'.format(subject, body)


def sms_encode(content):
    return SanitiseSMS.encode(content)


def strip_html(value):
    return bleach.clean(value, tags=[], strip=True)


def escape_html(value):
    if not value:
        return value
    value = str(value).replace('<', '&lt;')
    # If the content contains a HTML encoded non-breaking space then we
    # need to protect it from the escape/unescape process by temporarily
    # turning it into something else
    value = value.replace('&nbsp;', MAGIC_SEQUENCE)
    value = escape(unescape(value), quote=False)
    value = value.replace(MAGIC_SEQUENCE, '&nbsp;')
    return bleach.clean(value, tags=[], strip=False)


def url_encode_full_stops(value):
    return value.replace('.', '%2E')<commit_msg>Remove workaround for giving bleach HTML-like content<commit_after>import string
import re
import urllib

import mistune
import bleach
from html import escape, unescape
from itertools import count
from flask import Markup
from orderedset import OrderedSet
from . import email_with_smart_quotes_regex
from notifications_utils.sanitise_text import SanitiseSMS
import smartypants


LINK_STYLE = 'word-wrap: break-word; color: #1D70B8;'

OBSCURE_WHITESPACE = (
    '\u180E'  # Mongolian vowel separator
    '\u200B'  # zero width space
    '\u200C'  # zero width non-joiner
    '\u200D'  # zero width joiner
    '\u2060'  # word joiner
    '\u00A0'  # non breaking space
    '\uFEFF'  # zero width non-breaking space
)


mistune._block_quote_leading_pattern = re.compile(r'^ *\^ ?', flags=re.M)
mistune.BlockGrammar.block_quote = re.compile(r'^( *\^[^\n]+(\n[^\n]+)*\n*)+')
mistune.BlockGrammar.list_block = re.compile(
    r'^( *)([•*-]|\d+\.)[\s\S]+?'
    r'(?:'
    r'\n+(?=\1?(?:[-*_] *){3,}(?:\n+|$))'  # hrule
    r'|\n+(?=%s)'  # def links
    r'|\n+(?=%s)'  # def footnotes
    r'|\n{2,}'
    r'(?! )'
    r'(?!\1(?:[•*-]|\d+\.) )\n*'
    r'|'
    r'\s*$)' % (
        mistune._pure_pattern(mistune.BlockGrammar.def_links),
        mistune._pure_pattern(mistune.BlockGrammar.def_footnotes),
    )
)
mistune.BlockGrammar.list_item = re.compile(
    r'^(( *)(?:[•*-]|\d+\.)[^\n]*'
    r'(?:\n(?!\2(?:[•*-]|\d+\.))[^\n]*)*)',
    flags=re.M
)
mistune.BlockGrammar.list_bullet = re.compile(r'^ *(?:[•*-]|\d+\.)')
mistune.InlineGrammar.url = re.compile(r'''^(https?:\/\/[^\s<]+[^<.,:"')\]\s])''')

mistune.InlineLexer.default_rules = list(
    OrderedSet(mistune.InlineLexer.default_rules) - set((
        'emphasis',
        'double_emphasis',
    ))
)
mistune.InlineLexer.inline_html_rules = list(
    set(mistune.InlineLexer.inline_html_rules) - set((
        'emphasis',
        'double_emphasis',
    ))
)

govuk_not_a_link = re.compile(
    r'(^|\s)(#|\*|\^)?(GOV)\.(UK)(?!\/|\?|#)',
    re.IGNORECASE
)

smartypants.tags_to_skip = smartypants.tags_to_skip + ['a']

whitespace_before_punctuation = re.compile(r'[ \t]+([,\.])')

hyphens_surrounded_by_spaces = re.compile(r'\s+[-–—]{1,3}\s+')  # check three different unicode hyphens

multiple_newlines = re.compile(r'((\n)\2{2,})')

MAGIC_SEQUENCE = "🇬🇧🐦✉️"

magic_sequence_regex = re.compile(MAGIC_SEQUENCE)

# The Mistune URL regex only matches URLs at the start of a string,
# using `^`, so we slice that off and recompile
url = re.compile(mistune.InlineGrammar.url.pattern[1:])

multiple_spaces_in_a_row = re.compile(r'\s{2,}')

more_than_two_newlines_in_a_row = re.compile(r'\n{3,}')


def unlink_govuk_escaped(message):
    return re.sub(
        govuk_not_a_link,
        r'\1\2\3' + '.\u200B' + r'\4',  # Unicode zero-width space
        message
    )


def nl2br(value):
    return re.sub(r'\n|\r', '<br>', value.strip())


def add_prefix(body, prefix=None):
    if prefix:
        return "{}: {}".format(prefix.strip(), body)
    return body


def create_sanitised_html_for_url(link):
    """
    takes a link and returns an a tag to that link.  does the quote/unquote dance to ensure that " quotes are escaped
    correctly to prevent xss

    input: `http://foo.com/"bar"?x=1#2`
    output: `<a style=... href="http://foo.com/%22bar%22?x=1#2">http://foo.com/"bar"?x=1#2</a>`
    """
    return '<a style="{}" href="{}">{}</a>'.format(
        LINK_STYLE,
        urllib.parse.quote(
            urllib.parse.unquote(link),
            safe=':/?#=&;'
        ),
        link
    )


def autolink_sms(body):
    return url.sub(
        lambda match: create_sanitised_html_for_url(match.group(1)),
        body,
    )


def prepend_subject(body, subject):
    return '# {}\n\n{}'.format(subject, body)


def sms_encode(content):
    return SanitiseSMS.encode(content)


def strip_html(value):
    return bleach.clean(value, tags=[], strip=True)


def escape_html(value):
    if not value:
        return value
    value = str(value)
    # If the content contains a HTML encoded non-breaking space then we
    # need to protect it from the escape/unescape process by temporarily
    # turning it into something else
    value = value.replace('&nbsp;', MAGIC_SEQUENCE)
    value = escape(unescape(value), quote=False)
    value = value.replace(MAGIC_SEQUENCE, '&nbsp;')
    return bleach.clean(value, tags=[], strip=False)


def url_encode_full_stops(value):
    return value.replace('.', '%2E')
{'fraction_non_alphanumeric': 0.18981221185201844, 'fraction_numerical': 0.01236927920836613, 'mean_word_length': 3.4117063492063493, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2626991', 'n_tokens_mistral': 3553, 'n_tokens_neox': 3391, 'n_words': 642}