<commit_before>import re
from urllib2 import unquote

from django import forms
from django.forms.models import modelformset_factory

import happyforms
from tower import ugettext_lazy as _lazy

from quieter_formset.formset import BaseModelFormSet

import amo
import reviews
from .models import Review, ReviewFlag
from amo.utils import raise_required
from editors.models import ReviewerScore


class ReviewReplyForm(forms.Form):
    title = forms.CharField(required=False)
    body = forms.CharField(widget=forms.Textarea(attrs={'rows': 3}))

    def clean_body(self):
        body = self.cleaned_data.get('body', '')
        # Whitespace is not a review!
        if not body.strip():
            raise_required()
        return body


class ReviewForm(ReviewReplyForm):
    rating = forms.ChoiceField(zip(range(1, 6), range(1, 6)))
    flags = re.I | re.L | re.U | re.M
    # This matches the following three types of patterns:
    # http://... or https://..., RFC 3986 compliant host names, and IPv4
    # octets. It does not match IPv6 addresses or long strings such as
    # "example dot com".
    # This is much lighter weight than parsing and recompiling a string
    # then sending it through a DOM tree generator and searching for tokens.
    # Please note that bleach.linkify also currently recognizes only 23
    # potential patterns for TLDs, not the unlimited ICANN set.
    link_pattern = re.compile('((https?://[^\s]+)|(([a-z][0-9a-z\-%]+){1,63}'
            '\.)(([0-9a-z\-%]+){1,63}\.)*([\da-z\-]+){1,63})|((\d{1,3}\.){3}'
            '(\d{1,3}))', flags)

    def _post_clean(self):
        # Unquote the body in case someone tries 'example%2ecom'.
        data = unquote(self.cleaned_data.get('body', ''))
        if '<br>' in data:
            self.cleaned_data['body'] = re.sub('<br>', '\n', data)
        if self.link_pattern.search(data) is not None:
            self.cleaned_data['flag'] = True
            self.cleaned_data['editorreview'] = True<commit_msg>Better link regex for link detection (bug 785326)<commit_after>import re
from urllib2 import unquote

from django import forms
from django.forms.models import modelformset_factory

import happyforms
from tower import ugettext_lazy as _lazy

from quieter_formset.formset import BaseModelFormSet

import amo
import reviews
from .models import Review, ReviewFlag
from amo.utils import raise_required
from editors.models import ReviewerScore


class ReviewReplyForm(forms.Form):
    title = forms.CharField(required=False)
    body = forms.CharField(widget=forms.Textarea(attrs={'rows': 3}))

    def clean_body(self):
        body = self.cleaned_data.get('body', '')
        # Whitespace is not a review!
        if not body.strip():
            raise_required()
        return body


class ReviewForm(ReviewReplyForm):
    rating = forms.ChoiceField(zip(range(1, 6), range(1, 6)))
    flags = re.I | re.L | re.U | re.M
    # This matches the following three types of patterns:
    # http://... or https://..., generic domain names, and IPv4
    # octets. It does not match IPv6 addresses or long strings such as
    # "example dot com".
    link_pattern = re.compile('((://)|'  # Protocols (e.g.: http://)
                              '((\d{1,3}\.){3}(\d{1,3}))|'
                              '([0-9a-z\-%]+\.[0-9a-z\-]{2,10}))', flags)

    def _post_clean(self):
        # Unquote the body in case someone tries 'example%2ecom'.
        data = unquote(self.cleaned_data.get('body', ''))
        if '<br>' in data:
            self.cleaned_data['body'] = re.sub('<br>', '\n', data)
        if self.link_pattern.search(data) is not None:
            self.cleaned_data['flag'] = True
            self.cleaned_data['editorreview'] = True
{'fraction_non_alphanumeric': 0.13297442799461642, 'fraction_numerical': 0.016150740242261104, 'mean_word_length': 2.9322751322751324, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6234508', 'n_tokens_mistral': 1182, 'n_tokens_neox': 1139, 'n_words': 336}