<commit_before>from datetime import datetime, timedelta

from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.core.exceptions import ValidationError
from django.core.urlresolvers import reverse, get_callable
from django.db import models
from django.template.defaultfilters import striptags
from django.utils.http import base36_to_int, int_to_base36
from django.utils.translation import ugettext_lazy as _

from tcc.managers import (
    CurrentCommentManager, LimitedCurrentCommentManager,
    RemovedCommentManager, DisapprovedCommentManager,
    )
from tcc import signals
from tcc.settings import (
    STEPLEN, COMMENT_MAX_LENGTH, MODERATED, REPLY_LIMIT,
    MAX_DEPTH, MAX_REPLIES, ADMIN_CALLBACK, SUBSCRIBE_ON_POST
    )

SITE_ID = getattr(settings, 'SITE_ID', 1)

TWO_MINS = timedelta(minutes=2)


class Thread(models.Model):
    content_type = models.ForeignKey(
        ContentType, verbose_name=_('content type'),
        related_name="content_type_set_for_tcc_thread")
    object_pk = models.TextField(_('object id'))
    content_object = generic.GenericForeignKey(
        ct_field="content_type", fk_field="object_pk")
    site = models.ForeignKey(Site, default=SITE_ID,
                             related_name='tccthreads')
    is_open = models.BooleanField(_('Open'), default=True)
    is_moderated = models.BooleanField(_('Moderated'), default=MODERATED)


class Comment(models.Model):
    """ A comment table, aimed to be compatible with django.contrib.comments

    """
    # constants
    MAX_REPLIES = MAX_REPLIES
    REPLY_LIMIT = REPLY_LIMIT

    # From comments BaseCommentAbstractModel
    content_type = models.ForeignKey(
        ContentType, verbose_name=_('content type'),
        related_name="content_type_set_for_tcc_comment")
    object_pk = models.TextField(_('object id'))
    content_object = generic.GenericForeignKey(
        ct_field="content_type", fk_field="object_pk")
    site = models.ForeignKey(Site, default=SITE_ID,
                             related_name='tcccomments')
    # The actual comment fields
    parent = models.ForeignKey('self', verbose_name= _('Reply to'),
                               null=True, blank=True, related_name='parents')
    user = models.ForeignKey(User, verbose_name='Commenter')
    # These are here mainly for backwards compatibility
    ip_address = models.IPAddressField(default='127.0.0.1')
    user_name   = models.CharField(_("user's name"), max_length=50, blank=True)
    user_email  = models.EmailField(_("user's email address"), blank=True)
    user_url    = models.URLField(_("user's URL"), blank=True)
    submit_date = models.DateTimeField(_('Date'), default=datetime.utcnow)
    # Protip: Use postgres...
    comment = models.TextField(_('Comment'), max_length=COMMENT_MAX_LENGTH)
    comment_raw = models.TextField(_('Raw Comment'), max_length=COMMENT_MAX_LENGTH)
    # still accepting replies?
    is_open = models.BooleanField(_('Open'), default=True)
    is_removed = models.BooleanField(_('Removed'), default=False)
    is_approved = models.BooleanField(_('Approved'), default=not MODERATED)
    # is_public is rather pointless icw is_removed?
    # Keeping it for compatibility w/ contrib.comments
    is_public = models.BooleanField(_('Public'), default=True)
    path = models.CharField(_('Path'), unique=True, max_length=MAX_DEPTH*STEPLEN)
    limit = models.DateTimeField(
        _('Show replies from'), null=True, blank=True)
    # subscription (for notification)
    subscribers = models.ManyToManyField(User, related_name="thread_subscribers")
    # denormalized cache
    childcount = models.IntegerField(_('Reply count'), default=0)
    depth = models.IntegerField(_('Depth'), default=0)

    unfiltered = models.Manager()
    objects = CurrentCommentManager()
    limited = LimitedCurrentCommentManager()
    removed = RemovedCommentManager()
    disapproved = DisapprovedCommentManager()

    class Meta:
        ordering = ['path']

    def __unicode__(self):
        return u"%05d %s % 8s: %s" % (
            self.id, self.submit_date.isoformat(), self.user.username, self.comment[:20])

    def get_absolute_url(self):
        link = reverse('tcc_index',
                       args=(self.content_type.id, self.object_pk))
        return "%s#%s" % (link, self.get_base36())

    def clean(self):

        if self.parent:
            if not self.pk and self.parent.childcount >= self.MAX_REPLIES:
                raise ValidationError(_('Maximum number of replies reached'))
        if self.comment <> "" and striptags(self.comment).strip() == "":
            raise ValidationError(_("This field is required."))

        if not self.comment_raw:
            self.comment_raw = self.comment

        # Check for identical messages
        identical_msgs = Comment.objects.filter(
            content_type=self.content_type,
            object_pk=self.object_pk,
            user=self.user,
            comment_raw=self.comment_raw,
            submit_date__gte=(datetime.utcnow() - TWO_MINS),<commit_msg>Check all messages for the user to search for a (recent) duplicate message  (not just for the same object)<commit_after>from datetime import datetime, timedelta

from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.core.exceptions import ValidationError
from django.core.urlresolvers import reverse, get_callable
from django.db import models
from django.template.defaultfilters import striptags
from django.utils.http import base36_to_int, int_to_base36
from django.utils.translation import ugettext_lazy as _

from tcc.managers import (
    CurrentCommentManager, LimitedCurrentCommentManager,
    RemovedCommentManager, DisapprovedCommentManager,
    )
from tcc import signals
from tcc.settings import (
    STEPLEN, COMMENT_MAX_LENGTH, MODERATED, REPLY_LIMIT,
    MAX_DEPTH, MAX_REPLIES, ADMIN_CALLBACK, SUBSCRIBE_ON_POST
    )

SITE_ID = getattr(settings, 'SITE_ID', 1)

TWO_MINS = timedelta(minutes=2)


class Thread(models.Model):
    content_type = models.ForeignKey(
        ContentType, verbose_name=_('content type'),
        related_name="content_type_set_for_tcc_thread")
    object_pk = models.TextField(_('object id'))
    content_object = generic.GenericForeignKey(
        ct_field="content_type", fk_field="object_pk")
    site = models.ForeignKey(Site, default=SITE_ID,
                             related_name='tccthreads')
    is_open = models.BooleanField(_('Open'), default=True)
    is_moderated = models.BooleanField(_('Moderated'), default=MODERATED)


class Comment(models.Model):
    """ A comment table, aimed to be compatible with django.contrib.comments

    """
    # constants
    MAX_REPLIES = MAX_REPLIES
    REPLY_LIMIT = REPLY_LIMIT

    # From comments BaseCommentAbstractModel
    content_type = models.ForeignKey(
        ContentType, verbose_name=_('content type'),
        related_name="content_type_set_for_tcc_comment")
    object_pk = models.TextField(_('object id'))
    content_object = generic.GenericForeignKey(
        ct_field="content_type", fk_field="object_pk")
    site = models.ForeignKey(Site, default=SITE_ID,
                             related_name='tcccomments')
    # The actual comment fields
    parent = models.ForeignKey('self', verbose_name= _('Reply to'),
                               null=True, blank=True, related_name='parents')
    user = models.ForeignKey(User, verbose_name='Commenter')
    # These are here mainly for backwards compatibility
    ip_address = models.IPAddressField(default='127.0.0.1')
    user_name   = models.CharField(_("user's name"), max_length=50, blank=True)
    user_email  = models.EmailField(_("user's email address"), blank=True)
    user_url    = models.URLField(_("user's URL"), blank=True)
    submit_date = models.DateTimeField(_('Date'), default=datetime.utcnow)
    # Protip: Use postgres...
    comment = models.TextField(_('Comment'), max_length=COMMENT_MAX_LENGTH)
    comment_raw = models.TextField(_('Raw Comment'), max_length=COMMENT_MAX_LENGTH)
    # still accepting replies?
    is_open = models.BooleanField(_('Open'), default=True)
    is_removed = models.BooleanField(_('Removed'), default=False)
    is_approved = models.BooleanField(_('Approved'), default=not MODERATED)
    # is_public is rather pointless icw is_removed?
    # Keeping it for compatibility w/ contrib.comments
    is_public = models.BooleanField(_('Public'), default=True)
    path = models.CharField(_('Path'), unique=True, max_length=MAX_DEPTH*STEPLEN)
    limit = models.DateTimeField(
        _('Show replies from'), null=True, blank=True)
    # subscription (for notification)
    subscribers = models.ManyToManyField(User, related_name="thread_subscribers")
    # denormalized cache
    childcount = models.IntegerField(_('Reply count'), default=0)
    depth = models.IntegerField(_('Depth'), default=0)

    unfiltered = models.Manager()
    objects = CurrentCommentManager()
    limited = LimitedCurrentCommentManager()
    removed = RemovedCommentManager()
    disapproved = DisapprovedCommentManager()

    class Meta:
        ordering = ['path']

    def __unicode__(self):
        return u"%05d %s % 8s: %s" % (
            self.id, self.submit_date.isoformat(), self.user.username, self.comment[:20])

    def get_absolute_url(self):
        link = reverse('tcc_index',
                       args=(self.content_type.id, self.object_pk))
        return "%s#%s" % (link, self.get_base36())

    def clean(self):

        if self.parent:
            if not self.pk and self.parent.childcount >= self.MAX_REPLIES:
                raise ValidationError(_('Maximum number of replies reached'))
        if self.comment <> "" and striptags(self.comment).strip() == "":
            raise ValidationError(_("This field is required."))

        if not self.comment_raw:
            self.comment_raw = self.comment

        # Check for identical messages
        identical_msgs = Comment.objects.filter(
            user=self.user,
            comment_raw=self.comment_raw,
            submit_date__gte=(datetime.utcnow() - TWO_MINS),
{'fraction_non_alphanumeric': 0.10478840802226273, 'fraction_numerical': 0.0044141637078975145, 'mean_word_length': 3.7372727272727273, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4460468', 'n_tokens_mistral': 3091, 'n_tokens_neox': 2990, 'n_words': 661}