<commit_before>import datetime
import django
import logging
import warnings

from django.db import models
from django.db.models import signals

from sentry import conf
from sentry.helpers import construct_checksum, get_db_engine

assert not conf.DATABASE_USING or django.VERSION >= (1, 2), 'The `SENTRY_DATABASE_USING` setting requires Django >= 1.2'

logger = logging.getLogger('sentry.errors')

class ScoreClause(object):
    def __init__(self, group):
        self.group = group

    def prepare_database_save(self, unused):
        return self

    def prepare(self, evaluator, query, allow_joins):
        return

    def evaluate(self, node, qn, connection):
        engine = get_db_engine(getattr(connection, 'alias', 'default'))
        if engine.startswith('postgresql'):
            sql = 'log(times_seen) * 600 + last_seen::abstime::int'
        elif engine.startswith('mysql'):
            sql = 'log(times_seen) * 600 + unix_timestamp(last_seen)'
        else:
            # XXX: if we cant do it atomicly let's do it the best we can
            sql = self.group.get_score()
        
        return (sql, [])

class SentryManager(models.Manager):
    use_for_related_fields = True

    def get_query_set(self):
        qs = super(SentryManager, self).get_query_set()
        if conf.DATABASE_USING:
            qs = qs.using(conf.DATABASE_USING)
        return qs

    def from_kwargs(self, **kwargs):
        from sentry.models import Message, GroupedMessage, FilterValue
        
        URL_MAX_LENGTH = Message._meta.get_field_by_name('url')[0].max_length
        now = datetime.datetime.now()

        view = kwargs.pop('view', None)
        logger_name = kwargs.pop('logger', 'root')
        url = kwargs.pop('url', None)
        server_name = kwargs.pop('server_name', conf.CLIENT)
        site = kwargs.pop('site', None)
        data = kwargs.pop('data', {}) or {}
        message_id = kwargs.pop('message_id', None)

        if url:
            data['url'] = url
            url = url[:URL_MAX_LENGTH]

        checksum = kwargs.pop('checksum', None)
        if not checksum:
            checksum = construct_checksum(**kwargs)

        mail = False
        try:
            kwargs['data'] = {}

            if 'url' in data:
                kwargs['data']['url'] = data['url']
            if 'version' in data.get('__sentry__', {}):
                kwargs['data']['version'] = data['__sentry__']['version']
            if 'module' in data.get('__sentry__', {}):
                kwargs['data']['module'] = data['__sentry__']['module']

            group, created = GroupedMessage.objects.get_or_create(
                view=view,
                logger=logger_name,
                checksum=checksum,
                # we store some sample data for rendering
                defaults=kwargs
            )
            kwargs.pop('data', None)
            if not created:
                # HACK: maintain appeared state
                if group.status == 1:
                    mail = True
                group.status = 0
                group.last_seen = now
                group.times_seen += 1
                GroupedMessage.objects.filter(pk=group.pk).update(
                    times_seen=models.F('times_seen') + 1,
                    status=0,
                    last_seen=now,
                    score=ScoreClause(group),
                )
                signals.post_save.send(sender=GroupedMessage, instance=group, created=False)
            else:
                GroupedMessage.objects.filter(pk=group.pk).update(
                    score=ScoreClause(group),
                )
                mail = True
            instance = Message.objects.create(
                message_id=message_id,
                view=view,
                logger=logger_name,
                data=data,
                url=url,
                server_name=server_name,
                site=site,
                checksum=checksum,
                group=group,
                **kwargs
            )
            if server_name:
                FilterValue.objects.get_or_create(key='server_name', value=server_name)<commit_msg>Swap timestamp -> datetime in from_kwargs<commit_after>import datetime
import django
import logging
import warnings

from django.db import models
from django.db.models import signals

from sentry import conf
from sentry.helpers import construct_checksum, get_db_engine

assert not conf.DATABASE_USING or django.VERSION >= (1, 2), 'The `SENTRY_DATABASE_USING` setting requires Django >= 1.2'

logger = logging.getLogger('sentry.errors')

class ScoreClause(object):
    def __init__(self, group):
        self.group = group

    def prepare_database_save(self, unused):
        return self

    def prepare(self, evaluator, query, allow_joins):
        return

    def evaluate(self, node, qn, connection):
        engine = get_db_engine(getattr(connection, 'alias', 'default'))
        if engine.startswith('postgresql'):
            sql = 'log(times_seen) * 600 + last_seen::abstime::int'
        elif engine.startswith('mysql'):
            sql = 'log(times_seen) * 600 + unix_timestamp(last_seen)'
        else:
            # XXX: if we cant do it atomicly let's do it the best we can
            sql = self.group.get_score()
        
        return (sql, [])

class SentryManager(models.Manager):
    use_for_related_fields = True

    def get_query_set(self):
        qs = super(SentryManager, self).get_query_set()
        if conf.DATABASE_USING:
            qs = qs.using(conf.DATABASE_USING)
        return qs

    def from_kwargs(self, **kwargs):
        from sentry.models import Message, GroupedMessage, FilterValue
        
        URL_MAX_LENGTH = Message._meta.get_field_by_name('url')[0].max_length
        now = datetime.datetime.now()

        view = kwargs.pop('view', None)
        logger_name = kwargs.pop('logger', 'root')
        url = kwargs.pop('url', None)
        server_name = kwargs.pop('server_name', conf.CLIENT)
        site = kwargs.pop('site', None)
        data = kwargs.pop('data', {}) or {}
        message_id = kwargs.pop('message_id', None)
        timestamp = kwargs.pop('timestamp', None)
        
        if url:
            data['url'] = url
            url = url[:URL_MAX_LENGTH]

        checksum = kwargs.pop('checksum', None)
        if not checksum:
            checksum = construct_checksum(**kwargs)

        mail = False
        try:
            kwargs['data'] = {}

            if 'url' in data:
                kwargs['data']['url'] = data['url']
            if 'version' in data.get('__sentry__', {}):
                kwargs['data']['version'] = data['__sentry__']['version']
            if 'module' in data.get('__sentry__', {}):
                kwargs['data']['module'] = data['__sentry__']['module']

            if timestamp:
                kwargs['first_seen'] = timestamp

            group, created = GroupedMessage.objects.get_or_create(
                view=view,
                logger=logger_name,
                checksum=checksum,
                # we store some sample data for rendering
                defaults=kwargs
            )
            kwargs.pop('data', None)
            if not created:
                # HACK: maintain appeared state
                if group.status == 1:
                    mail = True
                group.status = 0
                group.last_seen = now
                group.times_seen += 1
                GroupedMessage.objects.filter(pk=group.pk).update(
                    times_seen=models.F('times_seen') + 1,
                    status=0,
                    last_seen=now,
                    score=ScoreClause(group),
                )
                signals.post_save.send(sender=GroupedMessage, instance=group, created=False)
            else:
                GroupedMessage.objects.filter(pk=group.pk).update(
                    score=ScoreClause(group),
                )
                mail = True

            if 'first_seen' in kwargs:
                kwargs['datetime'] = kwargs.pop('first_seen', None)
                
            instance = Message.objects.create(
                message_id=message_id,
                view=view,
                logger=logger_name,
                data=data,
                url=url,
                server_name=server_name,
                site=site,
                checksum=checksum,
                group=group,
                **kwargs
            )
            if server_name:
                FilterValue.objects.get_or_create(key='server_name', value=server_name)
{'fraction_non_alphanumeric': 0.10533068937361484, 'fraction_numerical': 0.00373264901434737, 'mean_word_length': 1.8047105004906772, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1630892', 'n_tokens_mistral': 2425, 'n_tokens_neox': 2325, 'n_words': 511}