<commit_before>import json
import traceback
from datetime import datetime, timedelta
from dateutil.parser import parse
from django.conf import settings
import math
from django.db import models
from django.db.models.aggregates import Count
from jsonfield.fields import JSONField

from pillowtop.feed.couch import change_from_couch_row
from pillowtop.feed.interface import ChangeMeta

ERROR_MESSAGE_LENGTH = 512


def _get_extra_args(limit, reduce, skip):
    extra_args = dict()
    if not reduce and limit is not None:
            extra_args.update(
                limit=limit,
                skip=skip
            )
    return extra_args


def path_from_object(obj):
    path = "{0}.{1}".format(obj.__class__.__module__, obj.__class__.__name__)
    return path


class PillowError(models.Model):
    doc_id = models.CharField(max_length=255, null=False, db_index=True)
    pillow = models.CharField(max_length=255, null=False)
    date_created = models.DateTimeField()
    date_last_attempt = models.DateTimeField()
    date_next_attempt = models.DateTimeField(db_index=True, null=True)
    total_attempts = models.IntegerField(default=0)
    current_attempt = models.IntegerField(default=0, db_index=True)
    error_type = models.CharField(max_length=255, null=True)
    error_traceback = models.TextField(null=True)
    change = JSONField(null=True)
    change_metadata = JSONField(null=True)
    queued = models.BooleanField(default=False)

    @property
    def change_object(self):
        change = change_from_couch_row(self.change if self.change else {'id': self.doc_id})
        if self.change_metadata:
            change.metadata = ChangeMeta.wrap(self.change_metadata)
        change.document = None
        return change

    class Meta:
        app_label = 'pillow_retry'
        unique_together = ('doc_id', 'pillow',)

    def add_attempt(self, exception, traceb, date=None):
        self.current_attempt += 1
        self.total_attempts += 1
        self.date_last_attempt = date or datetime.utcnow()
        self.error_type = path_from_object(exception)

        self.error_traceback = "{}\n\n{}".format(exception.message, "".join(traceback.format_tb(traceb)))

        if self.current_attempt <= settings.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS:
            time_till_next = settings.PILLOW_RETRY_REPROCESS_INTERVAL * math.pow(self.current_attempt, settings.PILLOW_RETRY_BACKOFF_FACTOR)
            self.date_next_attempt = self.date_last_attempt + timedelta(minutes=time_till_next)
        else:
            self.date_next_attempt = None

    def reset_attempts(self):
        self.current_attempt = 0
        self.date_next_attempt = datetime.utcnow()

    def has_next_attempt(self):
        return self.current_attempt == 0 or (
            self.total_attempts <= self.multi_attempts_cutoff() and
            self.current_attempt <= settings.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS
        )

    @classmethod
    def get_or_create(cls, change, pillow, date_next=None):
        change.document = None
        doc_id = change.id
        try:
            error = cls.objects.get(doc_id=doc_id, pillow=pillow.pillow_id)
        except cls.DoesNotExist:
            now = date_next or datetime.utcnow()
            error = PillowError(
                doc_id=doc_id,
                pillow=pillow.pillow_id,
                date_created=now,
                date_last_attempt=now,
                date_next_attempt=now,
                change=change.to_dict()
            )

            if change.metadata:
                error.change_metadata = change.metadata.to_json()

        return error

    @classmethod
    def get_errors_to_process(cls, utcnow, limit=None, skip=0, fetch_full=False):
        """
        Get errors according the following rules:

            date_next_attempt <= utcnow
            AND<commit_msg>Revert pillow retry change<commit_after>import json
import traceback
from datetime import datetime, timedelta
from dateutil.parser import parse
from django.conf import settings
import math
from django.db import models
from django.db.models.aggregates import Count
from jsonfield.fields import JSONField

from pillowtop.feed.couch import change_from_couch_row
from pillowtop.feed.interface import ChangeMeta

ERROR_MESSAGE_LENGTH = 512


def _get_extra_args(limit, reduce, skip):
    extra_args = dict()
    if not reduce and limit is not None:
            extra_args.update(
                limit=limit,
                skip=skip
            )
    return extra_args


def path_from_object(obj):
    path = "{0}.{1}".format(obj.__class__.__module__, obj.__class__.__name__)
    return path


class PillowError(models.Model):
    doc_id = models.CharField(max_length=255, null=False, db_index=True)
    pillow = models.CharField(max_length=255, null=False)
    date_created = models.DateTimeField()
    date_last_attempt = models.DateTimeField()
    date_next_attempt = models.DateTimeField(db_index=True, null=True)
    total_attempts = models.IntegerField(default=0)
    current_attempt = models.IntegerField(default=0, db_index=True)
    error_type = models.CharField(max_length=255, null=True)
    error_traceback = models.TextField(null=True)
    change = JSONField(null=True)
    change_metadata = JSONField(null=True)
    queued = models.BooleanField(default=False)

    @property
    def change_object(self):
        change = change_from_couch_row(self.change if self.change else {'id': self.doc_id})
        if self.change_metadata:
            change.metadata = ChangeMeta.wrap(self.change_metadata)
        change.document = None
        return change

    class Meta:
        app_label = 'pillow_retry'
        unique_together = ('doc_id', 'pillow',)

    def add_attempt(self, exception, traceb, date=None):
        self.current_attempt += 1
        self.total_attempts += 1
        self.date_last_attempt = date or datetime.utcnow()
        self.error_type = path_from_object(exception)

        self.error_traceback = "{}\n\n{}".format(exception.message, "".join(traceback.format_tb(traceb)))

        if self.current_attempt <= settings.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS:
            time_till_next = settings.PILLOW_RETRY_REPROCESS_INTERVAL * math.pow(self.current_attempt, settings.PILLOW_RETRY_BACKOFF_FACTOR)
            self.date_next_attempt = self.date_last_attempt + timedelta(minutes=time_till_next)
        else:
            self.date_next_attempt = None

    def reset_attempts(self):
        self.current_attempt = 0
        self.date_next_attempt = datetime.utcnow()

    def has_next_attempt(self):
        return self.current_attempt == 0 or (
            self.total_attempts <= self.multi_attempts_cutoff() and
            self.current_attempt <= settings.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS
        )

    @classmethod
    def get_or_create(cls, change, pillow):
        change.document = None
        doc_id = change.id
        try:
            error = cls.objects.get(doc_id=doc_id, pillow=pillow.pillow_id)
        except cls.DoesNotExist:
            now = datetime.utcnow()
            error = PillowError(
                doc_id=doc_id,
                pillow=pillow.pillow_id,
                date_created=now,
                date_last_attempt=now,
                date_next_attempt=now,
                change=change.to_dict()
            )

            if change.metadata:
                error.change_metadata = change.metadata.to_json()

        return error

    @classmethod
    def get_errors_to_process(cls, utcnow, limit=None, skip=0, fetch_full=False):
        """
        Get errors according the following rules:

            date_next_attempt <= utcnow
            AND
{'fraction_non_alphanumeric': 0.08506763787721124, 'fraction_numerical': 0.005463059313215401, 'mean_word_length': 2.873551637279597, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4728911', 'n_tokens_mistral': 2426, 'n_tokens_neox': 2295, 'n_words': 443}