<commit_before>import re
import time
import json
from datetime import datetime
from datetime import timedelta
import uuid
from traceback import format_tb
import logging

from django.db import models, transaction
from django.contrib.postgres.fields import JSONField
from django.utils import timezone
from django.core.exceptions import ValidationError
from django.core.cache import cache
from channels import Channel
from asgi_redis import RedisChannelLayer
from croniter import croniter

from .task import from_signature, to_signature, to_func_name, TaskFunc
from .managers import TaskManager
from .utils import import_attribute


logger = logging.getLogger('cq')


class Task(models.Model):
    """A persistent representation of a background task.
    """
    STATUS_PENDING = 'P'
    STATUS_RETRY = 'Y'
    STATUS_QUEUED = 'Q'
    STATUS_RUNNING = 'R'
    STATUS_FAILURE = 'F'
    STATUS_SUCCESS = 'S'
    STATUS_WAITING = 'W'
    STATUS_INCOMPLETE = 'I'
    STATUS_LOST = 'L'
    STATUS_REVOKED = 'E'
    STATUS_CHOICES = (
        (STATUS_PENDING, 'Pending'),
        (STATUS_RETRY, 'Retry'),
        (STATUS_QUEUED, 'Queued'),
        (STATUS_RUNNING, 'Running'),
        (STATUS_FAILURE, 'Failure'),
        (STATUS_SUCCESS, 'Success'),
        (STATUS_WAITING, 'Waiting'),
        (STATUS_INCOMPLETE, 'Incomplete'),
        (STATUS_LOST, 'Lost'),
        (STATUS_REVOKED, 'Revoked')
    )
    STATUS_DONE = {STATUS_FAILURE, STATUS_SUCCESS, STATUS_INCOMPLETE,
                   STATUS_LOST, STATUS_REVOKED}
    STATUS_ERROR = {STATUS_FAILURE, STATUS_LOST, STATUS_INCOMPLETE,
                    STATUS_REVOKED}
    STATUS_ACTIVE = {STATUS_PENDING, STATUS_QUEUED, STATUS_RUNNING,
                     STATUS_WAITING}

    AT_RISK_NONE = 'N'
    AT_RISK_QUEUED = 'Q'
    AT_RISK_RUNNING = 'R'
    AT_RISK_CHOICES = (
        (AT_RISK_NONE, 'None'),
        (AT_RISK_QUEUED, 'Queued'),
        (AT_RISK_RUNNING, 'Running'),
    )

    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    status = models.CharField(max_length=1, choices=STATUS_CHOICES,
                              default=STATUS_PENDING, db_index=True)
    signature = JSONField(default={}, blank=True)
    details = JSONField(default={}, blank=True)
    parent = models.ForeignKey('self', blank=True, null=True,
                               related_name='subtasks')
    previous = models.ForeignKey('self', related_name='next', blank=True,
                                 null=True)
    waiting_on = models.ForeignKey('self', blank=True, null=True)
    submitted = models.DateTimeField(auto_now_add=True)
    started = models.DateTimeField(null=True, blank=True)
    finished = models.DateTimeField(null=True, blank=True)
    result_ttl = models.PositiveIntegerField(default=1800, blank=True)
    result_expiry = models.DateTimeField(null=True, blank=True)
    at_risk = models.CharField(max_length=1, choices=AT_RISK_CHOICES,
                               default=AT_RISK_NONE)

    objects = TaskManager()

    class Meta:
        ordering = ('-submitted',)

    def __str__(self):
        return '{} - {}'.format(self.id, self.func_name)

    def retry(self):
        self.status = self.STATUS_PENDING
        self.started = None
        self.finished = None
        self.details = {}
        self.at_risk = self.AT_RISK_NONE
        self.submit()

    def submit(self, *pre_args):<commit_msg>Need to store the values during a retry before submitting.<commit_after>import re
import time
import json
from datetime import datetime
from datetime import timedelta
import uuid
from traceback import format_tb
import logging

from django.db import models, transaction
from django.contrib.postgres.fields import JSONField
from django.utils import timezone
from django.core.exceptions import ValidationError
from django.core.cache import cache
from channels import Channel
from asgi_redis import RedisChannelLayer
from croniter import croniter

from .task import from_signature, to_signature, to_func_name, TaskFunc
from .managers import TaskManager
from .utils import import_attribute


logger = logging.getLogger('cq')


class Task(models.Model):
    """A persistent representation of a background task.
    """
    STATUS_PENDING = 'P'
    STATUS_RETRY = 'Y'
    STATUS_QUEUED = 'Q'
    STATUS_RUNNING = 'R'
    STATUS_FAILURE = 'F'
    STATUS_SUCCESS = 'S'
    STATUS_WAITING = 'W'
    STATUS_INCOMPLETE = 'I'
    STATUS_LOST = 'L'
    STATUS_REVOKED = 'E'
    STATUS_CHOICES = (
        (STATUS_PENDING, 'Pending'),
        (STATUS_RETRY, 'Retry'),
        (STATUS_QUEUED, 'Queued'),
        (STATUS_RUNNING, 'Running'),
        (STATUS_FAILURE, 'Failure'),
        (STATUS_SUCCESS, 'Success'),
        (STATUS_WAITING, 'Waiting'),
        (STATUS_INCOMPLETE, 'Incomplete'),
        (STATUS_LOST, 'Lost'),
        (STATUS_REVOKED, 'Revoked')
    )
    STATUS_DONE = {STATUS_FAILURE, STATUS_SUCCESS, STATUS_INCOMPLETE,
                   STATUS_LOST, STATUS_REVOKED}
    STATUS_ERROR = {STATUS_FAILURE, STATUS_LOST, STATUS_INCOMPLETE,
                    STATUS_REVOKED}
    STATUS_ACTIVE = {STATUS_PENDING, STATUS_QUEUED, STATUS_RUNNING,
                     STATUS_WAITING}

    AT_RISK_NONE = 'N'
    AT_RISK_QUEUED = 'Q'
    AT_RISK_RUNNING = 'R'
    AT_RISK_CHOICES = (
        (AT_RISK_NONE, 'None'),
        (AT_RISK_QUEUED, 'Queued'),
        (AT_RISK_RUNNING, 'Running'),
    )

    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    status = models.CharField(max_length=1, choices=STATUS_CHOICES,
                              default=STATUS_PENDING, db_index=True)
    signature = JSONField(default={}, blank=True)
    details = JSONField(default={}, blank=True)
    parent = models.ForeignKey('self', blank=True, null=True,
                               related_name='subtasks')
    previous = models.ForeignKey('self', related_name='next', blank=True,
                                 null=True)
    waiting_on = models.ForeignKey('self', blank=True, null=True)
    submitted = models.DateTimeField(auto_now_add=True)
    started = models.DateTimeField(null=True, blank=True)
    finished = models.DateTimeField(null=True, blank=True)
    result_ttl = models.PositiveIntegerField(default=1800, blank=True)
    result_expiry = models.DateTimeField(null=True, blank=True)
    at_risk = models.CharField(max_length=1, choices=AT_RISK_CHOICES,
                               default=AT_RISK_NONE)

    objects = TaskManager()

    class Meta:
        ordering = ('-submitted',)

    def __str__(self):
        return '{} - {}'.format(self.id, self.func_name)

    def retry(self):
        self.status = self.STATUS_PENDING
        self.started = None
        self.finished = None
        self.details = {}
        self.at_risk = self.AT_RISK_NONE
        self.save(update_fields=('status', 'started', 'finished',
                                 'details', 'at_risk'))
        self.submit()

    def submit(self, *pre_args):
{'fraction_non_alphanumeric': 0.10236898779612348, 'fraction_numerical': 0.0020100502512562816, 'mean_word_length': 3.0265895953757225, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4063197', 'n_tokens_mistral': 2241, 'n_tokens_neox': 2198, 'n_words': 461}