<commit_before>from django.db import models
from django.db.models import Q
from django.conf import settings
from django.utils.encoding import python_2_unicode_compatible
import django
import inspect


from django.utils import timezone
from datetime import datetime, timedelta

from hashlib import sha1
import traceback
import logging
from compat import StringIO
from background_task.models_completed import CompletedTask 
import json

 

# inspired by http://github.com/tobi/delayed_job
# 

# Django 1.6 renamed Manager's get_query_set to get_queryset, and the old
# function will be removed entirely in 1.8. We work back to 1.4, so use a
# metaclass to not worry about it.
# from https://github.com/mysociety/mapit/blob/master/mapit/djangopatch.py#L14-L42

try:
    from django.utils import six
except ImportError:  # Django < 1.4.2
    import six 
    
    
if django.get_version() < '1.6':
    class GetQuerySetMetaclass(type):
        def __new__(cls, name, bases, attrs):
            new_class = super(GetQuerySetMetaclass, cls).__new__(cls, name, bases, attrs)

            old_method_name = 'get_query_set'
            new_method_name = 'get_queryset'
            for base in inspect.getmro(new_class):
                old_method = base.__dict__.get(old_method_name)
                new_method = base.__dict__.get(new_method_name)

                if not new_method and old_method:
                    setattr(base, new_method_name, old_method)
                if not old_method and new_method:
                    setattr(base, old_method_name, new_method)

            return new_class
elif django.get_version() < '1.8':
    # Nothing to do, make an empty metaclass
    from django.db.models.manager import RenameManagerMethods

    class GetQuerySetMetaclass(RenameManagerMethods):
        pass
else:
    class GetQuerySetMetaclass(type):
        pass
        
        
class TaskManager(six.with_metaclass(GetQuerySetMetaclass, models.Manager)):
    
    def find_available(self):
        now = timezone.now()
        qs = self.unlocked(now)
        ready = qs.filter(run_at__lte=now, failed_at=None)
        return ready.order_by('-priority', 'run_at')

    def unlocked(self, now):
        max_run_time = getattr(settings, 'MAX_RUN_TIME', 3600)
        qs = self.get_queryset()
        expires_at = now - timedelta(seconds=max_run_time)
        unlocked = Q(locked_by=None) | Q(locked_at__lt=expires_at)
        return qs.filter(unlocked)

    def new_task(self, task_name, args=None, kwargs=None,
                 run_at=None, priority=0):
        args = args or ()
        kwargs = kwargs or {}
        if run_at is None:
            run_at = timezone.now()

        task_params = json.dumps((args, kwargs))
        task_hash = sha1((task_name + task_params).encode()).hexdigest()

        return Task(task_name=task_name,
                    task_params=task_params,
                    task_hash=task_hash,
                    priority=priority,
                    run_at=run_at)

    def get_task(self, task_name, args=None, kwargs=None):<commit_msg>Fixed “TypeError: Unicode-objects must be encoded before hashing”<commit_after>from django.db import models
from django.db.models import Q
from django.conf import settings
from django.utils.encoding import python_2_unicode_compatible
import django
import inspect


from django.utils import timezone
from datetime import datetime, timedelta

from hashlib import sha1
import traceback
import logging
from compat import StringIO
from background_task.models_completed import CompletedTask 
import json

 

# inspired by http://github.com/tobi/delayed_job
# 

# Django 1.6 renamed Manager's get_query_set to get_queryset, and the old
# function will be removed entirely in 1.8. We work back to 1.4, so use a
# metaclass to not worry about it.
# from https://github.com/mysociety/mapit/blob/master/mapit/djangopatch.py#L14-L42

try:
    from django.utils import six
except ImportError:  # Django < 1.4.2
    import six 
    
    
if django.get_version() < '1.6':
    class GetQuerySetMetaclass(type):
        def __new__(cls, name, bases, attrs):
            new_class = super(GetQuerySetMetaclass, cls).__new__(cls, name, bases, attrs)

            old_method_name = 'get_query_set'
            new_method_name = 'get_queryset'
            for base in inspect.getmro(new_class):
                old_method = base.__dict__.get(old_method_name)
                new_method = base.__dict__.get(new_method_name)

                if not new_method and old_method:
                    setattr(base, new_method_name, old_method)
                if not old_method and new_method:
                    setattr(base, old_method_name, new_method)

            return new_class
elif django.get_version() < '1.8':
    # Nothing to do, make an empty metaclass
    from django.db.models.manager import RenameManagerMethods

    class GetQuerySetMetaclass(RenameManagerMethods):
        pass
else:
    class GetQuerySetMetaclass(type):
        pass
        
        
class TaskManager(six.with_metaclass(GetQuerySetMetaclass, models.Manager)):
    
    def find_available(self):
        now = timezone.now()
        qs = self.unlocked(now)
        ready = qs.filter(run_at__lte=now, failed_at=None)
        return ready.order_by('-priority', 'run_at')

    def unlocked(self, now):
        max_run_time = getattr(settings, 'MAX_RUN_TIME', 3600)
        qs = self.get_queryset()
        expires_at = now - timedelta(seconds=max_run_time)
        unlocked = Q(locked_by=None) | Q(locked_at__lt=expires_at)
        return qs.filter(unlocked)

    def new_task(self, task_name, args=None, kwargs=None,
                 run_at=None, priority=0):
        args = args or ()
        kwargs = kwargs or {}
        if run_at is None:
            run_at = timezone.now()

        task_params = json.dumps((args, kwargs))
        task_hash = sha1((task_name.encode('utf-8') + task_params.encode('utf-8')).encode()).hexdigest()

        return Task(task_name=task_name,
                    task_params=task_params,
                    task_hash=task_hash,
                    priority=priority,
                    run_at=run_at)

    def get_task(self, task_name, args=None, kwargs=None):
{'fraction_non_alphanumeric': 0.08486017357762778, 'fraction_numerical': 0.008357441337190614, 'mean_word_length': 2.6931750741839764, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3593462', 'n_tokens_mistral': 1955, 'n_tokens_neox': 1876, 'n_words': 464}