<commit_before>from datetime import timedelta
import re
from urllib import urlencode
from urlparse import parse_qs, urlparse, urlunparse

from django.conf import settings
from django.contrib.auth import (
    SESSION_KEY as CONTRIB_AUTH_SESSION_KEY,
    BACKEND_SESSION_KEY as CONTRIB_AUTH_BACKEND_SESSION_KEY,
)
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.db import models
from django.utils.importlib import import_module

from fulmine.settings import *
from fulmine.timeutils import utcnow
from fulmine.tokens import parse_bearer, random_b64, Token


def new_auth_code():
    return random_b64(AUTH_CODE_BYTES)

def new_access_token():
    return Token(ACCESS_TOKEN_BYTES)

def new_b64_access_token():
    return unicode(new_access_token())

def auth_code_expires():
    now = utcnow()
    delta = timedelta(seconds=AUTH_CODE_EXPIRE_SECONDS)
    return now + delta

class SeparatedValuesField(models.TextField):
    __metaclass__ = models.SubfieldBase

    def __init__(self, *args, **kwargs):
        self.separator = kwargs.pop('separator', ' ')
        super(SeparatedValuesField, self).__init__(*args, **kwargs)

    def to_python(self, value):
        if not value:
            return
        if isinstance(value, list):
            return value
        return value.split(self.separator)

    def get_db_prep_value(self, value, connection, prepared=False):
        if not value:
            return
        return self.separator.join([unicode(s) for s in value])

    def value_to_string(self, obj):
        value = self._get_val_from_obj(obj)
        return self.get_db_prep_value(value)

<commit_msg>fixed error with empty scope<commit_after>from datetime import timedelta
import re
from urllib import urlencode
from urlparse import parse_qs, urlparse, urlunparse

from django.conf import settings
from django.contrib.auth import (
    SESSION_KEY as CONTRIB_AUTH_SESSION_KEY,
    BACKEND_SESSION_KEY as CONTRIB_AUTH_BACKEND_SESSION_KEY,
)
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.db import models
from django.utils.importlib import import_module

from fulmine.settings import *
from fulmine.timeutils import utcnow
from fulmine.tokens import parse_bearer, random_b64, Token


def new_auth_code():
    return random_b64(AUTH_CODE_BYTES)

def new_access_token():
    return Token(ACCESS_TOKEN_BYTES)

def new_b64_access_token():
    return unicode(new_access_token())

def auth_code_expires():
    now = utcnow()
    delta = timedelta(seconds=AUTH_CODE_EXPIRE_SECONDS)
    return now + delta

class SeparatedValuesField(models.TextField):
    __metaclass__ = models.SubfieldBase

    def __init__(self, *args, **kwargs):
        self.separator = kwargs.pop('separator', ' ')
        super(SeparatedValuesField, self).__init__(*args, **kwargs)

    def to_python(self, value):
        if not value:
            return []
        if isinstance(value, list):
            return value
        return value.split(self.separator)

    def get_db_prep_value(self, value, connection, prepared=False):
        if not value:
            return
        return self.separator.join([unicode(s) for s in value])

    def value_to_string(self, obj):
        value = self._get_val_from_obj(obj)
        return self.get_db_prep_value(value)


{'fraction_non_alphanumeric': 0.07674003569303986, 'fraction_numerical': 0.003569303985722784, 'mean_word_length': 3.945588235294118, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3779705', 'n_tokens_mistral': 1046, 'n_tokens_neox': 1004, 'n_words': 260}