<commit_before>from __future__ import unicode_literals

from ..exceptions import ValidationError
from ..utils import digattr

from decimal import Decimal
from datetime import datetime

try:
    from django.utils.encoding import force_text
except ImportError:
    # For Django1.4
    from django.utils.encoding import force_unicode as force_text


class NoDefault(object):
    '''Indicates no default value was provided.'''


class Field(object):
    type_class = None

    def __init__(self, attribute=None, default=NoDefault, readonly=False,
                 null=True, virtual=False, **kwargs):
        self.attribute = attribute
        self.default = default
        self.readonly = readonly
        self.null = null
        self.virtual = virtual
        self.kwargs = kwargs

    def _get_attrname(self, name):
        return self.attribute if self.attribute else name

    def reduce(self, value, **kwargs):
        return value

    def restore(self, value, **kwargs):
        if self.type_class is not None:
            return self.type_class(value)
        return value

    def deflate(self, name, obj, data, **kwargs):
        src = self._get_attrname(name)
        value = digattr(obj, src, self.default)
        if value is NoDefault:
            if self.virtual:
                return
            raise AttributeError('No attribute "%s" found on %r' % (src, obj,))
        if value is not None:
            value = self.reduce(value, **kwargs)
        data[name] = value

    def inflate(self, name, data, obj, **kwargs):
        if self.readonly:
            return
        dest = self._get_attrname(name)
        try:
            value = data[name]
        except KeyError:
            if self.default is not NoDefault and 'instance' not in kwargs:
                obj[dest] = self.default
            return

        if value is not None:
            try:
                value = self.restore(value, **kwargs)
            except ValueError:
                raise ValidationError(
                    "Field '%s' (%r) received invalid value: %r" % (
                        name, self.__class__.__name__, value
                        )
                )
        elif not self.null:
            raise ValidationError("Field '%s' must not be None." % name)

        obj[dest] = value


class BooleanField(Field):
    type_class = bool


class IntegerField(Field):
    type_class = int


class DecimalField(Field):
    type_class = Decimal

    def reduce(self, value, **kwargs):
        return float(value)


class DateTimeField(Field):

    def reduce(self, value, **kwargs):
        return value.replace(microsecond=0).isoformat(str(' '))

    def restore(self, value, **kwargs):
        return datetime.strptime(value, '%Y-%m-%d %H:%M:%S')


class DateField(Field):

    def reduce(self, value, **kwargs):
        return value.isoformat()

    def restore(self, value, **kwargs):
        return datetime.strptime(value, '%Y-%m-%d').date()


class TimeField(Field):

    def reduce(self, value, **kwargs):
        return value.isoformat()

    def restore(self, value, **kwargs):
        return datetime.strptime(value, '%H:%M:%S').time()


class SerialiserField(Field):

    def __init__(self, *args, **kwargs):
        super(SerialiserField, self).__init__(*args, **kwargs)
        self.serialiser = kwargs['serialiser']

    def reduce(self, value, **kwargs):
        return self.serialiser.object_deflate(value, **kwargs)

    def restore(self, value, **kwargs):
        return self.serialiser.object_inflate(value, **kwargs)


class ManySerialiserField(Field):

    def __init__(self, *args, **kwargs):
        super(ManySerialiserField, self).__init__(*args, **kwargs)
        self.serialiser = kwargs['serialiser']

    def reduce(self, value, **kwargs):
        return self.serialiser.list_deflate(iter(value), **kwargs)

    def restore(self, value, **kwargs):
        return self.serialiser.list_inflate(value, **kwargs)<commit_msg>Don't call iter on value to many serialiser field -- it may destroy pre-cached queryset data<commit_after>from __future__ import unicode_literals

from ..exceptions import ValidationError
from ..utils import digattr

from decimal import Decimal
from datetime import datetime

try:
    from django.utils.encoding import force_text
except ImportError:
    # For Django1.4
    from django.utils.encoding import force_unicode as force_text


class NoDefault(object):
    '''Indicates no default value was provided.'''


class Field(object):
    type_class = None

    def __init__(self, attribute=None, default=NoDefault, readonly=False,
                 null=True, virtual=False, **kwargs):
        self.attribute = attribute
        self.default = default
        self.readonly = readonly
        self.null = null
        self.virtual = virtual
        self.kwargs = kwargs

    def _get_attrname(self, name):
        return self.attribute if self.attribute else name

    def reduce(self, value, **kwargs):
        return value

    def restore(self, value, **kwargs):
        if self.type_class is not None:
            return self.type_class(value)
        return value

    def deflate(self, name, obj, data, **kwargs):
        src = self._get_attrname(name)
        value = digattr(obj, src, self.default)
        if value is NoDefault:
            if self.virtual:
                return
            raise AttributeError('No attribute "%s" found on %r' % (src, obj,))
        if value is not None:
            value = self.reduce(value, **kwargs)
        data[name] = value

    def inflate(self, name, data, obj, **kwargs):
        if self.readonly:
            return
        dest = self._get_attrname(name)
        try:
            value = data[name]
        except KeyError:
            if self.default is not NoDefault and 'instance' not in kwargs:
                obj[dest] = self.default
            return

        if value is not None:
            try:
                value = self.restore(value, **kwargs)
            except ValueError:
                raise ValidationError(
                    "Field '%s' (%r) received invalid value: %r" % (
                        name, self.__class__.__name__, value
                        )
                )
        elif not self.null:
            raise ValidationError("Field '%s' must not be None." % name)

        obj[dest] = value


class BooleanField(Field):
    type_class = bool


class IntegerField(Field):
    type_class = int


class DecimalField(Field):
    type_class = Decimal

    def reduce(self, value, **kwargs):
        return float(value)


class DateTimeField(Field):

    def reduce(self, value, **kwargs):
        return value.replace(microsecond=0).isoformat(str(' '))

    def restore(self, value, **kwargs):
        return datetime.strptime(value, '%Y-%m-%d %H:%M:%S')


class DateField(Field):

    def reduce(self, value, **kwargs):
        return value.isoformat()

    def restore(self, value, **kwargs):
        return datetime.strptime(value, '%Y-%m-%d').date()


class TimeField(Field):

    def reduce(self, value, **kwargs):
        return value.isoformat()

    def restore(self, value, **kwargs):
        return datetime.strptime(value, '%H:%M:%S').time()


class SerialiserField(Field):

    def __init__(self, *args, **kwargs):
        super(SerialiserField, self).__init__(*args, **kwargs)
        self.serialiser = kwargs['serialiser']

    def reduce(self, value, **kwargs):
        return self.serialiser.object_deflate(value, **kwargs)

    def restore(self, value, **kwargs):
        return self.serialiser.object_inflate(value, **kwargs)


class ManySerialiserField(Field):

    def __init__(self, *args, **kwargs):
        super(ManySerialiserField, self).__init__(*args, **kwargs)
        self.serialiser = kwargs['serialiser']

    def reduce(self, value, **kwargs):
        return self.serialiser.list_deflate(value, **kwargs)

    def restore(self, value, **kwargs):
        return self.serialiser.list_inflate(value, **kwargs)
{'fraction_non_alphanumeric': 0.11131957473420888, 'fraction_numerical': 0.00075046904315197, 'mean_word_length': 2.468980477223427, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1844672', 'n_tokens_mistral': 2288, 'n_tokens_neox': 2199, 'n_words': 637}