<commit_before>import copy
import datetime
import inspect
import warnings

from django.core import validators
from django.core.exceptions import ValidationError
from django.conf import settings
from django.db import DEFAULT_DB_ALIAS
from django.db.models.related import RelatedObject
from django.utils.encoding import is_protected_type, smart_unicode
from django.utils.translation import ugettext_lazy as _
from rest_framework.compat import parse_date, parse_datetime
from rest_framework.compat import timezone


def is_simple_callable(obj):
    """
    True if the object is a callable that takes no arguments.
    """
    return (
        (inspect.isfunction(obj) and not inspect.getargspec(obj)[0]) or
        (inspect.ismethod(obj) and len(inspect.getargspec(obj)[0]) <= 1)
    )


class Field(object):
    creation_counter = 0
    default_validators = []
    default_error_messages = {
        'required': _('This field is required.'),
        'invalid': _('Invalid value.'),
    }
    empty = ''

    def __init__(self, source=None, readonly=False, required=None,
                 validators=[], error_messages=None):
        self.parent = None

        self.creation_counter = Field.creation_counter
        Field.creation_counter += 1

        self.source = source
        self.readonly = readonly
        if required is None:
            self.required = not(readonly)
        else:
            assert not readonly, "Cannot set required=True and readonly=True"
            self.required = required

        messages = {}
        for c in reversed(self.__class__.__mro__):
            messages.update(getattr(c, 'default_error_messages', {}))
        messages.update(error_messages or {})
        self.error_messages = messages

        self.validators = self.default_validators + validators

    def initialize(self, parent, model_field=None):
        """
        Called to set up a field prior to field_to_native or field_from_native.

        parent - The parent serializer.
        model_field - The model field this field corrosponds to, if one exists.
        """
        self.parent = parent
        self.root = parent.root or parent
        self.context = self.root.context
        if model_field:
            self.model_field = model_field

    def validate(self, value):
        if value in validators.EMPTY_VALUES and self.required:
            raise ValidationError(self.error_messages['required'])

    def run_validators(self, value):
        if value in validators.EMPTY_VALUES:
            return
        errors = []
        for v in self.validators:
            try:
                v(value)
            except ValidationError as e:
                if hasattr(e, 'code') and e.code in self.error_messages:
                    message = self.error_messages[e.code]
                    if e.params:
                        message = message % e.params
                    errors.append(message)
                else:
                    errors.extend(e.messages)
        if errors:
            raise ValidationError(errors)

    def field_from_native(self, data, field_name, into):
        """
        Given a dictionary and a field name, updates the dictionary `into`,
        with the field and it's deserialized value.
        """
        if self.readonly:
            return

        try:
            native = data[field_name]
        except KeyError:
            return  # TODO Consider validation behaviour, 'required' opt etc...

        value = self.from_native(native)
        if self.source == '*':
            if value:
                into.update(value)
        else:
            self.validate(value)
            self.run_validators(value)
            into[self.source or field_name] = value

    def from_native(self, value):
        """
        Reverts a simple representation back to the field's value.
        """
        if hasattr(self, 'model_field'):
            try:
                return self.model_field.rel.to._meta.get_field(self.model_field.rel.field_name).to_python(value)
            except:
                return self.model_field.to_python(value)
        return value

    def field_to_native(self, obj, field_name):
        """
        Given and object and a field name, returns the value that should be
        serialized for that field.
        """
        if obj is None:
            return self.empty

        if self.source == '*':
            return self.to_native(obj)

        self.obj = obj  # Need to hang onto this in the case of model fields
        if hasattr(self, 'model_field'):
            return self.to_native(self.model_field._get_val_from_obj(obj))

        if self.source:
            value = obj
            for component in self.source.split('.'):
                value = getattr(value, component)
        else:
            value = getattr(obj, field_name)
        return self.to_native(value)

    def to_native(self, value):
        """
        Converts the field's value into it's simple representation.
        """
        if is_simple_callable(value):
            value = value()

        if is_protected_type(value):
            return value
        elif hasattr(self, 'model_field'):
            return self.model_field.value_to_string(self.obj)
        elif hasattr(value, '__iter__') and not isinstance(value, (dict, basestring)):
            return [self.to_native(item) for item in value]
        return smart_unicode(value)

    def attributes(self):
        """
        Returns a dictionary of attributes to be used when serializing to xml.
        """
        try:
            return {
                "type": self.model_field.get_internal_type()
            }
        except AttributeError:
            return {}


class RelatedField(Field):
    """
    A base class for model related fields or related managers.

    Subclass this and override `convert` to define custom behaviour when
    serializing related objects.
    """
    def field_to_native(self, obj, field_name):
        obj = getattr(obj, self.source or field_name)
        if obj.__class__.__name__ in ('RelatedManager', 'ManyRelatedManager'):
            return [self.to_native(item) for item in obj.all()]
        return self.to_native(obj)

    def attributes(self):
        try:
            return {
                "rel": self.model_field.rel.__class__.__name__,
                "to": smart_unicode(self.model_field.rel.to._meta)
            }
        except AttributeError:
            return {}


class PrimaryKeyRelatedField(RelatedField):
    """
    Serializes a model related field or related manager to a pk value.
    """
    # Note the we use ModelRelatedField's implementation, as we want to get the
    # raw database value directly, since that won't involve another
    # database lookup.
    #
    # An alternative implementation would simply be this...
    #
    # class PrimaryKeyRelatedField(RelatedField):
    #     def to_native(self, obj):
    #         return obj.pk

    def to_native(self, pk):
        """
        Simply returns the object's pk.  You can subclass this method to
        provide different serialization behavior of the pk.
        (For example returning a URL based on the model's pk.)
        """
        return pk

    def field_to_native(self, obj, field_name):
        try:
            pk = obj.serializable_value(self.source or field_name)
        except AttributeError:
            # RelatedObject (reverse relationship)
            obj = getattr(obj, self.source or field_name)
            return self.to_native(obj.pk)
        # Forward relationship
        return self.to_native(pk)

    def field_from_native(self, data, field_name, into):
        value = data.get(field_name)
        into[field_name + '_id'] = self.from_native(value)


class ManyPrimaryKeyRelatedField(PrimaryKeyRelatedField):
    def field_to_native(self, obj, field_name):
        try:
            queryset = obj.serializable_value(self.source or field_name)
        except AttributeError:
            # RelatedManager (reverse relationship)
            queryset = getattr(obj, self.source or field_name)
            return [self.to_native(item.pk) for item in queryset.all()]
        # Forward relationship
        return [self.to_native(item.pk) for item in queryset.all()]

    def field_from_native(self, data, field_name, into):
        try:
            value = data.getlist(field_name)
        except:
            value = data.get(field_name)
        into[field_name] = [self.from_native(item) for item in value]


class NaturalKeyRelatedField(RelatedField):
    """
    Serializes a model related field or related manager to a natural key value.
    """
    is_natural_key = True  # XML renderer handles these differently

    def to_native(self, obj):
        if hasattr(obj, 'natural_key'):
            return obj.natural_key()<commit_msg>Fix for empty form fields<commit_after>import copy
import datetime
import inspect
import warnings

from django.core import validators
from django.core.exceptions import ValidationError
from django.conf import settings
from django.db import DEFAULT_DB_ALIAS
from django.db.models.related import RelatedObject
from django.utils.encoding import is_protected_type, smart_unicode
from django.utils.translation import ugettext_lazy as _
from rest_framework.compat import parse_date, parse_datetime
from rest_framework.compat import timezone


def is_simple_callable(obj):
    """
    True if the object is a callable that takes no arguments.
    """
    return (
        (inspect.isfunction(obj) and not inspect.getargspec(obj)[0]) or
        (inspect.ismethod(obj) and len(inspect.getargspec(obj)[0]) <= 1)
    )


class Field(object):
    creation_counter = 0
    default_validators = []
    default_error_messages = {
        'required': _('This field is required.'),
        'invalid': _('Invalid value.'),
    }
    empty = ''

    def __init__(self, source=None, readonly=False, required=None,
                 validators=[], error_messages=None):
        self.parent = None

        self.creation_counter = Field.creation_counter
        Field.creation_counter += 1

        self.source = source
        self.readonly = readonly
        if required is None:
            self.required = not(readonly)
        else:
            assert not readonly, "Cannot set required=True and readonly=True"
            self.required = required

        messages = {}
        for c in reversed(self.__class__.__mro__):
            messages.update(getattr(c, 'default_error_messages', {}))
        messages.update(error_messages or {})
        self.error_messages = messages

        self.validators = self.default_validators + validators

    def initialize(self, parent, model_field=None):
        """
        Called to set up a field prior to field_to_native or field_from_native.

        parent - The parent serializer.
        model_field - The model field this field corrosponds to, if one exists.
        """
        self.parent = parent
        self.root = parent.root or parent
        self.context = self.root.context
        if model_field:
            self.model_field = model_field

    def validate(self, value):
        if value in validators.EMPTY_VALUES and self.required:
            raise ValidationError(self.error_messages['required'])

    def run_validators(self, value):
        if value in validators.EMPTY_VALUES:
            return
        errors = []
        for v in self.validators:
            try:
                v(value)
            except ValidationError as e:
                if hasattr(e, 'code') and e.code in self.error_messages:
                    message = self.error_messages[e.code]
                    if e.params:
                        message = message % e.params
                    errors.append(message)
                else:
                    errors.extend(e.messages)
        if errors:
            raise ValidationError(errors)

    def field_from_native(self, data, field_name, into):
        """
        Given a dictionary and a field name, updates the dictionary `into`,
        with the field and it's deserialized value.
        """
        if self.readonly:
            return

        try:
            native = data[field_name]
        except KeyError:
            return  # TODO Consider validation behaviour, 'required' opt etc...

        value = self.from_native(native)
        if self.source == '*':
            if value:
                into.update(value)
        else:
            self.validate(value)
            self.run_validators(value)
            into[self.source or field_name] = value

    def from_native(self, value):
        """
        Reverts a simple representation back to the field's value.
        """
        if hasattr(self, 'model_field'):
            try:
                return self.model_field.rel.to._meta.get_field(self.model_field.rel.field_name).to_python(value)
            except:
                return self.model_field.to_python(value)
        return value

    def field_to_native(self, obj, field_name):
        """
        Given and object and a field name, returns the value that should be
        serialized for that field.
        """
        if obj is None:
            return self.empty

        if self.source == '*':
            return self.to_native(obj)

        self.obj = obj  # Need to hang onto this in the case of model fields
        if hasattr(self, 'model_field'):
            return self.to_native(self.model_field._get_val_from_obj(obj))

        if self.source:
            value = obj
            for component in self.source.split('.'):
                value = getattr(value, component)
        else:
            value = getattr(obj, field_name)
        return self.to_native(value)

    def to_native(self, value):
        """
        Converts the field's value into it's simple representation.
        """
        if is_simple_callable(value):
            value = value()

        if is_protected_type(value):
            return value
        elif hasattr(self, 'model_field'):
            return self.model_field.value_to_string(self.obj)
        elif hasattr(value, '__iter__') and not isinstance(value, (dict, basestring)):
            return [self.to_native(item) for item in value]
        return smart_unicode(value)

    def attributes(self):
        """
        Returns a dictionary of attributes to be used when serializing to xml.
        """
        try:
            return {
                "type": self.model_field.get_internal_type()
            }
        except AttributeError:
            return {}


class RelatedField(Field):
    """
    A base class for model related fields or related managers.

    Subclass this and override `convert` to define custom behaviour when
    serializing related objects.
    """
    def field_to_native(self, obj, field_name):
        obj = getattr(obj, self.source or field_name)
        if obj.__class__.__name__ in ('RelatedManager', 'ManyRelatedManager'):
            return [self.to_native(item) for item in obj.all()]
        return self.to_native(obj)

    def attributes(self):
        try:
            return {
                "rel": self.model_field.rel.__class__.__name__,
                "to": smart_unicode(self.model_field.rel.to._meta)
            }
        except AttributeError:
            return {}


class PrimaryKeyRelatedField(RelatedField):
    """
    Serializes a model related field or related manager to a pk value.
    """
    # Note the we use ModelRelatedField's implementation, as we want to get the
    # raw database value directly, since that won't involve another
    # database lookup.
    #
    # An alternative implementation would simply be this...
    #
    # class PrimaryKeyRelatedField(RelatedField):
    #     def to_native(self, obj):
    #         return obj.pk

    def to_native(self, pk):
        """
        Simply returns the object's pk.  You can subclass this method to
        provide different serialization behavior of the pk.
        (For example returning a URL based on the model's pk.)
        """
        return pk

    def field_to_native(self, obj, field_name):
        try:
            pk = obj.serializable_value(self.source or field_name)
        except AttributeError:
            # RelatedObject (reverse relationship)
            obj = getattr(obj, self.source or field_name)
            return self.to_native(obj.pk)
        # Forward relationship
        return self.to_native(pk)

    def field_from_native(self, data, field_name, into):
        value = data.get(field_name)
        into[field_name + '_id'] = self.from_native(value)


class ManyPrimaryKeyRelatedField(PrimaryKeyRelatedField):
    def field_to_native(self, obj, field_name):
        try:
            queryset = obj.serializable_value(self.source or field_name)
        except AttributeError:
            # RelatedManager (reverse relationship)
            queryset = getattr(obj, self.source or field_name)
            return [self.to_native(item.pk) for item in queryset.all()]
        # Forward relationship
        return [self.to_native(item.pk) for item in queryset.all()]

    def field_from_native(self, data, field_name, into):
        try:
            value = data.getlist(field_name)
        except:
            value = data.get(field_name)
        into[field_name] = [self.from_native(item) for item in value if item]


class NaturalKeyRelatedField(RelatedField):
    """
    Serializes a model related field or related manager to a natural key value.
    """
    is_natural_key = True  # XML renderer handles these differently

    def to_native(self, obj):
        if hasattr(obj, 'natural_key'):
            return obj.natural_key()
{'fraction_non_alphanumeric': 0.0834747200542925, 'fraction_numerical': 0.0005655468838366701, 'mean_word_length': 2.1650259531054235, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2066209', 'n_tokens_mistral': 4835, 'n_tokens_neox': 4670, 'n_words': 1383}