<commit_before>"""
Serializer fields perform validation on incoming data.

They are very similar to Django's form fields.
"""
from __future__ import unicode_literals

import copy
import datetime
import inspect
import re
import warnings
from decimal import Decimal, DecimalException
from django import forms
from django.core import validators
from django.core.exceptions import ValidationError
from django.conf import settings
from django.db.models.fields import BLANK_CHOICE_DASH
from django.http import QueryDict
from django.forms import widgets
from django.utils.encoding import is_protected_type
from django.utils.translation import ugettext_lazy as _
from django.utils.datastructures import SortedDict
from rest_framework import ISO_8601
from rest_framework.compat import (
    timezone, parse_date, parse_datetime, parse_time, BytesIO, six, smart_text,
    force_text, is_non_str_iterable
)
from rest_framework.settings import api_settings


def is_simple_callable(obj):
    """
    True if the object is a callable that takes no arguments.
    """
    function = inspect.isfunction(obj)
    method = inspect.ismethod(obj)

    if not (function or method):
        return False

    args, _, _, defaults = inspect.getargspec(obj)
    len_args = len(args) if function else len(args) - 1
    len_defaults = len(defaults) if defaults else 0
    return len_args <= len_defaults


def get_component(obj, attr_name):
    """
    Given an object, and an attribute name,
    return that attribute on the object.
    """
    if isinstance(obj, dict):
        val = obj.get(attr_name)
    else:
        val = getattr(obj, attr_name)

    if is_simple_callable(val):
        return val()
    return val


def readable_datetime_formats(formats):
    format = ', '.join(formats).replace(ISO_8601,
             'YYYY-MM-DDThh:mm[:ss[.uuuuuu]][+HHMM|-HHMM|Z]')
    return humanize_strptime(format)


def readable_date_formats(formats):
    format = ', '.join(formats).replace(ISO_8601, 'YYYY[-MM[-DD]]')
    return humanize_strptime(format)


def readable_time_formats(formats):
    format = ', '.join(formats).replace(ISO_8601, 'hh:mm[:ss[.uuuuuu]]')
    return humanize_strptime(format)


def humanize_strptime(format_string):
    # Note that we're missing some of the locale specific mappings that
    # don't really make sense.
    mapping = {
        "%Y": "YYYY",
        "%y": "YY",
        "%m": "MM",
        "%b": "[Jan-Dec]",
        "%B": "[January-December]",
        "%d": "DD",
        "%H": "hh",
        "%I": "hh",  # Requires '%p' to differentiate from '%H'.
        "%M": "mm",
        "%S": "ss",
        "%f": "uuuuuu",
        "%a": "[Mon-Sun]",
        "%A": "[Monday-Sunday]",
        "%p": "[AM|PM]",
        "%z": "[+HHMM|-HHMM]"
    }
    for key, val in mapping.items():
        format_string = format_string.replace(key, val)
    return format_string


def strip_multiple_choice_msg(help_text):
    """
    Remove the 'Hold down "control" ...' message that is Django enforces in
    select multiple fields on ModelForms.  (Required for 1.5 and earlier)

    See https://code.djangoproject.com/ticket/9321
    """
    multiple_choice_msg = _(' Hold down "Control", or "Command" on a Mac, to select more than one.')
    multiple_choice_msg = force_text(multiple_choice_msg)

    return help_text.replace(multiple_choice_msg, '')


class Field(object):
    read_only = True
    creation_counter = 0
    empty = ''
    type_name = None
    partial = False
    use_files = False
    form_field_class = forms.CharField
    type_label = 'field'
    widget = None

    def __init__(self, source=None, label=None, help_text=None):
        self.parent = None

        self.creation_counter = Field.creation_counter
        Field.creation_counter += 1

        self.source = source

        if label is not None:
            self.label = smart_text(label)
        else:
            self.label = None

        if help_text is not None:
            self.help_text = strip_multiple_choice_msg(smart_text(help_text))
        else:
            self.help_text = None

        self._errors = []
        self._value = None
        self._name = None

    @property
    def errors(self):
        return self._errors

    def widget_html(self):
        if not self.widget:
            return ''
        return self.widget.render(self._name, self._value)

    def label_tag(self):
        return '<label for="%s">%s:</label>' % (self._name, self.label)

    def initialize(self, parent, field_name):
        """
        Called to set up a field prior to field_to_native or field_from_native.

        parent - The parent serializer.
        model_field - The model field this field corresponds to, if one exists.
        """
        self.parent = parent
        self.root = parent.root or parent
        self.context = self.root.context
        self.partial = self.root.partial
        if self.partial:
            self.required = False

    def field_from_native(self, data, files, field_name, into):
        """
        Given a dictionary and a field name, updates the dictionary `into`,
        with the field and it's deserialized value.
        """
        return

    def field_to_native(self, obj, field_name):
        """
        Given and object and a field name, returns the value that should be
        serialized for that field.
        """
        if obj is None:
            return self.empty

        if self.source == '*':
            return self.to_native(obj)

        source = self.source or field_name
        value = obj

        for component in source.split('.'):
            value = get_component(value, component)
            if value is None:
                break

        return self.to_native(value)

    def to_native(self, value):
        """
        Converts the field's value into it's simple representation.
        """
        if is_simple_callable(value):
            value = value()

        if is_protected_type(value):
            return value
        elif (is_non_str_iterable(value) and
              not isinstance(value, (dict, six.string_types))):
            return [self.to_native(item) for item in value]
        elif isinstance(value, dict):
            # Make sure we preserve field ordering, if it exists
            ret = SortedDict()
            for key, val in value.items():
                ret[key] = self.to_native(val)
            return ret
        return force_text(value)

    def attributes(self):
        """
        Returns a dictionary of attributes to be used when serializing to xml.
        """
        if self.type_name:
            return {'type': self.type_name}
        return {}

    def metadata(self):
        metadata = SortedDict()
        metadata['type'] = self.type_label
        metadata['required'] = getattr(self, 'required', False)
        optional_attrs = ['read_only', 'label', 'help_text',
                          'min_length', 'max_length']
        for attr in optional_attrs:
            value = getattr(self, attr, None)
            if value is not None and value != '':
                metadata[attr] = force_text(value, strings_only=True)
        return metadata


class WritableField(Field):
    """
    Base for read/write fields.
    """
    default_validators = []
    default_error_messages = {
        'required': _('This field is required.'),
        'invalid': _('Invalid value.'),
    }
    widget = widgets.TextInput
    default = None

    def __init__(self, source=None, label=None, help_text=None,
                 read_only=False, required=None,
                 validators=[], error_messages=None, widget=None,
                 default=None, blank=None):

        # 'blank' is to be deprecated in favor of 'required'
        if blank is not None:
            warnings.warn('The `blank` keyword argument is deprecated. '
                          'Use the `required` keyword argument instead.',
                          DeprecationWarning, stacklevel=2)
            required = not(blank)

        super(WritableField, self).__init__(source=source, label=label, help_text=help_text)

        self.read_only = read_only
        if required is None:
            self.required = not(read_only)
        else:
            assert not (read_only and required), "Cannot set required=True and read_only=True"
            self.required = required

        messages = {}
        for c in reversed(self.__class__.__mro__):
            messages.update(getattr(c, 'default_error_messages', {}))
        messages.update(error_messages or {})
        self.error_messages = messages

        self.validators = self.default_validators + validators
        self.default = default if default is not None else self.default

        # Widgets are ony used for HTML forms.
        widget = widget or self.widget
        if isinstance(widget, type):
            widget = widget()
        self.widget = widget

    def __deepcopy__(self, memo):
        result = copy.copy(self)
        memo[id(self)] = result
        result.validators = self.validators[:]
        return result

    def validate(self, value):
        if value in validators.EMPTY_VALUES and self.required:
            raise ValidationError(self.error_messages['required'])

    def run_validators(self, value):
        if value in validators.EMPTY_VALUES:
            return
        errors = []
        for v in self.validators:
            try:
                v(value)
            except ValidationError as e:
                if hasattr(e, 'code') and e.code in self.error_messages:
                    message = self.error_messages[e.code]
                    if e.params:
                        message = message % e.params
                    errors.append(message)
                else:
                    errors.extend(e.messages)
        if errors:
            raise ValidationError(errors)

    def field_from_native(self, data, files, field_name, into):
        """
        Given a dictionary and a field name, updates the dictionary `into`,
        with the field and it's deserialized value.
        """
        if self.read_only:
            return

        try:
            data = data or {}
            if self.use_files:
                files = files or {}
                try:
                    native = files[field_name]
                except KeyError:
                    native = data[field_name]
            else:
                native = data[field_name]
        except KeyError:
            if self.default is not None and not self.partial:
                # Note: partial updates shouldn't set defaults
                if is_simple_callable(self.default):
                    native = self.default()
                else:
                    native = self.default
            else:
                if self.required:
                    raise ValidationError(self.error_messages['required'])
                return

        value = self.from_native(native)
        if self.source == '*':
            if value:
                into.update(value)
        else:
            self.validate(value)
            self.run_validators(value)
            into[self.source or field_name] = value

    def from_native(self, value):
        """
        Reverts a simple representation back to the field's value.
        """
        return value


class ModelField(WritableField):
    """
    A generic field that can be used against an arbitrary model field.
    """
    def __init__(self, *args, **kwargs):
        try:
            self.model_field = kwargs.pop('model_field')
        except KeyError:
            raise ValueError("ModelField requires 'model_field' kwarg")

        self.min_length = kwargs.pop('min_length',
                                     getattr(self.model_field, 'min_length', None))
        self.max_length = kwargs.pop('max_length',
                                     getattr(self.model_field, 'max_length', None))
        self.min_value = kwargs.pop('min_value',
                                    getattr(self.model_field, 'min_value', None))
        self.max_value = kwargs.pop('max_value',
                                    getattr(self.model_field, 'max_value', None))

        super(ModelField, self).__init__(*args, **kwargs)

        if self.min_length is not None:
            self.validators.append(validators.MinLengthValidator(self.min_length))
        if self.max_length is not None:
            self.validators.append(validators.MaxLengthValidator(self.max_length))
        if self.min_value is not None:
            self.validators.append(validators.MinValueValidator(self.min_value))
        if self.max_value is not None:
            self.validators.append(validators.MaxValueValidator(self.max_value))

    def from_native(self, value):
        rel = getattr(self.model_field, "rel", None)
        if rel is not None:
            return rel.to._meta.get_field(rel.field_name).to_python(value)
        else:
            return self.model_field.to_python(value)

    def field_to_native(self, obj, field_name):
        value = self.model_field._get_val_from_obj(obj)
        if is_protected_type(value):
            return value
        return self.model_field.value_to_string(obj)

    def attributes(self):
        return {
            "type": self.model_field.get_internal_type()
        }


##### Typed Fields #####

class BooleanField(WritableField):
    type_name = 'BooleanField'
    type_label = 'boolean'
    form_field_class = forms.BooleanField
    widget = widgets.CheckboxInput
    default_error_messages = {
        'invalid': _("'%s' value must be either True or False."),
    }
    empty = False

    def field_from_native(self, data, files, field_name, into):
        # HTML checkboxes do not explicitly represent unchecked as `False`
        # we deal with that here...
        if isinstance(data, QueryDict):
            self.default = False

        return super(BooleanField, self).field_from_native(
            data, files, field_name, into
        )

    def from_native(self, value):
        if value in ('true', 't', 'True', '1'):
            return True
        if value in ('false', 'f', 'False', '0'):
            return False
        return bool(value)


class CharField(WritableField):
    type_name = 'CharField'
    type_label = 'string'
    form_field_class = forms.CharField

    def __init__(self, max_length=None, min_length=None, *args, **kwargs):
        self.max_length, self.min_length = max_length, min_length
        super(CharField, self).__init__(*args, **kwargs)
        if min_length is not None:
            self.validators.append(validators.MinLengthValidator(min_length))
        if max_length is not None:
            self.validators.append(validators.MaxLengthValidator(max_length))

    def from_native(self, value):
        if isinstance(value, six.string_types) or value is None:
            return value
        return smart_text(value)


class URLField(CharField):
    type_name = 'URLField'
    type_label = 'url'

    def __init__(self, **kwargs):
        kwargs['validators'] = [validators.URLValidator()]
        super(URLField, self).__init__(**kwargs)


class SlugField(CharField):
    type_name = 'SlugField'
    type_label = 'slug'
    form_field_class = forms.SlugField

    default_error_messages = {
        'invalid': _("Enter a valid 'slug' consisting of letters, numbers,"
                     " underscores or hyphens."),
    }
    default_validators = [validators.validate_slug]

    def __init__(self, *args, **kwargs):
        super(SlugField, self).__init__(*args, **kwargs)


class ChoiceField(WritableField):
    type_name = 'ChoiceField'
    type_label = 'multiple choice'
    form_field_class = forms.ChoiceField
    widget = widgets.Select
    default_error_messages = {
        'invalid_choice': _('Select a valid choice. %(value)s is not one of '
                            'the available choices.'),
    }

    def __init__(self, choices=(), *args, **kwargs):
        self.empty = kwargs.pop('empty', '')
        super(ChoiceField, self).__init__(*args, **kwargs)
        self.choices = choices
        if not self.required:
            self.choices = BLANK_CHOICE_DASH + self.choices

    def _get_choices(self):
        return self._choices

    def _set_choices(self, value):
        # Setting choices also sets the choices on the widget.
        # choices can be any iterable, but we call list() on it because
        # it will be consumed more than once.
        self._choices = self.widget.choices = list(value)

    choices = property(_get_choices, _set_choices)

    def metadata(self):
        data = super(ChoiceField, self).metadata()
        data['choices'] = self.choices
        return data

    def validate(self, value):
        """
        Validates that the input is in self.choices.
        """<commit_msg>Add more context to the ChoiceField metadata.<commit_after>"""
Serializer fields perform validation on incoming data.

They are very similar to Django's form fields.
"""
from __future__ import unicode_literals

import copy
import datetime
import inspect
import re
import warnings
from decimal import Decimal, DecimalException
from django import forms
from django.core import validators
from django.core.exceptions import ValidationError
from django.conf import settings
from django.db.models.fields import BLANK_CHOICE_DASH
from django.http import QueryDict
from django.forms import widgets
from django.utils.encoding import is_protected_type
from django.utils.translation import ugettext_lazy as _
from django.utils.datastructures import SortedDict
from rest_framework import ISO_8601
from rest_framework.compat import (
    timezone, parse_date, parse_datetime, parse_time, BytesIO, six, smart_text,
    force_text, is_non_str_iterable
)
from rest_framework.settings import api_settings


def is_simple_callable(obj):
    """
    True if the object is a callable that takes no arguments.
    """
    function = inspect.isfunction(obj)
    method = inspect.ismethod(obj)

    if not (function or method):
        return False

    args, _, _, defaults = inspect.getargspec(obj)
    len_args = len(args) if function else len(args) - 1
    len_defaults = len(defaults) if defaults else 0
    return len_args <= len_defaults


def get_component(obj, attr_name):
    """
    Given an object, and an attribute name,
    return that attribute on the object.
    """
    if isinstance(obj, dict):
        val = obj.get(attr_name)
    else:
        val = getattr(obj, attr_name)

    if is_simple_callable(val):
        return val()
    return val


def readable_datetime_formats(formats):
    format = ', '.join(formats).replace(ISO_8601,
             'YYYY-MM-DDThh:mm[:ss[.uuuuuu]][+HHMM|-HHMM|Z]')
    return humanize_strptime(format)


def readable_date_formats(formats):
    format = ', '.join(formats).replace(ISO_8601, 'YYYY[-MM[-DD]]')
    return humanize_strptime(format)


def readable_time_formats(formats):
    format = ', '.join(formats).replace(ISO_8601, 'hh:mm[:ss[.uuuuuu]]')
    return humanize_strptime(format)


def humanize_strptime(format_string):
    # Note that we're missing some of the locale specific mappings that
    # don't really make sense.
    mapping = {
        "%Y": "YYYY",
        "%y": "YY",
        "%m": "MM",
        "%b": "[Jan-Dec]",
        "%B": "[January-December]",
        "%d": "DD",
        "%H": "hh",
        "%I": "hh",  # Requires '%p' to differentiate from '%H'.
        "%M": "mm",
        "%S": "ss",
        "%f": "uuuuuu",
        "%a": "[Mon-Sun]",
        "%A": "[Monday-Sunday]",
        "%p": "[AM|PM]",
        "%z": "[+HHMM|-HHMM]"
    }
    for key, val in mapping.items():
        format_string = format_string.replace(key, val)
    return format_string


def strip_multiple_choice_msg(help_text):
    """
    Remove the 'Hold down "control" ...' message that is Django enforces in
    select multiple fields on ModelForms.  (Required for 1.5 and earlier)

    See https://code.djangoproject.com/ticket/9321
    """
    multiple_choice_msg = _(' Hold down "Control", or "Command" on a Mac, to select more than one.')
    multiple_choice_msg = force_text(multiple_choice_msg)

    return help_text.replace(multiple_choice_msg, '')


class Field(object):
    read_only = True
    creation_counter = 0
    empty = ''
    type_name = None
    partial = False
    use_files = False
    form_field_class = forms.CharField
    type_label = 'field'
    widget = None

    def __init__(self, source=None, label=None, help_text=None):
        self.parent = None

        self.creation_counter = Field.creation_counter
        Field.creation_counter += 1

        self.source = source

        if label is not None:
            self.label = smart_text(label)
        else:
            self.label = None

        if help_text is not None:
            self.help_text = strip_multiple_choice_msg(smart_text(help_text))
        else:
            self.help_text = None

        self._errors = []
        self._value = None
        self._name = None

    @property
    def errors(self):
        return self._errors

    def widget_html(self):
        if not self.widget:
            return ''
        return self.widget.render(self._name, self._value)

    def label_tag(self):
        return '<label for="%s">%s:</label>' % (self._name, self.label)

    def initialize(self, parent, field_name):
        """
        Called to set up a field prior to field_to_native or field_from_native.

        parent - The parent serializer.
        model_field - The model field this field corresponds to, if one exists.
        """
        self.parent = parent
        self.root = parent.root or parent
        self.context = self.root.context
        self.partial = self.root.partial
        if self.partial:
            self.required = False

    def field_from_native(self, data, files, field_name, into):
        """
        Given a dictionary and a field name, updates the dictionary `into`,
        with the field and it's deserialized value.
        """
        return

    def field_to_native(self, obj, field_name):
        """
        Given and object and a field name, returns the value that should be
        serialized for that field.
        """
        if obj is None:
            return self.empty

        if self.source == '*':
            return self.to_native(obj)

        source = self.source or field_name
        value = obj

        for component in source.split('.'):
            value = get_component(value, component)
            if value is None:
                break

        return self.to_native(value)

    def to_native(self, value):
        """
        Converts the field's value into it's simple representation.
        """
        if is_simple_callable(value):
            value = value()

        if is_protected_type(value):
            return value
        elif (is_non_str_iterable(value) and
              not isinstance(value, (dict, six.string_types))):
            return [self.to_native(item) for item in value]
        elif isinstance(value, dict):
            # Make sure we preserve field ordering, if it exists
            ret = SortedDict()
            for key, val in value.items():
                ret[key] = self.to_native(val)
            return ret
        return force_text(value)

    def attributes(self):
        """
        Returns a dictionary of attributes to be used when serializing to xml.
        """
        if self.type_name:
            return {'type': self.type_name}
        return {}

    def metadata(self):
        metadata = SortedDict()
        metadata['type'] = self.type_label
        metadata['required'] = getattr(self, 'required', False)
        optional_attrs = ['read_only', 'label', 'help_text',
                          'min_length', 'max_length']
        for attr in optional_attrs:
            value = getattr(self, attr, None)
            if value is not None and value != '':
                metadata[attr] = force_text(value, strings_only=True)
        return metadata


class WritableField(Field):
    """
    Base for read/write fields.
    """
    default_validators = []
    default_error_messages = {
        'required': _('This field is required.'),
        'invalid': _('Invalid value.'),
    }
    widget = widgets.TextInput
    default = None

    def __init__(self, source=None, label=None, help_text=None,
                 read_only=False, required=None,
                 validators=[], error_messages=None, widget=None,
                 default=None, blank=None):

        # 'blank' is to be deprecated in favor of 'required'
        if blank is not None:
            warnings.warn('The `blank` keyword argument is deprecated. '
                          'Use the `required` keyword argument instead.',
                          DeprecationWarning, stacklevel=2)
            required = not(blank)

        super(WritableField, self).__init__(source=source, label=label, help_text=help_text)

        self.read_only = read_only
        if required is None:
            self.required = not(read_only)
        else:
            assert not (read_only and required), "Cannot set required=True and read_only=True"
            self.required = required

        messages = {}
        for c in reversed(self.__class__.__mro__):
            messages.update(getattr(c, 'default_error_messages', {}))
        messages.update(error_messages or {})
        self.error_messages = messages

        self.validators = self.default_validators + validators
        self.default = default if default is not None else self.default

        # Widgets are ony used for HTML forms.
        widget = widget or self.widget
        if isinstance(widget, type):
            widget = widget()
        self.widget = widget

    def __deepcopy__(self, memo):
        result = copy.copy(self)
        memo[id(self)] = result
        result.validators = self.validators[:]
        return result

    def validate(self, value):
        if value in validators.EMPTY_VALUES and self.required:
            raise ValidationError(self.error_messages['required'])

    def run_validators(self, value):
        if value in validators.EMPTY_VALUES:
            return
        errors = []
        for v in self.validators:
            try:
                v(value)
            except ValidationError as e:
                if hasattr(e, 'code') and e.code in self.error_messages:
                    message = self.error_messages[e.code]
                    if e.params:
                        message = message % e.params
                    errors.append(message)
                else:
                    errors.extend(e.messages)
        if errors:
            raise ValidationError(errors)

    def field_from_native(self, data, files, field_name, into):
        """
        Given a dictionary and a field name, updates the dictionary `into`,
        with the field and it's deserialized value.
        """
        if self.read_only:
            return

        try:
            data = data or {}
            if self.use_files:
                files = files or {}
                try:
                    native = files[field_name]
                except KeyError:
                    native = data[field_name]
            else:
                native = data[field_name]
        except KeyError:
            if self.default is not None and not self.partial:
                # Note: partial updates shouldn't set defaults
                if is_simple_callable(self.default):
                    native = self.default()
                else:
                    native = self.default
            else:
                if self.required:
                    raise ValidationError(self.error_messages['required'])
                return

        value = self.from_native(native)
        if self.source == '*':
            if value:
                into.update(value)
        else:
            self.validate(value)
            self.run_validators(value)
            into[self.source or field_name] = value

    def from_native(self, value):
        """
        Reverts a simple representation back to the field's value.
        """
        return value


class ModelField(WritableField):
    """
    A generic field that can be used against an arbitrary model field.
    """
    def __init__(self, *args, **kwargs):
        try:
            self.model_field = kwargs.pop('model_field')
        except KeyError:
            raise ValueError("ModelField requires 'model_field' kwarg")

        self.min_length = kwargs.pop('min_length',
                                     getattr(self.model_field, 'min_length', None))
        self.max_length = kwargs.pop('max_length',
                                     getattr(self.model_field, 'max_length', None))
        self.min_value = kwargs.pop('min_value',
                                    getattr(self.model_field, 'min_value', None))
        self.max_value = kwargs.pop('max_value',
                                    getattr(self.model_field, 'max_value', None))

        super(ModelField, self).__init__(*args, **kwargs)

        if self.min_length is not None:
            self.validators.append(validators.MinLengthValidator(self.min_length))
        if self.max_length is not None:
            self.validators.append(validators.MaxLengthValidator(self.max_length))
        if self.min_value is not None:
            self.validators.append(validators.MinValueValidator(self.min_value))
        if self.max_value is not None:
            self.validators.append(validators.MaxValueValidator(self.max_value))

    def from_native(self, value):
        rel = getattr(self.model_field, "rel", None)
        if rel is not None:
            return rel.to._meta.get_field(rel.field_name).to_python(value)
        else:
            return self.model_field.to_python(value)

    def field_to_native(self, obj, field_name):
        value = self.model_field._get_val_from_obj(obj)
        if is_protected_type(value):
            return value
        return self.model_field.value_to_string(obj)

    def attributes(self):
        return {
            "type": self.model_field.get_internal_type()
        }


##### Typed Fields #####

class BooleanField(WritableField):
    type_name = 'BooleanField'
    type_label = 'boolean'
    form_field_class = forms.BooleanField
    widget = widgets.CheckboxInput
    default_error_messages = {
        'invalid': _("'%s' value must be either True or False."),
    }
    empty = False

    def field_from_native(self, data, files, field_name, into):
        # HTML checkboxes do not explicitly represent unchecked as `False`
        # we deal with that here...
        if isinstance(data, QueryDict):
            self.default = False

        return super(BooleanField, self).field_from_native(
            data, files, field_name, into
        )

    def from_native(self, value):
        if value in ('true', 't', 'True', '1'):
            return True
        if value in ('false', 'f', 'False', '0'):
            return False
        return bool(value)


class CharField(WritableField):
    type_name = 'CharField'
    type_label = 'string'
    form_field_class = forms.CharField

    def __init__(self, max_length=None, min_length=None, *args, **kwargs):
        self.max_length, self.min_length = max_length, min_length
        super(CharField, self).__init__(*args, **kwargs)
        if min_length is not None:
            self.validators.append(validators.MinLengthValidator(min_length))
        if max_length is not None:
            self.validators.append(validators.MaxLengthValidator(max_length))

    def from_native(self, value):
        if isinstance(value, six.string_types) or value is None:
            return value
        return smart_text(value)


class URLField(CharField):
    type_name = 'URLField'
    type_label = 'url'

    def __init__(self, **kwargs):
        kwargs['validators'] = [validators.URLValidator()]
        super(URLField, self).__init__(**kwargs)


class SlugField(CharField):
    type_name = 'SlugField'
    type_label = 'slug'
    form_field_class = forms.SlugField

    default_error_messages = {
        'invalid': _("Enter a valid 'slug' consisting of letters, numbers,"
                     " underscores or hyphens."),
    }
    default_validators = [validators.validate_slug]

    def __init__(self, *args, **kwargs):
        super(SlugField, self).__init__(*args, **kwargs)


class ChoiceField(WritableField):
    type_name = 'ChoiceField'
    type_label = 'multiple choice'
    form_field_class = forms.ChoiceField
    widget = widgets.Select
    default_error_messages = {
        'invalid_choice': _('Select a valid choice. %(value)s is not one of '
                            'the available choices.'),
    }

    def __init__(self, choices=(), *args, **kwargs):
        self.empty = kwargs.pop('empty', '')
        super(ChoiceField, self).__init__(*args, **kwargs)
        self.choices = choices
        if not self.required:
            self.choices = BLANK_CHOICE_DASH + self.choices

    def _get_choices(self):
        return self._choices

    def _set_choices(self, value):
        # Setting choices also sets the choices on the widget.
        # choices can be any iterable, but we call list() on it because
        # it will be consumed more than once.
        self._choices = self.widget.choices = list(value)

    choices = property(_get_choices, _set_choices)

    def metadata(self):
        data = super(ChoiceField, self).metadata()
        data['choices'] = [{'value': v, 'name': n} for v, n in self.choices]
        return data

    def validate(self, value):
        """
        Validates that the input is in self.choices.
        """
{'fraction_non_alphanumeric': 0.09851676060516168, 'fraction_numerical': 0.0017205576980124592, 'mean_word_length': 2.203858582018628, 'pattern_counts': {'":': 32, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5252793', 'n_tokens_mistral': 9625, 'n_tokens_neox': 9350, 'n_words': 2582}