<commit_before># -*- coding: utf-8 -*-
"""
"""
from __future__ import print_function, unicode_literals
from __future__ import absolute_import, division
import collections
import datetime
from StringIO import StringIO
import six
from django import forms
from django.core import urlresolvers
from django.core.exceptions import ImproperlyConfigured
from django.db.models.fields import FieldDoesNotExist
from django.db.models.fields.related import RelatedField
from django.db.models.related import RelatedObject
from .. import utils
from .helpers import attribute as field_helper
from . import attributes


def _has(name, attrs, bases):
    """
    Determines if a property has been expliclty specified by this or a
    base class.
    """
    if name in attrs:
        # Attribute is explicitly declared on the class object.
        return attrs[name] is not None

    for base in bases:
        if name in base.__dict__:
            # Attribute has been declared explicitly in a base class.
            return base.__dict__[name] is not None

    # No attribute; tough luck.
    return False


def _collect(name, attrs, bases):
    """Merges the hashes for all bases plus the current class."""
    value = {}
    if name in attrs and attrs[name] is not None:
        value.update(**attrs[name])

    for base in bases:
        if name in base.__dict__ and base.__dict__[name] is not None:
            value.update(**base.__dict__[name])

    return value


def _config(self, name, config, attrs, bases):
    """Overrides a property by a config option if it isn't specified."""
    if not _has(name, attrs, bases):
        options = utils.config(config)
        if options is not None:
            setattr(self, name, options)


def _is_field_visible(obj, name):
    """Checks for the visibility in displaying of the attribute."""
    visible = not (obj.attributes and name not in obj.attributes)
    visible = visible and (not (obj.exclude and name in obj.exclude))
    return visible


def _is_field_editable(obj, name):
    """Checks if the attribute is declared to be editable."""
    visible = not (obj.fields and name not in obj.fields)
    visible = visible and (not (obj.exclude and name in obj.exclude))
    return visible


def _is_field_filterable(obj, name):
    """Checks if the specified attribute is declared to be filterable."""
    return not (obj.filterable and name not in obj.filterable)


def _is_field_collection(field):
    """Tests if the specified attribute is some type of collection."""
    if isinstance(field, RelatedObject):
        # This happens to be a reverse relation; test using the provided.
        return field.field.rel.multiple

    try:
        # Attempt to discover if the null value is some sort of
        # collection (and isn't a string) which would make it an collection.
        null = field.to_python(None)
        return (isinstance(null, collections.Iterable)
                and not isinstance(null, six.string_types))

    except forms.ValidationError:
        # None cannot be a valid value; definitely not an collection.
        return False


def _get_field_class(field):
    """Determines what class object to instantiate for the specified field."""
    if field is None:
        # No attribute passed; return the base class
        return attributes.Attribute

    try:
        # Attempt to handle date/times
        test = datetime.datetime.now()
        if field.to_python(test) == test:
            # Looks like we're a datetime attribute
            return attributes.DateTimeAttribute

    except (forms.ValidationError, AttributeError):
        # Times cannot be handled.
        pass

    try:
        # Attempt to handle times
        test = datetime.datetime.now().time()
        if field.to_python(test) == test:
            # Looks like we're a time field
            return attributes.TimeAttribute

    except (forms.ValidationError, AttributeError):
        # Times cannot be handled.
        pass

    try:
        # Attempt to handle dates
        test = datetime.datetime.now().date()
        if field.to_python(test) == test:
            # Looks like we're a date field
            return attributes.DateAttribute

    except (forms.ValidationError, AttributeError):
        # Dates cannot be handled.
        pass

    try:
        # Attempt to handle file streams
        test = StringIO()
        if field.to_python(test).getvalue() == '':
            # Looks like we're capable of dealing with file streams
            return attributes.FileAttribute

    except (forms.ValidationError, AttributeError):
        # File streams cannot be handled
        pass

    try:
        # Attempt to handle booleans
        test = True
        if field.to_python(test) is True:
            # Looks we can explicitly handle booleans
            return attributes.BooleanAttribute

    except (forms.ValidationError, AttributeError):
        # Booleans cannot be handled
        pass

    # We have no idea what we are; assume we're just text
    return attributes.Attribute


class DeclarativeResource(type):
    """Defines the metaclass for the Resource class.
    """

    def _get_field_object(self, name):
        try:
            # Check the form attribute dictionary for the attribute object
            return self.form_fields[name]
<commit_msg>Catching type error now; fixed gh-27<commit_after># -*- coding: utf-8 -*-
"""
"""
from __future__ import print_function, unicode_literals
from __future__ import absolute_import, division
import collections
import datetime
from StringIO import StringIO
import six
from django import forms
from django.core import urlresolvers
from django.core.exceptions import ImproperlyConfigured
from django.db.models.fields import FieldDoesNotExist
from django.db.models.fields.related import RelatedField
from django.db.models.related import RelatedObject
from .. import utils
from .helpers import attribute as field_helper
from . import attributes


def _has(name, attrs, bases):
    """
    Determines if a property has been expliclty specified by this or a
    base class.
    """
    if name in attrs:
        # Attribute is explicitly declared on the class object.
        return attrs[name] is not None

    for base in bases:
        if name in base.__dict__:
            # Attribute has been declared explicitly in a base class.
            return base.__dict__[name] is not None

    # No attribute; tough luck.
    return False


def _collect(name, attrs, bases):
    """Merges the hashes for all bases plus the current class."""
    value = {}
    if name in attrs and attrs[name] is not None:
        value.update(**attrs[name])

    for base in bases:
        if name in base.__dict__ and base.__dict__[name] is not None:
            value.update(**base.__dict__[name])

    return value


def _config(self, name, config, attrs, bases):
    """Overrides a property by a config option if it isn't specified."""
    if not _has(name, attrs, bases):
        options = utils.config(config)
        if options is not None:
            setattr(self, name, options)


def _is_field_visible(obj, name):
    """Checks for the visibility in displaying of the attribute."""
    visible = not (obj.attributes and name not in obj.attributes)
    visible = visible and (not (obj.exclude and name in obj.exclude))
    return visible


def _is_field_editable(obj, name):
    """Checks if the attribute is declared to be editable."""
    visible = not (obj.fields and name not in obj.fields)
    visible = visible and (not (obj.exclude and name in obj.exclude))
    return visible


def _is_field_filterable(obj, name):
    """Checks if the specified attribute is declared to be filterable."""
    return not (obj.filterable and name not in obj.filterable)


def _is_field_collection(field):
    """Tests if the specified attribute is some type of collection."""
    if isinstance(field, RelatedObject):
        # This happens to be a reverse relation; test using the provided.
        return field.field.rel.multiple

    try:
        # Attempt to discover if the null value is some sort of
        # collection (and isn't a string) which would make it an collection.
        null = field.to_python(None)
        return (isinstance(null, collections.Iterable)
                and not isinstance(null, six.string_types))

    except forms.ValidationError:
        # None cannot be a valid value; definitely not an collection.
        return False


def _get_field_class(field):
    """Determines what class object to instantiate for the specified field."""
    if field is None:
        # No attribute passed; return the base class
        return attributes.Attribute

    try:
        # Attempt to handle date/times
        test = datetime.datetime.now()
        if field.to_python(test) == test:
            # Looks like we're a datetime attribute
            return attributes.DateTimeAttribute

    except (forms.ValidationError, AttributeError, TypeError, ValueError):
        # Times cannot be handled.
        pass

    try:
        # Attempt to handle times
        test = datetime.datetime.now().time()
        if field.to_python(test) == test:
            # Looks like we're a time field
            return attributes.TimeAttribute

    except (forms.ValidationError, AttributeError, TypeError, ValueError):
        # Times cannot be handled.
        pass

    try:
        # Attempt to handle dates
        test = datetime.datetime.now().date()
        if field.to_python(test) == test:
            # Looks like we're a date field
            return attributes.DateAttribute

    except (forms.ValidationError, AttributeError, TypeError, ValueError):
        # Dates cannot be handled.
        pass

    try:
        # Attempt to handle file streams
        test = StringIO()
        if field.to_python(test).getvalue() == '':
            # Looks like we're capable of dealing with file streams
            return attributes.FileAttribute

    except (forms.ValidationError, AttributeError, TypeError, ValueError):
        # File streams cannot be handled
        pass

    try:
        # Attempt to handle booleans
        test = True
        if field.to_python(test) is True:
            # Looks we can explicitly handle booleans
            return attributes.BooleanAttribute

    except (forms.ValidationError, AttributeError, TypeError, ValueError):
        # Booleans cannot be handled
        pass

    # We have no idea what we are; assume we're just text
    return attributes.Attribute


class DeclarativeResource(type):
    """Defines the metaclass for the Resource class.
    """

    def _get_field_object(self, name):
        try:
            # Check the form attribute dictionary for the attribute object
            return self.form_fields[name]

{'fraction_non_alphanumeric': 0.07406719881195471, 'fraction_numerical': 0.00037126415444588826, 'mean_word_length': 2.653780942692438, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5973486', 'n_tokens_mistral': 2812, 'n_tokens_neox': 2735, 'n_words': 1125}