<commit_before>#!/usr/bin/env python

__author__ = '<NAME>, <NAME>, <NAME>'
__license__ = 'Apache 2.0'

import os
import re
import inspect
from collections import OrderedDict, Mapping, Iterable
import pprint
import StringIO

from pyon.util.log import log
from pyon.core.exception import BadRequest


built_in_attrs = set(['_id', '_rev', 'type_', 'blame_'])

class IonObjectBase(object):

    def __str__(self):
        return str(self.__dict__)

    def __eq__(self, other):
        if type(other) == type(self):
            if other.__dict__ == self.__dict__:
                return True
        return False

    def _validate(self):
        """
        Compare fields to the schema and raise AttributeError if mismatched.
        Named _validate instead of validate because the data may have a field named "validate".
        """
        fields, schema = self.__dict__, self._schema

        # Check for extra fields not defined in the schema
        extra_fields = fields.viewkeys() - schema.viewkeys() - built_in_attrs
        if len(extra_fields) > 0:
            raise AttributeError('Fields found that are not in the schema: %r' % (list(extra_fields)))

        required_decorator = 'Required'
        content_type_decorator = 'ContentType'
        content_count_decorator = 'ContentCount'
        value_range_decorator = 'ValueRange'
        value_pattern_decorator = 'ValuePattern'

        # Check required field criteria met
        for key in schema:
            if 'decorators' in schema[key] and required_decorator in schema[key]['decorators']:
                if not key in fields or fields[key] is None:
                    raise AttributeError('Required value "%s" not set' % key)

        # Check each attribute
        for key in fields.iterkeys():
            if key in built_in_attrs:
                continue

            schema_val = schema[key]

            # Correct any float or long types that got downgraded to int
            if isinstance(fields[key], int):
                if schema_val['type'] == 'float':
                    fields[key] = float(fields[key])
                elif schema_val['type'] == 'long':
                    fields[key] = long(fields[key])

            # argh, annoying work around for OrderedDict vs dict issue
            if type(fields[key]) == dict and schema_val['type'] == 'OrderedDict':
                fields[key] = OrderedDict(fields[key])

            # Basic type checking
            field_val = fields[key]
            if 'decorators' in schema_val:
                log.debug("Validating %s: %s: %s: %s" % (key, schema_val["type"], schema_val["decorators"], str(field_val)))
            else:
                log.debug("Validating %s: %s: %s" % (key, schema_val["type"], str(field_val)))
            if type(field_val).__name__ != schema_val['type']:

                # if the schema doesn't define a type, we can't very well validate it
                if schema_val['type'] == 'NoneType':
                    continue

                # Already checked for required above.  Assume optional and continue
                if field_val is None:
                    continue

                # IonObjects are ok for dict fields too!
                if isinstance(field_val, IonObjectBase) and schema_val['type'] == 'OrderedDict':
                    continue

                if not key in fields or fields[key] is None:
                    raise AttributeError('Required value "%s" not set' % key)

                # Check for inheritance
                if self.check_inheritance_chain(type(field_val), schema_val['type']):
                    continue

                # Check enum types
                from pyon.core.registry import enum_classes
                if isinstance(field_val, int) and schema_val['type'] in enum_classes:
                    if field_val not in enum_classes(schema_val['type'])._str_map:
                        raise AttributeError('Invalid enum value "%d" for field "%s.%s", should be between 1 and %d' %
                                     (fields[key], type(self).__name__, key, len(enum_classes(schema_val['type'])._str_map)))
                    else:
                        continue

                # TODO work around for msgpack issue
                if type(field_val) == tuple and schema_val['type'] == 'list':
                    continue

                # TODO remove this at some point
                if isinstance(field_val, IonObjectBase) and schema_val['type'] == 'dict':
                    log.warn('TODO: Please convert generic dict attribute type to abstract type for field "%s.%s"' % (type(self).__name__, key))
                    continue

                # Special case check for ION object being passed where default type is dict or str
                if 'decorators' in schema_val:
                    if content_type_decorator in schema_val['decorators']:
                        if isinstance(field_val, IonObjectBase) and schema_val['type'] == 'dict' or schema_val['type'] == 'str':
                            self.check_content(key, field_val, schema_val['decorators'][content_type_decorator])
                            continue

                raise AttributeError('Invalid type "%s" for field "%s.%s", should be "%s"' %
                                     (type(fields[key]), type(self).__name__, key, schema_val['type']))

            if type(field_val).__name__ == 'str':
                if value_pattern_decorator in schema_val['decorators']:
                    self.check_string_pattern_match(key, field_val, schema_val['decorators'][value_pattern_decorator])

            if type(field_val).__name__ in ['int', 'float', 'long']:
                if value_range_decorator in schema_val['decorators']:
                    self.check_numeric_value_range(key, field_val, schema_val['decorators'][value_range_decorator])

            if 'decorators' in schema_val:
                if content_type_decorator in schema_val['decorators']:
                    if schema_val['type'] == 'list':
                        self.check_collection_content(key, field_val, schema_val['decorators'][content_type_decorator])
                    elif schema_val['type'] == 'dict' or schema_val['type'] == 'OrderedDict':
                        self.check_collection_content(key, field_val.values(), schema_val['decorators'][content_type_decorator])
                    else:
                        self.check_content(key, field_val, schema_val['decorators'][content_type_decorator])
                if content_count_decorator in schema_val['decorators']:
                    if schema_val['type'] == 'list':
                        self.check_collection_length(key, field_val, schema_val['decorators'][content_count_decorator])
                    if schema_val['type'] == 'dict' or schema_val['type'] == 'OrderedDict':
                        self.check_collection_length(key, field_val.values(), schema_val['decorators'][content_count_decorator])

            if isinstance(field_val, IonObjectBase):
                field_val._validate()

            # Next validate only IonObjects found in child collections.
            # Note that this is non-recursive; only for first-level collections.
            elif isinstance(field_val, Mapping):
                for subkey in field_val:
                    subval = field_val[subkey]
                    if isinstance(subval, IonObjectBase):
                        subval._validate()
            elif isinstance(field_val, Iterable):
                for subval in field_val:
                    if isinstance(subval, IonObjectBase):
                        subval._validate()

    def _get_type(self):
        return self.__class__.__name__

    def _get_extends(self):
        parents = [parent.__name__ for parent in self.__class__.__mro__ if parent.__name__ not in ['IonObjectBase', 'object', self._get_type()]]
        return parents

    def __contains__(self, item):
        return hasattr(self, item)

    def update(self, other):
        """
        Method that allows self object attributes to be updated with other object.
        Other object must be of same type or super type.
        """
        if type(other) != type(self):
            bases = inspect.getmro(self.__class__)
            if other.__class__ not in bases:
                raise BadRequest("Object %s and %s do not have compatible types for update" % (type(self).__name__, type(other).__name__))
        for key in other.__dict__:
            setattr(self, key, other.__dict__[key])<commit_msg>Enable IonObjectBase to be subscriptable<commit_after>#!/usr/bin/env python

__author__ = '<NAME>, <NAME>, <NAME>'
__license__ = 'Apache 2.0'

import os
import re
import inspect
from collections import OrderedDict, Mapping, Iterable
import pprint
import StringIO

from pyon.util.log import log
from pyon.core.exception import BadRequest


built_in_attrs = set(['_id', '_rev', 'type_', 'blame_'])

class IonObjectBase(object):

    def __str__(self):
        return str(self.__dict__)

    def __eq__(self, other):
        if type(other) == type(self):
            if other.__dict__ == self.__dict__:
                return True
        return False

    def __getitem__(self, key):
        return getattr(self, key)

    def __setitem__(self, key, value):
        return setattr(self, key, value)

    def __contains__(self, item):
        return hasattr(self, item)

    def has_key(self, key):
        return hasattr(self, key)

    def _validate(self):
        """
        Compare fields to the schema and raise AttributeError if mismatched.
        Named _validate instead of validate because the data may have a field named "validate".
        """
        fields, schema = self.__dict__, self._schema

        # Check for extra fields not defined in the schema
        extra_fields = fields.viewkeys() - schema.viewkeys() - built_in_attrs
        if len(extra_fields) > 0:
            raise AttributeError('Fields found that are not in the schema: %r' % (list(extra_fields)))

        required_decorator = 'Required'
        content_type_decorator = 'ContentType'
        content_count_decorator = 'ContentCount'
        value_range_decorator = 'ValueRange'
        value_pattern_decorator = 'ValuePattern'

        # Check required field criteria met
        for key in schema:
            if 'decorators' in schema[key] and required_decorator in schema[key]['decorators']:
                if not key in fields or fields[key] is None:
                    raise AttributeError('Required value "%s" not set' % key)

        # Check each attribute
        for key in fields.iterkeys():
            if key in built_in_attrs:
                continue

            schema_val = schema[key]

            # Correct any float or long types that got downgraded to int
            if isinstance(fields[key], int):
                if schema_val['type'] == 'float':
                    fields[key] = float(fields[key])
                elif schema_val['type'] == 'long':
                    fields[key] = long(fields[key])

            # argh, annoying work around for OrderedDict vs dict issue
            if type(fields[key]) == dict and schema_val['type'] == 'OrderedDict':
                fields[key] = OrderedDict(fields[key])

            # Basic type checking
            field_val = fields[key]
            if 'decorators' in schema_val:
                log.debug("Validating %s: %s: %s: %s" % (key, schema_val["type"], schema_val["decorators"], str(field_val)))
            else:
                log.debug("Validating %s: %s: %s" % (key, schema_val["type"], str(field_val)))
            if type(field_val).__name__ != schema_val['type']:

                # if the schema doesn't define a type, we can't very well validate it
                if schema_val['type'] == 'NoneType':
                    continue

                # Already checked for required above.  Assume optional and continue
                if field_val is None:
                    continue

                # IonObjects are ok for dict fields too!
                if isinstance(field_val, IonObjectBase) and schema_val['type'] == 'OrderedDict':
                    continue

                if not key in fields or fields[key] is None:
                    raise AttributeError('Required value "%s" not set' % key)

                # Check for inheritance
                if self.check_inheritance_chain(type(field_val), schema_val['type']):
                    continue

                # Check enum types
                from pyon.core.registry import enum_classes
                if isinstance(field_val, int) and schema_val['type'] in enum_classes:
                    if field_val not in enum_classes(schema_val['type'])._str_map:
                        raise AttributeError('Invalid enum value "%d" for field "%s.%s", should be between 1 and %d' %
                                     (fields[key], type(self).__name__, key, len(enum_classes(schema_val['type'])._str_map)))
                    else:
                        continue

                # TODO work around for msgpack issue
                if type(field_val) == tuple and schema_val['type'] == 'list':
                    continue

                # TODO remove this at some point
                if isinstance(field_val, IonObjectBase) and schema_val['type'] == 'dict':
                    log.warn('TODO: Please convert generic dict attribute type to abstract type for field "%s.%s"' % (type(self).__name__, key))
                    continue

                # Special case check for ION object being passed where default type is dict or str
                if 'decorators' in schema_val:
                    if content_type_decorator in schema_val['decorators']:
                        if isinstance(field_val, IonObjectBase) and schema_val['type'] == 'dict' or schema_val['type'] == 'str':
                            self.check_content(key, field_val, schema_val['decorators'][content_type_decorator])
                            continue

                raise AttributeError('Invalid type "%s" for field "%s.%s", should be "%s"' %
                                     (type(fields[key]), type(self).__name__, key, schema_val['type']))

            if type(field_val).__name__ == 'str':
                if value_pattern_decorator in schema_val['decorators']:
                    self.check_string_pattern_match(key, field_val, schema_val['decorators'][value_pattern_decorator])

            if type(field_val).__name__ in ['int', 'float', 'long']:
                if value_range_decorator in schema_val['decorators']:
                    self.check_numeric_value_range(key, field_val, schema_val['decorators'][value_range_decorator])

            if 'decorators' in schema_val:
                if content_type_decorator in schema_val['decorators']:
                    if schema_val['type'] == 'list':
                        self.check_collection_content(key, field_val, schema_val['decorators'][content_type_decorator])
                    elif schema_val['type'] == 'dict' or schema_val['type'] == 'OrderedDict':
                        self.check_collection_content(key, field_val.values(), schema_val['decorators'][content_type_decorator])
                    else:
                        self.check_content(key, field_val, schema_val['decorators'][content_type_decorator])
                if content_count_decorator in schema_val['decorators']:
                    if schema_val['type'] == 'list':
                        self.check_collection_length(key, field_val, schema_val['decorators'][content_count_decorator])
                    if schema_val['type'] == 'dict' or schema_val['type'] == 'OrderedDict':
                        self.check_collection_length(key, field_val.values(), schema_val['decorators'][content_count_decorator])

            if isinstance(field_val, IonObjectBase):
                field_val._validate()

            # Next validate only IonObjects found in child collections.
            # Note that this is non-recursive; only for first-level collections.
            elif isinstance(field_val, Mapping):
                for subkey in field_val:
                    subval = field_val[subkey]
                    if isinstance(subval, IonObjectBase):
                        subval._validate()
            elif isinstance(field_val, Iterable):
                for subval in field_val:
                    if isinstance(subval, IonObjectBase):
                        subval._validate()

    def _get_type(self):
        return self.__class__.__name__

    def _get_extends(self):
        parents = [parent.__name__ for parent in self.__class__.__mro__ if parent.__name__ not in ['IonObjectBase', 'object', self._get_type()]]
        return parents

    def update(self, other):
        """
        Method that allows self object attributes to be updated with other object.
        Other object must be of same type or super type.
        """
        if type(other) != type(self):
            bases = inspect.getmro(self.__class__)
            if other.__class__ not in bases:
                raise BadRequest("Object %s and %s do not have compatible types for update" % (type(self).__name__, type(other).__name__))
        for key in other.__dict__:
            setattr(self, key, other.__dict__[key])
{'fraction_non_alphanumeric': 0.09619157530294288, 'fraction_numerical': 0.0004616272360069244, 'mean_word_length': 2.067433628318584, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4290698', 'n_tokens_mistral': 4630, 'n_tokens_neox': 4387, 'n_words': 1295}