<commit_before>#!/usr/bin/env python

__author__ = '<NAME>'
__license__ = 'Apache 2.0'

import copy
import re
import os
import fnmatch
import hashlib
from collections import OrderedDict, defaultdict, Mapping, Iterable
from weakref import WeakSet, WeakValueDictionary

import yaml

from pyon.util.log import log

class IonObjectError(Exception):
    pass

class IonYamlLoader(yaml.Loader):
    """ For ION-specific overrides of YAML loading behavior. """
    pass
class IonYamlDumper(yaml.Dumper):
    """ For ION-specific overrides of YAML dumping behavior. """
    pass

def service_name_from_file_name(file_name):
    file_name = os.path.basename(file_name).split('.', 1)[0]
    return file_name.title().replace('_', '').replace('-', '')

class IonObjectMetaType(type):
    """
    Metaclass that automatically generates subclasses of IonObject with the appropriate defaults for each field.
    """

    _type_cache = {}

    def __call__(cls, _def, _dict=None, *args, **kwargs):
        if _def in IonObjectMetaType._type_cache:
            cls_type = IonObjectMetaType._type_cache[_def]
        else:
            # Check that the object we are wrapping is an IonObjectDefinition
            if not isinstance(_def, IonObjectDefinition):
                raise IonObjectError('IonObject init first argument must be an IonObjectDefinition')

            # Generate a unique class name
            base_name = 'IonObject'
            cls_name = '%s_%s_%s' % (base_name, _def.type.name, _def.hash[:8])
            cls_dict = {'_def': _def}
            cls_dict.update(copy.deepcopy(_def.schema))
            #cls_dict['__slots__'] = cls_dict.keys() + ['__weakref__']

            cls_type = IonObjectMetaType.__new__(IonObjectMetaType, cls_name, (cls,), cls_dict)
            IonObjectMetaType._type_cache[_def] = cls_type

        # Auto-copy the schema so we can use __dict__ authoritatively and simplify the code
        __dict__ = copy.deepcopy(dict(_def.schema))
        if _dict is not None:
            __dict__.update(_dict)
            
        # Finally allow the instantiation to occur, but slip in our new class type
        obj = super(IonObjectMetaType, cls_type).__call__(__dict__, *args, **kwargs)
        return obj

class IonObjectBase(object):
    __metaclass__ = IonObjectMetaType
    
    def __init__(self, _dict=None, **kwargs):
        """
        Instead of instantiating these directly, you should go through an IonObjectRegistry.
        If you need to instantiate directly, set _def to an instance of IonObjectDefinition.
        """
        
        if _dict is not None:
            self.__dict__ = _dict       # Don't copy here, assume this came through the metaclass
        self.__dict__.update(kwargs)

    def __str__(self):
        """ This method will probably be too expensive to use frequently due to object allocation and YAML. """
        # TODO: Add a yaml representer for IonObjects to match their tag constructors

        _dict = self.__dict__
        #return '%s(%r)' % (self.__class__, _dict)
        # If the yaml is too slow revert to the line above
        name = '%s <%s>' % (self.__class__.__name__, id(self))
        return yaml.dump({name: _dict}, default_flow_style=False, Dumper=IonYamlDumper)

    def _validate(self):
        """
        Compare fields to the schema and raise AttributeError if mismatched.
        Named _validate instead of validate because the data may have a field named "validate".
        """
        fields, schema = self.__dict__, self._def.schema
        extra_fields = fields.viewkeys() - schema.viewkeys()
        if len(extra_fields) > 0:
            raise AttributeError('Fields found that are not in the schema: %r' % (list(extra_fields)))
        for key in fields.iterkeys():
            field_val, schema_val = fields[key], schema[key]
            if type(field_val) is not type(schema_val):
                raise AttributeError('Invalid type "%s" for field "%s", should be "%s"' %
                                     (type(fields[key]), key, type(schema[key])))
            if isinstance(field_val, IonObjectBase):
                field_val._validate()
            # Next validate only IonObjects found in child collections. Other than that, don't validate collections.
            # Note that this is non-recursive; only for first-level collections.
            elif isinstance(field_val, Mapping):
                for subkey in field_val:
                    subval = field_val[subkey]
                    if isinstance(subval, IonObjectBase):
                        subval._validate()
            elif isinstance(field_val, Iterable):
                for subval in field_val:
                    if isinstance(subval, IonObjectBase):
                        subval._validate()

def hashfunc(text):
    return hashlib.sha1(text).hexdigest()

_sha_re = re.compile('[0-9a-f]{40}')
def is_hash(val):
    return len(val) == 40 and _sha_re.match(val) is not None

class IonObjectDefinition(object):
    """ An ION object definition, with a single parent type and a specific version. """

    __slots__ = ('type', 'hash', 'schema', 'def_text')

    def __init__(self, _type, hash, schema, def_text):<commit_msg>Two fixes for validation.  First, ignore _id and _rev fields.  Second, allow the passing of int values for long or float<commit_after>#!/usr/bin/env python

__author__ = '<NAME>'
__license__ = 'Apache 2.0'

import copy
import re
import os
import fnmatch
import hashlib
from collections import OrderedDict, defaultdict, Mapping, Iterable
from weakref import WeakSet, WeakValueDictionary

import yaml

from pyon.util.log import log

class IonObjectError(Exception):
    pass

class IonYamlLoader(yaml.Loader):
    """ For ION-specific overrides of YAML loading behavior. """
    pass
class IonYamlDumper(yaml.Dumper):
    """ For ION-specific overrides of YAML dumping behavior. """
    pass

def service_name_from_file_name(file_name):
    file_name = os.path.basename(file_name).split('.', 1)[0]
    return file_name.title().replace('_', '').replace('-', '')

class IonObjectMetaType(type):
    """
    Metaclass that automatically generates subclasses of IonObject with the appropriate defaults for each field.
    """

    _type_cache = {}

    def __call__(cls, _def, _dict=None, *args, **kwargs):
        if _def in IonObjectMetaType._type_cache:
            cls_type = IonObjectMetaType._type_cache[_def]
        else:
            # Check that the object we are wrapping is an IonObjectDefinition
            if not isinstance(_def, IonObjectDefinition):
                raise IonObjectError('IonObject init first argument must be an IonObjectDefinition')

            # Generate a unique class name
            base_name = 'IonObject'
            cls_name = '%s_%s_%s' % (base_name, _def.type.name, _def.hash[:8])
            cls_dict = {'_def': _def}
            cls_dict.update(copy.deepcopy(_def.schema))
            #cls_dict['__slots__'] = cls_dict.keys() + ['__weakref__']

            cls_type = IonObjectMetaType.__new__(IonObjectMetaType, cls_name, (cls,), cls_dict)
            IonObjectMetaType._type_cache[_def] = cls_type

        # Auto-copy the schema so we can use __dict__ authoritatively and simplify the code
        __dict__ = copy.deepcopy(dict(_def.schema))
        if _dict is not None:
            __dict__.update(_dict)
            
        # Finally allow the instantiation to occur, but slip in our new class type
        obj = super(IonObjectMetaType, cls_type).__call__(__dict__, *args, **kwargs)
        return obj

class IonObjectBase(object):
    __metaclass__ = IonObjectMetaType
    
    def __init__(self, _dict=None, **kwargs):
        """
        Instead of instantiating these directly, you should go through an IonObjectRegistry.
        If you need to instantiate directly, set _def to an instance of IonObjectDefinition.
        """
        
        if _dict is not None:
            self.__dict__ = _dict       # Don't copy here, assume this came through the metaclass
        self.__dict__.update(kwargs)

    def __str__(self):
        """ This method will probably be too expensive to use frequently due to object allocation and YAML. """
        # TODO: Add a yaml representer for IonObjects to match their tag constructors

        _dict = self.__dict__
        #return '%s(%r)' % (self.__class__, _dict)
        # If the yaml is too slow revert to the line above
        name = '%s <%s>' % (self.__class__.__name__, id(self))
        return yaml.dump({name: _dict}, default_flow_style=False, Dumper=IonYamlDumper)

    def _validate(self):
        """
        Compare fields to the schema and raise AttributeError if mismatched.
        Named _validate instead of validate because the data may have a field named "validate".
        """
        id_and_rev_set = set(['_id','_rev'])
        fields, schema = self.__dict__, self._def.schema
        extra_fields = fields.viewkeys() - schema.viewkeys() - id_and_rev_set
        if len(extra_fields) > 0:
            raise AttributeError('Fields found that are not in the schema: %r' % (list(extra_fields)))
        for key in fields.iterkeys():
            if key in id_and_rev_set:
                continue
            field_val, schema_val = fields[key], schema[key]
            if type(field_val) is not type(schema_val):
                # Special handle numeric types.  Allow int to be
                # passed for long and float.  Auto convert to the
                # right type.
                if isinstance(field_val, int):
                    if isinstance(schema_val, float):
                        fields[key] = float(fields[key])
                        continue
                    elif isinstance(schema_val,long):
                        fields[key] = long(fields[key])
                        continue

                raise AttributeError('Invalid type "%s" for field "%s", should be "%s"' %
                                     (type(fields[key]), key, type(schema[key])))
            if isinstance(field_val, IonObjectBase):
                field_val._validate()
            # Next validate only IonObjects found in child collections. Other than that, don't validate collections.
            # Note that this is non-recursive; only for first-level collections.
            elif isinstance(field_val, Mapping):
                for subkey in field_val:
                    subval = field_val[subkey]
                    if isinstance(subval, IonObjectBase):
                        subval._validate()
            elif isinstance(field_val, Iterable):
                for subval in field_val:
                    if isinstance(subval, IonObjectBase):
                        subval._validate()

def hashfunc(text):
    return hashlib.sha1(text).hexdigest()

_sha_re = re.compile('[0-9a-f]{40}')
def is_hash(val):
    return len(val) == 40 and _sha_re.match(val) is not None

class IonObjectDefinition(object):
    """ An ION object definition, with a single parent type and a specific version. """

    __slots__ = ('type', 'hash', 'schema', 'def_text')

    def __init__(self, _type, hash, schema, def_text):
{'fraction_non_alphanumeric': 0.0927390366642703, 'fraction_numerical': 0.002336448598130841, 'mean_word_length': 2.495288944723618, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6676097', 'n_tokens_mistral': 3183, 'n_tokens_neox': 3032, 'n_words': 976}