<commit_before>from functools import partial

from stdnet.utils import zip, JSPLITTER, EMPTYJSON, iteritems
from stdnet.utils.exceptions import *

from .base import ModelType, ModelBase, raise_kwargs
from .session import Manager


__all__ = ['StdModel', 'create_model', 'model_to_dict']


class StdModel(ModelBase):
    '''A :class:`Model` which contains data in :class:`Field`. This represents
the main class of :mod:`stdnet.odm` module.'''
    _model_type = 'object'
    abstract = True
    _loadedfields = None

    def __init__(self, *args, **kwargs):
        meta = self._meta
        kwargs.pop(meta.pk.name, None)
        for field in meta.scalarfields:
            field.set_value(self, kwargs.pop(field.name, None))
        attributes = meta.attributes
        if args:
            N = len(args)
            if N > len(attributes):
                raise ValueError('Too many attributes')
            attrs, attributes = attributes[:N], attributes[N:]
            for name, value in zip(attrs, args):
                setattr(self, name, value)
        for name in attributes:
            setattr(self, name, kwargs.pop(name, None))
        if kwargs:
            raise_kwargs(self, kwargs)

    @property
    def has_all_data(self):
        '''``True`` if this :class:`StdModel` instance has all back-end data
loaded. This applies to persistent instances only. This property is used when
committing changes. If all data is available, the commit will replace the
previous object data entirely, otherwise it will only update it.'''
        return self.get_state().persistent and self._loadedfields is None
    
    def loadedfields(self):
        '''Generator of fields loaded from database'''
        if self._loadedfields is None:
            for field in self._meta.scalarfields:
                yield field
        else:
            fields = self._meta.dfields
            processed = set()
            for name in self._loadedfields:
                if name in processed:
                    continue
                if name in fields:
                    processed.add(name)
                    yield fields[name]
                else:
                    name = name.split(JSPLITTER)[0]
                    if name in fields and name not in processed:
                        field = fields[name]
                        if field.type == 'json object':
                            processed.add(name)
                            yield field

    def fieldvalue_pairs(self, exclude_cache=False):
        '''Generator of fields,values pairs. Fields correspond to
the ones which have been loaded (usually all of them) or
not loaded but modified.
Check the :ref:`load_only <performance-loadonly>` query function for more
details.

If *exclude_cache* evaluates to ``True``, fields with :attr:`Field.as_cache`
attribute set to ``True`` won't be included.

:rtype: a generator of two-elements tuples'''
        for field in self._meta.scalarfields:
            if exclude_cache and field.as_cache:
                continue
            name = field.attname
            if hasattr(self, name):
                yield field, getattr(self,name)

    def clear_cache_fields(self):
        '''Set cache fields to ``None``. Check :attr:`Field.as_cache`
for information regarding fields which are considered cache.'''
        for field in self._meta.scalarfields:
            if field.as_cache:
                setattr(self,field.name,None)

    def get_attr_value(self, name):
        '''Retrieve the ``value`` for the attribute ``name``. The ``name``
can be nested following the :ref:`double underscore <tutorial-underscore>`
notation, for example ``group__name``. If the attribute is not available it
raises :class:`AttributeError`.'''
        if name in self._meta.dfields:
            return self._meta.dfields[name].get_value(self)
        elif not name.startswith('__') and JSPLITTER in name:
            bits = name.split(JSPLITTER)
            fname = bits[0]
            if fname in self._meta.dfields:
                return self._meta.dfields[fname].get_value(self, *bits[1:])
            else:
                return getattr(self, name)
        else:
            return getattr(self, name)

    def clone(self, **data):
        '''Utility method for cloning the instance as a new object.

:parameter data: additional which override field data.
:rtype: a new instance of this class.
'''
        meta = self._meta
        session = self.session
        pkname = meta.pkname()
        pkvalue = data.pop(pkname, None)
        fields = self.todict(exclude_cache=True)
        fields.update(data)
        fields.pop('__dbdata__', None)
        obj = self._meta.make_object((pkvalue, None, fields))
        obj.session = session
        return obj

    def is_valid(self):
        '''Kick off the validation algorithm by checking all
:attr:`StdModel.loadedfields` against their respective validation algorithm.

:rtype: Boolean indicating if the model validates.'''
        return self._meta.is_valid(self)

    def todict(self, exclude_cache=False):
        '''Return a dictionary of serialised scalar field for pickling.
If the *exclude_cache* flag is ``True``, fields with :attr:`Field.as_cache`
attribute set to ``True`` will be excluded.'''
        odict = {}
        for field,value in self.fieldvalue_pairs(exclude_cache=exclude_cache):
            value = field.serialise(value)
            if value:
                odict[field.name] = value
        if 'id' in self._dbdata:
            odict['__dbdata__'] = {'id': self._dbdata['id']}
        return odict

    def _to_json(self, exclude_cache):
        pk = self.pkvalue()
        if pk:
            yield self._meta.pkname(),pk
            for field, value in self.fieldvalue_pairs(exclude_cache=\
                                                      exclude_cache):
                value = field.json_serialise(value)
                if value not in EMPTYJSON:
                    yield field.name, value<commit_msg>added set method to model<commit_after>from functools import partial

from stdnet.utils import zip, JSPLITTER, EMPTYJSON, iteritems
from stdnet.utils.exceptions import *

from .base import ModelType, ModelBase, raise_kwargs
from .session import Manager


__all__ = ['StdModel', 'create_model', 'model_to_dict']


class StdModel(ModelBase):
    '''A :class:`Model` which contains data in :class:`Field`. This represents
the main class of :mod:`stdnet.odm` module.'''
    _model_type = 'object'
    abstract = True
    _loadedfields = None

    def __init__(self, *args, **kwargs):
        meta = self._meta
        kwargs.pop(meta.pk.name, None)
        for field in meta.scalarfields:
            field.set_value(self, kwargs.pop(field.name, None))
        attributes = meta.attributes
        if args:
            N = len(args)
            if N > len(attributes):
                raise ValueError('Too many attributes')
            attrs, attributes = attributes[:N], attributes[N:]
            for name, value in zip(attrs, args):
                setattr(self, name, value)
        for name in attributes:
            setattr(self, name, kwargs.pop(name, None))
        if kwargs:
            raise_kwargs(self, kwargs)

    @property
    def has_all_data(self):
        '''``True`` if this :class:`StdModel` instance has all back-end data
loaded. This applies to persistent instances only. This property is used when
committing changes. If all data is available, the commit will replace the
previous object data entirely, otherwise it will only update it.'''
        return self.get_state().persistent and self._loadedfields is None
    
    def set(self, name, value):
        meta = self._meta
        if name in meta.dfields:
            meta.dfields[name].set_value(self, value)
        elif name in meta.attributes:
            setattr(self, name, value)
        else:
            raise AttributeError('Model has no field/attribute %s' % name)
        
    def loadedfields(self):
        '''Generator of fields loaded from database'''
        if self._loadedfields is None:
            for field in self._meta.scalarfields:
                yield field
        else:
            fields = self._meta.dfields
            processed = set()
            for name in self._loadedfields:
                if name in processed:
                    continue
                if name in fields:
                    processed.add(name)
                    yield fields[name]
                else:
                    name = name.split(JSPLITTER)[0]
                    if name in fields and name not in processed:
                        field = fields[name]
                        if field.type == 'json object':
                            processed.add(name)
                            yield field

    def fieldvalue_pairs(self, exclude_cache=False):
        '''Generator of fields,values pairs. Fields correspond to
the ones which have been loaded (usually all of them) or
not loaded but modified.
Check the :ref:`load_only <performance-loadonly>` query function for more
details.

If *exclude_cache* evaluates to ``True``, fields with :attr:`Field.as_cache`
attribute set to ``True`` won't be included.

:rtype: a generator of two-elements tuples'''
        for field in self._meta.scalarfields:
            if exclude_cache and field.as_cache:
                continue
            name = field.attname
            if hasattr(self, name):
                yield field, getattr(self,name)

    def clear_cache_fields(self):
        '''Set cache fields to ``None``. Check :attr:`Field.as_cache`
for information regarding fields which are considered cache.'''
        for field in self._meta.scalarfields:
            if field.as_cache:
                setattr(self,field.name,None)

    def get_attr_value(self, name):
        '''Retrieve the ``value`` for the attribute ``name``. The ``name``
can be nested following the :ref:`double underscore <tutorial-underscore>`
notation, for example ``group__name``. If the attribute is not available it
raises :class:`AttributeError`.'''
        if name in self._meta.dfields:
            return self._meta.dfields[name].get_value(self)
        elif not name.startswith('__') and JSPLITTER in name:
            bits = name.split(JSPLITTER)
            fname = bits[0]
            if fname in self._meta.dfields:
                return self._meta.dfields[fname].get_value(self, *bits[1:])
            else:
                return getattr(self, name)
        else:
            return getattr(self, name)

    def clone(self, **data):
        '''Utility method for cloning the instance as a new object.

:parameter data: additional which override field data.
:rtype: a new instance of this class.
'''
        meta = self._meta
        session = self.session
        pkname = meta.pkname()
        pkvalue = data.pop(pkname, None)
        fields = self.todict(exclude_cache=True)
        fields.update(data)
        fields.pop('__dbdata__', None)
        obj = self._meta.make_object((pkvalue, None, fields))
        obj.session = session
        return obj

    def is_valid(self):
        '''Kick off the validation algorithm by checking all
:attr:`StdModel.loadedfields` against their respective validation algorithm.

:rtype: Boolean indicating if the model validates.'''
        return self._meta.is_valid(self)

    def todict(self, exclude_cache=False):
        '''Return a dictionary of serialised scalar field for pickling.
If the *exclude_cache* flag is ``True``, fields with :attr:`Field.as_cache`
attribute set to ``True`` will be excluded.'''
        odict = {}
        for field,value in self.fieldvalue_pairs(exclude_cache=exclude_cache):
            value = field.serialise(value)
            if value:
                odict[field.name] = value
        if 'id' in self._dbdata:
            odict['__dbdata__'] = {'id': self._dbdata['id']}
        return odict

    def _to_json(self, exclude_cache):
        pk = self.pkvalue()
        if pk:
            yield self._meta.pkname(), pk
            for field, value in self.fieldvalue_pairs(exclude_cache=\
                                                      exclude_cache):
                value = field.json_serialise(value)
                if value not in EMPTYJSON:
                    yield field.name, value
{'fraction_non_alphanumeric': 0.09283798065197951, 'fraction_numerical': 0.00048776522233964716, 'mean_word_length': 2.173890608875129, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3908534', 'n_tokens_mistral': 3339, 'n_tokens_neox': 3236, 'n_words': 1079}