<commit_before># Copyright 2014 CloudFounders NV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
DataObject module
"""
import uuid
import copy
from ovs.dal.exceptions import ObjectNotFoundException, ConcurrencyException
from ovs.dal.helpers import Descriptor, Toolbox
from ovs.dal.relations.relations import RelationMapper
from ovs.dal.dataobjectlist import DataObjectList
from ovs.dal.datalist import DataList
from ovs.extensions.generic.volatilemutex import VolatileMutex
from ovs.extensions.storage.exceptions import KeyNotFoundException
from ovs.extensions.storage.persistentfactory import PersistentFactory
from ovs.extensions.storage.volatilefactory import VolatileFactory


class MetaClass(type):
    """
    This metaclass provides dynamic __doc__ generation feeding doc generators
    """
    def __new__(mcs, name, bases, dct):
        """
        Overrides instance creation of all DataObject instances
        """
        if name != 'DataObject':
            for attribute, default in dct['_blueprint'].iteritems():
                docstring = default[2] if len(default) == 3 else ''
                if isinstance(default[1], type):
                    itemtype = default[1].__name__
                    extra_info = ''
                else:
                    itemtype = 'Enum(%s)' % default[1][0].__class__.__name__
                    extra_info = '(enum values: %s)' % ', '.join([str(item) for item in default[1]])
                dct[attribute] = property(
                    doc='[persistent] %s %s\n@type: %s'
                        % (docstring, extra_info, itemtype)
                )
            for attribute, relation in dct['_relations'].iteritems():
                itemtype = relation[0].__name__ if relation[0] is not None else name
                dct[attribute] = property(
                    doc='[relation] one-to-many relation with %s.%s\n@type: %s'
                        % (itemtype, relation[1], itemtype)
                )
            for attribute, info in dct['_expiry'].iteritems():
                docstring = dct['_%s' % attribute].__doc__.strip()
                if isinstance(info[1], type):
                    itemtype = info[1].__name__
                    extra_info = ''
                else:
                    itemtype = 'Enum(%s)' % info[1][0].__class__.__name__
                    extra_info = '(enum values: %s)' % ', '.join([str(item) for item in info[1]])
                dct[attribute] = property(
                    fget=dct['_%s' % attribute],
                    doc='[dynamic] (%ds) %s %s\n@rtype: %s'
                        % (info[0], docstring, extra_info, itemtype)
                )

        return super(MetaClass, mcs).__new__(mcs, name, bases, dct)


class DataObject(object):
    """
    This base class contains all logic to support our multiple backends and the caching
      - Storage backends:
        - Persistent backend for persistent storage (key-value store)
        - Volatile backend for volatile but fast storage (key-value store)
        - Storage backends are abstracted and injected into this class, making it possible to use
          fake backends
      - Features:
        - Hybrid property access:
          - Persistent backend
          - 3rd party component for "live" properties
        - Individual cache settings for "live" properties
        - 1-n relations with automatic property propagation
        - Recursive save
    """
    __metaclass__ = MetaClass

    #######################
    ## Attributes
    #######################

    # Properties that needs to be overwritten by implementation
    _blueprint = None            # Blueprint data of the objec type
    _expiry = None               # Timeout of readonly object properties cache
    _relations = None            # Blueprint for relations

    #######################
    ## Constructor
    #######################

    def __init__(self, guid=None, data=None, datastore_wins=False):
        """
        Loads an object with a given guid. If no guid is given, a new object
        is generated with a new guid.
        * guid: The guid indicating which object should be loaded
        * datastoreWins: Optional boolean indicating save conflict resolve management.
        ** True: when saving, external modified fields will not be saved
        ** False: when saving, all changed data will be saved, regardless of external updates
        ** None: in case changed field were also changed externally, an error will be raised
        """

        # Initialize super class
        super(DataObject, self).__init__()

        # Initialize internal fields
        self._frozen = False
        self._datastore_wins = datastore_wins
        self._guid = None             # Guid identifier of the object
        self._original = {}           # Original data copy
        self._metadata = {}           # Some metadata, mainly used for unit testing
        self._data = {}               # Internal data storage
        self._objects = {}            # Internal objects storage

        # Initialize public fields
        self.dirty = False

        # Worker fields/objects
        self._name = self.__class__.__name__.lower()
        self._namespace = 'ovs_data'   # Namespace of the object
        self._mutex_listcache = VolatileMutex('listcache_%s' % self._name)

        # Init guid
        new = False
        if guid is None:
            self._guid = str(uuid.uuid4())
            new = True
        else:
            self._guid = str(guid)

        # Build base keys
        self._key = '%s_%s_%s' % (self._namespace, self._name, self._guid)

        # Load data from cache or persistent backend where appropriate
        self._volatile = VolatileFactory.get_client()
        self._persistent = PersistentFactory.get_client()
        self._metadata['cache'] = None
        if new:
            self._data = {}
        else:
            self._data = self._volatile.get(self._key)
            if self._data is None:
                Toolbox.log_cache_hit('object_load', False)
                self._metadata['cache'] = False
                try:
                    self._data = self._persistent.get(self._key)
                except KeyNotFoundException:
                    raise ObjectNotFoundException()
            else:
                Toolbox.log_cache_hit('object_load', True)
                self._metadata['cache'] = True

        # Set default values on new fields
        for key, default in self._blueprint.iteritems():
            if key not in self._data:
                self._data[key] = default[0]<commit_msg>OVS-337 - Better ObjectNotFoundExceptions, containing information in the exception message<commit_after># Copyright 2014 CloudFounders NV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
DataObject module
"""
import uuid
import copy
from ovs.dal.exceptions import ObjectNotFoundException, ConcurrencyException
from ovs.dal.helpers import Descriptor, Toolbox
from ovs.dal.relations.relations import RelationMapper
from ovs.dal.dataobjectlist import DataObjectList
from ovs.dal.datalist import DataList
from ovs.extensions.generic.volatilemutex import VolatileMutex
from ovs.extensions.storage.exceptions import KeyNotFoundException
from ovs.extensions.storage.persistentfactory import PersistentFactory
from ovs.extensions.storage.volatilefactory import VolatileFactory


class MetaClass(type):
    """
    This metaclass provides dynamic __doc__ generation feeding doc generators
    """
    def __new__(mcs, name, bases, dct):
        """
        Overrides instance creation of all DataObject instances
        """
        if name != 'DataObject':
            for attribute, default in dct['_blueprint'].iteritems():
                docstring = default[2] if len(default) == 3 else ''
                if isinstance(default[1], type):
                    itemtype = default[1].__name__
                    extra_info = ''
                else:
                    itemtype = 'Enum(%s)' % default[1][0].__class__.__name__
                    extra_info = '(enum values: %s)' % ', '.join([str(item) for item in default[1]])
                dct[attribute] = property(
                    doc='[persistent] %s %s\n@type: %s'
                        % (docstring, extra_info, itemtype)
                )
            for attribute, relation in dct['_relations'].iteritems():
                itemtype = relation[0].__name__ if relation[0] is not None else name
                dct[attribute] = property(
                    doc='[relation] one-to-many relation with %s.%s\n@type: %s'
                        % (itemtype, relation[1], itemtype)
                )
            for attribute, info in dct['_expiry'].iteritems():
                docstring = dct['_%s' % attribute].__doc__.strip()
                if isinstance(info[1], type):
                    itemtype = info[1].__name__
                    extra_info = ''
                else:
                    itemtype = 'Enum(%s)' % info[1][0].__class__.__name__
                    extra_info = '(enum values: %s)' % ', '.join([str(item) for item in info[1]])
                dct[attribute] = property(
                    fget=dct['_%s' % attribute],
                    doc='[dynamic] (%ds) %s %s\n@rtype: %s'
                        % (info[0], docstring, extra_info, itemtype)
                )

        return super(MetaClass, mcs).__new__(mcs, name, bases, dct)


class DataObject(object):
    """
    This base class contains all logic to support our multiple backends and the caching
      - Storage backends:
        - Persistent backend for persistent storage (key-value store)
        - Volatile backend for volatile but fast storage (key-value store)
        - Storage backends are abstracted and injected into this class, making it possible to use
          fake backends
      - Features:
        - Hybrid property access:
          - Persistent backend
          - 3rd party component for "live" properties
        - Individual cache settings for "live" properties
        - 1-n relations with automatic property propagation
        - Recursive save
    """
    __metaclass__ = MetaClass

    #######################
    ## Attributes
    #######################

    # Properties that needs to be overwritten by implementation
    _blueprint = None            # Blueprint data of the objec type
    _expiry = None               # Timeout of readonly object properties cache
    _relations = None            # Blueprint for relations

    #######################
    ## Constructor
    #######################

    def __init__(self, guid=None, data=None, datastore_wins=False):
        """
        Loads an object with a given guid. If no guid is given, a new object
        is generated with a new guid.
        * guid: The guid indicating which object should be loaded
        * datastoreWins: Optional boolean indicating save conflict resolve management.
        ** True: when saving, external modified fields will not be saved
        ** False: when saving, all changed data will be saved, regardless of external updates
        ** None: in case changed field were also changed externally, an error will be raised
        """

        # Initialize super class
        super(DataObject, self).__init__()

        # Initialize internal fields
        self._frozen = False
        self._datastore_wins = datastore_wins
        self._guid = None             # Guid identifier of the object
        self._original = {}           # Original data copy
        self._metadata = {}           # Some metadata, mainly used for unit testing
        self._data = {}               # Internal data storage
        self._objects = {}            # Internal objects storage

        # Initialize public fields
        self.dirty = False

        # Worker fields/objects
        self._name = self.__class__.__name__.lower()
        self._namespace = 'ovs_data'   # Namespace of the object
        self._mutex_listcache = VolatileMutex('listcache_%s' % self._name)

        # Init guid
        new = False
        if guid is None:
            self._guid = str(uuid.uuid4())
            new = True
        else:
            self._guid = str(guid)

        # Build base keys
        self._key = '%s_%s_%s' % (self._namespace, self._name, self._guid)

        # Load data from cache or persistent backend where appropriate
        self._volatile = VolatileFactory.get_client()
        self._persistent = PersistentFactory.get_client()
        self._metadata['cache'] = None
        if new:
            self._data = {}
        else:
            self._data = self._volatile.get(self._key)
            if self._data is None:
                Toolbox.log_cache_hit('object_load', False)
                self._metadata['cache'] = False
                try:
                    self._data = self._persistent.get(self._key)
                except KeyNotFoundException:
                    raise ObjectNotFoundException('%s with guid \'%s\' could not be found' %
                                                  (self.__class__.__name__, self._guid))
            else:
                Toolbox.log_cache_hit('object_load', True)
                self._metadata['cache'] = True

        # Set default values on new fields
        for key, default in self._blueprint.iteritems():
            if key not in self._data:
                self._data[key] = default[0]
{'fraction_non_alphanumeric': 0.09674056570991729, 'fraction_numerical': 0.004100354437417472, 'mean_word_length': 2.092628411777348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4899022', 'n_tokens_mistral': 3700, 'n_tokens_neox': 3552, 'n_words': 1198}