<commit_before>import copy
import yaml
import types

import numpy as np
import opentis as ops
import simtk.unit as u

# TODO: Combine the cache and all_names to be stored in one bis dict

class ObjectStorage(object):
    """
    Base Class for storing complex objects in a netCDF4 file. It holds a reference to the store file.
    """

    def __init__(self, storage, obj, named=False, json=False, identifier=None, dimension_units=None, enable_caching=True, load_lazy=False):
        """
        Attributes
        ----------

        storage : Storage
        content_class : class
            a reference to the class type to be stored using this Storage
        idx_dimension : str
            name of the dimension used for major numbering the stored object in the netCDF file.
            This is usually the lowercase class name
        cache : dict {int : object}
            A dictionary pointing to the actual stored object by index. It is filled at saving or loading
        named : bool
            Set, if objects can also be loaded by a string identifier/name
        json : string
            if already computed a JSON Serialized string of the object
        all_names : dict
            same as cache but for names
        simplifier : util.StorableObjectJSON
            an instance of a JSON Serializer
        identifier : str
            name of the netCDF variable that contains the string to be identified by
        enable_caching : bool
            if True (default) this will add caching to this storage which keeps once loaded
            or saved objects in memory for faster later access


        """
        self.storage = storage
        self.content_class = obj
        self.idx_dimension = obj.__name__.lower()
        self.db = obj.__name__.lower()
        self.cache = dict()
        self.named = named
        self.json = json
        self.all_names = None
        self.simplifier = ops.storage.StorableObjectJSON(storage)
        self._names_loaded = False
        if identifier is not None:
            self.identifier = self.idx_dimension + '_' + identifier
        else:
            self.identifier = None

        if dimension_units is not None:
            self.dimension_units = dimension_units
        else:
            self.dimension_units = {}

        if load_lazy:
            # wrap load to load lazy. Can now be used with all load functions.
            # I effectively creates a Stub that is replaced by the real object
            # once an attribute is accessed
            # This should be called first so that the cache superseeds loading

            _load = self.load
            self.load = types.MethodType(loadlazy(_load), self)


        if enable_caching:
            # wrap load/save to make this work. I use MethodType here to bind the
            # wrapped function to this instance. An alternative would be to
            # add the wrapper to the class itself, which would mean that all
            # instances have the same setting and a change would be present in all
            # instances. E.g., first instance has caching and the second not
            # when the second instance is created the change in the class would
            # also disable caching in the first instance. The present way with
            # bound methods is more flexible
            # Should be not really important, since there will be mostly only one
            # storage, but this way it is cleaner

            _save = self.save
            self.save = types.MethodType(savecache(_save), self)

            _load = self.load
            self.load = types.MethodType(loadcache(_load), self)


    def __str__(self):
        """
        Make a nice output of the store
        """

        return "Store '" + self.db + "'"

    def idx(self, obj):
        if hasattr(obj, 'idx'):
            if self in obj.idx:
                return obj.idx[self]

        return None

    @property
    def units(self):
        """
        Return the units dictionary used in the attached storage

        Returns
        -------
        units : dict of {str : simtk.unit.Unit }
            representing a dict of string representing a dimension ('length', 'velocity', 'energy')
            pointing to the simtk.unit.Unit to be used

        """
        return self.storage.units

    def register(self, nestable = False):
        self.storage._storages[self.content_class] = self
        self.storage._storages[self.content_class.__name__] = self
        self.storage._storages[self.content_class.__name__.lower()] = self

        # Add here the logic to change the actual class and add the decorator
        # this is the same as add the  decorator (without default_storage,
        # I removed this since it is not used anyway)
        # all it does is make sure that there is a .idx property and the base_cls is known

        self.content_class.base_cls_name = self.content_class.__name__
        self.content_class.base_cls = self.content_class

        # add a property idx that keeps the storage reference

#        print 'REGISTER ', self.content_class.__name__

        def _idx(this):
            if not hasattr(this, '_idx'):
                this._idx = dict()

            return this._idx

        def _save(this, storage):
            storage.save(this)

        if nestable:
            self.content_class.nestable = True

        self.content_class.save = _save
        self.content_class.idx = property(_idx)

        # register as a base_class for storable objects
        self.storage.links.append(self)

        return self

    def attach_partial_loading(self):
        # fix the object class to allow partial loading

        cls = self.content_class

        def _getattr(self, item):
            if hasattr(cls, '_delayed_loading'):
                if item in cls._delayed_loading:
                    if self.__dict__[item] is None:
                        # apparently the part has not yet been loaded so get it
                        _loader = cls._delayed_loading['item']
                        setattr(self, item, _loader(self))

            return self.__dict__[item]

        setattr(cls, '__getattr__', _getattr)

    def set_variable_partial_loading(self, variable, loader):
        cls = self.content_class
        cls._delayed_loading[variable] = loader

    def copy(self):
        """
        Create a deep copy of the ObjectStorage instance

        Returns
        -------
        ObjectStorage()
            the copied instance
        """<commit_msg>add automatic partial loading of objects<commit_after>import copy
import yaml
import types

import numpy as np
import opentis as ops
import simtk.unit as u

# TODO: Combine the cache and all_names to be stored in one bis dict

class ObjectStorage(object):
    """
    Base Class for storing complex objects in a netCDF4 file. It holds a reference to the store file.
    """

    def __init__(self, storage, obj, named=False, json=False, identifier=None, dimension_units=None, enable_caching=True, load_lazy=False):
        """
        Attributes
        ----------

        storage : Storage
        content_class : class
            a reference to the class type to be stored using this Storage
        idx_dimension : str
            name of the dimension used for major numbering the stored object in the netCDF file.
            This is usually the lowercase class name
        cache : dict {int : object}
            A dictionary pointing to the actual stored object by index. It is filled at saving or loading
        named : bool
            Set, if objects can also be loaded by a string identifier/name
        json : string
            if already computed a JSON Serialized string of the object
        all_names : dict
            same as cache but for names
        simplifier : util.StorableObjectJSON
            an instance of a JSON Serializer
        identifier : str
            name of the netCDF variable that contains the string to be identified by
        enable_caching : bool
            if True (default) this will add caching to this storage which keeps once loaded
            or saved objects in memory for faster later access


        """
        self.storage = storage
        self.content_class = obj
        self.idx_dimension = obj.__name__.lower()
        self.db = obj.__name__.lower()
        self.cache = dict()
        self.named = named
        self.json = json
        self.all_names = None
        self.simplifier = ops.storage.StorableObjectJSON(storage)
        self._names_loaded = False
        if identifier is not None:
            self.identifier = self.idx_dimension + '_' + identifier
        else:
            self.identifier = None

        if dimension_units is not None:
            self.dimension_units = dimension_units
        else:
            self.dimension_units = {}

        if load_lazy:
            # wrap load to load lazy. Can now be used with all load functions.
            # I effectively creates a Stub that is replaced by the real object
            # once an attribute is accessed
            # This should be called first so that the cache superseeds loading

            _load = self.load
            self.load = types.MethodType(loadlazy(_load), self)


        if enable_caching:
            # wrap load/save to make this work. I use MethodType here to bind the
            # wrapped function to this instance. An alternative would be to
            # add the wrapper to the class itself, which would mean that all
            # instances have the same setting and a change would be present in all
            # instances. E.g., first instance has caching and the second not
            # when the second instance is created the change in the class would
            # also disable caching in the first instance. The present way with
            # bound methods is more flexible
            # Should be not really important, since there will be mostly only one
            # storage, but this way it is cleaner

            _save = self.save
            self.save = types.MethodType(savecache(_save), self)

            _load = self.load
            self.load = types.MethodType(loadcache(_load), self)


    def __str__(self):
        """
        Make a nice output of the store
        """

        return "Store '" + self.db + "'"

    def idx(self, obj):
        if hasattr(obj, 'idx'):
            if self in obj.idx:
                return obj.idx[self]

        return None

    @property
    def units(self):
        """
        Return the units dictionary used in the attached storage

        Returns
        -------
        units : dict of {str : simtk.unit.Unit }
            representing a dict of string representing a dimension ('length', 'velocity', 'energy')
            pointing to the simtk.unit.Unit to be used

        """
        return self.storage.units

    def register(self, nestable = False):
        self.storage._storages[self.content_class] = self
        self.storage._storages[self.content_class.__name__] = self
        self.storage._storages[self.content_class.__name__.lower()] = self

        # Add here the logic to change the actual class and add the decorator
        # this is the same as add the  decorator (without default_storage,
        # I removed this since it is not used anyway)
        # all it does is make sure that there is a .idx property and the base_cls is known

        self.content_class.base_cls_name = self.content_class.__name__
        self.content_class.base_cls = self.content_class

        # add a property idx that keeps the storage reference

#        print 'REGISTER ', self.content_class.__name__

        def _idx(this):
            if not hasattr(this, '_idx'):
                this._idx = dict()

            return this._idx

        def _save(this, storage):
            storage.save(this)

        if nestable:
            self.content_class.nestable = True

        self.content_class.save = _save
        self.content_class.idx = property(_idx)

        # register as a base_class for storable objects
        self.storage.links.append(self)

        return self

    def attach_partial_loading(self):
        # fix the object class to allow partial loading

        cls = self.content_class

        def _getattr(self, item):
            if hasattr(cls, '_delayed_loading'):
                if item in cls._delayed_loading:
                    if self.__dict__[item] is None:
                        # apparently the part has not yet been loaded so get it
                        _loader = cls._delayed_loading['item']
                        _loader(self)

            return self.__dict__[item]

        setattr(cls, '__getattr__', _getattr)

    def set_variable_partial_loading(self, variable, loader):
        cls = self.content_class
        cls._delayed_loading[variable] = loader

    def copy(self):
        """
        Create a deep copy of the ObjectStorage instance

        Returns
        -------
        ObjectStorage()
            the copied instance
        """
{'fraction_non_alphanumeric': 0.06337058013163938, 'fraction_numerical': 0.00015306903413439462, 'mean_word_length': 1.9337674000898069, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3083357', 'n_tokens_mistral': 3289, 'n_tokens_neox': 3176, 'n_words': 1303}