manage shelves of pickled objects a shelf is a persistent dictionarylike object the difference with dbm databases is that the values not the keys in a shelf can be essentially arbitrary python objects anything that the pickle module can handle this includes most class instances recursive data types and objects containing lots of shared subobjects the keys are ordinary strings to summarize the interface key is a string data is an arbitrary object import shelve d shelve openfilename open with gdbm filename no suffix dkey data store data at key overwrites old data if using an existing key data dkey retrieve a copy of the data at key raise keyerror if no such key note that this access returns a copy of the entry del dkey delete data stored at key raises keyerror if no such key flag key in d true if the key exists list d keys a list of all existing keys slow d close close it dependent on the implementation closing a persistent dictionary may or may not be necessary to flush changes to disk normally dkey returns a copy of the entry this needs care when mutable entries are mutated for example if dkey is a list dkey appendanitem does not modify the entry dkey itself as stored in the persistent mapping it only modifies the copy which is then immediately discarded so that the append has no effect whatsoever to append an item to dkey in a way that will affect the persistent mapping use data dkey data appendanitem dkey data to avoid the problem with mutable entries you may pass the keyword argument writebacktrue in the call to shelve open when you use d shelve openfilename writebacktrue then d keeps a cache of all entries you access and writes them all back to the persistent mapping when you call d close this ensures that such usage as dkey appendanitem works as intended however using keyword argument writebacktrue may consume vast amount of memory for the cache and it may make d close very slow if you access many of d s entries after opening it in this way d has no way to check which of the entries you access are mutable andor which ones you actually mutate so it must cache and write back at close all of the entries that you access you can call d sync to write back all the entries in the cache and empty the cache d sync also synchronizes the persistent dictionary on disk if feasible base class for shelf implementations this is initialized with a dictionarylike object see the module s doc string for an overview of the interface catch errors that may happen when close is called from del because cpython is in interpreter shutdown init didn t succeed so don t bother closing see http bugs python orgissue1339007 for details shelf implementation using the bsd db interface this adds methods first next previous last and setlocation that have no counterpart in gdbm databases the actual database must be opened using one of the bsddb modules open routines i e bsddb hashopen bsddb btopen or bsddb rnopen and passed to the constructor see the module s doc string for an overview of the interface shelf implementation using the dbm generic dbm interface this is initialized with the filename for the dbm database see the module s doc string for an overview of the interface remove all items from the shelf call through to the clear method on dbmbacked shelves see https github compythoncpythonissues107089 self cache clear self dict clear def openfilename flag c protocolnone writebackfalse return dbfilenameshelffilename flag protocol writeback base class for shelf implementations this is initialized with a dictionary like object see the module s __doc__ string for an overview of the interface catch errors that may happen when close is called from __del__ because cpython is in interpreter shutdown __init__ didn t succeed so don t bother closing see http bugs python org issue1339007 for details shelf implementation using the bsd db interface this adds methods first next previous last and set_location that have no counterpart in g dbm databases the actual database must be opened using one of the bsddb modules open routines i e bsddb hashopen bsddb btopen or bsddb rnopen and passed to the constructor see the module s __doc__ string for an overview of the interface shelf implementation using the dbm generic dbm interface this is initialized with the filename for the dbm database see the module s __doc__ string for an overview of the interface remove all items from the shelf call through to the clear method on dbm backed shelves see https github com python cpython issues 107089 open a persistent dictionary for reading and writing the filename parameter is the base filename for the underlying database as a side effect an extension may be added to the filename and more than one file may be created the optional flag parameter has the same interpretation as the flag parameter of dbm open the optional protocol parameter specifies the version of the pickle protocol see the module s __doc__ string for an overview of the interface
from pickle import DEFAULT_PROTOCOL, Pickler, Unpickler
from io import BytesIO

import collections.abc

__all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"]

class _ClosedDict(collections.abc.MutableMapping):
    'Marker for a closed dict.  Access attempts raise a ValueError.'

    def closed(self, *args):
        raise ValueError('invalid operation on closed shelf')
    __iter__ = __len__ = __getitem__ = __setitem__ = __delitem__ = keys = closed

    def __repr__(self):
        return '<Closed Dictionary>'


class Shelf(collections.abc.MutableMapping):
    

    def __init__(self, dict, protocol=None, writeback=False,
                 keyencoding="utf-8"):
        self.dict = dict
        if protocol is None:
            protocol = DEFAULT_PROTOCOL
        self._protocol = protocol
        self.writeback = writeback
        self.cache = {}
        self.keyencoding = keyencoding

    def __iter__(self):
        for k in self.dict.keys():
            yield k.decode(self.keyencoding)

    def __len__(self):
        return len(self.dict)

    def __contains__(self, key):
        return key.encode(self.keyencoding) in self.dict

    def get(self, key, default=None):
        if key.encode(self.keyencoding) in self.dict:
            return self[key]
        return default

    def __getitem__(self, key):
        try:
            value = self.cache[key]
        except KeyError:
            f = BytesIO(self.dict[key.encode(self.keyencoding)])
            value = Unpickler(f).load()
            if self.writeback:
                self.cache[key] = value
        return value

    def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = BytesIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key.encode(self.keyencoding)] = f.getvalue()

    def __delitem__(self, key):
        del self.dict[key.encode(self.keyencoding)]
        try:
            del self.cache[key]
        except KeyError:
            pass

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.close()

    def close(self):
        if self.dict is None:
            return
        try:
            self.sync()
            try:
                self.dict.close()
            except AttributeError:
                pass
        finally:
            
            
            try:
                self.dict = _ClosedDict()
            except:
                self.dict = None

    def __del__(self):
        if not hasattr(self, 'writeback'):
            
            
            return
        self.close()

    def sync(self):
        if self.writeback and self.cache:
            self.writeback = False
            for key, entry in self.cache.items():
                self[key] = entry
            self.writeback = True
            self.cache = {}
        if hasattr(self.dict, 'sync'):
            self.dict.sync()


class BsdDbShelf(Shelf):
    

    def __init__(self, dict, protocol=None, writeback=False,
                 keyencoding="utf-8"):
        Shelf.__init__(self, dict, protocol, writeback, keyencoding)

    def set_location(self, key):
        (key, value) = self.dict.set_location(key)
        f = BytesIO(value)
        return (key.decode(self.keyencoding), Unpickler(f).load())

    def next(self):
        (key, value) = next(self.dict)
        f = BytesIO(value)
        return (key.decode(self.keyencoding), Unpickler(f).load())

    def previous(self):
        (key, value) = self.dict.previous()
        f = BytesIO(value)
        return (key.decode(self.keyencoding), Unpickler(f).load())

    def first(self):
        (key, value) = self.dict.first()
        f = BytesIO(value)
        return (key.decode(self.keyencoding), Unpickler(f).load())

    def last(self):
        (key, value) = self.dict.last()
        f = BytesIO(value)
        return (key.decode(self.keyencoding), Unpickler(f).load())


class DbfilenameShelf(Shelf):
    

    def __init__(self, filename, flag='c', protocol=None, writeback=False):
        import dbm
        Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback)

    def clear(self):
        
        
        
        self.cache.clear()
        self.dict.clear()


def open(filename, flag='c', protocol=None, writeback=False):
    

    return DbfilenameShelf(filename, flag, protocol, writeback)