<commit_before>#!/bin/env python
#------------------------------------------------------------------------
#           Copyright (c) 1997-2001 by Total Control Software
#                         All Rights Reserved
#------------------------------------------------------------------------
#
# Module Name:  dbShelve.py
#
# Description:  A reimplementation of the standard shelve.py that
#               forces the use of cPickle, and DB.
#
# Creation Date:    11/3/97 3:39:04PM
#
# License:      This is free software.  You may use this software for any
#               purpose including modification/redistribution, so long as
#               this header remains intact and that you do not claim any
#               rights of ownership or authorship of this software.  This
#               software has been tested, but no warranty is expressed or
#               implied.
#
# 13-Dec-2000:  Updated to be used with the new bsddb3 package.
#               Added DBShelfCursor class.
#
#------------------------------------------------------------------------

"""Manage shelves of pickled objects using bsddb database files for the
storage.
"""

#------------------------------------------------------------------------

import cPickle
try:
    from UserDict import DictMixin
except ImportError:
    # DictMixin is new in Python 2.3
    class DictMixin: pass
import db

#------------------------------------------------------------------------


def open(filename, flags=db.DB_CREATE, mode=0660, filetype=db.DB_HASH,
         dbenv=None, dbname=None):
    """
    A simple factory function for compatibility with the standard
    shleve.py module.  It can be used like this, where key is a string
    and data is a pickleable object:

        from bsddb import dbshelve
        db = dbshelve.open(filename)

        db[key] = data

        db.close()
    """
    if type(flags) == type(''):
        sflag = flags
        if sflag == 'r':
            flags = db.DB_RDONLY
        elif sflag == 'rw':
            flags = 0
        elif sflag == 'w':
            flags =  db.DB_CREATE
        elif sflag == 'c':
            flags =  db.DB_CREATE
        elif sflag == 'n':
            flags = db.DB_TRUNCATE | db.DB_CREATE
        else:
            raise db.DBError, "flags should be one of 'r', 'w', 'c' or 'n' or use the bsddb.db.DB_* flags"

    d = DBShelf(dbenv)
    d.open(filename, dbname, filetype, flags, mode)
    return d

#---------------------------------------------------------------------------

class DBShelveError(db.DBError): pass


class DBShelf(DictMixin):
    """A shelf to hold pickled objects, built upon a bsddb DB object.  It
    automatically pickles/unpickles data objects going to/from the DB.
    """
    def __init__(self, dbenv=None):
        self.db = db.DB(dbenv)
        self.binary = 1


    def __del__(self):
        self.close()


    def __getattr__(self, name):
        """Many methods we can just pass through to the DB object.
        (See below)
        """
        return getattr(self.db, name)


    #-----------------------------------
    # Dictionary access methods

    def __len__(self):
        return len(self.db)


    def __getitem__(self, key):
        data = self.db[key]
        return cPickle.loads(data)


    def __setitem__(self, key, value):
        data = cPickle.dumps(value, self.binary)
        self.db[key] = data


    def __delitem__(self, key):
        del self.db[key]


    def keys(self, txn=None):
        if txn != None:
            return self.db.keys(txn)
        else:
            return self.db.keys()


    def items(self, txn=None):
        if txn != None:
            items = self.db.items(txn)
        else:
            items = self.db.items()
        newitems = []

        for k, v in items:
            newitems.append( (k, cPickle.loads(v)) )
        return newitems

    def values(self, txn=None):
        if txn != None:
            values = self.db.values(txn)
        else:
            values = self.db.values()

        return map(cPickle.loads, values)

    #-----------------------------------
    # Other methods

    def __append(self, value, txn=None):
        data = cPickle.dumps(value, self.binary)
        return self.db.append(data, txn)

    def append(self, value, txn=None):
        if self.get_type() == db.DB_RECNO:
            self.append = self.__append
            return self.append(value, txn=txn)
        raise DBShelveError, "append() only supported when dbshelve opened with filetype=dbshelve.db.DB_RECNO"


    def associate(self, secondaryDB, callback, flags=0):
        def _shelf_callback(priKey, priData, realCallback=callback):
            data = cPickle.loads(priData)
            return realCallback(priKey, data)
        return self.db.associate(secondaryDB, _shelf_callback, flags)


    #def get(self, key, default=None, txn=None, flags=0):
    def get(self, *args, **kw):<commit_msg>Backport 58450: fix uncollectable reference leak in bsddb.db.DBShelf.append<commit_after>#!/bin/env python
#------------------------------------------------------------------------
#           Copyright (c) 1997-2001 by Total Control Software
#                         All Rights Reserved
#------------------------------------------------------------------------
#
# Module Name:  dbShelve.py
#
# Description:  A reimplementation of the standard shelve.py that
#               forces the use of cPickle, and DB.
#
# Creation Date:    11/3/97 3:39:04PM
#
# License:      This is free software.  You may use this software for any
#               purpose including modification/redistribution, so long as
#               this header remains intact and that you do not claim any
#               rights of ownership or authorship of this software.  This
#               software has been tested, but no warranty is expressed or
#               implied.
#
# 13-Dec-2000:  Updated to be used with the new bsddb3 package.
#               Added DBShelfCursor class.
#
#------------------------------------------------------------------------

"""Manage shelves of pickled objects using bsddb database files for the
storage.
"""

#------------------------------------------------------------------------

import cPickle
try:
    from UserDict import DictMixin
except ImportError:
    # DictMixin is new in Python 2.3
    class DictMixin: pass
import db

#------------------------------------------------------------------------


def open(filename, flags=db.DB_CREATE, mode=0660, filetype=db.DB_HASH,
         dbenv=None, dbname=None):
    """
    A simple factory function for compatibility with the standard
    shleve.py module.  It can be used like this, where key is a string
    and data is a pickleable object:

        from bsddb import dbshelve
        db = dbshelve.open(filename)

        db[key] = data

        db.close()
    """
    if type(flags) == type(''):
        sflag = flags
        if sflag == 'r':
            flags = db.DB_RDONLY
        elif sflag == 'rw':
            flags = 0
        elif sflag == 'w':
            flags =  db.DB_CREATE
        elif sflag == 'c':
            flags =  db.DB_CREATE
        elif sflag == 'n':
            flags = db.DB_TRUNCATE | db.DB_CREATE
        else:
            raise db.DBError, "flags should be one of 'r', 'w', 'c' or 'n' or use the bsddb.db.DB_* flags"

    d = DBShelf(dbenv)
    d.open(filename, dbname, filetype, flags, mode)
    return d

#---------------------------------------------------------------------------

class DBShelveError(db.DBError): pass


class DBShelf(DictMixin):
    """A shelf to hold pickled objects, built upon a bsddb DB object.  It
    automatically pickles/unpickles data objects going to/from the DB.
    """
    def __init__(self, dbenv=None):
        self.db = db.DB(dbenv)
        self.binary = 1


    def __del__(self):
        self.close()


    def __getattr__(self, name):
        """Many methods we can just pass through to the DB object.
        (See below)
        """
        return getattr(self.db, name)


    #-----------------------------------
    # Dictionary access methods

    def __len__(self):
        return len(self.db)


    def __getitem__(self, key):
        data = self.db[key]
        return cPickle.loads(data)


    def __setitem__(self, key, value):
        data = cPickle.dumps(value, self.binary)
        self.db[key] = data


    def __delitem__(self, key):
        del self.db[key]


    def keys(self, txn=None):
        if txn != None:
            return self.db.keys(txn)
        else:
            return self.db.keys()


    def items(self, txn=None):
        if txn != None:
            items = self.db.items(txn)
        else:
            items = self.db.items()
        newitems = []

        for k, v in items:
            newitems.append( (k, cPickle.loads(v)) )
        return newitems

    def values(self, txn=None):
        if txn != None:
            values = self.db.values(txn)
        else:
            values = self.db.values()

        return map(cPickle.loads, values)

    #-----------------------------------
    # Other methods

    def __append(self, value, txn=None):
        data = cPickle.dumps(value, self.binary)
        return self.db.append(data, txn)

    def append(self, value, txn=None):
        if self.get_type() == db.DB_RECNO:
            return self.__append(value, txn=txn)
        raise DBShelveError, "append() only supported when dbshelve opened with filetype=dbshelve.db.DB_RECNO"


    def associate(self, secondaryDB, callback, flags=0):
        def _shelf_callback(priKey, priData, realCallback=callback):
            data = cPickle.loads(priData)
            return realCallback(priKey, data)
        return self.db.associate(secondaryDB, _shelf_callback, flags)


    #def get(self, key, default=None, txn=None, flags=0):
    def get(self, *args, **kw):
{'fraction_non_alphanumeric': 0.1921357447673237, 'fraction_numerical': 0.007620402357244462, 'mean_word_length': 1.5460424211070873, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2842110', 'n_tokens_mistral': 2922, 'n_tokens_neox': 2764, 'n_words': 848}