<commit_before>from rdflib.backends import Backend

from rdflib.util import from_bits
from rdflib import BNode

from bsddb import db
from base64 import b64encode
from base64 import b64decode
from os import mkdir
from os.path import exists

# TODO: tool to convert old Sleepycat DBs to this version.

class Sleepycat(Backend):

    def __init__(self):
        super(Sleepycat, self).__init__()
        self.__open = False
	self.identifier = BNode() # TODO: move to create and persisit
        
    def open(self, path, create=True):
        homeDir = path        
        envsetflags  = db.DB_CDB_ALLDB
        envflags = db.DB_INIT_MPOOL | db.DB_INIT_CDB | db.DB_THREAD
        try:
            if not exists(homeDir):
                mkdir(homeDir)
        except Exception, e:
            print e
        self.env = env = db.DBEnv()
        env.set_cachesize(0, 1024*1024*50) # TODO
        #env.set_lg_max(1024*1024)
        env.set_flags(envsetflags, 1)
        env.open(homeDir, envflags | db.DB_CREATE)

        self.__open = True
        
        dbname = None
        dbtype = db.DB_BTREE
        dbopenflags = db.DB_THREAD
        
        dbmode = 0660
        dbsetflags   = 0

        # create and open the DBs
        self.__spo = db.DB(env)
        self.__spo.set_flags(dbsetflags)
        self.__spo.open("spo", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__pos = db.DB(env)
        self.__pos.set_flags(dbsetflags)
        self.__pos.open("pos", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__osp = db.DB(env)
        self.__osp.set_flags(dbsetflags)
        self.__osp.open("osp", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__cspo = db.DB(env)
        self.__cspo.set_flags(dbsetflags)
        self.__cspo.open("cspo", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__cpos = db.DB(env)
        self.__cpos.set_flags(dbsetflags)
        self.__cpos.open("cpos", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__cosp = db.DB(env)
        self.__cosp.set_flags(dbsetflags)
        self.__cosp.open("cosp", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__contexts = db.DB(env)
        self.__contexts.set_flags(dbsetflags)
        self.__contexts.open("contexts", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__namespace = db.DB(env)
        self.__namespace.set_flags(dbsetflags)
        self.__namespace.open("namespace", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__prefix = db.DB(env)
        self.__prefix.set_flags(dbsetflags)
        self.__prefix.open("prefix", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)
        
        self.__pending_sync = None
	self.__syncing = False

    def _schedule_sync(self):
        from threading import Timer
        if self.__open and self.__pending_sync is None:
            t = Timer(60.0, self.sync)
            self.__pending_sync = t
            t.setDaemon(True)
            t.start()
    
    def sync(self):
        if self.__open:
	    self.__syncing = True
            self.__spo.sync()
            self.__pos.sync()
            self.__osp.sync()

            self.__cspo.sync()
            self.__cpos.sync()
            self.__cosp.sync()

            self.__contexts.sync()
            self.__namespace.sync()
            self.__prefix.sync()
        self.__pending_sync = None
	self.__syncing = False

    def close(self):
        self.__open = False

	if self.__pending_sync and self.__syncing:
	    self.__pending_sync.join()
	else:
	    self.__pending_sync.cancel()

        self.__spo.close()
        self.__pos.close()
        self.__osp.close()

        self.__cspo.close()
        self.__cpos.close()
        self.__cosp.close()

        self.__contexts.close()
        self.__namespace.close()
        self.__prefix.close()
        self.env.close()

    def add(self, (subject, predicate, object), context, quoted=False):
        """\
        Add a triple to the store of triples.
        """
        assert self.__open, "The InformationStore must be open."

        _to_string = self._to_string
        _split = self._split

        s = _to_string(subject)
        p = _to_string(predicate)
        o = _to_string(object)
        c = _to_string(context)
        
        self.__contexts.put(c, "")        
        self.__cspo.put("%s^%s^%s^%s" % (c, s, p, o), "")
        self.__cpos.put("%s^%s^%s^%s" % (c, p, o, s), "")
        self.__cosp.put("%s^%s^%s^%s" % (c, o, s, p), "")<commit_msg>fixed for case __pending_sync is None<commit_after>from rdflib.backends import Backend

from rdflib.util import from_bits
from rdflib import BNode

from bsddb import db
from base64 import b64encode
from base64 import b64decode
from os import mkdir
from os.path import exists

# TODO: tool to convert old Sleepycat DBs to this version.

class Sleepycat(Backend):

    def __init__(self):
        super(Sleepycat, self).__init__()
        self.__open = False
	self.identifier = BNode() # TODO: move to create and persisit
        
    def open(self, path, create=True):
        homeDir = path        
        envsetflags  = db.DB_CDB_ALLDB
        envflags = db.DB_INIT_MPOOL | db.DB_INIT_CDB | db.DB_THREAD
        try:
            if not exists(homeDir):
                mkdir(homeDir)
        except Exception, e:
            print e
        self.env = env = db.DBEnv()
        env.set_cachesize(0, 1024*1024*50) # TODO
        #env.set_lg_max(1024*1024)
        env.set_flags(envsetflags, 1)
        env.open(homeDir, envflags | db.DB_CREATE)

        self.__open = True
        
        dbname = None
        dbtype = db.DB_BTREE
        dbopenflags = db.DB_THREAD
        
        dbmode = 0660
        dbsetflags   = 0

        # create and open the DBs
        self.__spo = db.DB(env)
        self.__spo.set_flags(dbsetflags)
        self.__spo.open("spo", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__pos = db.DB(env)
        self.__pos.set_flags(dbsetflags)
        self.__pos.open("pos", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__osp = db.DB(env)
        self.__osp.set_flags(dbsetflags)
        self.__osp.open("osp", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__cspo = db.DB(env)
        self.__cspo.set_flags(dbsetflags)
        self.__cspo.open("cspo", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__cpos = db.DB(env)
        self.__cpos.set_flags(dbsetflags)
        self.__cpos.open("cpos", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__cosp = db.DB(env)
        self.__cosp.set_flags(dbsetflags)
        self.__cosp.open("cosp", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__contexts = db.DB(env)
        self.__contexts.set_flags(dbsetflags)
        self.__contexts.open("contexts", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__namespace = db.DB(env)
        self.__namespace.set_flags(dbsetflags)
        self.__namespace.open("namespace", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)

        self.__prefix = db.DB(env)
        self.__prefix.set_flags(dbsetflags)
        self.__prefix.open("prefix", dbname, dbtype, dbopenflags|db.DB_CREATE, dbmode)
        
        self.__pending_sync = None
	self.__syncing = False

    def _schedule_sync(self):
        from threading import Timer
        if self.__open and self.__pending_sync is None:
            t = Timer(60.0, self.sync)
            self.__pending_sync = t
            t.setDaemon(True)
            t.start()
    
    def sync(self):
        if self.__open:
	    self.__syncing = True
            self.__spo.sync()
            self.__pos.sync()
            self.__osp.sync()

            self.__cspo.sync()
            self.__cpos.sync()
            self.__cosp.sync()

            self.__contexts.sync()
            self.__namespace.sync()
            self.__prefix.sync()
        self.__pending_sync = None
	self.__syncing = False

    def close(self):
        self.__open = False
	if self.__pending_sync:
            if self.__syncing:
                self.__pending_sync.join()
            else:
                self.__pending_sync.cancel()
        self.__spo.close()
        self.__pos.close()
        self.__osp.close()

        self.__cspo.close()
        self.__cpos.close()
        self.__cosp.close()

        self.__contexts.close()
        self.__namespace.close()
        self.__prefix.close()
        self.env.close()

    def add(self, (subject, predicate, object), context, quoted=False):
        """\
        Add a triple to the store of triples.
        """
        assert self.__open, "The InformationStore must be open."

        _to_string = self._to_string
        _split = self._split

        s = _to_string(subject)
        p = _to_string(predicate)
        o = _to_string(object)
        c = _to_string(context)
        
        self.__contexts.put(c, "")        
        self.__cspo.put("%s^%s^%s^%s" % (c, s, p, o), "")
        self.__cpos.put("%s^%s^%s^%s" % (c, p, o, s), "")
        self.__cosp.put("%s^%s^%s^%s" % (c, o, s, p), "")
{'fraction_non_alphanumeric': 0.11415725408487773, 'fraction_numerical': 0.00789560258800307, 'mean_word_length': 2.105209397344229, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '95519', 'n_tokens_mistral': 3096, 'n_tokens_neox': 3001, 'n_words': 593}