<commit_before>import sqlitedict as sqd
from os.path import abspath, join, exists
import os
import sys
import shutil
import traceback

if sys.version_info[0] == 2:
    # fixes keyword problems with python 2.x
    os_remove = os.remove
    def new_remove(path):
        os_remove(path)
    os.remove = new_remove
    
    os_rmdir = os.rmdir
    def new_rmdir(path):
        os_rmdir(path)
    os.rmdir = new_rmdir

MAGIC_SIGN = 0xff4a87

KEY_COUNTER = '0'
KEY_SUB_DATA_KEYS = '1'

RESERVED_KEYS = (KEY_COUNTER, KEY_SUB_DATA_KEYS)

class PersistentDataStructure(object):
    """
        Note: avoid using pickled dictionaries as binary keys! The problem with dicts is
        that the order of the keys, when returned as list, depends on the hash value of
        the keys. If the keys are strings, the hash value will be randomly seeded for
        each python session, which may lead to different binary representations of the
        same dict. Therefore the same dict may actually be considered as distinct keys.
        
        The same hold true when using classes with default pickler routine as binary keys
        (because the pickler will essentially pickle the dictionary self.__dict__).
        If you want to use "complicated" python objects as binary keys make sure you
        implement your own pickle behavior without the need of dictionaries.   
    """
    def __init__(self, name, path="./", verbose=1):
        self._open = False
        self._name = name
        self._path = abspath(path)
        if not exists(self._path):
            raise RuntimeError("given path does not exists ({} -> {})".format(path, self._path))
        
        self.verbose = verbose
        
        # create directory to hold sub structures
        self._dirname = join(self._path, "__" + self._name)
        if not exists(self._dirname):
            os.mkdir(self._dirname)
        
        # open actual sqltedict
        self._filename = join(self._dirname, self._name + '.db')
        self.open()
        
        if KEY_COUNTER in self.db:
           self.counter = self.db[KEY_COUNTER] 
        else:
            self.counter = 0
            
        if KEY_SUB_DATA_KEYS in self.db:
            self.sub_data_keys = self.db[KEY_SUB_DATA_KEYS]
        else:
            self.sub_data_keys = set()
            
    def _consistency_check(self):
        self.need_open()
        
        c = 0
        
        for key in self.db:
            value = self.db[key]
            if self.__is_sub_data(value):
                c += 1
                assert key in self.sub_data_keys
        
        assert len(self.sub_data_keys) == c
        
    def __enter__(self):
        return self
    
    def __exit__(self, exc_type, exc_value, traceback):
        if self.verbose > 1:
            print("exit called for        {} in {}".format(self._name, self._dirname))
        self.close()
        
    def open(self):
        """
            open the SQL database at self._filename = <path>/__<name>/<name>.db
            as sqlitedict
        """
        if self.verbose > 1:
            print("open db                {} in {}".format(self._name, self._dirname))             
        self.db = sqd.SqliteDict(filename = self._filename, autocommit=False)
        self._open = True
        
    def is_open(self):
        return self._open
        
    def is_closed(self):
        return not self._open
    
    def need_open(self):
        if self.is_closed():
            raise RuntimeError("PersistentDataStructure needs to be open")
        
    def close(self):
        """
            close the sqligtedict ans therefore the SQL database
        """
        try:
            self.db.close()
            self._open = False
            if self.verbose > 1:
                print("closed db              {} in {}".format(self._name, self._dirname))
        except:
            if self.verbose > 1:
                print("db seem already closed {} in {}".format(self._name, self._dirname))
            
    def erase(self):
        """
            removed the database file from the disk
            
            this is called recursively for all sub PersistentDataStructure
        """
        if self.verbose > 1:
            print("erase db               {} in {}".format(self._name, self._dirname))        

        if self.is_closed():
            self.open()
            
        try:

            if self.verbose > 1:
                print("sub_data_keys:", self.sub_data_keys)
            for key in self.sub_data_keys:
                if self.verbose > 1:
                    print("call erase for key:", key, "on file", self._filename)
                sub_data = self.getData(key)
                sub_data.erase()
        except:
            traceback.print_exc()
        finally:
            self.close()

        os.remove(path = self._filename)
        try:
            os.rmdir(path = self._dirname)
        except OSError as e:
            if self.verbose > 0:
                print("Warning: directory structure can not be deleted")
                print("         {}".format(e))

    def show_stat(self, recursive = False, prepend = ""):
        prepend += self._name
        print("{}: I'm a pds called {}".format(prepend, self._name))
        print("{}:     dirname  :{}".format(prepend, self._dirname))
        print("{}:     filename :{}".format(prepend, self._filename))
        
        self.need_open()
        
        str_key = 0
        bin_key = 0
        oth_key = 0
        
        for k in self:
            if isinstance(k, str):
                str_key += 1
            elif isinstance(k, bytes):
                bin_key += 1
            else:
                oth_key += 1
                
        print("{}:     number of string keys:".format(prepend, str_key))
        print("{}:     number of byte   keys:".format(prepend, bin_key))
        if oth_key > 0:
            print("{}:     number of other  keys:".format(prepend, oth_key))
        print("{}:     number of subdata: {}".format(prepend, len(self.sub_data_keys)))
        
        if recursive:
            for k in self.sub_data_keys:
                with self.getData(k) as subdata:
                    subdata.show_stat(recursive = recursive,
                                      prepend = prepend + "->")
            
       
    def __check_key(self, key):
        """<commit_msg>minor bugfix to push to tqo<commit_after>import sqlitedict as sqd
from os.path import abspath, join, exists
import os
import sys
import shutil
import traceback

if sys.version_info[0] == 2:
    # fixes keyword problems with python 2.x
    os_remove = os.remove
    def new_remove(path):
        os_remove(path)
    os.remove = new_remove
    
    os_rmdir = os.rmdir
    def new_rmdir(path):
        os_rmdir(path)
    os.rmdir = new_rmdir

MAGIC_SIGN = 0xff4a87

KEY_COUNTER = '0'
KEY_SUB_DATA_KEYS = '1'

RESERVED_KEYS = (KEY_COUNTER, KEY_SUB_DATA_KEYS)

class PersistentDataStructure(object):
    """
        Note: avoid using pickled dictionaries as binary keys! The problem with dicts is
        that the order of the keys, when returned as list, depends on the hash value of
        the keys. If the keys are strings, the hash value will be randomly seeded for
        each python session, which may lead to different binary representations of the
        same dict. Therefore the same dict may actually be considered as distinct keys.
        
        The same hold true when using classes with default pickler routine as binary keys
        (because the pickler will essentially pickle the dictionary self.__dict__).
        If you want to use "complicated" python objects as binary keys make sure you
        implement your own pickle behavior without the need of dictionaries.   
    """
    def __init__(self, name, path="./", verbose=1):
        self._open = False
        self._name = name
        self._path = abspath(path)
        if not exists(self._path):
            raise RuntimeError("given path does not exists ({} -> {})".format(path, self._path))
        
        self.verbose = verbose
        
        # create directory to hold sub structures
        self._dirname = join(self._path, "__" + self._name)
        if not exists(self._dirname):
            os.mkdir(self._dirname)
        
        # open actual sqltedict
        self._filename = join(self._dirname, self._name + '.db')
        self.open()
        
        if KEY_COUNTER in self.db:
           self.counter = self.db[KEY_COUNTER] 
        else:
            self.counter = 0
            
        if KEY_SUB_DATA_KEYS in self.db:
            self.sub_data_keys = self.db[KEY_SUB_DATA_KEYS]
        else:
            self.sub_data_keys = set()
            
    def _consistency_check(self):
        self.need_open()
        
        c = 0
        
        for key in self.db:
            value = self.db[key]
            if self.__is_sub_data(value):
                c += 1
                assert key in self.sub_data_keys
        
        assert len(self.sub_data_keys) == c
        
    def __enter__(self):
        return self
    
    def __exit__(self, exc_type, exc_value, traceback):
        if self.verbose > 1:
            print("exit called for        {} in {}".format(self._name, self._dirname))
        self.close()
        
    def open(self):
        """
            open the SQL database at self._filename = <path>/__<name>/<name>.db
            as sqlitedict
        """
        if self.verbose > 1:
            print("open db                {} in {}".format(self._name, self._dirname))             
        self.db = sqd.SqliteDict(filename = self._filename, autocommit=False)
        self._open = True
        
    def is_open(self):
        return self._open
        
    def is_closed(self):
        return not self._open
    
    def need_open(self):
        if self.is_closed():
            raise RuntimeError("PersistentDataStructure needs to be open")
        
    def close(self):
        """
            close the sqligtedict ans therefore the SQL database
        """
        try:
            self.db.close()
            self._open = False
            if self.verbose > 1:
                print("closed db              {} in {}".format(self._name, self._dirname))
        except:
            if self.verbose > 1:
                print("db seem already closed {} in {}".format(self._name, self._dirname))
            
    def erase(self):
        """
            removed the database file from the disk
            
            this is called recursively for all sub PersistentDataStructure
        """
        if self.verbose > 1:
            print("erase db               {} in {}".format(self._name, self._dirname))        

        if self.is_closed():
            self.open()
            
        try:

            if self.verbose > 1:
                print("sub_data_keys:", self.sub_data_keys)
            for key in self.sub_data_keys:
                if self.verbose > 1:
                    print("call erase for key:", key, "on file", self._filename)
                sub_data = self.getData(key)
                sub_data.erase()
        except:
            traceback.print_exc()
        finally:
            self.close()

        os.remove(path = self._filename)
        try:
            os.rmdir(path = self._dirname)
        except OSError as e:
            if self.verbose > 0:
                print("Warning: directory structure can not be deleted")
                print("         {}".format(e))

    def show_stat(self, recursive = False, prepend = ""):
        prepend += self._name
        print("{}: I'm a pds called {}".format(prepend, self._name))
        print("{}:     dirname  :{}".format(prepend, self._dirname))
        print("{}:     filename :{}".format(prepend, self._filename))
        
        self.need_open()
        
        str_key = 0
        bin_key = 0
        oth_key = 0
        
        for k in self:
            if isinstance(k, str):
                str_key += 1
            elif isinstance(k, bytes):
                bin_key += 1
            else:
                oth_key += 1
                
        print("{}:     number of string keys: {}".format(prepend, str_key))
        print("{}:     number of byte   keys: {}".format(prepend, bin_key))
        if oth_key > 0:
            print("{}:     number of other  keys: {}".format(prepend, oth_key))
        print("{}:     number of subdata: {}".format(prepend, len(self.sub_data_keys)))
        print()
        sys.stdout.flush()
        if recursive:
            for k in self.sub_data_keys:
                with self.getData(k) as subdata:
                    subdata.show_stat(recursive = recursive,
                                      prepend = prepend + "->")
            
       
    def __check_key(self, key):
        """
{'fraction_non_alphanumeric': 0.09481481481481481, 'fraction_numerical': 0.0043664717348927875, 'mean_word_length': 1.6186198448346263, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1180621', 'n_tokens_mistral': 3653, 'n_tokens_neox': 3541, 'n_words': 1042}