<commit_before># the alternative for store_file
import sys
import sqlite3
import hashlib


class data_base():
    # laxect.data_base.1.0.0
    def __init__(self, sid):
        self.sid = sid.replace('.', '_')
        self.status = False
        self.path = sys.path[0] + '/laxect.data_base.tmp'
        self.hash = hashlib.new('ripemd160')

    def _table_build_up(self):
        cur = sqlite3.connect(self.path).cursor()
        cur.execute(f'create table {self.sid} (id text, hash text)')

    def __enter__(self):
        self.conn = sqlite3.connect(self.path)
        self.status = True
        return self

    def check_up_to_date(self, cid, content):
        if not self.status:
            raise RuntimeError('Error: status not correct.')
        cur = self.conn.cursor()
        try:
            cur.execute(f'select hash from \'{self.sid}\' where id=\'{cid}\'')
        except sqlite3.OperationalError:
            self._table_build_up()
        res = cur.fetchall()
        if res:
            return False
        self.hash.update(str(content).encode('utf-8'))
        _hash = self.hash.hexdigest()
        cur.execute(f'insert into {self.sid} values(\'{cid}\', \'{_hash}\')')
        return True

    def __exit__(self, exc_ty, exc_val, tb):
        self.conn.commit()
        self.conn.close()
        self.status = False<commit_msg>fix bug of data_base and update it to 1.0.1<commit_after># the alternative for store_file
import sys
import sqlite3
import hashlib


class data_base():
    # laxect.data_base.1.0.1
    def __init__(self, sid):
        self.sid = sid.replace('.', '_')
        self.status = False
        self.path = sys.path[0] + '/laxect.data_base.tmp'
        self.hash = hashlib.new('ripemd160')

    def _table_build_up(self):
        cur = sqlite3.connect(self.path).cursor()
        cur.execute(f'create table {self.sid} (id text, hash text)')

    def __enter__(self):
        self.conn = sqlite3.connect(self.path)
        self.status = True
        return self

    def check_up_to_date(self, cid, content):
        if not self.status:
            raise RuntimeError('Error: status not correct.')
        cur = self.conn.cursor()
        try:
            cur.execute(f'select hash from \'{self.sid}\' where id=\'{cid}\'')
        except sqlite3.OperationalError:
            self._table_build_up()
        res = cur.fetchall()
        self.hash.update(str(content).encode('utf-8'))
        _hash = self.hash.hexdigest()
        if res and res[0][0] == _hash:
            return False
        cur.execute(f'insert into {self.sid} values(\'{cid}\', \'{_hash}\')')
        return True

    def __exit__(self, exc_ty, exc_val, tb):
        self.conn.commit()
        self.conn.close()
        self.status = False
{'fraction_non_alphanumeric': 0.1368765926465235, 'fraction_numerical': 0.010556971241354204, 'mean_word_length': 2.2520710059171596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7109155', 'n_tokens_mistral': 904, 'n_tokens_neox': 876, 'n_words': 198}