<commit_before>import threading
import os
import time
import sqlite3

import BoardManager
import PostEntry
import Config
from Log import Log
from Util import Util

INDEX_INTERVAL = 15
INDEX_DB = "index.db"

class State(object):
    def __init__(self):
        self.locks = {}

class IndexBoardInfo(object):
    def __init__(self, last_idx):
        self.last_idx = last_idx

class FastIndexer(threading.Thread):
    def __init__(self, state):
        threading.Thread.__init__(self)
        self.stopped = False
        self.conn = sqlite3.connect(os.path.join(Config.BBS_ROOT, INDEX_DB),
                detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES)
        self.conn.row_factory = sqlite3.Row
        self.board_info = {}
        self.state = state
        try:
            self.index_boards()
        except Exception as exc:
            Log.error("Exception caught initializing FastIndexer: %r" % exc)
            raise exc

    def run(self):
        while True:
            if self.stopped:
                break
            try:
                self.index_boards()
            except Exception as exc:
                Log.error("Exception caught in FastIndexer: %r" % exc)

            time.sleep(INDEX_INTERVAL)

    def init_db(self, board):
        self.conn.execute("drop table if exists %s" % self.table_name(board))
        self.conn.execute("create table %s("\
                "id int, xid int, tid int, rid int, time int"\
                ")" % self.table_name(board))

    def index_boards(self):
        boards = BoardManager.BoardManager.boards.keys()
        for board in boards:
            try:
                self.index_board(board)
            except Exception as exc:
                Log.error("Exception caught when indexing %s: %r"
                        % (board, exc))

    def index_board(self, board):
        boardobj = BoardManager.BoardManager.GetBoard(board)
        if not boardobj:
            Log.error("Error loading board %s" % board)
            return

        if board in self.board_info:
            idx_obj = self.board_info[board]
        else:
            idx_obj = IndexBoardInfo(0)
            self.board_info[board] = idx_obj

        bdir_path = boardobj.GetDirPath()
        with open(bdir_path, 'rb') as bdir:
            Util.FLock(bdir, shared=True)
            try:
                st = os.stat(bdir_path)
                if st.st_mtime <= idx_obj.last_idx:
                    # why <? anyway...
                    return

                if not board in self.state.locks:
                    self.state.locks[board] = threading.Lock()

                self.state.locks[board].acquire()
                try:
                    self.init_db(board)
                    for idx in xrange(st.st_size / PostEntry.PostEntry.size):
                        pe = PostEntry.PostEntry(
                                bdir.read(PostEntry.PostEntry.size))
                        self.insert_entry(board, pe, idx)
                    idx_obj.last_idx = st.st_mtime
                finally:
                    self.state.locks[board].release()
            finally:
                Util.FUnlock(bdir)

    def insert_entry(self, board, pe, idx):
        self.conn.execute("insert into %s values (?, ?, ?, ?, ?)"
                % self.table_name(board),
                (idx, pe.id, pe.groupid, pe.reid, pe.GetPostTime()))

    def table_name(self, board):
        return "idx_" + board

<commit_msg>save index status<commit_after>import threading
import os
import time
import sqlite3

import BoardManager
import PostEntry
import Config
from Log import Log
from Util import Util

INDEX_INTERVAL = 15
INDEX_DB = "index.db"

class State(object):
    def __init__(self):
        self.locks = {}

class IndexBoardInfo(object):
    def __init__(self, board, last_idx):
        self.board = board
        self.last_idx = last_idx

class FastIndexer(threading.Thread):
    def __init__(self, state):
        threading.Thread.__init__(self)
        self.stopped = False
        self.conn = sqlite3.connect(os.path.join(Config.BBS_ROOT, INDEX_DB),
                detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES)
        self.conn.row_factory = sqlite3.Row
        self.board_info = {}
        self.load_idx_status()
        self.state = state
        try:
            self.index_boards()
        except Exception as exc:
            Log.error("Exception caught initializing FastIndexer: %r" % exc)
            raise exc

    def run(self):
        while True:
            if self.stopped:
                break
            try:
                self.index_boards()
            except Exception as exc:
                Log.error("Exception caught in FastIndexer: %r" % exc)

            time.sleep(INDEX_INTERVAL)

    def init_db(self, board):
        self.conn.execute("drop table if exists %s" % self.table_name(board))
        self.conn.execute("create table %s("\
                "id int, xid int, tid int, rid int, time int"\
                ")" % self.table_name(board))
        self.conn.commit()

    def load_idx_status(self):
        self.conn.execute("create table if not exists status("\
                "board text, last_idx int)")
        self.conn.commit()
        try:
            for row in self.conn.execute("select * from status"):
                idx_info = IndexBoardInfo(**row)
                self.board_info[idx_info.board] = idx_info
        except:
            Log.info("Index info not present")

    def insert_idx_status(self, idx_obj):
        self.conn.execute("insert into status values (?, ?)",
                (idx_obj.board, idx_obj.last_idx))
        self.conn.commit()

    def remove_idx_status(self, idx_obj):
        self.conn.execute("delete from status where board=?",
                (idx_obj.board, ))
        self.conn.commit()

    def index_boards(self):
        boards = BoardManager.BoardManager.boards.keys()
        for board in boards:
            try:
                self.index_board(board)
            except Exception as exc:
                Log.error("Exception caught when indexing %s: %r"
                        % (board, exc))

    def index_board(self, board):
        boardobj = BoardManager.BoardManager.GetBoard(board)
        if not boardobj:
            Log.error("Error loading board %s" % board)
            return

        if board in self.board_info:
            idx_obj = self.board_info[board]
        else:
            idx_obj = IndexBoardInfo(board, 0)
            self.board_info[board] = idx_obj

        bdir_path = boardobj.GetDirPath()
        with open(bdir_path, 'rb') as bdir:
            Util.FLock(bdir, shared=True)
            try:
                st = os.stat(bdir_path)
                if st.st_mtime <= idx_obj.last_idx:
                    # why <? anyway...
                    return

                if not board in self.state.locks:
                    self.state.locks[board] = threading.Lock()

                self.state.locks[board].acquire()
                try:
                    self.remove_idx_status(idx_obj)
                    self.init_db(board)
                    for idx in xrange(st.st_size / PostEntry.PostEntry.size):
                        pe = PostEntry.PostEntry(
                                bdir.read(PostEntry.PostEntry.size))
                        self.insert_entry(board, pe, idx)
                    idx_obj.last_idx = st.st_mtime
                    self.insert_idx_status(idx_obj)
                finally:
                    self.state.locks[board].release()
            finally:
                Util.FUnlock(bdir)

    def insert_entry(self, board, pe, idx):
        self.conn.execute("insert into %s values (?, ?, ?, ?, ?)"
                % self.table_name(board),
                (idx, pe.id, pe.groupid, pe.reid, pe.GetPostTime()))
        # batch commit later

    def table_name(self, board):
        return "idx_" + board


{'fraction_non_alphanumeric': 0.09808371154815936, 'fraction_numerical': 0.002017145738779627, 'mean_word_length': 1.6755480607082631, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1596447', 'n_tokens_mistral': 2332, 'n_tokens_neox': 2220, 'n_words': 514}