<commit_before>from datetime import datetime, date
import time
from multiprocessing import Pool

from trac.env import Environment
from trac.db.api import get_column_names

from buildbot_api import BuildbotConnection, BuildbotException
import environmentSetup
from tools import Singleton


last_cached_query = "SELECT MAX(num) FROM buildbot_builds WHERE builder=%s"
save_build_query = "INSERT INTO buildbot_builds (%s) VALUES (%s)"
get_builds_query = """SELECT *,finish-start AS duration FROM buildbot_builds
                      WHERE finish >= %s AND finish <= %s AND builder IN (%s)
                   """

get_builds_query2 = """SELECT *,finish-start AS duration FROM buildbot_builds
                      WHERE finish >= %s AND finish <= %s
                   """


class BuildbotCacheException(BuildbotException):
    pass


class BuildbotCache(Singleton):
    def __init__(self, env):
        self.env = env
        self.connection = BuildbotConnection()
        self.fields = [col.name for col in environmentSetup.table.columns]

    def connect_to(self, address):
        self.connection.connect_to(address)

    def cache(self, builders):
        for builder in builders:
            self.cache_builder(builder)

    def cache_builder(self, builder):
        last_build_num = self.connection.get_build(builder, -1)['num']
        with self.env.db_transaction as db:
            cursor = db.cursor()

            cursor.execute(last_cached_query, [builder])
            last_cached_num = None
            try:
                last_cached_num = iter(cursor).next()[0]
            except StopIteration:
                pass

            if not last_cached_num: last_cached_num = -1

            for num in xrange(last_cached_num + 1, last_build_num + 1):
                build = self.connection.get_build(builder, num)

                query_build = dict()
                try:
                    for key, val in build.items():
                        if not key in self.fields: continue

                        if type(val) in (str, unicode):
                            query_build[key] = "'%s'" % val
                        elif type(val) == int:
                            query_build[key] = str(int(val))
                        elif type(val) == datetime:
                            query_build[key] = str(time.mktime(val.timetuple()))
                        else:
                            raise BuildbotCacheException('unknown type %s - %s' % (key, val))
                except BuildbotCacheException as e:
                    env.log.error(e)
                    print(e)
                    continue

                cursor.execute(save_build_query % (','.join(query_build.keys()),
                                                  ','.join(query_build.values())))

    def get_builds(self, builders, start, stop):
        start = str(int(time.mktime(start.timetuple())))
        stop = str(int(time.mktime(stop.timetuple())))
        with self.env.db_transaction as db:
            cursor = db.cursor()
            cursor.execute(get_builds_query %
                           (start, stop, ','.join(["'%s'" % builder for builder in builders])))
            fields = get_column_names(cursor)
            builds = []
            for build in cursor:
                builds.append(dict(zip(fields, build)))
            return builds

    def clear_cache(self):
        with self.env.db_transaction as db:
            cursor = db.cursor()
            cursor.execute("DELETE FROM buildbot_builds")


def async_buildbot_cache_init(env_path):
    global cache
    cache = BuildbotCache(Environment(env_path))

def async_buildbot_cache_worker(address, builders):
    global cache
    try:
        cache.env.log.error('cache')
        cache.connect_to(address)
        cache.cache(builders)
        cache.env.log.error('cache finished')
    except BuildbotException as e:
        cache.env.log.error(e)
        raise e<commit_msg>tracbuildbot/buildbot_cache.py: fixed type of log messages.<commit_after>from datetime import datetime, date
import time
from multiprocessing import Pool

from trac.env import Environment
from trac.db.api import get_column_names

from buildbot_api import BuildbotConnection, BuildbotException
import environmentSetup
from tools import Singleton


last_cached_query = "SELECT MAX(num) FROM buildbot_builds WHERE builder=%s"
save_build_query = "INSERT INTO buildbot_builds (%s) VALUES (%s)"
get_builds_query = """SELECT *,finish-start AS duration FROM buildbot_builds
                      WHERE finish >= %s AND finish <= %s AND builder IN (%s)
                   """

get_builds_query2 = """SELECT *,finish-start AS duration FROM buildbot_builds
                      WHERE finish >= %s AND finish <= %s
                   """


class BuildbotCacheException(BuildbotException):
    pass


class BuildbotCache(Singleton):
    def __init__(self, env):
        self.env = env
        self.connection = BuildbotConnection()
        self.fields = [col.name for col in environmentSetup.table.columns]

    def connect_to(self, address):
        self.connection.connect_to(address)

    def cache(self, builders):
        for builder in builders:
            self.cache_builder(builder)

    def cache_builder(self, builder):
        last_build_num = self.connection.get_build(builder, -1)['num']
        with self.env.db_transaction as db:
            cursor = db.cursor()

            cursor.execute(last_cached_query, [builder])
            last_cached_num = None
            try:
                last_cached_num = iter(cursor).next()[0]
            except StopIteration:
                pass

            if not last_cached_num: last_cached_num = -1

            for num in xrange(last_cached_num + 1, last_build_num + 1):
                build = self.connection.get_build(builder, num)

                query_build = dict()
                try:
                    for key, val in build.items():
                        if not key in self.fields: continue

                        if type(val) in (str, unicode):
                            query_build[key] = "'%s'" % val
                        elif type(val) == int:
                            query_build[key] = str(int(val))
                        elif type(val) == datetime:
                            query_build[key] = str(time.mktime(val.timetuple()))
                        else:
                            raise BuildbotCacheException('unknown type %s - %s' % (key, val))
                except BuildbotCacheException as e:
                    env.log.error(e)
                    print(e)
                    continue

                cursor.execute(save_build_query % (','.join(query_build.keys()),
                                                  ','.join(query_build.values())))

    def get_builds(self, builders, start, stop):
        start = str(int(time.mktime(start.timetuple())))
        stop = str(int(time.mktime(stop.timetuple())))
        with self.env.db_transaction as db:
            cursor = db.cursor()
            cursor.execute(get_builds_query %
                           (start, stop, ','.join(["'%s'" % builder for builder in builders])))
            fields = get_column_names(cursor)
            builds = []
            for build in cursor:
                builds.append(dict(zip(fields, build)))
            return builds

    def clear_cache(self):
        with self.env.db_transaction as db:
            cursor = db.cursor()
            cursor.execute("DELETE FROM buildbot_builds")


def async_buildbot_cache_init(env_path):
    global cache
    cache = BuildbotCache(Environment(env_path))

def async_buildbot_cache_worker(address, builders):
    global cache
    try:
        cache.env.log.debug('cache')
        cache.connect_to(address)
        cache.cache(builders)
        cache.env.log.debug('cache finished')
    except BuildbotException as e:
        cache.env.log.error(e)
        raise e
{'fraction_non_alphanumeric': 0.09521410579345088, 'fraction_numerical': 0.0015113350125944584, 'mean_word_length': 1.9077261076528744, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2862133', 'n_tokens_mistral': 2177, 'n_tokens_neox': 2082, 'n_words': 526}