<commit_before>import time
import traceback
import ConfigParser

from lib.pipeline import *
from lib.utils import *
from lib.cache import *


SYSTEM_CONF_FILE = "conf/system.conf"
PIPELINE_CONF_FILE = "conf/pipeline.conf"
BOTS_CONF_FILE = "conf/bots.conf"


class Bot(object):

    def __init__(self, bot_id):
        self.bot_id = bot_id
        self.logger = self.get_logger()
        self.logger.info('Bot is starting')

        self.get_bot_configurations()

        src_queue, dest_queues = self.get_queues()
        self.pipeline = self.get_pipeline(src_queue, dest_queues)

        if self.parameters.cached:
            cache_db_index = 10 #FIXME
            self.cache = self.create_cache(cache_db_index, self.parameters.cache_ttl)


    def get_bot_configurations(self):
        self.parameters = Parameters()
        config = ConfigParser.ConfigParser()
        config.read(BOTS_CONF_FILE)
        
        default_section = "default"
        self.logger.debug("Loading configuration in default section from '%s' file" % BOTS_CONF_FILE)
        
        if config.has_section(default_section):
            for option in config.options(default_section):
                setattr(self.parameters, option, config.get(default_section, option))
                self.logger.debug("Parameter '%s' loaded with the value '%s'" % (option, config.get(default_section, option)))
        
        self.logger.debug("Loading configuration in %s section from '%s' file" % (self.bot_id, BOTS_CONF_FILE))
        
        if config.has_section(self.bot_id):
            for option in config.options(self.bot_id):
                setattr(self.parameters, option, config.get(self.bot_id, option))
                self.logger.debug("Parameter '%s' loaded with the value '%s'" % (option, config.get(self.bot_id, option)))


    def get_logger(self):
        config = ConfigParser.ConfigParser()
        config.read(SYSTEM_CONF_FILE)
        loglevel = config.get('Logging','level')
        return log(self.bot_id, loglevel)


    def create_cache(self, cache_id, ttl):
        config = ConfigParser.ConfigParser()
        config.read(SYSTEM_CONF_FILE)

        host = config.get('Redis', 'host')
        port = int(config.get('Redis', 'port'))
        self.logger.debug("Connecting to Redis cache '%s:%s'" % (host, port))
        return Cache(host, port, cache_id, ttl)


    def get_pipeline(self, src_queue, dest_queues):
        self.logger.debug("Connecting to pipeline queues")
        return Pipeline(src_queue, dest_queues)


    def get_queues(self):
        config = ConfigParser.ConfigParser()
        config.read(PIPELINE_CONF_FILE)
        self.logger.debug("Loading pipeline queues from '%s' file" % PIPELINE_CONF_FILE)
        
        for option in config.options("Pipeline"):
            if option == self.bot_id:
                queues = config.get("Pipeline", self.bot_id)
                queues = queues.split('|')
                
                if len(queues) == 2:
                    src_queue = queues[0].strip()
                    dest_queues = queues[1].strip()
                    
                    if src_queue == "None":
                        src_queue = None
                    if dest_queues == "None":
                        dest_queue = None

                    dest_queues_list = list()
                    for queue in dest_queues.split(','):
                        dest_queues_list.append(queue.strip())
                    
                    self.logger.info("Source queue '%s'" % src_queue)
                    self.logger.info("Destination queue(s) '%s'" % dest_queues_list)
                    
                    return [src_queue, dest_queues_list]
        
        self.logger.error("Failed to load queues")
        self.exit()


    def send_message(self, message):
        self.pipeline.send(message)

    def receive_message(self):
        raw_message = self.pipeline.receive()
        try:
            message = Event.from_unicode(raw_message)
        except:
            message = raw_message
        return message

    def acknowledge_message(self):
        self.pipeline.acknowledge()
            

    def start(self):
        self.logger.info('Bot start processing')

        while True:
            try:
                self.process()
                time.sleep(int(self.parameters.processing_interval))
            except:
                self.logger.error(traceback.format_exc())
                self.exit()
    
    def exit(self):
        self.logger.error("Bot found an error. Exiting")
        exit(-1)

<commit_msg>Update bot.py<commit_after>import time
import traceback
import ConfigParser

from lib.pipeline import *
from lib.utils import *
from lib.cache import *


SYSTEM_CONF_FILE = "conf/system.conf"
PIPELINE_CONF_FILE = "conf/pipeline.conf"
BOTS_CONF_FILE = "conf/bots.conf"


class Bot(object):

    def __init__(self, bot_id):
        self.bot_id = bot_id
        self.logger = self.load_logger()
        self.logger.info('Bot is starting')

        self.load_configurations()

        src_queue, dest_queues = self.load_queues()
        self.pipeline = self.load_pipeline(src_queue, dest_queues)

        if self.parameters.cached:
            cache_db_index = 10 #FIXME
            self.cache = self.create_cache(cache_db_index, self.parameters.cache_ttl)


    def start(self):
        self.logger.info('Bot start processing')

        while True:
            try:
                self.process()
                time.sleep(int(self.parameters.processing_interval))
            except:
                self.logger.error(traceback.format_exc())
                self.exit()

    
    def exit(self): # FIXME: rename the method
        self.logger.error("Bot found an error. Exiting")
        exit(-1)


    def load_configurations(self):
        self.parameters = Parameters()
        config = ConfigParser.ConfigParser()
        config.read(BOTS_CONF_FILE)
        
        default_section = "default"
        self.logger.debug("Loading configuration in default section from '%s' file" % BOTS_CONF_FILE)
        
        if config.has_section(default_section):
            for option in config.options(default_section):
                setattr(self.parameters, option, config.get(default_section, option))
                self.logger.debug("Parameter '%s' loaded with the value '%s'" % (option, config.get(default_section, option)))
        
        self.logger.debug("Loading configuration in %s section from '%s' file" % (self.bot_id, BOTS_CONF_FILE))
        
        if config.has_section(self.bot_id):
            for option in config.options(self.bot_id):
                setattr(self.parameters, option, config.get(self.bot_id, option))
                self.logger.debug("Parameter '%s' loaded with the value '%s'" % (option, config.get(self.bot_id, option)))


    def load_logger(self):
        config = ConfigParser.ConfigParser()
        config.read(SYSTEM_CONF_FILE)
        loglevel = config.get('Logging','level')
        return log(self.bot_id, loglevel)


    def load_pipeline(self, src_queue, dest_queues):
        self.logger.debug("Connecting to pipeline queues")
        return Pipeline(src_queue, dest_queues)


    def load_queues(self):
        config = ConfigParser.ConfigParser()
        config.read(PIPELINE_CONF_FILE)
        self.logger.debug("Loading pipeline queues from '%s' file" % PIPELINE_CONF_FILE)
        
        for option in config.options("Pipeline"):
            if option == self.bot_id:
                queues = config.get("Pipeline", self.bot_id)
                queues = queues.split('|')
                
                if len(queues) == 2:
                    src_queue = queues[0].strip()
                    dest_queues = queues[1].strip()
                    
                    if src_queue == "None":
                        src_queue = None
                    if dest_queues == "None":
                        dest_queue = None

                    dest_queues_list = list()
                    for queue in dest_queues.split(','):
                        dest_queues_list.append(queue.strip())
                    
                    self.logger.info("Source queue '%s'" % src_queue)
                    self.logger.info("Destination queue(s) '%s'" % dest_queues_list)
                    
                    return [src_queue, dest_queues_list]
        
        self.logger.error("Failed to load queues")
        self.exit()


    def send_message(self, message):
        self.pipeline.send(message)


    def receive_message(self):
        raw_message = self.pipeline.receive()
        try:
            message = Event.from_unicode(raw_message)
        except:
            message = raw_message
        return message


    def acknowledge_message(self):
        self.pipeline.acknowledge()
            

    def create_cache(self, cache_id, ttl):
        config = ConfigParser.ConfigParser()
        config.read(SYSTEM_CONF_FILE)

        host = config.get('Redis', 'host')
        port = int(config.get('Redis', 'port'))
        self.logger.debug("Connecting to Redis cache '%s:%s'" % (host, port))
        return Cache(host, port, cache_id, ttl)



{'fraction_non_alphanumeric': 0.10458156491457177, 'fraction_numerical': 0.001305909239307868, 'mean_word_length': 2.017071569271175, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6909369', 'n_tokens_mistral': 2655, 'n_tokens_neox': 2561, 'n_words': 537}