<commit_before>import logging
import inspect
import importlib
import imp
import os
import re
import sys
import time
import traceback
from os.path import abspath, curdir, dirname
from multiprocessing import Process

from gevent import monkey
# Monkeypatch has to come before bottle
monkey.patch_all()
import bottle

from listener import WillXMPPClientMixin
from mixins import ScheduleMixin, StorageMixin, ErrorMixin, HipChatMixin, RoomMixin, PluginModulesLibraryMixin, \
    EmailMixin
from scheduler import Scheduler
import settings


# Force UTF8
if sys.version_info < (3, 0):
    reload(sys)
    sys.setdefaultencoding('utf8')
else:
    raw_input = input


# Update path
PROJECT_ROOT = abspath(os.path.join(dirname(__file__)))
PLUGINS_ROOT = abspath(os.path.join(PROJECT_ROOT, "plugins"))
TEMPLATES_ROOT = abspath(os.path.join(PROJECT_ROOT, "templates"))
sys.path.append(PROJECT_ROOT)
sys.path.append(os.path.join(PROJECT_ROOT, "will"))


class WillBot(EmailMixin, WillXMPPClientMixin, StorageMixin, ScheduleMixin, ErrorMixin, RoomMixin, HipChatMixin, PluginModulesLibraryMixin):

    def __init__(self, plugins_dirs=[], template_dirs=[]):
        logging.basicConfig(level=logging.ERROR, format='%(levelname)-8s %(message)s')

        self.plugins_dirs = [PLUGINS_ROOT, ]
        for plugin_dir in plugins_dirs:
            p = os.path.abspath(plugin_dir)
            if p not in self.plugins_dirs:
                self.plugins_dirs.append(p)

        full_path_template_dirs = []
        for t in template_dirs:
            full_path_template_dirs.append(os.path.abspath(t))
        if not TEMPLATES_ROOT in full_path_template_dirs:
            full_path_template_dirs += [TEMPLATES_ROOT, ]

        os.environ["WILL_TEMPLATE_DIRS_PICKLED"] = ";;".join(full_path_template_dirs)

    def bootstrap(self):
        self.bootstrap_storage()
        self.bootstrap_plugins()

        # Scheduler
        scheduler_thread = Process(target=self.bootstrap_scheduler)
        # scheduler_thread.daemon = True
        
        # Bottle
        bottle_thread = Process(target=self.bootstrap_bottle)
        # bottle_thread.daemon = True

        # XMPP Listener
        xmpp_thread = Process(target=self.bootstrap_xmpp)
        # xmpp_thread.daemon = True

        try:
            # Start up threads.
            xmpp_thread.start()
            scheduler_thread.start()
            bottle_thread.start()

            errors = self.get_startup_errors()
            if len(errors) > 0:
                default_room = self.get_room_from_name_or_id(settings.WILL_DEFAULT_ROOM)["room_id"]
                error_message = "FYI, I had some errors starting up:"
                for err in errors:
                    error_message += "\n%s\n" % err
                self.send_room_message(default_room, error_message)
                

            while True: time.sleep(100)
        except (KeyboardInterrupt, SystemExit):
            scheduler_thread.terminate()
            bottle_thread.terminate()
            xmpp_thread.terminate()
            print '\n\nReceived keyboard interrupt, quitting threads.',
            while scheduler_thread.is_alive() or\
                  bottle_thread.is_alive() or\
                  xmpp_thread.is_alive():
                    sys.stdout.write(".")
                    sys.stdout.flush()
                    time.sleep(0.5)

    def bootstrap_scheduler(self):
        print "Bootstrapping scheduler..."
        bootstrapped = False
        try:
            
            self.save("plugin_modules_library", self._plugin_modules_library)
            Scheduler.clear_locks(self)
            self.scheduler = Scheduler()
            
            for plugin_info, fn, function_name in self.periodic_tasks:
                self.add_periodic_task(plugin_info["full_module_name"], plugin_info["name"], function_name, fn.sched_args, fn.sched_kwargs, fn.function_name,)
            for plugin_info, fn, function_name in self.random_tasks:
                self.add_random_tasks(plugin_info["full_module_name"], plugin_info["name"], function_name, fn.start_hour, fn.end_hour, fn.day_of_week, fn.num_times_per_day)
            bootstrapped = True
        except Exception, e:
            self.startup_error("Error bootstrapping scheduler", e)
        if bootstrapped:
            self.scheduler.start_loop(self)
           

    def bootstrap_bottle(self):
        print "Bootstrapping bottle..."
        bootstrapped = False
        try:
            for cls, function_name in self.bottle_routes:
                instantiated_cls = cls()
                instantiated_fn = getattr(instantiated_cls, function_name)
                bottle.route(instantiated_fn.bottle_route)(instantiated_fn)
            bootstrapped = True
        except Exception, e:
            self.startup_error("Error bootstrapping bottle", e)
        if bootstrapped:
            bottle.run(host='0.0.0.0', port=settings.WILL_HTTPSERVER_PORT, server='gevent')

    def bootstrap_xmpp(self):
        print "Bootstrapping xmpp..."
        bootstrapped = False
        try:
            self.start_xmpp_client()
            self.save("all_listener_regexes", self.all_listener_regexes)
            self.connect()
            bootstrapped = True
        except Exception, e:
            self.startup_error("Error bootstrapping bottle", e)
        if bootstrapped:
            self.process(block=True)

    def bootstrap_plugins(self):
        print "Bootstrapping plugins..."
        plugin_modules = {}
        plugin_modules_library = {}

        # NOTE: You can't access self.storage here, or it will deadlock when the threads try to access redis.

        # Sure does feel like this should be a solved problem somehow.
        for plugin_root in self.plugins_dirs:<commit_msg>Fix typo in raised exception<commit_after>import logging
import inspect
import importlib
import imp
import os
import re
import sys
import time
import traceback
from os.path import abspath, curdir, dirname
from multiprocessing import Process

from gevent import monkey
# Monkeypatch has to come before bottle
monkey.patch_all()
import bottle

from listener import WillXMPPClientMixin
from mixins import ScheduleMixin, StorageMixin, ErrorMixin, HipChatMixin, RoomMixin, PluginModulesLibraryMixin, \
    EmailMixin
from scheduler import Scheduler
import settings


# Force UTF8
if sys.version_info < (3, 0):
    reload(sys)
    sys.setdefaultencoding('utf8')
else:
    raw_input = input


# Update path
PROJECT_ROOT = abspath(os.path.join(dirname(__file__)))
PLUGINS_ROOT = abspath(os.path.join(PROJECT_ROOT, "plugins"))
TEMPLATES_ROOT = abspath(os.path.join(PROJECT_ROOT, "templates"))
sys.path.append(PROJECT_ROOT)
sys.path.append(os.path.join(PROJECT_ROOT, "will"))


class WillBot(EmailMixin, WillXMPPClientMixin, StorageMixin, ScheduleMixin, ErrorMixin, RoomMixin, HipChatMixin, PluginModulesLibraryMixin):

    def __init__(self, plugins_dirs=[], template_dirs=[]):
        logging.basicConfig(level=logging.ERROR, format='%(levelname)-8s %(message)s')

        self.plugins_dirs = [PLUGINS_ROOT, ]
        for plugin_dir in plugins_dirs:
            p = os.path.abspath(plugin_dir)
            if p not in self.plugins_dirs:
                self.plugins_dirs.append(p)

        full_path_template_dirs = []
        for t in template_dirs:
            full_path_template_dirs.append(os.path.abspath(t))
        if not TEMPLATES_ROOT in full_path_template_dirs:
            full_path_template_dirs += [TEMPLATES_ROOT, ]

        os.environ["WILL_TEMPLATE_DIRS_PICKLED"] = ";;".join(full_path_template_dirs)

    def bootstrap(self):
        self.bootstrap_storage()
        self.bootstrap_plugins()

        # Scheduler
        scheduler_thread = Process(target=self.bootstrap_scheduler)
        # scheduler_thread.daemon = True
        
        # Bottle
        bottle_thread = Process(target=self.bootstrap_bottle)
        # bottle_thread.daemon = True

        # XMPP Listener
        xmpp_thread = Process(target=self.bootstrap_xmpp)
        # xmpp_thread.daemon = True

        try:
            # Start up threads.
            xmpp_thread.start()
            scheduler_thread.start()
            bottle_thread.start()

            errors = self.get_startup_errors()
            if len(errors) > 0:
                default_room = self.get_room_from_name_or_id(settings.WILL_DEFAULT_ROOM)["room_id"]
                error_message = "FYI, I had some errors starting up:"
                for err in errors:
                    error_message += "\n%s\n" % err
                self.send_room_message(default_room, error_message)
                

            while True: time.sleep(100)
        except (KeyboardInterrupt, SystemExit):
            scheduler_thread.terminate()
            bottle_thread.terminate()
            xmpp_thread.terminate()
            print '\n\nReceived keyboard interrupt, quitting threads.',
            while scheduler_thread.is_alive() or\
                  bottle_thread.is_alive() or\
                  xmpp_thread.is_alive():
                    sys.stdout.write(".")
                    sys.stdout.flush()
                    time.sleep(0.5)

    def bootstrap_scheduler(self):
        print "Bootstrapping scheduler..."
        bootstrapped = False
        try:
            
            self.save("plugin_modules_library", self._plugin_modules_library)
            Scheduler.clear_locks(self)
            self.scheduler = Scheduler()
            
            for plugin_info, fn, function_name in self.periodic_tasks:
                self.add_periodic_task(plugin_info["full_module_name"], plugin_info["name"], function_name, fn.sched_args, fn.sched_kwargs, fn.function_name,)
            for plugin_info, fn, function_name in self.random_tasks:
                self.add_random_tasks(plugin_info["full_module_name"], plugin_info["name"], function_name, fn.start_hour, fn.end_hour, fn.day_of_week, fn.num_times_per_day)
            bootstrapped = True
        except Exception, e:
            self.startup_error("Error bootstrapping scheduler", e)
        if bootstrapped:
            self.scheduler.start_loop(self)
           

    def bootstrap_bottle(self):
        print "Bootstrapping bottle..."
        bootstrapped = False
        try:
            for cls, function_name in self.bottle_routes:
                instantiated_cls = cls()
                instantiated_fn = getattr(instantiated_cls, function_name)
                bottle.route(instantiated_fn.bottle_route)(instantiated_fn)
            bootstrapped = True
        except Exception, e:
            self.startup_error("Error bootstrapping bottle", e)
        if bootstrapped:
            bottle.run(host='0.0.0.0', port=settings.WILL_HTTPSERVER_PORT, server='gevent')

    def bootstrap_xmpp(self):
        print "Bootstrapping xmpp..."
        bootstrapped = False
        try:
            self.start_xmpp_client()
            self.save("all_listener_regexes", self.all_listener_regexes)
            self.connect()
            bootstrapped = True
        except Exception, e:
            self.startup_error("Error bootstrapping xmpp", e)
        if bootstrapped:
            self.process(block=True)

    def bootstrap_plugins(self):
        print "Bootstrapping plugins..."
        plugin_modules = {}
        plugin_modules_library = {}

        # NOTE: You can't access self.storage here, or it will deadlock when the threads try to access redis.

        # Sure does feel like this should be a solved problem somehow.
        for plugin_root in self.plugins_dirs:
{'fraction_non_alphanumeric': 0.08601404664874707, 'fraction_numerical': 0.002601231249458077, 'mean_word_length': 2.4730502860584163, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6384099', 'n_tokens_mistral': 3478, 'n_tokens_neox': 3381, 'n_words': 705}