<commit_before>#!/usr/bin/env python
from __future__ import print_function
import copy
import functools
from glob import glob
import importlib
import logging
import os
import re
import sqlite3
import sys
import time
import traceback
import json

from slackrtm import SlackClient
from slackrtm.server import SlackConnectionError, SlackLoginError

from beepboop import resourcer
from beepboop import bot_manager

from .server import LimboServer
from .fakeserver import FakeServer

from .handlers import handle_event, bb_handlers, run_hook
from .utils import (decode,
                    encode,
                    relevant_environ,
                    strip_extension)
from .settings import CONFIG


CURDIR = os.path.abspath(os.path.dirname(__file__))
DIR = functools.partial(os.path.join, CURDIR)

logger = logging.getLogger(__name__)


class InvalidPluginDir(Exception):
    def __init__(self, plugindir):
        message = "Unable to find plugin dir {0}".format(plugindir)
        super(InvalidPluginDir, self).__init__(message)


class Slackbot(object):
    def __init__(self, bot_token=None, ServerClass=LimboServer, Client=SlackClient, config=CONFIG):
        self.resource = None
        self.server = None
        if bot_token:
            # using resource if no token
            self.token = bot_token

        self.ServerClass = ServerClass
        self.Client = Client
        self.config = config

        self._init_log(config)
        logger.debug("config: {0}".format(config))
        config_plugins = config.get("plugins")
        plugins_to_load = config_plugins.split(",") if config_plugins else []

        # disable custom plugindir for now. In the future, input that through
        # config instead. Replace None with something else later
        self.hooks = self._init_plugins(None, plugins_to_load)

    def _init_log(self, config):
        loglevel = config.get("loglevel", logging.INFO)
        logformat = config.get("logformat", '%(asctime)s:%(levelname)s:%(name)s:%(message)s')
        if config.get("logfile"):
            logging.basicConfig(filename=config.get("logfile"), format=logformat, level=loglevel)
        else:
            logging.basicConfig(format=logformat, level=loglevel)

    def _init_plugins(self, plugindir, plugins_to_load=None):
        if plugindir and not os.path.isdir(plugindir):
            raise InvalidPluginDir(plugindir)

        if not plugindir:
            plugindir = DIR("plugins")

        logger.debug("plugindir: {0}".format(plugindir))

        if os.path.isdir(plugindir):
            pluginfiles = glob(os.path.join(plugindir, "[!_]*.py"))
            plugins = strip_extension(os.path.basename(p) for p in pluginfiles)
        else:
            # we might be in an egg; try to get the files that way
            logger.debug("trying pkg_resources")
            import pkg_resources
            try:
                plugins = strip_extension(
                        pkg_resources.resource_listdir(__name__, "plugins"))
            except OSError:
                raise InvalidPluginDir(plugindir)

        hooks = {}

        oldpath = copy.deepcopy(sys.path)
        sys.path.insert(0, plugindir)

        for plugin in plugins:
            if plugins_to_load and plugin not in plugins_to_load:
                logger.debug("skipping plugin {0}, not in plugins_to_load {1}".format(plugin, plugins_to_load))
                continue

            logger.debug("plugin: {0}".format(plugin))
            try:
                mod = importlib.import_module(plugin)
                modname = mod.__name__
                for hook in re.findall("on_(\w+)", " ".join(dir(mod))):
                    hookfun = getattr(mod, "on_" + hook)
                    logger.debug("plugin: attaching %s hook for %s", hook, modname)
                    hooks.setdefault(hook, []).append(hookfun)

                if mod.__doc__:
                    # firstline = mod.__doc__.split('\n')[0]
                    part_attachment = json.loads(mod.__doc__)
                    hooks.setdefault('help', {})[modname] = part_attachment
                    hooks.setdefault('extendedhelp', {})[modname] = mod.__doc__

            # bare except, because the modules could raise any number of errors
            # on import, and we want them not to kill our server
            except:
                logger.warning("import failed on module {0}, module not loaded".format(plugin))
                logger.warning("{0}".format(sys.exc_info()[0]))
                logger.warning("{0}".format(traceback.format_exc()))

        sys.path = oldpath
        return hooks

    def start(self, resource=None):
        if resource:
            self.resource = resource
            logger.debug("Started Bot for ResourceID: {}".format(
                self.resource['resourceID'])
            )
            slack = self.Client(self.resource['resource']['SlackBotAccessToken'])
            self.config['resource'] = self.resource['resource']
        else:
            slack = self.Client(self.token)

        # Currently not supporting a database, might do later
        self.server = self.ServerClass(slack, self.config, self.hooks, None)
        self.server.slack.rtm_connect()
        self.loop()

    def stop(self, resource):
        logger.debug("Stopped Bot for ResourceID: {}".format(
            self.resource['resourceID'])
        )
        self.server.slack = None<commit_msg>Fix shared dictionary between bot instances<commit_after>#!/usr/bin/env python
from __future__ import print_function
import copy
import functools
from glob import glob
import importlib
import logging
import os
import re
import sqlite3
import sys
import time
import traceback
import json

from slackrtm import SlackClient
from slackrtm.server import SlackConnectionError, SlackLoginError

from beepboop import resourcer
from beepboop import bot_manager

from .server import LimboServer
from .fakeserver import FakeServer

from .handlers import handle_event, bb_handlers, run_hook
from .utils import (decode,
                    encode,
                    relevant_environ,
                    strip_extension)
from .settings import CONFIG


CURDIR = os.path.abspath(os.path.dirname(__file__))
DIR = functools.partial(os.path.join, CURDIR)

logger = logging.getLogger(__name__)


class InvalidPluginDir(Exception):
    def __init__(self, plugindir):
        message = "Unable to find plugin dir {0}".format(plugindir)
        super(InvalidPluginDir, self).__init__(message)


class Slackbot(object):
    def __init__(self, bot_token=None, ServerClass=LimboServer, Client=SlackClient, config=CONFIG):
        self.resource = None
        self.server = None
        if bot_token:
            # using resource if no token
            self.token = bot_token

        self.ServerClass = ServerClass
        self.Client = Client
        self.config = config

        self._init_log(config)
        logger.debug("config: {0}".format(config))
        config_plugins = config.get("plugins")
        plugins_to_load = config_plugins.split(",") if config_plugins else []

        # disable custom plugindir for now. In the future, input that through
        # config instead. Replace None with something else later
        self.hooks = self._init_plugins(None, plugins_to_load)

    def _init_log(self, config):
        loglevel = config.get("loglevel", logging.INFO)
        logformat = config.get("logformat", '%(asctime)s:%(levelname)s:%(name)s:%(message)s')
        if config.get("logfile"):
            logging.basicConfig(filename=config.get("logfile"), format=logformat, level=loglevel)
        else:
            logging.basicConfig(format=logformat, level=loglevel)

    def _init_plugins(self, plugindir, plugins_to_load=None):
        if plugindir and not os.path.isdir(plugindir):
            raise InvalidPluginDir(plugindir)

        if not plugindir:
            plugindir = DIR("plugins")

        logger.debug("plugindir: {0}".format(plugindir))

        if os.path.isdir(plugindir):
            pluginfiles = glob(os.path.join(plugindir, "[!_]*.py"))
            plugins = strip_extension(os.path.basename(p) for p in pluginfiles)
        else:
            # we might be in an egg; try to get the files that way
            logger.debug("trying pkg_resources")
            import pkg_resources
            try:
                plugins = strip_extension(
                        pkg_resources.resource_listdir(__name__, "plugins"))
            except OSError:
                raise InvalidPluginDir(plugindir)

        hooks = {}

        oldpath = copy.deepcopy(sys.path)
        sys.path.insert(0, plugindir)

        for plugin in plugins:
            if plugins_to_load and plugin not in plugins_to_load:
                logger.debug("skipping plugin {0}, not in plugins_to_load {1}".format(plugin, plugins_to_load))
                continue

            logger.debug("plugin: {0}".format(plugin))
            try:
                mod = importlib.import_module(plugin)
                modname = mod.__name__
                for hook in re.findall("on_(\w+)", " ".join(dir(mod))):
                    hookfun = getattr(mod, "on_" + hook)
                    logger.debug("plugin: attaching %s hook for %s", hook, modname)
                    hooks.setdefault(hook, []).append(hookfun)

                if mod.__doc__:
                    # firstline = mod.__doc__.split('\n')[0]
                    part_attachment = json.loads(mod.__doc__)
                    hooks.setdefault('help', {})[modname] = part_attachment
                    hooks.setdefault('extendedhelp', {})[modname] = mod.__doc__

            # bare except, because the modules could raise any number of errors
            # on import, and we want them not to kill our server
            except:
                logger.warning("import failed on module {0}, module not loaded".format(plugin))
                logger.warning("{0}".format(sys.exc_info()[0]))
                logger.warning("{0}".format(traceback.format_exc()))

        sys.path = oldpath
        return hooks

    def start(self, resource=None):
        if resource:
            self.resource = resource
            logger.debug("Started Bot for ResourceID: {}".format(
                self.resource['resourceID'])
            )
            slack = self.Client(self.resource['resource']['SlackBotAccessToken'])
            config = dict(self.config)
            config['resource'] = self.resource['resource']
        else:
            slack = self.Client(self.token)
            config = dict(self.config)

        # Currently not supporting a database, might do later
        self.server = self.ServerClass(slack, config, self.hooks, None)
        self.server.slack.rtm_connect()
        self.loop()

    def stop(self, resource):
        logger.debug("Stopped Bot for ResourceID: {}".format(
            self.resource['resourceID'])
        )
        self.server.slack = None
{'fraction_non_alphanumeric': 0.10042070605450887, 'fraction_numerical': 0.0023779037863544904, 'mean_word_length': 2.3277541083384055, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '453801', 'n_tokens_mistral': 3068, 'n_tokens_neox': 2979, 'n_words': 754}