<commit_before>import logging
from datetime import datetime, timedelta
from glob import glob
from importlib import import_module
from operator import itemgetter
from time import sleep

import yaml
from slackclient import SlackClient


LOGGER = logging.getLogger(__name__)


class NewsSlackBot(object):

    def __init__(self, secrets_yaml='secrets.yml'):
        with open(secrets_yaml, 'rb') as secrets_file:
            self.config = config = yaml.load(secrets_file)
        self.bot = bot = {
            'token': config['slackbot']['token'],
            'id': config['slackbot'].get('id'),
            'name': config['slackbot'].get('name'),
        }

        # Instantiate Slack client.
        self.slack_client = SlackClient(bot['token'])

        # Set bot id.
        if not bot['id']:
            if not bot['name']:
                raise EnvironmentError

            users_list = self.get('users.list')
            if not users_list.get('ok'):
                raise EnvironmentError
            users = [u for u in users_list.get('members') if 'name' in u]
            bot['id'] = next(u.get('id') for u in users if u['name'] == bot['name'])
        bot['at'] = '<@{id}>'.format(id=bot['id'])

        self._channels = None, None

    def get(self, name, key=None):
        result = self.slack_client.api_call(name)
        return result if key is None else result[key]

    @property
    def channels(self):
        last_update, channels = self._channels

        now = datetime.now()
        if last_update is None or now - last_update > timedelta(minutes=10):
            last_update = now
            self._channels = last_update, channels = now, [
                c
                for c in self.get('channels.list', 'channels') if c['is_member']]

        return channels

    def loop_forever(self):
        if self.slack_client.rtm_connect():
            LOGGER.info('StarterBot connected and running!')
            interval = timedelta(minutes=60)
            last_news_update = datetime.now() - 2*interval
            while True:
                now = datetime.now()
                command, channel = self.parse_rtm_events(self.slack_client.rtm_read())
                if command and channel:
                    # Respond to messages.
                    self.handle_command(command, channel)
                elif now - last_news_update > interval:
                    # Send news updates.
                    last_news_update = now
                    self.send_news()

                sleep(1)
        else:
            LOGGER.critical('Connection failed. Invalid Slack token or bot ID?')

    def parse_rtm_events(self, events):
        """
        The Slack Real Time Messaging API is an events firehose.
        This parsing function returns any for any non-empty message.

        """
        at_bot = self.bot['at']
        for evt in events:
            LOGGER.debug(evt)
            channel, text = evt.get('channel'), evt.get('text')
            if evt.get('user') == self.bot['id'] or not text or evt['type'] != 'message':
                continue
            elif at_bot in text \
                    or channel not in map(itemgetter('id'), self.channels):
                # Handle command only if @mention or direct channel (IM).
                tokens = filter(None, (t.strip() for t in text.split(at_bot)))
                # return text, whitespace and @mention removed
                return ' '.join(t.lower() for t in tokens), channel
        return None, None

    def handle_command(self, cmd, channel):
        cmd = cmd.split()[0]
        response = 'I am unable to process commands at this time...'
        self.slack_client.api_call('chat.postMessage',
                                   channel=channel,
                                   text=response,
                                   as_user=True)

    @property
    def newsclients(self):
        for path in glob('newsclients/[!_]*.py'):
            module = import_module(path.rstrip('.py').replace('/', '.'))
            if hasattr(module, 'NewsClient'):
                yield module.NewsClient(self.config)

    def send_news(self):
        for client in self.newsclients:
            for channel in self.channels:
                for article in client.fetch(topic=channel['topic']['value']<commit_msg>refactor(bot): Get news clients programmatically<commit_after>import logging
from datetime import datetime, timedelta
from importlib import import_module
from operator import itemgetter
from pkgutil import iter_modules
from time import sleep

import yaml
from slackclient import SlackClient


LOGGER = logging.getLogger(__name__)


class NewsSlackBot(object):

    def __init__(self, secrets_yaml='secrets.yml'):
        with open(secrets_yaml, 'rb') as secrets_file:
            self.config = config = yaml.load(secrets_file)
        self.bot = bot = {
            'token': config['slackbot']['token'],
            'id': config['slackbot'].get('id'),
            'name': config['slackbot'].get('name'),
        }

        # Instantiate Slack client.
        self.slack_client = SlackClient(bot['token'])

        # Set bot id.
        if not bot['id']:
            if not bot['name']:
                raise EnvironmentError

            users_list = self.get('users.list')
            if not users_list.get('ok'):
                raise EnvironmentError
            users = [u for u in users_list.get('members') if 'name' in u]
            bot['id'] = next(u.get('id') for u in users if u['name'] == bot['name'])
        bot['at'] = '<@{id}>'.format(id=bot['id'])

        self._channels = None, None

    def get(self, name, key=None):
        result = self.slack_client.api_call(name)
        return result if key is None else result[key]

    @property
    def channels(self):
        last_update, channels = self._channels

        now = datetime.now()
        if last_update is None or now - last_update > timedelta(minutes=10):
            last_update = now
            self._channels = last_update, channels = now, [
                c
                for c in self.get('channels.list', 'channels') if c['is_member']]

        return channels

    def loop_forever(self):
        if self.slack_client.rtm_connect():
            LOGGER.info('StarterBot connected and running!')
            interval = timedelta(minutes=60)
            last_news_update = datetime.now() - 2*interval
            while True:
                now = datetime.now()
                command, channel = self.parse_rtm_events(self.slack_client.rtm_read())
                if command and channel:
                    # Respond to messages.
                    self.handle_command(command, channel)
                elif now - last_news_update > interval:
                    # Send news updates.
                    last_news_update = now
                    self.send_news()

                sleep(1)
        else:
            LOGGER.critical('Connection failed. Invalid Slack token or bot ID?')

    def parse_rtm_events(self, events):
        """
        The Slack Real Time Messaging API is an events firehose.
        This parsing function returns any for any non-empty message.

        """
        at_bot = self.bot['at']
        for evt in events:
            LOGGER.debug(evt)
            channel, text = evt.get('channel'), evt.get('text')
            if evt.get('user') == self.bot['id'] or not text or evt['type'] != 'message':
                continue
            elif at_bot in text \
                    or channel not in map(itemgetter('id'), self.channels):
                # Handle command only if @mention or direct channel (IM).
                tokens = filter(None, (t.strip() for t in text.split(at_bot)))
                # return text, whitespace and @mention removed
                return ' '.join(t.lower() for t in tokens), channel
        return None, None

    def handle_command(self, cmd, channel):
        cmd = cmd.split()[0]
        response = 'I am unable to process commands at this time...'
        self.slack_client.api_call('chat.postMessage',
                                   channel=channel,
                                   text=response,
                                   as_user=True)

    @staticmethod
    def get_client_classes(pkg='newsclients'):
        """
        Yield all of the NewsClients existing in non-private modules of the
        given package.

        """
        for _, name, is_pkg in iter_modules([pkg]):
            if not is_pkg and not name.startswith('_'):
                module = import_module('{pkg}.{client}'.format(pkg=pkg, client=name))
                if hasattr(module, 'NewsClient'):
                    yield module.NewsClient

    def send_news(self):
        for client_cls in self.get_client_classes():
            client = client_cls(self.config)
            for channel in self.channels:
                for article in client.fetch(topic=channel['topic']['value']
{'fraction_non_alphanumeric': 0.10646387832699619, 'fraction_numerical': 0.00156564526951465, 'mean_word_length': 1.8829787234042554, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1006709', 'n_tokens_mistral': 2472, 'n_tokens_neox': 2360, 'n_words': 680}