<commit_before>#
# client.py
#
# statbot - Store Discord records for later analysis
# Copyright (c) 2017-2018 <NAME>
#
# statbot is available free of charge under the terms of the MIT
# License. You are free to redistribute and/or modify it under those
# terms. It is distributed in the hopes that it will be useful, but
# WITHOUT ANY WARRANTY. See the LICENSE file for more details.
#

import asyncio
import discord

from .emoji import EmojiData
from .util import null_logger

__all__ = [
    'EventIngestionClient',
]

def member_needs_update(before, after):
    '''
    See if the given member update is something
    we care about.

    Returns 'False' for no difference or
    change we will ignore.
    '''

    for attr in ('name', 'discriminator', 'nick', 'avatar', 'roles'):
        if getattr(before, attr) != getattr(after, attr):
            return True
    return False

class EventIngestionClient(discord.Client):
    __slots__ = (
        'config',
        'logger',
        'sql',
        'ready',
        'sql_init',
        'hooks',
    )

    def __init__(self, config, sql, logger=null_logger):
        super().__init__()
        self.config = config
        self.logger = logger
        self.sql = sql
        self.ready = asyncio.Event()
        self.sql_init = False
        self.hooks = {
            'on_guild_channel_create': None,
            'on_guild_channel_delete': None,
            'on_guild_channel_update': None,
        }

    def run_with_token(self):
        return self.run(self.config['bot']['token'])

    async def wait_until_ready(self):
        # Override wait method to wait until SQL data is also ready
        # At least as long as "await super().wait_until_ready()"
        await self.ready.wait()

    async def _accept_message(self, message):
        await self.wait_until_ready()

        if not hasattr(message, 'guild'):
            self._log_ignored("Message not from a guild.")
            self._log_ignored("Ignoring message.")
            return False
        elif getattr(message.guild, 'id', None) not in self.config['guild-ids']:
            self._log_ignored("Message from a guild we don't care about.")
            self._log_ignored("Ignoring message.")
            return False
        elif message.type != discord.MessageType.default:
            self._log_ignored("Special type of message receieved.")
            self._log_ignored("Ignoring message.")
        else:
            return True

    async def _accept_channel(self, channel):
        await self.wait_until_ready()

        if not hasattr(channel, 'guild'):
            self._log_ignored("Channel not in a guild.")
            self._log_ignored("Ignoring message.")
        elif getattr(channel.guild, 'id', None) not in self.config['guild-ids']:
            self._log_ignored("Event from a guild we don't care about.")
            self._log_ignored("Ignoring message.")
            return False
        else:
            return True

    async def _accept_guild(self, guild):
        await self.wait_until_ready()

        if getattr(guild, 'id', None) not in self.config['guild-ids']:
            self._log_ignored("Event from a guild we don't care about.")
            self._log_ignored("Ignoring message.")
            return False
        else:
            return True

    def _log(self, message, action):
        name = message.author.display_name
        guild = message.guild.name
        chan = message.channel.name

        self.logger.info(f"Message {action} by {name} in {guild} #{chan}")
        if self.config['logger']['full-messages']:
            self.logger.info("<bom>")
            self.logger.info(message.content)
            self.logger.info("<eom>")

    def _log_typing(self, channel, user):
        name = user.display_name
        guild = channel.guild.name
        chan = channel.name

        self.logger.info(f"Typing by {name} on {guild} #{chan}")

    def _log_react(self, reaction, user, action):
        name = user.display_name
        emote = EmojiData(reaction.emoji)
        count = reaction.count
        id = reaction.message.id

        self.logger.info(f"{name} {action} {emote} (total {count}) on message id {id}")

    def _log_ignored(self, message):
        if self.config['logger']['ignored-events']:
            self.logger.debug(message)

    def _init_sql(self, trans):
        self.logger.info(f"Processing {len(self.users)} users...")
        for user in self.users:
            self.sql.upsert_user(trans, user)

        self.logger.info(f"Processing {len(self.guilds)} guilds...")
        for guild in self.guilds:
            self.sql.upsert_guild(trans, guild)

            self.logger.info(f"Processing {len(guild.roles)} roles...")
            for role in guild.roles:
                self.sql.upsert_role(trans, role)

            self.logger.info(f"Processing {len(guild.emojis)} emojis...")
            for emoji in guild.emojis:
                self.sql.upsert_emoji(trans, emoji)

            self.logger.info(f"Processing {len(guild.members)} members...")
            for member in guild.members:
                self.sql.upsert_member(trans, member)

            # In case people left while the bot was down
            self.sql.remove_old_members(trans, guild)

            text_channels = []
            voice_channels = []
            categories = []
            for channel in guild.channels:
                if isinstance(channel, discord.TextChannel):
                    text_channels.append(channel)
                elif isinstance(channel, discord.VoiceChannel):
                    voice_channels.append(channel)
                elif isinstance(channel, discord.CategoryChannel):
                    categories.append(channel)

            self.logger.info(f"Processing {len(categories)} channel categories...")
            for category in categories:
                self.sql.upsert_channel_category(trans, category)

            self.logger.info(f"Processing {len(text_channels)} channels...")
            for channel in text_channels:
                self.sql.upsert_channel(trans, channel)

            self.logger.info(f"Processing {len(voice_channels)} voice channels...")
            for channel in voice_channels:
                self.sql.upsert_voice_channel(trans, channel)

    async def on_ready(self):
        # Print welcome string
        self.logger.info(f"Logged in as {self.user.name} ({self.user.id})")
        self.logger.info("Recording activity in the following guilds:")
        for id in self.config['guild-ids']:
            guild = self.get_guild(id)
            self.logger.info(f"* {guild.name} ({id})")

        if not self.sql_init:
            self.logger.info("Initializing SQL lookup tables...")
            with self.sql.transaction() as trans:
                self._init_sql(trans)
                self.sql_init = True

        # All done setting up
        self.logger.info("")
        self.logger.info("Ready!")
        self.ready.set()

    async def on_message(self, message):
        self._log_ignored(f"Message id {message.id} created")
        if not await self._accept_message(message):
            return

        self._log(message, 'created')

        with self.sql.transaction() as trans:
            self.sql.add_message(trans, message)<commit_msg>Print error on missing config guild.<commit_after>#
# client.py
#
# statbot - Store Discord records for later analysis
# Copyright (c) 2017-2018 <NAME>
#
# statbot is available free of charge under the terms of the MIT
# License. You are free to redistribute and/or modify it under those
# terms. It is distributed in the hopes that it will be useful, but
# WITHOUT ANY WARRANTY. See the LICENSE file for more details.
#

import asyncio
import discord

from .emoji import EmojiData
from .util import null_logger

__all__ = [
    'EventIngestionClient',
]

def member_needs_update(before, after):
    '''
    See if the given member update is something
    we care about.

    Returns 'False' for no difference or
    change we will ignore.
    '''

    for attr in ('name', 'discriminator', 'nick', 'avatar', 'roles'):
        if getattr(before, attr) != getattr(after, attr):
            return True
    return False

class EventIngestionClient(discord.Client):
    __slots__ = (
        'config',
        'logger',
        'sql',
        'ready',
        'sql_init',
        'hooks',
    )

    def __init__(self, config, sql, logger=null_logger):
        super().__init__()
        self.config = config
        self.logger = logger
        self.sql = sql
        self.ready = asyncio.Event()
        self.sql_init = False
        self.hooks = {
            'on_guild_channel_create': None,
            'on_guild_channel_delete': None,
            'on_guild_channel_update': None,
        }

    def run_with_token(self):
        return self.run(self.config['bot']['token'])

    async def wait_until_ready(self):
        # Override wait method to wait until SQL data is also ready
        # At least as long as "await super().wait_until_ready()"
        await self.ready.wait()

    async def _accept_message(self, message):
        await self.wait_until_ready()

        if not hasattr(message, 'guild'):
            self._log_ignored("Message not from a guild.")
            self._log_ignored("Ignoring message.")
            return False
        elif getattr(message.guild, 'id', None) not in self.config['guild-ids']:
            self._log_ignored("Message from a guild we don't care about.")
            self._log_ignored("Ignoring message.")
            return False
        elif message.type != discord.MessageType.default:
            self._log_ignored("Special type of message receieved.")
            self._log_ignored("Ignoring message.")
        else:
            return True

    async def _accept_channel(self, channel):
        await self.wait_until_ready()

        if not hasattr(channel, 'guild'):
            self._log_ignored("Channel not in a guild.")
            self._log_ignored("Ignoring message.")
        elif getattr(channel.guild, 'id', None) not in self.config['guild-ids']:
            self._log_ignored("Event from a guild we don't care about.")
            self._log_ignored("Ignoring message.")
            return False
        else:
            return True

    async def _accept_guild(self, guild):
        await self.wait_until_ready()

        if getattr(guild, 'id', None) not in self.config['guild-ids']:
            self._log_ignored("Event from a guild we don't care about.")
            self._log_ignored("Ignoring message.")
            return False
        else:
            return True

    def _log(self, message, action):
        name = message.author.display_name
        guild = message.guild.name
        chan = message.channel.name

        self.logger.info(f"Message {action} by {name} in {guild} #{chan}")
        if self.config['logger']['full-messages']:
            self.logger.info("<bom>")
            self.logger.info(message.content)
            self.logger.info("<eom>")

    def _log_typing(self, channel, user):
        name = user.display_name
        guild = channel.guild.name
        chan = channel.name

        self.logger.info(f"Typing by {name} on {guild} #{chan}")

    def _log_react(self, reaction, user, action):
        name = user.display_name
        emote = EmojiData(reaction.emoji)
        count = reaction.count
        id = reaction.message.id

        self.logger.info(f"{name} {action} {emote} (total {count}) on message id {id}")

    def _log_ignored(self, message):
        if self.config['logger']['ignored-events']:
            self.logger.debug(message)

    def _init_sql(self, trans):
        self.logger.info(f"Processing {len(self.users)} users...")
        for user in self.users:
            self.sql.upsert_user(trans, user)

        self.logger.info(f"Processing {len(self.guilds)} guilds...")
        for guild in self.guilds:
            self.sql.upsert_guild(trans, guild)

            self.logger.info(f"Processing {len(guild.roles)} roles...")
            for role in guild.roles:
                self.sql.upsert_role(trans, role)

            self.logger.info(f"Processing {len(guild.emojis)} emojis...")
            for emoji in guild.emojis:
                self.sql.upsert_emoji(trans, emoji)

            self.logger.info(f"Processing {len(guild.members)} members...")
            for member in guild.members:
                self.sql.upsert_member(trans, member)

            # In case people left while the bot was down
            self.sql.remove_old_members(trans, guild)

            text_channels = []
            voice_channels = []
            categories = []
            for channel in guild.channels:
                if isinstance(channel, discord.TextChannel):
                    text_channels.append(channel)
                elif isinstance(channel, discord.VoiceChannel):
                    voice_channels.append(channel)
                elif isinstance(channel, discord.CategoryChannel):
                    categories.append(channel)

            self.logger.info(f"Processing {len(categories)} channel categories...")
            for category in categories:
                self.sql.upsert_channel_category(trans, category)

            self.logger.info(f"Processing {len(text_channels)} channels...")
            for channel in text_channels:
                self.sql.upsert_channel(trans, channel)

            self.logger.info(f"Processing {len(voice_channels)} voice channels...")
            for channel in voice_channels:
                self.sql.upsert_voice_channel(trans, channel)

    async def on_ready(self):
        # Print welcome string
        self.logger.info(f"Logged in as {self.user.name} ({self.user.id})")
        self.logger.info("Recording activity in the following guilds:")
        for id in self.config['guild-ids']:
            guild = self.get_guild(id)
            if guild is not None:
                self.logger.info(f"* {guild.name} ({id})")
            else:
                self.logger.error(f"Unable to find guild ID {id}")

        if not self.sql_init:
            self.logger.info("Initializing SQL lookup tables...")
            with self.sql.transaction() as trans:
                self._init_sql(trans)
                self.sql_init = True

        # All done setting up
        self.logger.info("")
        self.logger.info("Ready!")
        self.ready.set()

    async def on_message(self, message):
        self._log_ignored(f"Message id {message.id} created")
        if not await self._accept_message(message):
            return

        self._log(message, 'created')

        with self.sql.transaction() as trans:
            self.sql.add_message(trans, message)
{'fraction_non_alphanumeric': 0.1092944369063772, 'fraction_numerical': 0.001085481682496608, 'mean_word_length': 2.153155080213904, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '304123', 'n_tokens_mistral': 4279, 'n_tokens_neox': 4135, 'n_words': 1080}