<commit_before>"""
Defines :class:`~.State`.

.. currentmodule:: curious.core.state
"""

import collections
import logging

import curio
import pprint
import typing
from types import MappingProxyType

from curious.core import gateway
from curious.dataclasses.channel import Channel, ChannelType
from curious.dataclasses.emoji import Emoji
from curious.dataclasses.guild import Guild
from curious.dataclasses.member import Member
from curious.dataclasses.message import Message
from curious.dataclasses.permissions import Permissions
from curious.dataclasses.reaction import Reaction
from curious.dataclasses.role import Role
from curious.dataclasses.presence import Game, Status, Presence
from curious.dataclasses.user import BotUser, User, FriendType, UserSettings
from curious.dataclasses.user import RelationshipUser
from curious.dataclasses.voice_state import VoiceState
from curious.dataclasses.webhook import Webhook

UserType = typing.TypeVar("U", bound=User)


class GuildStore(collections.MutableMapping):
    """
    A store for guilds in the state.
    """

    def __init__(self):
        #: The internal actual guilds.
        self.guilds = {}

        #: The order of the guilds, as specified by the READY packet.
        self.order = []

    def view(self) -> typing.Mapping[int, Guild]:
        """
        :return: A :class:`mappingproxy` of the internal guilds. 
        """
        return MappingProxyType(self.guilds)

    @property
    def with_order(self):
        """
        :return: A mapping of the guilds with the order specified in the ready packet.
        """
        if not self.order:
            return self.view()

        o = collections.OrderedDict()
        for guild in map(int, self.order):
            o[guild] = self.guilds[guild]

        return MappingProxyType(o)

    # abc methods
    def __setitem__(self, key, value):
        return self.guilds.__setitem__(key, value)

    def __getitem__(self, key):
        return self.guilds.__getitem__(key)

    def __delitem__(self, key):
        return self.guilds.__delitem__(key)

    def __iter__(self):
        return self.guilds.__iter__()

    def __len__(self):
        return self.guilds.__len__()


class State(object):
    """
    This represents the state of the Client - in other libraries, the cache.

    It is also used to parse objects into their representations.
    """

    def __init__(self, client, max_messages: int = 500):
        #: The current user of this bot.
        #: This is automatically set after login.
        self._user = None  # type: BotUser

        #: The client associated with this connection.
        self.client = client

        #: The private channel cache.
        self._private_channels = {}

        #: The guilds the bot can see.
        self._guilds = GuildStore()

        #: This user's friends.
        self._friends = {}  # type: typing.Dict[int, RelationshipUser]

        #: This user's blocked users.
        self._blocked = {}  # type: typing.Dict[int, RelationshipUser]

        #: The current user cache.
        self._users = {}

        #: The state logger.
        self.logger = logging.getLogger("curious.state")

        #: The deque of messages.
        #: This is bounded to prevent the message cache from growing infinitely.
        self._messages = collections.deque(maxlen=max_messages)

        self.__shards_is_ready = collections.defaultdict(lambda *args, **kwargs: curio.Event())
        self.__voice_state_crap = collections.defaultdict(lambda *args, **kwargs: ((curio.Event(), curio.Event()), {}))

    def is_ready(self, shard_id: int) -> curio.Event:
        """
        Checks if a shard is ready.
        
        :param shard_id: The shard ID to check.
        :return: A :class:`curio.Event` signifying if this shard is ready or not.
        """
        return self.__shards_is_ready[shard_id]

    def _reset(self, shard_id: int):
        """
        Called after session is invalidated, to reset our state.
        """
        self.__shards_is_ready.pop(shard_id, None)

        for guild in self.guilds_for_shard(shard_id):
            guild._finished_chunking.clear()

    def _mark_for_chunking(self, gw: 'gateway.Gateway', guild: Guild):
        """
        Marks a guild for chunking.
        """
        if all(not g.unavailable for g in self.guilds_for_shard(guild.shard_id)):
            gw._enqueued_guilds.append(guild)
            # we have all guilds anyway, so raise ChunkGuilds NOW
            raise gateway.ChunkGuilds

        if len(gw._enqueued_guilds) >= 74:
            # bump it up to 74
            gw._enqueued_guilds.append(guild)
            raise gateway.ChunkGuilds

        gw._enqueued_guilds.append(guild)

    @property
    def guilds(self) -> typing.Mapping[int, Guild]:
        """
        :return: A mapping of int -> :class:`~.Guild`. 
        """
        return MappingProxyType(self._guilds)

    def have_all_chunks(self, shard_id: int):
        """
        Checks if we have all the chunks for the specified shard.

        :param shard_id: The shard ID to check.
        """
        return all(guild._finished_chunking.is_set()
                   for guild in self.guilds.values()
                   if guild.shard_id == shard_id)

    def guilds_for_shard(self, shard_id: int):
        """
        Gets all the guilds for a particular shard.
        """
        return [guild for guild in self.guilds.values() if guild.shard_id == shard_id]

    async def _check_ready(self, gw: 'gateway.Gateway', guild: Guild):
        """
        Checks if we should dispatch ready for this shard.
        """
        if self.is_ready(gw.shard_id).is_set():
            # Already ready, don't bother.
            return

        if self.have_all_chunks(gw.shard_id):
            # Have all chunks anyway, dispatch now.
            self.logger.info("All guilds fully chunked on shard {}, dispatching READY.".format(gw.shard_id))
            await self.is_ready(gw.shard_id).set()
            await self.client.fire_event("ready", gateway=gw)

            # check for userbots
            if not self._user.bot:
                # request a guild sync
                self.logger.info("Requesting GUILD_SYNC to update our presences.")
                await gw.send_guild_sync(self.guilds.values())

            return

        if self._user.bot:
            # Check if we need to forcibly fire ChunkGuilds.
            if all(g.unavailable is False for g in self.guilds_for_shard(gw.shard_id)):
                raise gateway.ChunkGuilds

    # get_all_* methods
    def get_all_channels(self) -> typing.Generator[Channel, None, None]:
        """
        :return: A generator that yields all :class:`~.Channel`s the bot can see.
        """
        for guild in self._guilds.values():
            for channel in guild.channels.values():
                yield channel

    def get_all_members(self) -> typing.Generator[Member, None, None]:
        """
        :return: A generator that yields all :class:`~.Member`s the bot can see.
        """
        for guild in self.guilds.values():
            for member in guild.members.values():
                yield member

    def get_all_roles(self) -> typing.Generator[Role, None, None]:
        """
        :return: A generator that yields all :class:`~.Role`s the bot can see. 
        """
        for guild in self.guilds.values():
            for role in guild.roles.values():
                yield role

    def find_channel(self, channel_id: int) -> typing.Union[Channel, None]:
        """
        Finds a channel by ID.  
        This will search all guild channels, as well as private channels.
        
        :param channel_id: The ID of the channel to find.
        :return: A :class:`~.Channel` that represents the channel, or None if no channel was found.
        """
        # default channel_id == guild id
        if channel_id in self._guilds:
            try:
                return self._guilds[channel_id].default_channel
            except KeyError:
                return None

        if channel_id in self._private_channels:
            return self._private_channels[channel_id]

        for guild in self._guilds.values():
            if channel_id in guild._channels:
                return guild._channels[channel_id]

    def _find_message(self, message_id: int) -> Message:
        for message in reversed(self._messages):
            if message.id == message_id:
                return message

    def _check_decache_user(self, id: int):
        """
        Checks if we should decache a user.

        This will check if there is any guild with a reference to the user.
        """
        if id in self._friends or id in self._blocked:
            return

        for guild in self._guilds.values():
            if id in guild.members:
                return

        for channel in self._private_channels.values():
            if id in [m.id for m in channel.recipients.values()]:
                return

        # didn't return, so no references
        self._users.pop(id, None)

    def make_webhook(self, event_data: dict) -> Webhook:
        """
        Creates a new webhook object from the event data.

        :param event_data: The event data.
        :return: A :class:`~.Webhook`.
        """
        if "content" in event_data:
            # message object, so we have to do a minor bit of remapping
            user = event_data.get("author", {})
            webhook_id = int(event_data.pop("webhook_id", 0))
            owner = {}
        else:
            # make a "fake" user
            webhook_id = int(event_data.get("id", 0))
            user = {
                "id": webhook_id,
                "discriminator": "0000",
                "avatar": event_data.get("avatar"),
                "username": event_data.get("username")
            }
            owner = event_data.get("user", {})

        channel = self.find_channel(int(event_data.get("channel_id")))
        user = User(self.client, **user)
        user.bot = True
        webhook = Webhook(client=self.client, webhook_id=webhook_id, **event_data)
        webhook.channel = channel
        webhook.user = user
        webhook.token = event_data.get("token", None)

        if owner:
            # only create Owner if the data was returned
            webhook.owner = self.make_user(owner)

        # default fields, these are lazily loaded by properties
        webhook._default_name = event_data.get("name", None)
        webhook._default_avatar = event_data.get("avatar", None)

        return webhook

    def make_private_channel(self, channel_data: dict) -> Channel:
        """
        Creates a new private channel and caches it.

        :param channel_data: The channel data to cache.
        :return: A new :class:`~.Channel`.
        """
        channel = Channel(self.client, **channel_data)
        self._private_channels[channel.id] = channel

        return channel

    def make_user(self, user_data: dict, *,
                  user_klass: typing.Type[UserType] = User,
                  override_cache: bool = False) -> UserType:
        """
        Creates a new user and caches it.

        :param user_data: The user data to use to create.
        :param user_klass: The type of user to create.
        :param override_cache: Should the cache be overridden?
        :return: A new :class`~.User` (hopefully).
        """
        id = int(user_data.get("id", 0))
        if id in self._users and not override_cache:
            return self._users[id]

        user = user_klass(self.client, **user_data)
        self._users[user.id] = user

        return user

    def make_message(self, event_data: dict, cache: bool = True) -> Message:
        """
        Constructs a new message object.
        
        :param event_data: The message data to use to create.
        :param cache: Should this message be cached?
        :return: A new :class:`~.Message` object for the message.
        """
        message = Message(self.client, **event_data)
        # discord won't give us the Guild id
        # so we have to search it from the channels
        channel_id = int(event_data.get("channel_id"))
        channel = self.find_channel(channel_id)

        author_id = int(event_data.get("author", {}).get("id", 0))

        message.channel = channel
        message.guild = channel.guild
        if message.channel.type == ChannelType.PRIVATE:
            if author_id == self._user.id:
                message.author = self._user
            else:
                message.author = message.channel.user
        elif message.channel.type == ChannelType.GROUP:
            message.author = message.channel.recipients.get(author_id, None)
        else:
            # Webhooks also exist.
            if event_data.get("webhook_id") is not None:
                message.author = self.make_webhook(event_data)
            else:
                message.author = message.guild.members.get(author_id)

        for reaction_data in event_data.get("reactions", []):
            emoji = reaction_data.get("emoji", {})
            reaction = Reaction(**reaction_data)

            if "id" in emoji and emoji["id"] is not None:
                emoji_obb = message.guild.emojis.get(int(emoji["id"]))
                if emoji_obb is None:
                    emoji_obb = Emoji(id=emoji["id"], name=emoji["name"])
            else:
                emoji_obb = emoji.get("name", None)

            reaction.emoji = emoji_obb
            message.reactions.append(reaction)

        if cache and message not in self._messages:
            self._messages.append(message)

        return message

    async def wait_for_voice_data(self, guild_id: int):
        """
        Waits for the two voice data packets to be received for the specified guild.
        """
        events, state = self.__voice_state_crap[guild_id]  # state is a pointer

        for event in events:
            await event.wait()

        # pop out the event data for laeter reconnects
        self.__voice_state_crap.pop(guild_id)

        return state

    # Event handlers.
    # These parse the events and deconstruct them.
    async def handle_ready(self, gw: 'gateway.Gateway', event_data: dict):
        """
        Called when READY is dispatched.
        """
        gw.session_id = event_data.get("session_id")

        # Create our bot user.
        self._user = BotUser(self.client, **event_data.get("user"))
        # cache ourselves
        self._users[self._user.id] = self._user

        self.logger.info("Parsing ready for `{}#{}` ({})".format(self._user.username, self._user.discriminator,
                                                                 self._user.id))

        self.logger.info("Logged in as a userbot: {}".format(not self._user.bot))

        # User-bots only.
        # Parse relationships first before processing anything else.
        # This ensures all friends are calculated and assigned.
        if not self._user.bot:
            # Parse settings.
            settings = event_data.get("user_settings", {})
            self._user.settings = UserSettings(self.client, **settings)
            self._user.settings.status = event_data.get("status", None)

            # Parse friends and blocked users.
            for item in event_data.get("relationships", []):
                # make user to cache it
                user = self.make_user(item.get("user", {}), user_klass=RelationshipUser)
                user.type_ = FriendType(item["type"])
                if user.type_ == FriendType.FRIEND:
                    self._friends[int(item["id"])] = user
                else:
                    self._blocked[int(item["id"])] = user

            for presence in event_data.get("presences", []):
                u = int(presence["user"]["id"])
                fr = self._friends.get(u)

                # eventual consistency
                if not fr:
                    continue

                fr.presence = Presence(**presence)

            self._guilds.order = list(map(int, self._user.settings.guild_positions))

            self.logger.info("Processed {} friends "
                             "and {} blocked users.".format(len(self._friends), len(self._blocked)))

        # Create all of the private channels.
        for channel in event_data.get("private_channels"):
            self.make_private_channel(channel)

        self.logger.info("Processed {} private channels.".format(len(self._private_channels)))

        # Create all of the guilds.
        for guild in event_data.get("guilds"):
            new_guild = Guild(self.client, **guild)
            new_guild.shard_id = gw.shard_id
            self._guilds[new_guild.id] = new_guild

            if not self._user.bot and len(event_data.get("guilds")) <= 100:
                # this might as well be a GUILD_CREATE, so treat it as one
                new_guild.start_chunking()
                await self.handle_guild_create(gw, guild)
                gw._dispatches_handled["GUILD_CREATE"] += 1

        if not self._user.bot and len(event_data.get("guilds")) <= 100:
            # Chunk now, sync later.
            await gw.request_chunks([g for g in self.guilds.values()])
            self.logger.info("Chunking {} guilds immediately.".format(len(self.guilds)))

        self.logger.info("Ready processed for shard {}. Delaying until all guilds are chunked.".format(gw.shard_id))

        # Don't fire `_ready` here, because we don't have all guilds.
        await self.client.fire_event("connect", gateway=gw)

        # event_data.pop("guilds")
        # pprint.pprint(event_data)

        # However, if the client has no guilds, we DO want to fire ready.
        if len(event_data.get("guilds", {})) == 0:
            await self.is_ready(gw.shard_id).set()
            self.logger.info("No more guilds to get for shard {}, or client is user. "
                             "Dispatching READY.".format(gw.shard_id))
            await self.client.fire_event("ready", gateway=gw)

    async def handle_resumed(self, gw: 'gateway.Gateway', event_data: dict):
        self.logger.info("Successfully resumed session on shard ID {}.".format(gw.shard_id))
        await self.client.fire_event("resumed", gateway=gw)

    async def handle_user_update(self, gw: 'gateway.Gateway', event_data: dict):
        """
        Called when the bot's user is updated.
        """
        id = event_data.get("id")

        self._user.id = int(id)
        self._user.username = event_data.get("username", self._user.username)
        self._user.discriminator = event_data.get("discriminator", self._user.discriminator)
        self._user._avatar_hash = event_data.get("avatar", self._user._avatar_hash)

        await self.client.fire_event("user_update", gateway=gw)

    async def handle_presence_update(self, gw: 'gateway.Gateway', event_data: dict):
        """
        Called when a member changes game.
        """
        guild_id = int(event_data.get("guild_id", 0))
        guild = self._guilds.get(guild_id)

        if not guild:
            # user presence update
            user_id = int(event_data["user"]["id"])
            fr = self._friends.get(user_id)

            if not fr:
                # wtf
                return

            # re-create the user object
            if "username" in event_data["user"]:
                # full user object
                self._friends[user_id] = self.make_user(event_data["user"], user_klass=RelationshipUser,
                                                        override_cache=True)
                fr = self._friends[user_id]

            fr.presence.status = event_data.get("status")
            game = event_data.get("game")
            fr.presence.game = game

            await self.client.fire_event("friend_update", fr, gateway=gw)
            return

        member_id = int(event_data["user"]["id"])
        member = guild.members.get(member_id)

        if member is None:
            # thanks discord
            return

        old_member = member._copy()

        game = event_data.get("game", {})
        member.presence.game = game

        roles = event_data.get("roles", [])
        for role_id in roles:
            role_id = int(role_id)
            role = guild.roles.get(role_id)

            if not role:
                continue

            member._roles[role.id] = role

        member.presence.status = event_data.get("status")

        if not isinstance(member.user, RelationshipUser):
            # recreate the user object
            if "username" in event_data["user"]:
                self.make_user(event_data["user"], override_cache=True)

        await self.client.fire_event("member_update", old_member, member, gateway=gw)

    async def handle_guild_members_chunk(self, gw: 'gateway.Gateway', event_data: dict):
        """
        Called when a chunk of members has arrived.
        """
        id = int(event_data.get("guild_id"))
        guild = self._guilds.get(id)

        if not guild:
            self.logger.warning("Got a chunk for a Guild that doesn't exist...")
            return

        members = event_data.get("members", [])
        self.logger.info("Got a chunk of {} members in guild {} "
                         "on shard {}".format(len(members), guild.name or guild.id, guild.shard_id))

        guild._handle_member_chunk(event_data.get("members"))
        await self.client.fire_event("guild_chunk", guild, gateway=gw)
<commit_msg>Clear roles when parsing from PRESENCE_UPDATE<commit_after>"""
Defines :class:`~.State`.

.. currentmodule:: curious.core.state
"""

import collections
import logging

import curio
import pprint
import typing
from types import MappingProxyType

from curious.core import gateway
from curious.dataclasses.channel import Channel, ChannelType
from curious.dataclasses.emoji import Emoji
from curious.dataclasses.guild import Guild
from curious.dataclasses.member import Member
from curious.dataclasses.message import Message
from curious.dataclasses.permissions import Permissions
from curious.dataclasses.reaction import Reaction
from curious.dataclasses.role import Role
from curious.dataclasses.presence import Game, Status, Presence
from curious.dataclasses.user import BotUser, User, FriendType, UserSettings
from curious.dataclasses.user import RelationshipUser
from curious.dataclasses.voice_state import VoiceState
from curious.dataclasses.webhook import Webhook

UserType = typing.TypeVar("U", bound=User)


class GuildStore(collections.MutableMapping):
    """
    A store for guilds in the state.
    """

    def __init__(self):
        #: The internal actual guilds.
        self.guilds = {}

        #: The order of the guilds, as specified by the READY packet.
        self.order = []

    def view(self) -> typing.Mapping[int, Guild]:
        """
        :return: A :class:`mappingproxy` of the internal guilds. 
        """
        return MappingProxyType(self.guilds)

    @property
    def with_order(self):
        """
        :return: A mapping of the guilds with the order specified in the ready packet.
        """
        if not self.order:
            return self.view()

        o = collections.OrderedDict()
        for guild in map(int, self.order):
            o[guild] = self.guilds[guild]

        return MappingProxyType(o)

    # abc methods
    def __setitem__(self, key, value):
        return self.guilds.__setitem__(key, value)

    def __getitem__(self, key):
        return self.guilds.__getitem__(key)

    def __delitem__(self, key):
        return self.guilds.__delitem__(key)

    def __iter__(self):
        return self.guilds.__iter__()

    def __len__(self):
        return self.guilds.__len__()


class State(object):
    """
    This represents the state of the Client - in other libraries, the cache.

    It is also used to parse objects into their representations.
    """

    def __init__(self, client, max_messages: int = 500):
        #: The current user of this bot.
        #: This is automatically set after login.
        self._user = None  # type: BotUser

        #: The client associated with this connection.
        self.client = client

        #: The private channel cache.
        self._private_channels = {}

        #: The guilds the bot can see.
        self._guilds = GuildStore()

        #: This user's friends.
        self._friends = {}  # type: typing.Dict[int, RelationshipUser]

        #: This user's blocked users.
        self._blocked = {}  # type: typing.Dict[int, RelationshipUser]

        #: The current user cache.
        self._users = {}

        #: The state logger.
        self.logger = logging.getLogger("curious.state")

        #: The deque of messages.
        #: This is bounded to prevent the message cache from growing infinitely.
        self._messages = collections.deque(maxlen=max_messages)

        self.__shards_is_ready = collections.defaultdict(lambda *args, **kwargs: curio.Event())
        self.__voice_state_crap = collections.defaultdict(lambda *args, **kwargs: ((curio.Event(), curio.Event()), {}))

    def is_ready(self, shard_id: int) -> curio.Event:
        """
        Checks if a shard is ready.
        
        :param shard_id: The shard ID to check.
        :return: A :class:`curio.Event` signifying if this shard is ready or not.
        """
        return self.__shards_is_ready[shard_id]

    def _reset(self, shard_id: int):
        """
        Called after session is invalidated, to reset our state.
        """
        self.__shards_is_ready.pop(shard_id, None)

        for guild in self.guilds_for_shard(shard_id):
            guild._finished_chunking.clear()

    def _mark_for_chunking(self, gw: 'gateway.Gateway', guild: Guild):
        """
        Marks a guild for chunking.
        """
        if all(not g.unavailable for g in self.guilds_for_shard(guild.shard_id)):
            gw._enqueued_guilds.append(guild)
            # we have all guilds anyway, so raise ChunkGuilds NOW
            raise gateway.ChunkGuilds

        if len(gw._enqueued_guilds) >= 74:
            # bump it up to 74
            gw._enqueued_guilds.append(guild)
            raise gateway.ChunkGuilds

        gw._enqueued_guilds.append(guild)

    @property
    def guilds(self) -> typing.Mapping[int, Guild]:
        """
        :return: A mapping of int -> :class:`~.Guild`. 
        """
        return MappingProxyType(self._guilds)

    def have_all_chunks(self, shard_id: int):
        """
        Checks if we have all the chunks for the specified shard.

        :param shard_id: The shard ID to check.
        """
        return all(guild._finished_chunking.is_set()
                   for guild in self.guilds.values()
                   if guild.shard_id == shard_id)

    def guilds_for_shard(self, shard_id: int):
        """
        Gets all the guilds for a particular shard.
        """
        return [guild for guild in self.guilds.values() if guild.shard_id == shard_id]

    async def _check_ready(self, gw: 'gateway.Gateway', guild: Guild):
        """
        Checks if we should dispatch ready for this shard.
        """
        if self.is_ready(gw.shard_id).is_set():
            # Already ready, don't bother.
            return

        if self.have_all_chunks(gw.shard_id):
            # Have all chunks anyway, dispatch now.
            self.logger.info("All guilds fully chunked on shard {}, dispatching READY.".format(gw.shard_id))
            await self.is_ready(gw.shard_id).set()
            await self.client.fire_event("ready", gateway=gw)

            # check for userbots
            if not self._user.bot:
                # request a guild sync
                self.logger.info("Requesting GUILD_SYNC to update our presences.")
                await gw.send_guild_sync(self.guilds.values())

            return

        if self._user.bot:
            # Check if we need to forcibly fire ChunkGuilds.
            if all(g.unavailable is False for g in self.guilds_for_shard(gw.shard_id)):
                raise gateway.ChunkGuilds

    # get_all_* methods
    def get_all_channels(self) -> typing.Generator[Channel, None, None]:
        """
        :return: A generator that yields all :class:`~.Channel`s the bot can see.
        """
        for guild in self._guilds.values():
            for channel in guild.channels.values():
                yield channel

    def get_all_members(self) -> typing.Generator[Member, None, None]:
        """
        :return: A generator that yields all :class:`~.Member`s the bot can see.
        """
        for guild in self.guilds.values():
            for member in guild.members.values():
                yield member

    def get_all_roles(self) -> typing.Generator[Role, None, None]:
        """
        :return: A generator that yields all :class:`~.Role`s the bot can see. 
        """
        for guild in self.guilds.values():
            for role in guild.roles.values():
                yield role

    def find_channel(self, channel_id: int) -> typing.Union[Channel, None]:
        """
        Finds a channel by ID.  
        This will search all guild channels, as well as private channels.
        
        :param channel_id: The ID of the channel to find.
        :return: A :class:`~.Channel` that represents the channel, or None if no channel was found.
        """
        # default channel_id == guild id
        if channel_id in self._guilds:
            try:
                return self._guilds[channel_id].default_channel
            except KeyError:
                return None

        if channel_id in self._private_channels:
            return self._private_channels[channel_id]

        for guild in self._guilds.values():
            if channel_id in guild._channels:
                return guild._channels[channel_id]

    def _find_message(self, message_id: int) -> Message:
        for message in reversed(self._messages):
            if message.id == message_id:
                return message

    def _check_decache_user(self, id: int):
        """
        Checks if we should decache a user.

        This will check if there is any guild with a reference to the user.
        """
        if id in self._friends or id in self._blocked:
            return

        for guild in self._guilds.values():
            if id in guild.members:
                return

        for channel in self._private_channels.values():
            if id in [m.id for m in channel.recipients.values()]:
                return

        # didn't return, so no references
        self._users.pop(id, None)

    def make_webhook(self, event_data: dict) -> Webhook:
        """
        Creates a new webhook object from the event data.

        :param event_data: The event data.
        :return: A :class:`~.Webhook`.
        """
        if "content" in event_data:
            # message object, so we have to do a minor bit of remapping
            user = event_data.get("author", {})
            webhook_id = int(event_data.pop("webhook_id", 0))
            owner = {}
        else:
            # make a "fake" user
            webhook_id = int(event_data.get("id", 0))
            user = {
                "id": webhook_id,
                "discriminator": "0000",
                "avatar": event_data.get("avatar"),
                "username": event_data.get("username")
            }
            owner = event_data.get("user", {})

        channel = self.find_channel(int(event_data.get("channel_id")))
        user = User(self.client, **user)
        user.bot = True
        webhook = Webhook(client=self.client, webhook_id=webhook_id, **event_data)
        webhook.channel = channel
        webhook.user = user
        webhook.token = event_data.get("token", None)

        if owner:
            # only create Owner if the data was returned
            webhook.owner = self.make_user(owner)

        # default fields, these are lazily loaded by properties
        webhook._default_name = event_data.get("name", None)
        webhook._default_avatar = event_data.get("avatar", None)

        return webhook

    def make_private_channel(self, channel_data: dict) -> Channel:
        """
        Creates a new private channel and caches it.

        :param channel_data: The channel data to cache.
        :return: A new :class:`~.Channel`.
        """
        channel = Channel(self.client, **channel_data)
        self._private_channels[channel.id] = channel

        return channel

    def make_user(self, user_data: dict, *,
                  user_klass: typing.Type[UserType] = User,
                  override_cache: bool = False) -> UserType:
        """
        Creates a new user and caches it.

        :param user_data: The user data to use to create.
        :param user_klass: The type of user to create.
        :param override_cache: Should the cache be overridden?
        :return: A new :class`~.User` (hopefully).
        """
        id = int(user_data.get("id", 0))
        if id in self._users and not override_cache:
            return self._users[id]

        user = user_klass(self.client, **user_data)
        self._users[user.id] = user

        return user

    def make_message(self, event_data: dict, cache: bool = True) -> Message:
        """
        Constructs a new message object.
        
        :param event_data: The message data to use to create.
        :param cache: Should this message be cached?
        :return: A new :class:`~.Message` object for the message.
        """
        message = Message(self.client, **event_data)
        # discord won't give us the Guild id
        # so we have to search it from the channels
        channel_id = int(event_data.get("channel_id"))
        channel = self.find_channel(channel_id)

        author_id = int(event_data.get("author", {}).get("id", 0))

        message.channel = channel
        message.guild = channel.guild
        if message.channel.type == ChannelType.PRIVATE:
            if author_id == self._user.id:
                message.author = self._user
            else:
                message.author = message.channel.user
        elif message.channel.type == ChannelType.GROUP:
            message.author = message.channel.recipients.get(author_id, None)
        else:
            # Webhooks also exist.
            if event_data.get("webhook_id") is not None:
                message.author = self.make_webhook(event_data)
            else:
                message.author = message.guild.members.get(author_id)

        for reaction_data in event_data.get("reactions", []):
            emoji = reaction_data.get("emoji", {})
            reaction = Reaction(**reaction_data)

            if "id" in emoji and emoji["id"] is not None:
                emoji_obb = message.guild.emojis.get(int(emoji["id"]))
                if emoji_obb is None:
                    emoji_obb = Emoji(id=emoji["id"], name=emoji["name"])
            else:
                emoji_obb = emoji.get("name", None)

            reaction.emoji = emoji_obb
            message.reactions.append(reaction)

        if cache and message not in self._messages:
            self._messages.append(message)

        return message

    async def wait_for_voice_data(self, guild_id: int):
        """
        Waits for the two voice data packets to be received for the specified guild.
        """
        events, state = self.__voice_state_crap[guild_id]  # state is a pointer

        for event in events:
            await event.wait()

        # pop out the event data for laeter reconnects
        self.__voice_state_crap.pop(guild_id)

        return state

    # Event handlers.
    # These parse the events and deconstruct them.
    async def handle_ready(self, gw: 'gateway.Gateway', event_data: dict):
        """
        Called when READY is dispatched.
        """
        gw.session_id = event_data.get("session_id")

        # Create our bot user.
        self._user = BotUser(self.client, **event_data.get("user"))
        # cache ourselves
        self._users[self._user.id] = self._user

        self.logger.info("Parsing ready for `{}#{}` ({})".format(self._user.username, self._user.discriminator,
                                                                 self._user.id))

        self.logger.info("Logged in as a userbot: {}".format(not self._user.bot))

        # User-bots only.
        # Parse relationships first before processing anything else.
        # This ensures all friends are calculated and assigned.
        if not self._user.bot:
            # Parse settings.
            settings = event_data.get("user_settings", {})
            self._user.settings = UserSettings(self.client, **settings)
            self._user.settings.status = event_data.get("status", None)

            # Parse friends and blocked users.
            for item in event_data.get("relationships", []):
                # make user to cache it
                user = self.make_user(item.get("user", {}), user_klass=RelationshipUser)
                user.type_ = FriendType(item["type"])
                if user.type_ == FriendType.FRIEND:
                    self._friends[int(item["id"])] = user
                else:
                    self._blocked[int(item["id"])] = user

            for presence in event_data.get("presences", []):
                u = int(presence["user"]["id"])
                fr = self._friends.get(u)

                # eventual consistency
                if not fr:
                    continue

                fr.presence = Presence(**presence)

            self._guilds.order = list(map(int, self._user.settings.guild_positions))

            self.logger.info("Processed {} friends "
                             "and {} blocked users.".format(len(self._friends), len(self._blocked)))

        # Create all of the private channels.
        for channel in event_data.get("private_channels"):
            self.make_private_channel(channel)

        self.logger.info("Processed {} private channels.".format(len(self._private_channels)))

        # Create all of the guilds.
        for guild in event_data.get("guilds"):
            new_guild = Guild(self.client, **guild)
            new_guild.shard_id = gw.shard_id
            self._guilds[new_guild.id] = new_guild

            if not self._user.bot and len(event_data.get("guilds")) <= 100:
                # this might as well be a GUILD_CREATE, so treat it as one
                new_guild.start_chunking()
                await self.handle_guild_create(gw, guild)
                gw._dispatches_handled["GUILD_CREATE"] += 1

        if not self._user.bot and len(event_data.get("guilds")) <= 100:
            # Chunk now, sync later.
            await gw.request_chunks([g for g in self.guilds.values()])
            self.logger.info("Chunking {} guilds immediately.".format(len(self.guilds)))

        self.logger.info("Ready processed for shard {}. Delaying until all guilds are chunked.".format(gw.shard_id))

        # Don't fire `_ready` here, because we don't have all guilds.
        await self.client.fire_event("connect", gateway=gw)

        # event_data.pop("guilds")
        # pprint.pprint(event_data)

        # However, if the client has no guilds, we DO want to fire ready.
        if len(event_data.get("guilds", {})) == 0:
            await self.is_ready(gw.shard_id).set()
            self.logger.info("No more guilds to get for shard {}, or client is user. "
                             "Dispatching READY.".format(gw.shard_id))
            await self.client.fire_event("ready", gateway=gw)

    async def handle_resumed(self, gw: 'gateway.Gateway', event_data: dict):
        self.logger.info("Successfully resumed session on shard ID {}.".format(gw.shard_id))
        await self.client.fire_event("resumed", gateway=gw)

    async def handle_user_update(self, gw: 'gateway.Gateway', event_data: dict):
        """
        Called when the bot's user is updated.
        """
        id = event_data.get("id")

        self._user.id = int(id)
        self._user.username = event_data.get("username", self._user.username)
        self._user.discriminator = event_data.get("discriminator", self._user.discriminator)
        self._user._avatar_hash = event_data.get("avatar", self._user._avatar_hash)

        await self.client.fire_event("user_update", gateway=gw)

    async def handle_presence_update(self, gw: 'gateway.Gateway', event_data: dict):
        """
        Called when a member changes game.
        """
        guild_id = int(event_data.get("guild_id", 0))
        guild = self._guilds.get(guild_id)

        if not guild:
            # user presence update
            user_id = int(event_data["user"]["id"])
            fr = self._friends.get(user_id)

            if not fr:
                # wtf
                return

            # re-create the user object
            if "username" in event_data["user"]:
                # full user object
                self._friends[user_id] = self.make_user(event_data["user"], user_klass=RelationshipUser,
                                                        override_cache=True)
                fr = self._friends[user_id]

            fr.presence.status = event_data.get("status")
            game = event_data.get("game")
            fr.presence.game = game

            await self.client.fire_event("friend_update", fr, gateway=gw)
            return

        member_id = int(event_data["user"]["id"])
        member = guild.members.get(member_id)

        if member is None:
            # thanks discord
            return

        old_member = member._copy()

        game = event_data.get("game", {})
        member.presence.game = game

        roles = event_data.get("roles", [])

        if roles:
            # clear roles
            member._roles = {}
            for role_id in roles:
                role_id = int(role_id)
                role = guild.roles.get(role_id)

                if not role:
                    continue

                member._roles[role.id] = role

        member.presence.status = event_data.get("status")

        if not isinstance(member.user, RelationshipUser):
            # recreate the user object
            if "username" in event_data["user"]:
                self.make_user(event_data["user"], override_cache=True)

        await self.client.fire_event("member_update", old_member, member, gateway=gw)

    async def handle_guild_members_chunk(self, gw: 'gateway.Gateway', event_data: dict):
        """
        Called when a chunk of members has arrived.
        """
        id = int(event_data.get("guild_id"))
        guild = self._guilds.get(id)

        if not guild:
            self.logger.warning("Got a chunk for a Guild that doesn't exist...")
            return

        members = event_data.get("members", [])
        self.logger.info("Got a chunk of {} members in guild {} "
                         "on shard {}".format(len(members), guild.name or guild.id, guild.shard_id))

        guild._handle_member_chunk(event_data.get("members"))
        await self.client.fire_event("guild_chunk", guild, gateway=gw)

{'fraction_non_alphanumeric': 0.10295890919209721, 'fraction_numerical': 0.001113069288563213, 'mean_word_length': 2.0637254901960786, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '729958', 'n_tokens_mistral': 12812, 'n_tokens_neox': 12180, 'n_words': 3302}