<commit_before>import threading
import discord
import logging
import asyncio
import janus
import yaml
import irc3
import os

logging.basicConfig(level=logging.INFO)

server_cache = {}
client = discord.Client()
irc_to_discord = janus.Queue()
discord_to_irc = janus.Queue()


def format_message(user, msg):
    return "<%s>: %s" % (user, msg)


@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('--------')
    servers = client.servers
    for server in servers:
        print('Joined server %s' % server)


@client.event
async def on_message(message):
    if message.author == client.user:
        return

    await discord_to_irc.async_q.put(
        (format_message(message.author.name, message.content),
         message.channel.id)
    )


def get_discord_server(channel_mapping, channel):
    server_id = channel_mapping[channel]
    if server_id not in server_cache:
        server_cache[server_id] = discord.Server(id=server_id)

    return server_cache[server_id]


async def dequeue_irc(config):
    await client.wait_until_ready()
    while True:
        msg, channel = await irc_to_discord.async_q.get()
        try:
            server = get_discord_server(config['channel_mapping'], channel)
        except KeyError:
            logging.debug(
                'Skipping %s because %s isnt in the mapping' % (msg, channel)
            )
        # Prepend the message with zero width white space char to
        # avoid bot loops.
        await client.send_message(server, '\u200B' + msg)

<commit_msg>use clean_content property to convert mentions.<commit_after>import threading
import discord
import logging
import asyncio
import janus
import yaml
import irc3
import os

logging.basicConfig(level=logging.INFO)

server_cache = {}
client = discord.Client()
irc_to_discord = janus.Queue()
discord_to_irc = janus.Queue()


def format_message(user, msg):
    return "<%s>: %s" % (user, msg)


@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('--------')
    servers = client.servers
    for server in servers:
        print('Joined server %s' % server)


@client.event
async def on_message(message):
    if message.author == client.user:
        return

    await discord_to_irc.async_q.put(
        (format_message(message.author.name, message.clean_content),
         message.channel.id)
    )


def get_discord_server(channel_mapping, channel):
    server_id = channel_mapping[channel]
    if server_id not in server_cache:
        server_cache[server_id] = discord.Server(id=server_id)

    return server_cache[server_id]


async def dequeue_irc(config):
    await client.wait_until_ready()
    while True:
        msg, channel = await irc_to_discord.async_q.get()
        try:
            server = get_discord_server(config['channel_mapping'], channel)
        except KeyError:
            logging.debug(
                'Skipping %s because %s isnt in the mapping' % (msg, channel)
            )
        # Prepend the message with zero width white space char to
        # avoid bot loops.
        await client.send_message(server, '\u200B' + msg)


{'fraction_non_alphanumeric': 0.10174237710018669, 'fraction_numerical': 0.002489110143123833, 'mean_word_length': 3.0237797246558196, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2504649', 'n_tokens_mistral': 1019, 'n_tokens_neox': 990, 'n_words': 253}