<commit_before>import json
import re
import sys
from random import randint, choice

import discord
import requests
from discord import Message
from discord.ext.commands import Bot
from requests import Response

from api import newline_separator, directions, regions, statuses, release_types
from api.request import ApiRequest
from bot_config import latest_limit, newest_header, invalid_command_text, oldest_header, boot_up_message
from bot_utils import get_code
from math_parse import NumericStringParser
from math_utils import limit_int
from phases import LogAnalyzer
from stream_handlers import stream_text_log, stream_gzip_decompress

channel_id = "291679908067803136"
bot_spam_id = "319224795785068545"
rpcs3Bot = Bot(command_prefix="!")
id_pattern = '[A-z]{4}\\d{5}'
nsp = NumericStringParser()

file_handlers = (
    # {
    #     'ext': '.zip'
    # },
    {
        'ext': '.log',
        'handler': stream_text_log
    },
    {
        'ext': '.gz',
        'handler': stream_gzip_decompress
    },
    # {
    #     'ext': '.7z',
    #     'handler': stream_7z_decompress
    # }
)


@rpcs3Bot.event
async def on_message(message: Message):
    """
    OnMessage event listener
    :param message: message
    """
    # Self reply detect
    if message.author.name == "RPCS3 Bot":
        return
    # Command detect
    try:
        if message.content[0] == "!":
            return await rpcs3Bot.process_commands(message)
    except IndexError as ie:
        print("Empty message! Could still have attachments.")

    # Code reply
    code_list = []
    for matcher in re.finditer(id_pattern, message.content):
        code = str(matcher.group(0)).upper()
        if code not in code_list:
            code_list.append(code)
            print(code)
    if len(code_list) > 0:
        for code in code_list:
            info = get_code(code)
            if info is not None:
                await rpcs3Bot.send_message(message.channel, '```{}```'.format(info))
            else:
                await rpcs3Bot.send_message(message.channel, '```Serial not found in compatibility database, possibly '
                                                             'untested!```')
        return

    # Log Analysis!
    if len(message.attachments) > 0:
        log = LogAnalyzer()
        print("Attachments present, looking for log file...")
        for attachment in filter(lambda a: any(e['ext'] in a['url'] for e in file_handlers), message.attachments):
            for handler in file_handlers:
                if attachment['url'].endswith(handler['ext']):
                    print("Found log attachment, name: {name}".format(name=attachment['filename']))
                    with requests.get(attachment['url'], stream=True) as response:
                        print("Opened request stream!")
                        # noinspection PyTypeChecker
                        for row in stream_line_by_line_safe(response, handler['handler']):
                            error_code = log.feed(row)
                            if error_code == LogAnalyzer.ERROR_SUCCESS:
                                continue
                            elif error_code == LogAnalyzer.ERROR_PIRACY:
                                await piracy_alert(message, log.get_trigger())
                                break
                            elif error_code == LogAnalyzer.ERROR_OVERFLOW:
                                print("Possible Buffer Overflow Attack Detected!")
                                break
                            elif error_code == LogAnalyzer.ERROR_STOP:
                                await rpcs3Bot.send_message(
                                    message.channel,
                                    log.get_report()
                                )
                                break
                            elif error_code == LogAnalyzer.ERROR_FAIL:
                                break
                    print("Stopping stream!")
        del log


async def piracy_alert(message: Message, trigger: str):
    await rpcs3Bot.send_message(
        message.channel,
        "Pirated release detected {author}!\n"
        "Please note that the RPCS3 community and it's developers do not support piracy!\n"
        "Most of the issues caused by pirated dumps is because they have been tampered with in such a way "
        "and therefore act unpredictably on RPCS3.\n"
        "If you need help obtaining legal dumps please read <https://rpcs3.net/quickstart>\n"
        "The trigger phrase was `{trigger}`, if you believe this was detected wrongly please contact a mod "
        "or {bot_admin}".format(
            author=message.author.mention,
            trigger=mask(trigger),
            bot_admin=message.server.get_member(267367850706993152).mention
        )
    )


def mask(string: str):
    return ''.join("*" if i % 2 == 0 else char for i, char in enumerate(string, 1))


def stream_line_by_line_safe(stream: Response, func: staticmethod):
    buffer = ''
    chunk_buffer = b''<commit_msg>Apostrophes, seriously?<commit_after>import json
import re
import sys
from random import randint, choice

import discord
import requests
from discord import Message
from discord.ext.commands import Bot
from requests import Response

from api import newline_separator, directions, regions, statuses, release_types
from api.request import ApiRequest
from bot_config import latest_limit, newest_header, invalid_command_text, oldest_header, boot_up_message
from bot_utils import get_code
from math_parse import NumericStringParser
from math_utils import limit_int
from phases import LogAnalyzer
from stream_handlers import stream_text_log, stream_gzip_decompress

channel_id = "291679908067803136"
bot_spam_id = "319224795785068545"
rpcs3Bot = Bot(command_prefix="!")
id_pattern = '[A-z]{4}\\d{5}'
nsp = NumericStringParser()

file_handlers = (
    # {
    #     'ext': '.zip'
    # },
    {
        'ext': '.log',
        'handler': stream_text_log
    },
    {
        'ext': '.gz',
        'handler': stream_gzip_decompress
    },
    # {
    #     'ext': '.7z',
    #     'handler': stream_7z_decompress
    # }
)


@rpcs3Bot.event
async def on_message(message: Message):
    """
    OnMessage event listener
    :param message: message
    """
    # Self reply detect
    if message.author.name == "RPCS3 Bot":
        return
    # Command detect
    try:
        if message.content[0] == "!":
            return await rpcs3Bot.process_commands(message)
    except IndexError as ie:
        print("Empty message! Could still have attachments.")

    # Code reply
    code_list = []
    for matcher in re.finditer(id_pattern, message.content):
        code = str(matcher.group(0)).upper()
        if code not in code_list:
            code_list.append(code)
            print(code)
    if len(code_list) > 0:
        for code in code_list:
            info = get_code(code)
            if info is not None:
                await rpcs3Bot.send_message(message.channel, '```{}```'.format(info))
            else:
                await rpcs3Bot.send_message(message.channel, '```Serial not found in compatibility database, possibly '
                                                             'untested!```')
        return

    # Log Analysis!
    if len(message.attachments) > 0:
        log = LogAnalyzer()
        print("Attachments present, looking for log file...")
        for attachment in filter(lambda a: any(e['ext'] in a['url'] for e in file_handlers), message.attachments):
            for handler in file_handlers:
                if attachment['url'].endswith(handler['ext']):
                    print("Found log attachment, name: {name}".format(name=attachment['filename']))
                    with requests.get(attachment['url'], stream=True) as response:
                        print("Opened request stream!")
                        # noinspection PyTypeChecker
                        for row in stream_line_by_line_safe(response, handler['handler']):
                            error_code = log.feed(row)
                            if error_code == LogAnalyzer.ERROR_SUCCESS:
                                continue
                            elif error_code == LogAnalyzer.ERROR_PIRACY:
                                await piracy_alert(message, log.get_trigger())
                                break
                            elif error_code == LogAnalyzer.ERROR_OVERFLOW:
                                print("Possible Buffer Overflow Attack Detected!")
                                break
                            elif error_code == LogAnalyzer.ERROR_STOP:
                                await rpcs3Bot.send_message(
                                    message.channel,
                                    log.get_report()
                                )
                                break
                            elif error_code == LogAnalyzer.ERROR_FAIL:
                                break
                    print("Stopping stream!")
        del log


async def piracy_alert(message: Message, trigger: str):
    print(message.author.id)
    await rpcs3Bot.send_message(
        message.channel,
        "Pirated release detected {author}!\n"
        "Please note that the RPCS3 community and it's developers do not support piracy!\n"
        "Most of the issues caused by pirated dumps is because they have been tampered with in such a way "
        "and therefore act unpredictably on RPCS3.\n"
        "If you need help obtaining legal dumps please read <https://rpcs3.net/quickstart>\n"
        "The trigger phrase was `{trigger}`, if you believe this was detected wrongly please contact a mod "
        "or {bot_admin}".format(
            author=message.author.mention,
            trigger=mask(trigger),
            bot_admin=message.server.get_member('267367850706993152').mention
        )
    )


def mask(string: str):
    return ''.join("*" if i % 2 == 0 else char for i, char in enumerate(string, 1))


def stream_line_by_line_safe(stream: Response, func: staticmethod):
    buffer = ''
    chunk_buffer = b''
{'fraction_non_alphanumeric': 0.08894159501927068, 'fraction_numerical': 0.01502124715881016, 'mean_word_length': 1.9307848247900377, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2377793', 'n_tokens_mistral': 2827, 'n_tokens_neox': 2676, 'n_words': 747}