<commit_before>import random
import time
from threading import Thread
from parsing import *
from datahandling import *
from metasmoke import Metasmoke
from globalvars import GlobalVars
import os
import re
from datetime import datetime
from utcdate import UtcDate
from apigetpost import api_get_post
from spamhandling import handle_spam
from termcolor import colored
from findspam import FindSpam


# Please note: If new !!/ commands are added or existing ones are modified, don't forget to
# update the wiki at https://github.com/Charcoal-SE/SmokeDetector/wiki/Commands.


def post_message_in_room(room_id_str, msg, length_check=True):
    if room_id_str == GlobalVars.charcoal_room_id:
        GlobalVars.charcoal_hq.send_message(msg, length_check)
    elif room_id_str == GlobalVars.meta_tavern_room_id:
        GlobalVars.tavern_on_the_meta.send_message(msg, length_check)
    elif room_id_str == GlobalVars.socvr_room_id:
        GlobalVars.socvr.send_message(msg, length_check)


def is_smokedetector_message(user_id, room_id):
    return user_id == GlobalVars.smokeDetector_user_id[room_id]


def print_chat_message(ev):
    message = colored("Chat message in " + ev.data["room_name"] + " (" + str(ev.data["room_id"]) + "): \"", attrs=['bold'])
    message += ev.data['content']
    message += "\""
    message += colored(" - " + ev.data['user_name'], attrs=['bold'])
    print message


def watcher(ev, wrap2):
    if ev.type_id != 1:
        return

    print_chat_message(ev)

    ev_room = str(ev.data["room_id"])
    ev_user_id = str(ev.data["user_id"])
    ev_room_name = ev.data["room_name"].encode('utf-8')
    content_source = ev.message.content_source
    message_id = ev.message.id
    if is_smokedetector_message(ev_user_id, ev_room):
        add_latest_smokedetector_message(ev_room, message_id)
    message_parts = content_source.split(" ")

    ev_user_name = ev.data["user_name"].encode('utf-8')
    ev_user_link = "//chat." + wrap2.host + "/users/" + str(ev.user.id)
    if ev_user_name != "SmokeDetector":
        GlobalVars.tavern_users_chatting.append((ev_user_name, ev_user_link))

    shortcut_messages = []
    if message_parts[0].lower() == "sd":
        message_parts = preprocess_shortcut_command(content_source).split(" ")
        latest_smokedetector_messages = GlobalVars.latest_smokedetector_messages[ev_room]
        commands = message_parts[1:]
        if len(latest_smokedetector_messages) == 0:
            ev.message.reply("I don't have any messages posted after the latest reboot.")
            return
        if len(commands) > len(latest_smokedetector_messages):
            ev.message.reply("I've only posted {} messages since the latest reboot; that's not enough to execute all commands. No commands were executed.".format(len(latest_smokedetector_messages)))
            return
        for i in range(0, len(commands)):
            shortcut_messages.append(":" + str(latest_smokedetector_messages[-(i + 1)]) + " " + commands[i])
        reply = ""
        amount_none = 0
        amount_skipped = 0
        length = len(shortcut_messages)
        for i in range(0, length):
            current_message = shortcut_messages[i]
            if length > 1:
                reply += str(i + 1) + ". "
            reply += "[" + current_message.split(" ")[0] + "] "
            if current_message.split(" ")[1] != "-":
                result = handle_commands(current_message.lower(), current_message.split(" "), ev_room, ev_room_name, ev_user_id, ev_user_name, wrap2, current_message, message_id)
                if result is not None:
                    reply += result + os.linesep
                else:
                    reply += "<processed without return value>" + os.linesep
                    amount_none += 1
            else:
                reply += "<skipped>" + os.linesep
                amount_skipped += 1
        if amount_none + amount_skipped == length:
            reply = ""
        else:
            reply = reply.strip()
        if reply != "":
            ev.message.reply(reply)
    else:
        r = handle_commands(content_source.lower(), message_parts, ev_room, ev_room_name, ev_user_id, ev_user_name, wrap2, content_source, message_id)
        if r is not None:
            message_with_reply = u":{} {}".format(message_id, r)
            if len(message_with_reply) <= 500 or "\n" in r:
                ev.message.reply(r, False)


def handle_commands(content_lower, message_parts, ev_room, ev_room_name, ev_user_id, ev_user_name, wrap2, content, message_id):
    message_url = "//chat." + wrap2.host + "/transcript/message/" + str(message_id)
    second_part_lower = "" if len(message_parts) < 2 else message_parts[1].lower()
    if second_part_lower == "f":
        second_part_lower = "fp-"
    if second_part_lower == "k":
        second_part_lower = "tpu-"
    if re.compile("^:[0-9]+$").search(message_parts[0]):
        msg_id = int(message_parts[0][1:])
        msg = wrap2.get_message(msg_id)
        msg_content = msg.content_source
        quiet_action = ("-" in second_part_lower)
        if str(msg.owner.id) != GlobalVars.smokeDetector_user_id[ev_room] or msg_content is None:
            return
        post_url = fetch_post_url_from_msg_content(msg_content)
        post_site_id = fetch_post_id_and_site_from_msg_content(msg_content)
        if post_site_id is not None:
            post_type = post_site_id[2]
        else:
            post_type = None
        if (second_part_lower.startswith("false") or second_part_lower.startswith("fp") or second_part_lower.startswith("ignore")) \
                and is_privileged(ev_room, ev_user_id, wrap2):
            if post_site_id is None:
                return "That message is not a report."

            t_metasmoke = Thread(target=Metasmoke.send_feedback_for_post,
                                 args=(post_url, second_part_lower, ev_user_name, ))
            t_metasmoke.start()

            add_false_positive((post_site_id[0], post_site_id[1]))
            user_added = False
            if second_part_lower.startswith("falseu") or second_part_lower.startswith("fpu") or second_part_lower.startswith("ignoreu"):
                url_from_msg = fetch_owner_url_from_msg_content(msg_content)
                if url_from_msg is not None:
                    user = get_user_from_url(url_from_msg)
                    if user is not None:
                        add_whitelisted_user(user)
                        user_added = True
            if post_type == "question":
                if user_added and not quiet_action:
                    return "Registered question as false positive and whitelisted user. I will not post any reports anymore about that question, and also not about that user *if they would get reported for their username*."
                elif not quiet_action:
                    return "Registered question as false positive. I will not post any reports anymore about that question."
            elif post_type == "answer":
                if user_added and not quiet_action:
                    return "Registered answer as false positive and whitelisted user. I will not post any reports anymore about that answer, and also not about that user *if they would get reported for their username*."
                elif not quiet_action:
                    return "Registered answer as false positive. I will not post any reports anymore about that answer."
            try:
                msg.delete()
            except:
                pass
        if (second_part_lower.startswith("true") or second_part_lower.startswith("tp")) \
                and is_privileged(ev_room, ev_user_id, wrap2):
            if post_site_id is None:
                return "That message is not a report."

            t_metasmoke = Thread(target=Metasmoke.send_feedback_for_post,
                                 args=(post_url, second_part_lower, ev_user_name, ))
            t_metasmoke.start()

            user_added = False
            if second_part_lower.startswith("trueu") or second_part_lower.startswith("tpu"):
                url_from_msg = fetch_owner_url_from_msg_content(msg_content)
                if url_from_msg is not None:
                    user = get_user_from_url(url_from_msg)
                    if user is not None:
                        add_blacklisted_user(user, message_url, "http:" + post_url)
                        user_added = True
            if post_type == "question":
                if not quiet_action:
                    if user_added:
                        return "Blacklisted user and registered question as true positive."
                    return "Recorded question as true positive in metasmoke. Use `tpu` or `trueu` if you want to blacklist a user."
            elif post_type == "answer":
                if not quiet_action:
                    if user_added:
                        return "Blacklisted user."
                    return "Recorded answer as true positive in metasmoke. If you want to blacklist the poster of the answer, use `trueu` or `tpu`."
        if (second_part_lower.startswith("delete") or second_part_lower.startswith("remove") or second_part_lower.startswith("gone") or second_part_lower.startswith("poof")<commit_msg>Revert "Merge fp and ignore"<commit_after>import random
import time
from threading import Thread
from parsing import *
from datahandling import *
from metasmoke import Metasmoke
from globalvars import GlobalVars
import os
import re
from datetime import datetime
from utcdate import UtcDate
from apigetpost import api_get_post
from spamhandling import handle_spam
from termcolor import colored
from findspam import FindSpam


# Please note: If new !!/ commands are added or existing ones are modified, don't forget to
# update the wiki at https://github.com/Charcoal-SE/SmokeDetector/wiki/Commands.


def post_message_in_room(room_id_str, msg, length_check=True):
    if room_id_str == GlobalVars.charcoal_room_id:
        GlobalVars.charcoal_hq.send_message(msg, length_check)
    elif room_id_str == GlobalVars.meta_tavern_room_id:
        GlobalVars.tavern_on_the_meta.send_message(msg, length_check)
    elif room_id_str == GlobalVars.socvr_room_id:
        GlobalVars.socvr.send_message(msg, length_check)


def is_smokedetector_message(user_id, room_id):
    return user_id == GlobalVars.smokeDetector_user_id[room_id]


def print_chat_message(ev):
    message = colored("Chat message in " + ev.data["room_name"] + " (" + str(ev.data["room_id"]) + "): \"", attrs=['bold'])
    message += ev.data['content']
    message += "\""
    message += colored(" - " + ev.data['user_name'], attrs=['bold'])
    print message


def watcher(ev, wrap2):
    if ev.type_id != 1:
        return

    print_chat_message(ev)

    ev_room = str(ev.data["room_id"])
    ev_user_id = str(ev.data["user_id"])
    ev_room_name = ev.data["room_name"].encode('utf-8')
    content_source = ev.message.content_source
    message_id = ev.message.id
    if is_smokedetector_message(ev_user_id, ev_room):
        add_latest_smokedetector_message(ev_room, message_id)
    message_parts = content_source.split(" ")

    ev_user_name = ev.data["user_name"].encode('utf-8')
    ev_user_link = "//chat." + wrap2.host + "/users/" + str(ev.user.id)
    if ev_user_name != "SmokeDetector":
        GlobalVars.tavern_users_chatting.append((ev_user_name, ev_user_link))

    shortcut_messages = []
    if message_parts[0].lower() == "sd":
        message_parts = preprocess_shortcut_command(content_source).split(" ")
        latest_smokedetector_messages = GlobalVars.latest_smokedetector_messages[ev_room]
        commands = message_parts[1:]
        if len(latest_smokedetector_messages) == 0:
            ev.message.reply("I don't have any messages posted after the latest reboot.")
            return
        if len(commands) > len(latest_smokedetector_messages):
            ev.message.reply("I've only posted {} messages since the latest reboot; that's not enough to execute all commands. No commands were executed.".format(len(latest_smokedetector_messages)))
            return
        for i in range(0, len(commands)):
            shortcut_messages.append(":" + str(latest_smokedetector_messages[-(i + 1)]) + " " + commands[i])
        reply = ""
        amount_none = 0
        amount_skipped = 0
        length = len(shortcut_messages)
        for i in range(0, length):
            current_message = shortcut_messages[i]
            if length > 1:
                reply += str(i + 1) + ". "
            reply += "[" + current_message.split(" ")[0] + "] "
            if current_message.split(" ")[1] != "-":
                result = handle_commands(current_message.lower(), current_message.split(" "), ev_room, ev_room_name, ev_user_id, ev_user_name, wrap2, current_message, message_id)
                if result is not None:
                    reply += result + os.linesep
                else:
                    reply += "<processed without return value>" + os.linesep
                    amount_none += 1
            else:
                reply += "<skipped>" + os.linesep
                amount_skipped += 1
        if amount_none + amount_skipped == length:
            reply = ""
        else:
            reply = reply.strip()
        if reply != "":
            ev.message.reply(reply)
    else:
        r = handle_commands(content_source.lower(), message_parts, ev_room, ev_room_name, ev_user_id, ev_user_name, wrap2, content_source, message_id)
        if r is not None:
            message_with_reply = u":{} {}".format(message_id, r)
            if len(message_with_reply) <= 500 or "\n" in r:
                ev.message.reply(r, False)


def handle_commands(content_lower, message_parts, ev_room, ev_room_name, ev_user_id, ev_user_name, wrap2, content, message_id):
    message_url = "//chat." + wrap2.host + "/transcript/message/" + str(message_id)
    second_part_lower = "" if len(message_parts) < 2 else message_parts[1].lower()
    if second_part_lower == "f":
        second_part_lower = "fp-"
    if second_part_lower == "k":
        second_part_lower = "tpu-"
    if re.compile("^:[0-9]+$").search(message_parts[0]):
        msg_id = int(message_parts[0][1:])
        msg = wrap2.get_message(msg_id)
        msg_content = msg.content_source
        quiet_action = ("-" in second_part_lower)
        if str(msg.owner.id) != GlobalVars.smokeDetector_user_id[ev_room] or msg_content is None:
            return
        post_url = fetch_post_url_from_msg_content(msg_content)
        post_site_id = fetch_post_id_and_site_from_msg_content(msg_content)
        if post_site_id is not None:
            post_type = post_site_id[2]
        else:
            post_type = None
        if (second_part_lower.startswith("false") or second_part_lower.startswith("fp")) \
                and is_privileged(ev_room, ev_user_id, wrap2):
            if post_site_id is None:
                return "That message is not a report."

            t_metasmoke = Thread(target=Metasmoke.send_feedback_for_post,
                                 args=(post_url, second_part_lower, ev_user_name, ))
            t_metasmoke.start()

            add_false_positive((post_site_id[0], post_site_id[1]))
            user_added = False
            if second_part_lower.startswith("falseu") or second_part_lower.startswith("fpu"):
                url_from_msg = fetch_owner_url_from_msg_content(msg_content)
                if url_from_msg is not None:
                    user = get_user_from_url(url_from_msg)
                    if user is not None:
                        add_whitelisted_user(user)
                        user_added = True
            if post_type == "question":
                if user_added and not quiet_action:
                    return "Registered question as false positive and whitelisted user."
                elif not quiet_action:
                    return "Registered question as false positive."
            elif post_type == "answer":
                if user_added and not quiet_action:
                    return "Registered answer as false positive and whitelisted user."
                elif not quiet_action:
                    return "Registered answer as false positive."
            try:
                msg.delete()
            except:
                pass
        if (second_part_lower.startswith("true") or second_part_lower.startswith("tp")) \
                and is_privileged(ev_room, ev_user_id, wrap2):
            if post_site_id is None:
                return "That message is not a report."

            t_metasmoke = Thread(target=Metasmoke.send_feedback_for_post,
                                 args=(post_url, second_part_lower, ev_user_name, ))
            t_metasmoke.start()

            user_added = False
            if second_part_lower.startswith("trueu") or second_part_lower.startswith("tpu"):
                url_from_msg = fetch_owner_url_from_msg_content(msg_content)
                if url_from_msg is not None:
                    user = get_user_from_url(url_from_msg)
                    if user is not None:
                        add_blacklisted_user(user, message_url, "http:" + post_url)
                        user_added = True
            if post_type == "question":
                if not quiet_action:
                    if user_added:
                        return "Blacklisted user and registered question as true positive."
                    return "Recorded question as true positive in metasmoke. Use `tpu` or `trueu` if you want to blacklist a user."
            elif post_type == "answer":
                if not quiet_action:
                    if user_added:
                        return "Blacklisted user."
                    return "Recorded answer as true positive in metasmoke. If you want to blacklist the poster of the answer, use `trueu` or `tpu`."
        if second_part_lower.startswith("ignore") and is_privileged(ev_room, ev_user_id, wrap2):
            if post_site_id is None:
                return "That message is not a report."

            t_metasmoke = Thread(target=Metasmoke.send_feedback_for_post,
                                 args=(post_url, second_part_lower, ev_user_name, ))
            t_metasmoke.start()

            add_ignored_post(post_site_id[0:2])
            if not quiet_action:
                return "Post ignored; alerts about it will no longer be posted."
        if (second_part_lower.startswith("delete") or second_part_lower.startswith("remove") or second_part_lower.startswith("gone") or second_part_lower.startswith("poof")
{'fraction_non_alphanumeric': 0.09282949903246614, 'fraction_numerical': 0.004353902386583531, 'mean_word_length': 2.4168962350780534, 'pattern_counts': {'":': 24, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7346541', 'n_tokens_mistral': 5584, 'n_tokens_neox': 5413, 'n_words': 1319}