<commit_before>#requires https://pypi.python.org/pypi/websocket-client/
import websocket
import sys
import threading
import json,os,getpass,time
from findspam import FindSpam
from ChatExchange.chatexchange.client import *
import HTMLParser
import random
from bayesian.classify import Classify
from bayesian.learn import Learn
import re
import pickle
from datetime import datetime
import traceback
import requests

class UtcDate:
    startup_utc_date = datetime.utcnow()

# !! Important! Be careful when adding code before the add_custom_print_exception function. Any errors thrown there won't be catched, so only insert code here if you are really sure it works fine.

def uncaught_exception(exctype, value, tb):
    now = datetime.utcnow()
    delta = UtcDate.startup_utc_date - now
    seconds = delta.total_seconds()
    tr = os.linesep.join(traceback.format_tb(tb))
    print(tr)
    with open("errorLogs.txt", "a") as f:
        f.write(str(now) + " UTC" + os.linesep + tr + os.linesep + os.linesep)
    if(seconds < 180):
        os._exit(4)
    else:
        os._exit(1)

sys.excepthook = uncaught_exception

def installThreadExcepthook():
    """
    Workaround for sys.excepthook thread bug
    From
    http://spyced.blogspot.com/2007/06/workaround-for-sysexcepthook-bug.html
    (https://sourceforge.net/tracker/?func=detail&atid=105470&aid=1230540&group_id=5470).
    Call once from __main__ before creating any threads.
    If using psyco, call psyco.cannotcompile(threading.Thread.run)
    since this replaces a new-style class method.
    """
    init_old = threading.Thread.__init__
    def init(self, *args, **kwargs):
        init_old(self, *args, **kwargs)
        run_old = self.run
        def run_with_except_hook(*args, **kw):
            try:
                run_old(*args, **kw)
            except (KeyboardInterrupt, SystemExit):
                raise
            except:
                sys.excepthook(*sys.exc_info())
        self.run = run_with_except_hook
    threading.Thread.__init__ = init

installThreadExcepthook()

class GlobalVars:
    false_positives = []
    whitelisted_users = []
    blacklisted_users = []
    ignored_posts = []
    auto_ignored_posts = []
    startup_utc = datetime.utcnow().strftime("%H:%M:%S")
    latest_questions = []
    blockedTime = 0
    charcoal_room_id = "11540"
    meta_tavern_room_id = "89"
    privileged_users = {}
    smokeDetector_user_id = {}
    site_filename = { "electronics.stackexchange.com" : "ElectronicsGood.txt", "gaming.stackexchange.com" : "GamingGood.txt", "german.stackexchange.com" : "GermanGood.txt",
                      "italian.stackexchange.com" : "ItalianGood.txt", "math.stackexchange.com" : "MathematicsGood.txt", "spanish.stackexchange.com" : "SpanishGood.txt",
                      "stats.stackexchange.com" : "StatsGood.txt" }
    parser=HTMLParser.HTMLParser()
    wrap=Client("stackexchange.com")
    wrapm=Client("meta.stackexchange.com")
    commit = os.popen("git log --pretty=format:'%h' -n 1").read()
    commit_with_author = os.popen("git log --pretty=format:'%h (%cn: *%s*)' -n 1").read()
    on_master = os.popen("git rev-parse --abbrev-ref HEAD").read().strip() == "master"
    charcoal_hq = None
    tavern_on_the_meta = None
    s = ""
    s_reverted = ""
    specialrooms = []
    bayesian_testroom = None

GlobalVars.privileged_users = { GlobalVars.charcoal_room_id: ["117490", "66258", "31768","103081","73046","88521","59776", "31465"], GlobalVars.meta_tavern_room_id: ["259867", "244519","244382","194047","158100","178438","237685","215468","229438","180276", "161974", "244382", "186281", "266094", "245167", "230261", "213575", "241919", "203389"] }
GlobalVars.smokeDetector_user_id = { GlobalVars.charcoal_room_id: "120914", GlobalVars.meta_tavern_room_id: "266345" }

def load_files():
    if(os.path.isfile("falsePositives.txt")):
        with open("falsePositives.txt", "r") as f:
            GlobalVars.false_positives = pickle.load(f)
    if(os.path.isfile("whitelistedUsers.txt")):
        with open("whitelistedUsers.txt", "r") as f:
            GlobalVars.whitelisted_users = pickle.load(f)
    if(os.path.isfile("blacklistedUsers.txt")):
        with open("blacklistedUsers.txt", "r") as f:
            GlobalVars.blacklisted_users = pickle.load(f)
    if(os.path.isfile("ignoredPosts.txt")):
        with open("ignoredPosts.txt", "r") as f:
            GlobalVars.ignored_posts = pickle.load(f)
    if(os.path.isfile("autoIgnoredPosts.txt")):
        with open("autoIgnoredPosts.txt", "r") as f:
            GlobalVars.auto_ignored_posts = pickle.load(f)

def filter_auto_ignored_posts():
    todayDate = datetime.today()
    to_remove = []
    for aip in GlobalVars.auto_ignored_posts:
        dayIgnored = aip[2]
        dayDiff = (todayDate - dayIgnored).days
        if(dayDiff > 7):
            to_remove.append(aip)
    for tr in to_remove:
        GlobalVars.auto_ignored_posts.remove(tr)
    with open("autoIgnoredPosts.txt", "w") as f:
        pickle.dump(GlobalVars.auto_ignored_posts, f)

if("ChatExchangeU" in os.environ):
    username=os.environ["ChatExchangeU"]
else:
    print "Username: "
    username=raw_input()
if("ChatExchangeP" in os.environ):
    password=os.environ["Chat<PASSWORD>P"]
else:
    password=<PASSWORD>("Password: ")

load_files()
filter_auto_ignored_posts()

GlobalVars.wrap.login(username,password)
GlobalVars.wrapm.login(username,password)
GlobalVars.s="[ [SmokeDetector](https://github.com/Charcoal-SE/SmokeDetector) ] SmokeDetector started at [rev " + GlobalVars.commit_with_author + "](https://github.com/Charcoal-SE/SmokeDetector/commit/"+ GlobalVars.commit +") (hosted by Undo)"
GlobalVars.s_reverted="[ [SmokeDetector](https://github.com/Charcoal-SE/SmokeDetector) ] SmokeDetector started in [reverted mode](https://github.com/Charcoal-SE/SmokeDetector/blob/master/RevertedMode.md) at [rev " + GlobalVars.commit_with_author + "](https://github.com/Charcoal-SE/SmokeDetector/commit/"+ GlobalVars.commit +") (hosted by Undo)"
GlobalVars.charcoal_hq = GlobalVars.wrap.get_room(GlobalVars.charcoal_room_id)
GlobalVars.tavern_on_the_meta = GlobalVars.wrapm.get_room(GlobalVars.meta_tavern_room_id)

GlobalVars.specialrooms = [{ "sites": ["english.stackexchange.com"], "room": GlobalVars.wrap.get_room("95"), "unwantedReasons": [] }, { "sites": ["askubuntu.com"], "room": GlobalVars.wrap.get_room("201"), "unwantedReasons": ["All-caps title", "Phone number detected"] }]

GlobalVars.bayesian_testroom = GlobalVars.wrap.get_room("17251")
if "first_start" in sys.argv and GlobalVars.on_master:
    GlobalVars.charcoal_hq.send_message(GlobalVars.s)
    GlobalVars.bayesian_testroom.send_message(GlobalVars.s)
    #GlobalVars.tavern_on_the_meta.send_message(GlobalVars.s)
    #Commented out because the Tavern folk don't really need to see when it starts
elif "first_start" in sys.argv and not GlobalVars.on_master:
    GlobalVars.charcoal_hq.send_message(GlobalVars.s_reverted)
    GlobalVars.bayesian_testroom.send_message(GlobalVars.s_reverted)

def restart_automatically(time_in_seconds):
    time.sleep(time_in_seconds)
    os._exit(1)

threading.Thread(target=restart_automatically,args=(3600,)).start()

def get_user_from_url(url):
    m = re.compile(r"https?://([\w.]+)/users/(\d+)/.+/?").search(url)
    site = m.group(1)
    user_id = m.group(2)
    return (user_id, site)

def postMessageInRoom(room_id_str, msg):
    if room_id_str == GlobalVars.charcoal_room_id:
        GlobalVars.charcoal_hq.send_message(msg)
    elif room_id_str == GlobalVars.meta_tavern_room_id:
        GlobalVars.tavern_on_the_meta.send_message(msg)

def is_whitelisted_user(user):
    return user in GlobalVars.whitelisted_users

def is_blacklisted_user(user):
    return user in GlobalVars.blacklisted_users

def is_ignored_post(postid_site_tuple):
    return postid_site_tuple in GlobalVars.ignored_posts

def is_auto_ignored_post(postid_site_tuple):
    return postid_site_tuple in GlobalVars.auto_ignored_posts

def add_whitelisted_user(user):
    if user in GlobalVars.whitelisted_users or user is None:
        return
    GlobalVars.whitelisted_users.append(user)
    with open("whitelistedUsers.txt", "w") as f:
        pickle.dump(GlobalVars.whitelisted_users, f)

def add_blacklisted_user(user):
    if user in GlobalVars.blacklisted_users or user is None:
        return
    GlobalVars.blacklisted_users.append(user)<commit_msg>Revert "Simplify"<commit_after>#requires https://pypi.python.org/pypi/websocket-client/
import websocket
import sys
import threading
import json,os,getpass,time
from findspam import FindSpam
from ChatExchange.chatexchange.client import *
import HTMLParser
import random
from bayesian.classify import Classify
from bayesian.learn import Learn
import re
import pickle
from datetime import datetime
import traceback
import requests

class UtcDate:
    startup_utc_date = datetime.utcnow()

# !! Important! Be careful when adding code before the add_custom_print_exception function. Any errors thrown there won't be catched, so only insert code here if you are really sure it works fine.

def uncaught_exception(exctype, value, tb):
    now = datetime.utcnow()
    delta = UtcDate.startup_utc_date - now
    seconds = delta.total_seconds()
    tr = os.linesep.join(traceback.format_tb(tb))
    print(tr)
    with open("errorLogs.txt", "a") as f:
        f.write(str(now) + " UTC" + os.linesep + tr + os.linesep + os.linesep)
    if(seconds < 180):
        os._exit(4)
    else:
        os._exit(1)

sys.excepthook = uncaught_exception

def installThreadExcepthook():
    """
    Workaround for sys.excepthook thread bug
    From
    http://spyced.blogspot.com/2007/06/workaround-for-sysexcepthook-bug.html
    (https://sourceforge.net/tracker/?func=detail&atid=105470&aid=1230540&group_id=5470).
    Call once from __main__ before creating any threads.
    If using psyco, call psyco.cannotcompile(threading.Thread.run)
    since this replaces a new-style class method.
    """
    init_old = threading.Thread.__init__
    def init(self, *args, **kwargs):
        init_old(self, *args, **kwargs)
        run_old = self.run
        def run_with_except_hook(*args, **kw):
            try:
                run_old(*args, **kw)
            except (KeyboardInterrupt, SystemExit):
                raise
            except:
                sys.excepthook(*sys.exc_info())
        self.run = run_with_except_hook
    threading.Thread.__init__ = init

installThreadExcepthook()

class GlobalVars:
    false_positives = []
    whitelisted_users = []
    blacklisted_users = []
    ignored_posts = []
    auto_ignored_posts = []
    startup_utc = datetime.utcnow().strftime("%H:%M:%S")
    latest_questions = []
    blockedTime = 0
    charcoal_room_id = "11540"
    meta_tavern_room_id = "89"
    privileged_users = {}
    smokeDetector_user_id = {}
    site_filename = { "electronics.stackexchange.com" : "ElectronicsGood.txt", "gaming.stackexchange.com" : "GamingGood.txt", "german.stackexchange.com" : "GermanGood.txt",
                      "italian.stackexchange.com" : "ItalianGood.txt", "math.stackexchange.com" : "MathematicsGood.txt", "spanish.stackexchange.com" : "SpanishGood.txt",
                      "stats.stackexchange.com" : "StatsGood.txt" }
    parser=HTMLParser.HTMLParser()
    wrap=Client("stackexchange.com")
    wrapm=Client("meta.stackexchange.com")
    commit = os.popen("git log --pretty=format:'%h' -n 1").read()
    commit_with_author = os.popen("git log --pretty=format:'%h (%cn: *%s*)' -n 1").read()
    on_master = os.popen("git rev-parse --abbrev-ref HEAD").read().strip() == "master"
    charcoal_hq = None
    tavern_on_the_meta = None
    s = ""
    s_reverted = ""
    specialrooms = []
    bayesian_testroom = None

GlobalVars.privileged_users = { GlobalVars.charcoal_room_id: ["117490", "66258", "31768","103081","73046","88521","59776", "31465"], GlobalVars.meta_tavern_room_id: ["259867", "244519","244382","194047","158100","178438","237685","215468","229438","180276", "161974", "244382", "186281", "266094", "245167", "230261", "213575", "241919", "203389"] }
GlobalVars.smokeDetector_user_id = { GlobalVars.charcoal_room_id: "120914", GlobalVars.meta_tavern_room_id: "266345" }

def load_files():
    if(os.path.isfile("falsePositives.txt")):
        with open("falsePositives.txt", "r") as f:
            GlobalVars.false_positives = pickle.load(f)
    if(os.path.isfile("whitelistedUsers.txt")):
        with open("whitelistedUsers.txt", "r") as f:
            GlobalVars.whitelisted_users = pickle.load(f)
    if(os.path.isfile("blacklistedUsers.txt")):
        with open("blacklistedUsers.txt", "r") as f:
            GlobalVars.blacklisted_users = pickle.load(f)
    if(os.path.isfile("ignoredPosts.txt")):
        with open("ignoredPosts.txt", "r") as f:
            GlobalVars.ignored_posts = pickle.load(f)
    if(os.path.isfile("autoIgnoredPosts.txt")):
        with open("autoIgnoredPosts.txt", "r") as f:
            GlobalVars.auto_ignored_posts = pickle.load(f)

def filter_auto_ignored_posts():
    todayDate = datetime.today()
    to_remove = []
    for aip in GlobalVars.auto_ignored_posts:
        dayIgnored = aip[2]
        dayDiff = (todayDate - dayIgnored).days
        if(dayDiff > 7):
            to_remove.append(aip)
    for tr in to_remove:
        GlobalVars.auto_ignored_posts.remove(tr)
    with open("autoIgnoredPosts.txt", "w") as f:
        pickle.dump(GlobalVars.auto_ignored_posts, f)

if("ChatExchangeU" in os.environ):
    username=os.environ["ChatExchangeU"]
else:
    print "Username: "
    username=raw_input()
if("ChatExchangeP" in os.environ):
    password=os.environ["ChatExchangeP"]
else:
    password=get<PASSWORD>("Password: ")

load_files()
filter_auto_ignored_posts()

GlobalVars.wrap.login(username,password)
GlobalVars.wrapm.login(username,password)
GlobalVars.s="[ [SmokeDetector](https://github.com/Charcoal-SE/SmokeDetector) ] SmokeDetector started at [rev " + GlobalVars.commit_with_author + "](https://github.com/Charcoal-SE/SmokeDetector/commit/"+ GlobalVars.commit +") (hosted by Undo)"
GlobalVars.s_reverted="[ [SmokeDetector](https://github.com/Charcoal-SE/SmokeDetector) ] SmokeDetector started in [reverted mode](https://github.com/Charcoal-SE/SmokeDetector/blob/master/RevertedMode.md) at [rev " + GlobalVars.commit_with_author + "](https://github.com/Charcoal-SE/SmokeDetector/commit/"+ GlobalVars.commit +") (hosted by Undo)"
GlobalVars.charcoal_hq = GlobalVars.wrap.get_room(GlobalVars.charcoal_room_id)
GlobalVars.tavern_on_the_meta = GlobalVars.wrapm.get_room(GlobalVars.meta_tavern_room_id)

GlobalVars.specialrooms = [{ "sites": ["english.stackexchange.com"], "room": GlobalVars.wrap.get_room("95"), "unwantedReasons": [] }, { "sites": ["askubuntu.com"], "room": GlobalVars.wrap.get_room("201"), "unwantedReasons": ["All-caps title", "Phone number detected"] }]

GlobalVars.bayesian_testroom = GlobalVars.wrap.get_room("17251")
if "first_start" in sys.argv and GlobalVars.on_master:
    GlobalVars.charcoal_hq.send_message(GlobalVars.s)
    GlobalVars.bayesian_testroom.send_message(GlobalVars.s)
    #GlobalVars.tavern_on_the_meta.send_message(GlobalVars.s)
    #Commented out because the Tavern folk don't really need to see when it starts
elif "first_start" in sys.argv and not GlobalVars.on_master:
    GlobalVars.charcoal_hq.send_message(GlobalVars.s_reverted)
    GlobalVars.bayesian_testroom.send_message(GlobalVars.s_reverted)

def restart_automatically(time_in_seconds):
    time.sleep(time_in_seconds)
    os._exit(1)

threading.Thread(target=restart_automatically,args=(3600,)).start()

def get_user_from_url(url):
    m = re.compile(r"https?://([\w.]+)/users/(\d+)/.+/?").search(url)
    site = m.group(1)
    user_id = m.group(2)
    return (user_id, site)

def postMessageInRoom(room_id_str, msg):
    if room_id_str == GlobalVars.charcoal_room_id:
        GlobalVars.charcoal_hq.send_message(msg)
    elif room_id_str == GlobalVars.meta_tavern_room_id:
        GlobalVars.tavern_on_the_meta.send_message(msg)

def is_whitelisted_user(user):
    return user in GlobalVars.whitelisted_users

def is_blacklisted_user(user):
    return user in GlobalVars.blacklisted_users

def is_ignored_post(postid_site_tuple):
    return postid_site_tuple in GlobalVars.ignored_posts

def is_auto_ignored_post(postid_site_tuple):
    for p in GlobalVars.auto_ignored_posts:
        if p[0] == postid_site_tuple[0] and p[1] == postid_site_tuple[1]:
            return True
    return False

def add_whitelisted_user(user):
    if user in GlobalVars.whitelisted_users or user is None:
        return
    GlobalVars.whitelisted_users.append(user)
    with open("whitelistedUsers.txt", "w") as f:
        pickle.dump(GlobalVars.whitelisted_users, f)

def add_blacklisted_user(user):
    if user in GlobalVars.blacklisted_users or user is None:
        return
    GlobalVars.blacklisted_users.append(user)
{'fraction_non_alphanumeric': 0.12937973484848486, 'fraction_numerical': 0.026870265151515152, 'mean_word_length': 4.12030303030303, 'pattern_counts': {'":': 12, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 14, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '147272', 'n_tokens_mistral': 6179, 'n_tokens_neox': 5914, 'n_words': 1020}