<commit_before>#
# DeepSmokeListener.py
# PulseMonitor
#
# Created by <NAME> on 3rd October 2017.
# This file is licensed under the MIT License.
#

from WebsocketListener import WebsocketListener
import json
import pprint

class DeepSmokeListener:
    def __init__(self, error_room, report_rooms):
        self.error_room = error_room
        self.report_rooms = report_rooms
        self.ws_link = "ws://smokey-deepsmoke2903.cloudapp.net:8888/"
        self.ws_listener = WebsocketListener(self.ws_link, self.on_message_handler)
        
    def report(self, message):
        for each_room in self.report_rooms:
            each_room.send_message("[ [DeepSmoke](https://git.io/vdlxx) | [PM](https://git.io/vdlx5) ] " + message)

    def get_link(self, data):
        return "https://{0}/q/{1}".format(data["site"], data["question_id"])
       
    def on_message_handler(self, ws, message):
        data = json.loads(message)
       # print("-------------------------------------------")
       # print("               RESTART                     ")
       # print("-------------------------------------------")
       # pprint.pprint(data)
        
        ds = data['deepsmoke'][0]
        ds_response = data['deepsmoke'][1]

        if ds and not data['site'] == "ru.stackoverflow.com" and not data['site'] == "ja.stackoverflow.com" and not data['site'] == "rus.stackexchange.com":
            self.report("Potential spam because of deepsmoke analysis: [" + data['title'] + "]("+ self.get_link(data) + ") on `" + data['site'] + "` with score `" + str(ds_response['score']) + "`")
            return

    def start(self):
        self.ws_listener.start()

    def stop(self):
        self.ws_listener.stop()<commit_msg>DeepSmokeListener.py: attempt to fix multilingual check<commit_after>#
# DeepSmokeListener.py
# PulseMonitor
#
# Created by <NAME>huja on 3rd October 2017.
# This file is licensed under the MIT License.
#

from WebsocketListener import WebsocketListener
import json
import pprint

class DeepSmokeListener:
    def __init__(self, error_room, report_rooms):
        self.error_room = error_room
        self.report_rooms = report_rooms
        self.ws_link = "ws://smokey-deepsmoke2903.cloudapp.net:8888/"
        self.ws_listener = WebsocketListener(self.ws_link, self.on_message_handler)
        
    def report(self, message):
        for each_room in self.report_rooms:
            each_room.send_message("[ [DeepSmoke](https://git.io/vdlxx) | [PM](https://git.io/vdlx5) ] " + message)

    def get_link(self, data):
        return "https://{0}/q/{1}".format(data["site"], data["question_id"])
       
    def on_message_handler(self, ws, message):
        data = json.loads(message)
       # print("-------------------------------------------")
       # print("               RESTART                     ")
       # print("-------------------------------------------")
       # pprint.pprint(data)
        
        ds = data['deepsmoke'][0]
        ds_response = data['deepsmoke'][1]

        if ds and data['site'] not in [
                "ru.stackoverflow.com", "ja.stackoverflow.com",
                "rus.stackexchange.com"]:
            self.report("Potential spam because of deepsmoke analysis: [" + data['title'] + "]("+ self.get_link(data) + ") on `" + data['site'] + "` with score `" + str(ds_response['score']) + "`")
            return

    def start(self):
        self.ws_listener.start()

    def stop(self):
        self.ws_listener.stop()
{'fraction_non_alphanumeric': 0.18385521401896007, 'fraction_numerical': 0.010341855788566504, 'mean_word_length': 2.2181146025878005, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '634949', 'n_tokens_mistral': 1092, 'n_tokens_neox': 1060, 'n_words': 213}