<commit_before>#!/usr/bin/python

import json
import signal

from redis import StrictRedis
from sqlalchemy import and_

from charat2.helpers.chat import send_message
from charat2.model import sm, ChatUser, Message
from charat2.model.connections import redis_pool


db = sm()
redis = StrictRedis(connection_pool=redis_pool)


class Silence(Exception):
    pass


def on_ps(ps_message):

    try:
        chat_id = int(ps_message["channel"].split(":")[1])
        data = json.loads(ps_message["data"])
    except (IndexError, KeyError, ValueError):
        return

    if "messages" not in data or len(data["messages"]) == 0:
        return

    for message in data["messages"]:

        if message["user_number"] is None:
            continue

        try:
            check_connection_spam(chat_id, message)
        except Silence, e:
            db.query(Message).filter(Message.id == message["id"]).update({"spam_flag": e.message})
            db.query(ChatUser).filter(and_(
                ChatUser.chat_id == chat_id,
                ChatUser.number == message["user_number"]
            )).update({"group": "silent"})
            send_message(db, redis, Message(
                chat_id=chat_id,
                type="spamless",
                name="The Spamless",
                acronym=u"\u264b",
                text="Spam has been detected and silenced.",
                color="626262"
            ), True)
            db.commit()


def check_connection_spam(chat_id, message):
    if message["type"] not in ("join", "disconnect", "timeout", "user_info"):
        return
    attempts = increx("antispam:join:%s:%s" % (chat_id, message["user_number"]), expire=5)
    if attempts <= 10:
        return
    raise Silence("connection_spam")


def increx(key, expire=60, incr=1):
    result = redis.incr(key, incr)
    redis.expire(key, expire)
    return result


def halt(signal=None, frame=None):
    print "Caught signal %s" % signal<commit_msg>Check names against the name blacklist. #77<commit_after>#!/usr/bin/python

import json
import signal

from redis import StrictRedis
from sqlalchemy import and_

from charat2.helpers.chat import send_message
from charat2.model import sm, ChatUser, Message
from charat2.model.connections import redis_pool


db = sm()
redis = StrictRedis(connection_pool=redis_pool)


lists = {
    "name_blacklist": redis.smembers("spamless:name_blacklist"),
}


class Silence(Exception):
    pass


def on_ps(ps_message):

    try:
        chat_id = int(ps_message["channel"].split(":")[1])
        data = json.loads(ps_message["data"])
    except (IndexError, KeyError, ValueError):
        return

    if "messages" not in data or len(data["messages"]) == 0:
        return

    for message in data["messages"]:

        if message["user_number"] is None:
            continue

        try:
            check_connection_spam(chat_id, message)
            check_name_blacklist(chat_id, message)
        except Silence, e:
            db.query(Message).filter(Message.id == message["id"]).update({"spam_flag": e.message})
            db.query(ChatUser).filter(and_(
                ChatUser.chat_id == chat_id,
                ChatUser.number == message["user_number"]
            )).update({"group": "silent"})
            send_message(db, redis, Message(
                chat_id=chat_id,
                type="spamless",
                name="The Spamless",
                acronym=u"\u264b",
                text="Spam has been detected and silenced.",
                color="626262"
            ), True)
            db.commit()


def check_connection_spam(chat_id, message):
    if message["type"] not in ("join", "disconnect", "timeout", "user_info"):
        return
    attempts = increx("antispam:join:%s:%s" % (chat_id, message["user_number"]), expire=5)
    if attempts <= 10:
        return
    raise Silence("connection_spam")


def check_name_blacklist(chat_id, message):
    if not message["name"]:
        return
    lower_name = message["name"].lower()
    for name in lists["name_blacklist"]:
        if name in lower_name:
            raise Silence("name")


def increx(key, expire=60, incr=1):
    result = redis.incr(key, incr)
    redis.expire(key, expire)
    return result


def halt(signal=None, frame=None):
    print "Caught signal %s" % signal
{'fraction_non_alphanumeric': 0.12320185614849188, 'fraction_numerical': 0.00974477958236659, 'mean_word_length': 2.628787878787879, 'pattern_counts': {'":': 7, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '785919', 'n_tokens_mistral': 1387, 'n_tokens_neox': 1317, 'n_words': 307}