<commit_before>import re
import socket
import opencollab.wiki

from idiokit import threado, timer
from abusehelper.core import bot, rules, events
from abusehelper.core.runtime import RuntimeBot, Room, Session

room_prefix = ""

sources_room = Room(room_prefix + "sources")

def raw_room(name):
    return Room(room_prefix + "raw." + name)

def sanitized_room(name):
    return Room(room_prefix + "sanitized." + name)

def customer_room(customer):
    return Room(room_prefix + "customer." + customer)

def fallback_room(name):
    return Room(room_prefix + "fallback." + name)

def parse_asn(string):
    try:
        return unicode(int(string))
    except ValueError:
        return None

def map_asn(asn):
    return rules.MATCH(u"asn", unicode(int(asn)))

def map_netblock(netblock):
    split = netblock.split("/", 1)
    ip = split[0]

    try:
        socket.inet_pton(socket.AF_INET, ip)
    except socket.error:
        try:
            socket.inet_pton(socket.AF_INET6, ip)
        except socket.error:
            return None
        bits = 128
    else:
        bits = 32

    if len(split) == 2:
        try:
            bits = int(split[1])
        except ValueError:
            return None
    return rules.NETBLOCK(ip, bits)

def map_alias_rule((alias, ruleset)):
    return rules.AND(rules.MATCH("feed", alias),
                     combine_rules(rules.OR, ruleset))

def filter_value(string):
    return string.strip() not in ("", "-")

def filter_email(string):
    return "@" in string

def combine_rules(base, ruleset):
    if not ruleset:
        return None
    if len(ruleset) == 1:
        return list(ruleset)[0]
    return base(*ruleset)

class Source(object):
    def __init__(self, wiki_name, bot_name, alias, **attrs):
        self.wiki_name = wiki_name
        self.bot_name = bot_name
        self.alias = alias
        self.attrs = attrs

    def __iter__(self):
        yield (Session(self.bot_name, **self.attrs)
               | raw_room(self.alias)
               | Session(self.bot_name + ".sanitizer")
               | sanitized_room(self.alias)
               | sources_room)
        yield raw_room(self.alias) | Session("archive")
        yield sanitized_room(self.alias) | Session("archive")

class Customer(object):
    mail_to = []
    mail_cc = []
    mail_times = [5*3600]

    def __init__(self, name, template, rule, **attrs):
        self.name = name
        self.template = template
        self.rule = rule

        for key, value in attrs.items():
            setattr(self, key, value)

    def __iter__(self):
        template = "Subject: AbuseHelper report for " + self.name + "\n"
        template += self.template

        yield (sources_room<commit_msg>Added extra linefeed after headers so that the first line is included.<commit_after>import re
import socket
import opencollab.wiki

from idiokit import threado, timer
from abusehelper.core import bot, rules, events
from abusehelper.core.runtime import RuntimeBot, Room, Session

room_prefix = ""

sources_room = Room(room_prefix + "sources")

def raw_room(name):
    return Room(room_prefix + "raw." + name)

def sanitized_room(name):
    return Room(room_prefix + "sanitized." + name)

def customer_room(customer):
    return Room(room_prefix + "customer." + customer)

def fallback_room(name):
    return Room(room_prefix + "fallback." + name)

def parse_asn(string):
    try:
        return unicode(int(string))
    except ValueError:
        return None

def map_asn(asn):
    return rules.MATCH(u"asn", unicode(int(asn)))

def map_netblock(netblock):
    split = netblock.split("/", 1)
    ip = split[0]

    try:
        socket.inet_pton(socket.AF_INET, ip)
    except socket.error:
        try:
            socket.inet_pton(socket.AF_INET6, ip)
        except socket.error:
            return None
        bits = 128
    else:
        bits = 32

    if len(split) == 2:
        try:
            bits = int(split[1])
        except ValueError:
            return None
    return rules.NETBLOCK(ip, bits)

def map_alias_rule((alias, ruleset)):
    return rules.AND(rules.MATCH("feed", alias),
                     combine_rules(rules.OR, ruleset))

def filter_value(string):
    return string.strip() not in ("", "-")

def filter_email(string):
    return "@" in string

def combine_rules(base, ruleset):
    if not ruleset:
        return None
    if len(ruleset) == 1:
        return list(ruleset)[0]
    return base(*ruleset)

class Source(object):
    def __init__(self, wiki_name, bot_name, alias, **attrs):
        self.wiki_name = wiki_name
        self.bot_name = bot_name
        self.alias = alias
        self.attrs = attrs

    def __iter__(self):
        yield (Session(self.bot_name, **self.attrs)
               | raw_room(self.alias)
               | Session(self.bot_name + ".sanitizer")
               | sanitized_room(self.alias)
               | sources_room)
        yield raw_room(self.alias) | Session("archive")
        yield sanitized_room(self.alias) | Session("archive")

class Customer(object):
    mail_to = []
    mail_cc = []
    mail_times = [5*3600]

    def __init__(self, name, template, rule, **attrs):
        self.name = name
        self.template = template
        self.rule = rule

        for key, value in attrs.items():
            setattr(self, key, value)

    def __iter__(self):
        template = "Subject: AbuseHelper report for " + self.name + "\n\n"
        template += self.template

        yield (sources_room
{'fraction_non_alphanumeric': 0.11279430553020624, 'fraction_numerical': 0.006205511954736266, 'mean_word_length': 2.5446313065976716, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5088920', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1682, 'n_words': 424}