<commit_before>import random
import socket
import threading
import time

from .formatting import *


class _Backend:
    def __init__(self):
        self.storage = None
        self.event_listeners = []

    def add_event_listener(self, name, callback):
        self.event_listeners.append((name, callback))

    def dispatch_event(self, name, *event):
        for listener in self.event_listeners:
            if listener[0] == name:
                listener[1](*event)


class IRC(_Backend):
    def __init__(self, hostname, port=6667, username=None, realname=None):
        super().__init__()
        self.hostname = hostname
        self.port = port
        self.nick = None
        self.username = username
        self.realname = realname
        self.channels = []

        self.lock = threading.Lock()
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    def ping_thread(self):
        while True:
            time.sleep(60)
            msg = "ping-" + str(random.randint(0, 100))
            self.write("PING", msg)

    def write(self, *args):
        def convert_part(x):
            x = bytes(x, "utf-8").strip()
            if b" " in x:
                return b":" + x
            else:
                return x

        with self.lock:
            line = b" ".join([convert_part(x) for x in args])
            line += b"\r\n"
            self.socket.send(line)

    def parse_usermask(self, usermask):
        return usermask.split("!")[0]

    def parse(self, args):
        if args[0] == "PING":
            self.write("PONG", args[1])
        elif args[1] == "001":
            self.dispatch_event("ready")
        elif args[1] == "JOIN":
            nick = self.parse_usermask(args[0])
            self.channels.append(args[2])
            self.dispatch_event("join", {
                "user": nick, "channel": args[2], "is_me": nick == self.nick
            })
        elif args[1] == "KICK":
            nick = self.parse_usermask(args[3])
            if nick == self.nick:
                self.join(args[2])
        elif args[1] == "PRIVMSG":
            sender = self.parse_usermask(args[0])
            target = args[2]
            message = args[3]

            reply_to = target
            if target == self.nick:
                reply_to = sender

            self.dispatch_event("message", {
                "sender": sender, "target": target,
                "message": message, "reply_to": reply_to
            })
        else:
            print(args)

    def encode_string(self, s):
        try:
            return str(s, "utf-8")
        except UnicodeError:
            return str(s, "ISO-8859-1", "ignore")

    def run(self, name):
        self.nick = name
        if not self.username:
            self.username = self.nick
        if not self.realname:
            self.realname = self.nick

        self.socket.connect((self.hostname, self.port))

        self.dispatch_event("connect")

        thread = threading.Thread(target=self.ping_thread)
        thread.daemon = True
        thread.start()

        self.write("NICK", self.nick)
        self.write("USER", self.username, "8", "*", self.realname)

        buf = b""
        while True:
            data = self.socket.recv(4096)
            if not data:
                break

            array = (buf + data).split(b"\n")
            for line in array[:-1]:
                args = line.strip().split(b" ")
                line = []
                for i, p in enumerate(args):
                    if p.startswith(b":"):
                        if i != 0:
                            line.append((b" ".join(args[i:]))[1:])
                            break
                        elif i == 0:
                            line.append(p[1:])
                    else:
                        line.append(p)

                self.parse([self.encode_string(x) for x in line])

            buf = array[-1]

        self.dispatch_event("disconnect")

    def join(self, channel):
        self.write("JOIN", channel)

    def send(self, target, message):
        for msg in message.splitlines():
            self.write("PRIVMSG", target, msg)
            time.sleep(1)

    def format(self, text, properties):
        values = []

        if Style.bold in properties:
            values.append("\x02")
        if Style.italic in properties:
            values.append("\x1D")
        if Style.underline in properties:
            values.append("\x1F")

        fg_colour = None

        if Colour.fg_white in properties:
            fg_colour = "00"
        if Colour.fg_black in properties:
            fg_colour = "01"
        if Colour.fg_blue in properties:
            fg_colour = "02"
        if Colour.fg_green in properties:
            fg_colour = "03"
        if Colour.fg_red in properties:
            fg_colour = "04"
        if Colour.fg_brown in properties:
            fg_colour = "05"
        if Colour.fg_purple in properties:
            fg_colour = "06"
        if Colour.fg_orange in properties:
            fg_colour = "07"
        if Colour.fg_yellow in properties:
            fg_colour = "08"
        if Colour.fg_light_green in properties:
            fg_colour = "09"
        if Colour.fg_teal in properties:
            fg_colour = "10"
        if Colour.fg_light_cyan in properties:
            fg_colour = "11"
        if Colour.fg_light_blue in properties:
            fg_colour = "12"
        if Colour.fg_pink in properties:
            fg_colour = "13"
        if Colour.fg_grey in properties:
            fg_colour = "14"
        if Colour.fg_light_grey in properties:
            fg_colour = "15"

        bg_colour = None

        if Colour.bg_white in properties:
            bg_colour = "00"
        if Colour.bg_black in properties:
            bg_colour = "01"
        if Colour.bg_blue in properties:
            bg_colour = "02"
        if Colour.bg_green in properties:
            bg_colour = "03"
        if Colour.bg_red in properties:
            bg_colour = "04"
        if Colour.bg_brown in properties:
            bg_colour = "05"
        if Colour.bg_purple in properties:
            bg_colour = "06"
        if Colour.bg_orange in properties:
            bg_colour = "07"
        if Colour.bg_yellow in properties:
            bg_colour = "08"
        if Colour.bg_light_green in properties:
            bg_colour = "09"
        if Colour.bg_teal in properties:
            bg_colour = "10"
        if Colour.bg_light_cyan in properties:
            bg_colour = "11"
        if Colour.bg_light_blue in properties:
            bg_colour = "12"
        if Colour.bg_pink in properties:
            bg_colour = "13"
        if Colour.bg_grey in properties:
            bg_colour = "14"
        if Colour.bg_light_grey in properties:
            bg_colour = "15"

        if bg_colour and not fg_colour:
            fg_colour = "01" # default to black

        if fg_colour or bg_colour:
            values.append("\x03")
            values.append(fg_colour)
            if bg_colour:
                values.append(",")
                values.append(bg_colour)

        return "{}{}\x0F".format("".join(values), text)


class CommandLine(_Backend):
    def __init__(self):
        super().__init__()

    def run(self, name):
        self.dispatch_event("connect")
        self.dispatch_event("ready")

        while True:
            line = input("> ").strip()
            self.dispatch_event("message", {
                "sender": "stdin", "target": "stdout",
                "message": line, "reply_to": "stdin"
            })

        self.dispatch_event("disconnect")

    def join(self, channel):
        print("Joining", channel)

    def send(self, target, message):
        print(target + ":", message)

    def format(self, text, properties):
        values = []

        if Style.bold in properties:
            values.append("1")
        if Style.italic in properties:
            values.append("3")
        if Style.underline in properties:
            values.append("4")
        if Colour.fg_white in properties:
            values.append("37")
        if Colour.fg_black in properties:
            values.append("30")
        if Colour.fg_blue in properties:
            values.append("34")
        if Colour.fg_green in properties:
            values.append("32")
        if Colour.fg_red in properties:
            values.append("31")
        if Colour.fg_purple in properties:
            values.append("35")
        if Colour.fg_yellow in properties:
            values.append("33")
        if Colour.fg_light_cyan in properties:
            values.append("36")
        if Colour.bg_white in properties:
            values.append("47")
        if Colour.bg_black in properties:
            values.append("40")
        if Colour.bg_blue in properties:
            values.append("44")
        if Colour.bg_green in properties:
            values.append("42")
        if Colour.bg_red in properties:
            values.append("41")
        if Colour.bg_purple in properties:
            values.append("45")
        if Colour.bg_yellow in properties:
            values.append("43")
        if Colour.bg_light_cyan in properties:
            values.append("46")

        return "\033[{}m{}\033[0m".format(";".join(values), text)
<commit_msg>Avoid flooding with large messages<commit_after>import random
import socket
import threading
import time

from .formatting import *


class _Backend:
    def __init__(self):
        self.storage = None
        self.event_listeners = []

    def add_event_listener(self, name, callback):
        self.event_listeners.append((name, callback))

    def dispatch_event(self, name, *event):
        for listener in self.event_listeners:
            if listener[0] == name:
                listener[1](*event)


class IRC(_Backend):
    def __init__(self, hostname, port=6667, username=None, realname=None):
        super().__init__()
        self.hostname = hostname
        self.port = port
        self.nick = None
        self.username = username
        self.realname = realname
        self.channels = []

        self.lock = threading.Lock()
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    def ping_thread(self):
        while True:
            time.sleep(60)
            msg = "ping-" + str(random.randint(0, 100))
            self.write("PING", msg)

    def write(self, *args):
        def convert_part(x):
            x = bytes(x, "utf-8").strip()
            if b" " in x:
                return b":" + x
            else:
                return x

        with self.lock:
            line = b" ".join([convert_part(x) for x in args])
            line += b"\r\n"
            self.socket.send(line)

    def parse_usermask(self, usermask):
        return usermask.split("!")[0]

    def parse(self, args):
        if args[0] == "PING":
            self.write("PONG", args[1])
        elif args[1] == "001":
            self.dispatch_event("ready")
        elif args[1] == "JOIN":
            nick = self.parse_usermask(args[0])
            self.channels.append(args[2])
            self.dispatch_event("join", {
                "user": nick, "channel": args[2], "is_me": nick == self.nick
            })
        elif args[1] == "KICK":
            nick = self.parse_usermask(args[3])
            if nick == self.nick:
                self.join(args[2])
        elif args[1] == "PRIVMSG":
            sender = self.parse_usermask(args[0])
            target = args[2]
            message = args[3]

            reply_to = target
            if target == self.nick:
                reply_to = sender

            self.dispatch_event("message", {
                "sender": sender, "target": target,
                "message": message, "reply_to": reply_to
            })
        else:
            print(args)

    def encode_string(self, s):
        try:
            return str(s, "utf-8")
        except UnicodeError:
            return str(s, "ISO-8859-1", "ignore")

    def run(self, name):
        self.nick = name
        if not self.username:
            self.username = self.nick
        if not self.realname:
            self.realname = self.nick

        self.socket.connect((self.hostname, self.port))

        self.dispatch_event("connect")

        thread = threading.Thread(target=self.ping_thread)
        thread.daemon = True
        thread.start()

        self.write("NICK", self.nick)
        self.write("USER", self.username, "8", "*", self.realname)

        buf = b""
        while True:
            data = self.socket.recv(4096)
            if not data:
                break

            array = (buf + data).split(b"\n")
            for line in array[:-1]:
                args = line.strip().split(b" ")
                line = []
                for i, p in enumerate(args):
                    if p.startswith(b":"):
                        if i != 0:
                            line.append((b" ".join(args[i:]))[1:])
                            break
                        elif i == 0:
                            line.append(p[1:])
                    else:
                        line.append(p)

                self.parse([self.encode_string(x) for x in line])

            buf = array[-1]

        self.dispatch_event("disconnect")

    def join(self, channel):
        self.write("JOIN", channel)

    def send(self, target, message):
        for msg in message.splitlines():
            self.write("PRIVMSG", target, msg[:400])  # to avoid flooding
            time.sleep(1)

    def format(self, text, properties):
        values = []

        if Style.bold in properties:
            values.append("\x02")
        if Style.italic in properties:
            values.append("\x1D")
        if Style.underline in properties:
            values.append("\x1F")

        fg_colour = None

        if Colour.fg_white in properties:
            fg_colour = "00"
        if Colour.fg_black in properties:
            fg_colour = "01"
        if Colour.fg_blue in properties:
            fg_colour = "02"
        if Colour.fg_green in properties:
            fg_colour = "03"
        if Colour.fg_red in properties:
            fg_colour = "04"
        if Colour.fg_brown in properties:
            fg_colour = "05"
        if Colour.fg_purple in properties:
            fg_colour = "06"
        if Colour.fg_orange in properties:
            fg_colour = "07"
        if Colour.fg_yellow in properties:
            fg_colour = "08"
        if Colour.fg_light_green in properties:
            fg_colour = "09"
        if Colour.fg_teal in properties:
            fg_colour = "10"
        if Colour.fg_light_cyan in properties:
            fg_colour = "11"
        if Colour.fg_light_blue in properties:
            fg_colour = "12"
        if Colour.fg_pink in properties:
            fg_colour = "13"
        if Colour.fg_grey in properties:
            fg_colour = "14"
        if Colour.fg_light_grey in properties:
            fg_colour = "15"

        bg_colour = None

        if Colour.bg_white in properties:
            bg_colour = "00"
        if Colour.bg_black in properties:
            bg_colour = "01"
        if Colour.bg_blue in properties:
            bg_colour = "02"
        if Colour.bg_green in properties:
            bg_colour = "03"
        if Colour.bg_red in properties:
            bg_colour = "04"
        if Colour.bg_brown in properties:
            bg_colour = "05"
        if Colour.bg_purple in properties:
            bg_colour = "06"
        if Colour.bg_orange in properties:
            bg_colour = "07"
        if Colour.bg_yellow in properties:
            bg_colour = "08"
        if Colour.bg_light_green in properties:
            bg_colour = "09"
        if Colour.bg_teal in properties:
            bg_colour = "10"
        if Colour.bg_light_cyan in properties:
            bg_colour = "11"
        if Colour.bg_light_blue in properties:
            bg_colour = "12"
        if Colour.bg_pink in properties:
            bg_colour = "13"
        if Colour.bg_grey in properties:
            bg_colour = "14"
        if Colour.bg_light_grey in properties:
            bg_colour = "15"

        if bg_colour and not fg_colour:
            fg_colour = "01" # default to black

        if fg_colour or bg_colour:
            values.append("\x03")
            values.append(fg_colour)
            if bg_colour:
                values.append(",")
                values.append(bg_colour)

        return "{}{}\x0F".format("".join(values), text)


class CommandLine(_Backend):
    def __init__(self):
        super().__init__()

    def run(self, name):
        self.dispatch_event("connect")
        self.dispatch_event("ready")

        while True:
            line = input("> ").strip()
            self.dispatch_event("message", {
                "sender": "stdin", "target": "stdout",
                "message": line, "reply_to": "stdin"
            })

        self.dispatch_event("disconnect")

    def join(self, channel):
        print("Joining", channel)

    def send(self, target, message):
        print(target + ":", message)

    def format(self, text, properties):
        values = []

        if Style.bold in properties:
            values.append("1")
        if Style.italic in properties:
            values.append("3")
        if Style.underline in properties:
            values.append("4")
        if Colour.fg_white in properties:
            values.append("37")
        if Colour.fg_black in properties:
            values.append("30")
        if Colour.fg_blue in properties:
            values.append("34")
        if Colour.fg_green in properties:
            values.append("32")
        if Colour.fg_red in properties:
            values.append("31")
        if Colour.fg_purple in properties:
            values.append("35")
        if Colour.fg_yellow in properties:
            values.append("33")
        if Colour.fg_light_cyan in properties:
            values.append("36")
        if Colour.bg_white in properties:
            values.append("47")
        if Colour.bg_black in properties:
            values.append("40")
        if Colour.bg_blue in properties:
            values.append("44")
        if Colour.bg_green in properties:
            values.append("42")
        if Colour.bg_red in properties:
            values.append("41")
        if Colour.bg_purple in properties:
            values.append("45")
        if Colour.bg_yellow in properties:
            values.append("43")
        if Colour.bg_light_cyan in properties:
            values.append("46")

        return "\033[{}m{}\033[0m".format(";".join(values), text)

{'fraction_non_alphanumeric': 0.10925628678437667, 'fraction_numerical': 0.017710005350454788, 'mean_word_length': 1.6097458810388159, 'pattern_counts': {'":': 38, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2838962', 'n_tokens_mistral': 5763, 'n_tokens_neox': 5340, 'n_words': 1313}