<commit_before>#! /usr/bin/python

import logging
import re
import select
import socket
import subprocess
import sys
import time

from ircbot import SingleServerIRCBot, nm_to_n as get_nick, parse_channel_modes

# FIXME figure out async subprocess
# FIXME use optparse and/or configreader

logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(message)s")

CHANNEL = '#foo'
PATTERN = '^[\!\?][^ ]+'

class PyCatBot(SingleServerIRCBot):
    def __init__(self, server_list, nick, real, channel):
        SingleServerIRCBot.__init__(self, server_list, nick, real)

        self.channel = channel

        self.sockets = []
        self.recivers = []
        self.processes = []
        self.buffers = {}
        self.listener = self.get_listener()

        self.last_seen = time.time()
        self.ircobj.fn_to_add_socket = self.sockets.append
        self.ircobj.fn_to_remove_socket = self.sockets.remove

        self.setup_logging()

    def setup_logging(self):
        orignial_send_raw = self.connection.send_raw

        def send_raw(string):
            logging.debug(string.decode('utf-8'))
            orignial_send_raw(string)

        def logger(conn, event):
            line = u' '.join(event.arguments())
            logging.debug(line)

        self.connection.add_global_handler('all_raw_messages', logger)
        self.connection.send_raw = send_raw

    def get_listener(self, addr=('', 12345)):
        listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        listener.setblocking(0)
        listener.bind(addr)
        listener.listen(5)

        return listener

    def on_welcome(self, conn, event):
        conn.join(self.channel)

    def on_nicknameinuse(self, conn, event):
        conn.nick(conn.get_nickname() + '_')

    def on_invite(self, conn, event):
        if event.arguments()[0] == self.channel:
            conn.join(self.channel)

    def on_mode(self, conn, event):
        if event.target() != self.channel:
            return

        nick = conn.get_nickname()
        modes = parse_channel_modes(' '.join(event.arguments()))

        if ['+', 'o', nick] in modes:
            conn.mode(self.channel, '+v-o %s %s' % (nick, nick))

    def on_pubmsg(self, conn, event):
        channel = event.target()
        nick = get_nick(event.source())
        message = self.decode(event.arguments()[0])

        p = subprocess.Popen(['./test.sh', channel, nick, message],
            bufsize=1024, stdout=subprocess.PIPE)

        self.processes.append(p.stdout)

    def start(self):
        self._connect()

        while 1:
            sockets = self.sockets + self.recivers + \
                self.processes + [self.listener]
            self.process_sockets(sockets)

    def stop(self):
        if self.connection.is_connected():
            self.connection.disconnect('Bye :)')

    def process_sockets(self, sockets):
        for sock in select.select(sockets, [], [], 0.2)[0]:
            if sock in self.recivers:
                self.handle_reciver(sock)
            elif sock is self.listener:
                self.handle_listener(sock)
            elif sock in self.sockets:
                self.handle_irc(sock)
            elif sock in self.processes:
                self.handle_process(sock)

        self.handle_timeout()

    def handle_process(self, sock):
        if sock not in self.buffers:
            self.buffers[sock] = u''

        data = sock.read(512)

        if len(data) == 0:
            self.processes.remove(sock)
            sock.close()
        else:
            self.buffers[sock] += self.decode(data)

        while '\n' in self.buffers[sock]:
            message, trailing = self.buffers[sock].split('\n', 1)
            self.buffers[sock] = trailing

            self.handle_reciver_message(message)

        if len(data) == 0:
            del self.buffers[sock]

    def handle_reciver(self, sock):
        if sock not in self.buffers:
            self.buffers[sock] = u''

        data = sock.recv(512)

        if len(data) == 0:
            self.recivers.remove(sock)
            sock.close()
            logging.debug('%s disconnected', sock.getpeername()[0])
        else:
            self.buffers[sock] += self.decode(data)

        while '\n' in self.buffers[sock]:
            message, trailing = self.buffers[sock].split('\n', 1)
            self.buffers[sock] = trailing

            self.handle_reciver_message(message)
            logging.debug('%s %s', sock.getpeername()[0], message)

        if len(data) == 0:
            del self.buffers[sock]

    def handle_reciver_message(self, message):
        message = message.encode('utf-8')

        if not message.strip() or not self.connection.is_connected():
            return
        elif message.startswith('/me '):
            self.connection.action(CHANNEL, message[len('/me '):])
        elif message.startswith('/notice '):
            self.connection.notice(CHANNEL, message[len('/notice '):])
        else:
            self.connection.privmsg(CHANNEL, message)

    def handle_listener(self, sock):
        conn, addr = sock.accept()
        logging.debug('%s connected', addr[0])
        self.recivers.append(conn)
<commit_msg>Get peer info before closing socket<commit_after>#! /usr/bin/python

import logging
import re
import select
import socket
import subprocess
import sys
import time

from ircbot import SingleServerIRCBot, nm_to_n as get_nick, parse_channel_modes

# FIXME figure out async subprocess
# FIXME use optparse and/or configreader

logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(message)s")

CHANNEL = '#foo'
PATTERN = '^[\!\?][^ ]+'

class PyCatBot(SingleServerIRCBot):
    def __init__(self, server_list, nick, real, channel):
        SingleServerIRCBot.__init__(self, server_list, nick, real)

        self.channel = channel

        self.sockets = []
        self.recivers = []
        self.processes = []
        self.buffers = {}
        self.listener = self.get_listener()

        self.last_seen = time.time()
        self.ircobj.fn_to_add_socket = self.sockets.append
        self.ircobj.fn_to_remove_socket = self.sockets.remove

        self.setup_logging()

    def setup_logging(self):
        orignial_send_raw = self.connection.send_raw

        def send_raw(string):
            logging.debug(string.decode('utf-8'))
            orignial_send_raw(string)

        def logger(conn, event):
            line = u' '.join(event.arguments())
            logging.debug(line)

        self.connection.add_global_handler('all_raw_messages', logger)
        self.connection.send_raw = send_raw

    def get_listener(self, addr=('', 12345)):
        listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        listener.setblocking(0)
        listener.bind(addr)
        listener.listen(5)

        return listener

    def on_welcome(self, conn, event):
        conn.join(self.channel)

    def on_nicknameinuse(self, conn, event):
        conn.nick(conn.get_nickname() + '_')

    def on_invite(self, conn, event):
        if event.arguments()[0] == self.channel:
            conn.join(self.channel)

    def on_mode(self, conn, event):
        if event.target() != self.channel:
            return

        nick = conn.get_nickname()
        modes = parse_channel_modes(' '.join(event.arguments()))

        if ['+', 'o', nick] in modes:
            conn.mode(self.channel, '+v-o %s %s' % (nick, nick))

    def on_pubmsg(self, conn, event):
        channel = event.target()
        nick = get_nick(event.source())
        message = self.decode(event.arguments()[0])

        p = subprocess.Popen(['./test.sh', channel, nick, message],
            bufsize=1024, stdout=subprocess.PIPE)

        self.processes.append(p.stdout)

    def start(self):
        self._connect()

        while 1:
            sockets = self.sockets + self.recivers + \
                self.processes + [self.listener]
            self.process_sockets(sockets)

    def stop(self):
        if self.connection.is_connected():
            self.connection.disconnect('Bye :)')

    def process_sockets(self, sockets):
        for sock in select.select(sockets, [], [], 0.2)[0]:
            if sock in self.recivers:
                self.handle_reciver(sock)
            elif sock is self.listener:
                self.handle_listener(sock)
            elif sock in self.sockets:
                self.handle_irc(sock)
            elif sock in self.processes:
                self.handle_process(sock)

        self.handle_timeout()

    def handle_process(self, sock):
        if sock not in self.buffers:
            self.buffers[sock] = u''

        data = sock.read(512)

        if len(data) == 0:
            self.processes.remove(sock)
            sock.close()
        else:
            self.buffers[sock] += self.decode(data)

        while '\n' in self.buffers[sock]:
            message, trailing = self.buffers[sock].split('\n', 1)
            self.buffers[sock] = trailing

            self.handle_reciver_message(message)

        if len(data) == 0:
            del self.buffers[sock]

    def handle_reciver(self, sock):
        if sock not in self.buffers:
            self.buffers[sock] = u''

        data = sock.recv(512)
        peer = sock.getpeername()[0]

        if len(data) == 0:
            self.recivers.remove(sock)
            logging.debug('%s disconnected', peer)
            sock.close()
        else:
            self.buffers[sock] += self.decode(data)

        while '\n' in self.buffers[sock]:
            message, trailing = self.buffers[sock].split('\n', 1)
            self.buffers[sock] = trailing

            self.handle_reciver_message(message)
            logging.debug('%s %s', peer, message)

        if len(data) == 0:
            del self.buffers[sock]

    def handle_reciver_message(self, message):
        message = message.encode('utf-8')

        if not message.strip() or not self.connection.is_connected():
            return
        elif message.startswith('/me '):
            self.connection.action(CHANNEL, message[len('/me '):])
        elif message.startswith('/notice '):
            self.connection.notice(CHANNEL, message[len('/notice '):])
        else:
            self.connection.privmsg(CHANNEL, message)

    def handle_listener(self, sock):
        conn, addr = sock.accept()
        logging.debug('%s connected', addr[0])
        self.recivers.append(conn)

{'fraction_non_alphanumeric': 0.12431654676258992, 'fraction_numerical': 0.006426858513189448, 'mean_word_length': 2.1922841396203308, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '147509', 'n_tokens_mistral': 3222, 'n_tokens_neox': 3073, 'n_words': 637}