<commit_before>#!/usr/bin/env python
#Let's keep this file in particular as clean and neatly organized as possible.
#If this is nice and organized, then writing new modules will be a snap and this
#file should rarely have to be edited.

import ConfigParser
DEFAULT_CONFIG = "default.conf"
CONFIG_NAME = ".scrappyrc"

import logging
import os, os.path
import ssl
import sys
import thread, threading
import time

from irclib import client as ircclient


# Logging guidelines and levels
# DEBUG - Python debugging (e.g., printing list of connected servers on startup)
# INFO - Anything we would want to see during normal operation (e.g., startup/reboot messages)
# WARNING - Things we would rather not happen (e.g., using a non-SSL connection)
# ERROR - Deviations from normal operation (e.g., failed to connect to a server)
# CRITICAL - Scrappy gonna die! But not before spitting out a final goodbye! (e.g., SIGINT)
logging.basicConfig(level=logging.DEBUG)
#logging.basicConfig(level=5)


irc_logger = logging.getLogger("irclib.client")
# Change me if you want to see IRC DEBUG lines
irc_logger.setLevel(logging.INFO)
#irc_logger.setLevel(logging.DEBUG)

# State classes

class User(object):
    def __init__(self, server, nick, host=None):
        self.server = server
        self.nick = nick
        self._host = host

        self.channels = {}

    @property
    def host(self):
        if self._host is None:
            self.server.whois((self.nick,))
            timeout = 1
            time_spent = 0
            while self._host is None and time_spent < timeout:
                time_spent += .01
                time.sleep(.01)
        return self._host

    @host.setter
    def host(self, value):
        self._host = value

    def join(self, channel):
        channel.usercount += 1
        self.channels[channel.name] = channel

    def part(self, channel):
        channel.usercount -= 1
        del self.channels[channel.name]

    def __str__(self):
        return "%s@%s" % (self.nick, self._host)

    def __repr__(self):
        return self.__str__()

#TODO: use weakref dictionaries for the users
class Channel(object):
    def __init__(self, server, channel_name):
        self.logger = server.logger.getChild(channel_name)
        self.server = server
        self.name = channel_name
        self.usercount = 0

class ServerState(ircclient.ServerConnection):
    def __init__(self, name, config, bot, irclibobj):
        super(ServerState, self).__init__(irclibobj)
        irclibobj.add_connection(self)

        self.logger = bot.logger.getChild(name)
        self.server_name = name

        # Register handlers
        self.add_global_handler("all_events", bot.process_event)
        # One second tick for timed functions
        self.execute_every(1, bot.on_tick, arguments=(self,))

        bot.register_event("server-%s"%name,"welcome", self.join_defaults)
        bot.register_event("server-%s"%name,"join", self.update_channel)
        bot.register_event("server-%s"%name,"part", self.update_channel)
        bot.register_event("server-%s"%name,"kick", self.update_channel)
        bot.register_event("server-%s"%name,"quit", self.update_channel)
        bot.register_event("server-%s"%name,"namreply", self.on_namereply)
        bot.register_event("server-%s"%name,"whoisuser", self.on_whoisreply)

        if "password" in config:
            password = config["password"]
        else:
            password = None

        if config["ssl"].lower() == "yes":
            factory = ircclient.connection.Factory(wrapper=ssl.wrap_socket)
        else:
            self.logger.warning("Hey, we really don't like you not using SSL!")
            factory = ircclient.connection.Factory()

        try:
            self.connect(config["server"], int(config["port"]), config["nickname"], password=password,
                        username=config["username"], ircname=config["realname"], connect_factory=factory)
        except ircclient.ServerConnectionError, err:
            self.logger.exception("Failed to connect on port %s" % (config["port"]))

        self.cmdchar = config["cmdchar"]
        self.config = config

        self.channels = {}
        self.initial_channels = config["channels"].split()

        self.users = {}

        self.temp_state = {}

        #self.channels = [Channel(x) for x in config["channels"].split()]

    def join_defaults(self, server, event, bot):
        if server == self:
            for channel in self.initial_channels:
                self.join(channel)

    def update_channel(self, server, event, bot):
        if server == self:
            if event.type == "kick":
                nick = event.arguments[0]
                host = None
            elif event.type =="part" or event.type == "join" or event.type == "quit":
                nick = event.source.nick
                host = event.source.host

            if nick not in self.users:
                self.users[nick] = User(self, nick, host)
            user = self.users[nick]

            if event.type == "quit":
                for channel in user.channels:
                    channel.usercount -= 1
                    if channel.user_count == 0:
                        del self.channels[channel.name]
                del self.users[nick]
                return

            if event.target not in self.channels:
                self.channels[event.target] = Channel(self, event.target)
            channel = Channel(self, event.target)

            if event.type == "join":
                user.join(channel)
            elif event.type == "part" or event.type == "kick":
                # TODO: Need to take care of the part when _I_ part a channel, shouldn't track its state anymore
                user.part(channel)
                if len(user.channels) == 0:
                    del self.users[user.nick]
                if channel.user_count == 0:
                    del self.channels[channel.name]

    def on_namereply(self, server, event, bot):
        if server == self:
            # Hoping that the namreply is only additive! In theory, we shouldn't have missed any users leaving channels though
            if event.type == "namreply":
                channel_name = event.arguments[1]
                try:
                    channel = self.channels[channel_name]
                except KeyError:
                    return #ignore it, we're not in the channel

                for nick in event.arguments[2].strip().split(" "):
                    if nick not in self.users:
                        self.users[nick] = User(self, nick)<commit_msg>Minor change to User.host timeout<commit_after>#!/usr/bin/env python
#Let's keep this file in particular as clean and neatly organized as possible.
#If this is nice and organized, then writing new modules will be a snap and this
#file should rarely have to be edited.

import ConfigParser
DEFAULT_CONFIG = "default.conf"
CONFIG_NAME = ".scrappyrc"

import logging
import os, os.path
import ssl
import sys
import thread, threading
import time

from irclib import client as ircclient


# Logging guidelines and levels
# DEBUG - Python debugging (e.g., printing list of connected servers on startup)
# INFO - Anything we would want to see during normal operation (e.g., startup/reboot messages)
# WARNING - Things we would rather not happen (e.g., using a non-SSL connection)
# ERROR - Deviations from normal operation (e.g., failed to connect to a server)
# CRITICAL - Scrappy gonna die! But not before spitting out a final goodbye! (e.g., SIGINT)
logging.basicConfig(level=logging.DEBUG)
#logging.basicConfig(level=5)


irc_logger = logging.getLogger("irclib.client")
# Change me if you want to see IRC DEBUG lines
irc_logger.setLevel(logging.INFO)
#irc_logger.setLevel(logging.DEBUG)

# State classes

class User(object):
    def __init__(self, server, nick, host=None):
        self.server = server
        self.nick = nick
        self._host = host

        self.channels = {}

    @property
    def host(self):
        if self._host is None:
            self.server.whois((self.nick,))
            timeout = 1
            time_spent = 0
            while self._host is None and timeout > 0:
                timeout -= .01
                time.sleep(.01)
        return self._host

    @host.setter
    def host(self, value):
        self._host = value

    def join(self, channel):
        channel.usercount += 1
        self.channels[channel.name] = channel

    def part(self, channel):
        channel.usercount -= 1
        del self.channels[channel.name]

    def __str__(self):
        return "%s@%s" % (self.nick, self._host)

    def __repr__(self):
        return self.__str__()

#TODO: use weakref dictionaries for the users
class Channel(object):
    def __init__(self, server, channel_name):
        self.logger = server.logger.getChild(channel_name)
        self.server = server
        self.name = channel_name
        self.usercount = 0

class ServerState(ircclient.ServerConnection):
    def __init__(self, name, config, bot, irclibobj):
        super(ServerState, self).__init__(irclibobj)
        irclibobj.add_connection(self)

        self.logger = bot.logger.getChild(name)
        self.server_name = name

        # Register handlers
        self.add_global_handler("all_events", bot.process_event)
        # One second tick for timed functions
        self.execute_every(1, bot.on_tick, arguments=(self,))

        bot.register_event("server-%s"%name,"welcome", self.join_defaults)
        bot.register_event("server-%s"%name,"join", self.update_channel)
        bot.register_event("server-%s"%name,"part", self.update_channel)
        bot.register_event("server-%s"%name,"kick", self.update_channel)
        bot.register_event("server-%s"%name,"quit", self.update_channel)
        bot.register_event("server-%s"%name,"namreply", self.on_namereply)
        bot.register_event("server-%s"%name,"whoisuser", self.on_whoisreply)

        if "password" in config:
            password = config["password"]
        else:
            password = <PASSWORD>

        if config["ssl"].lower() == "yes":
            factory = ircclient.connection.Factory(wrapper=ssl.wrap_socket)
        else:
            self.logger.warning("Hey, we really don't like you not using SSL!")
            factory = ircclient.connection.Factory()

        try:
            self.connect(config["server"], int(config["port"]), config["nickname"], password=password,
                        username=config["username"], ircname=config["realname"], connect_factory=factory)
        except ircclient.ServerConnectionError, err:
            self.logger.exception("Failed to connect on port %s" % (config["port"]))

        self.cmdchar = config["cmdchar"]
        self.config = config

        self.channels = {}
        self.initial_channels = config["channels"].split()

        self.users = {}

        self.temp_state = {}

        #self.channels = [Channel(x) for x in config["channels"].split()]

    def join_defaults(self, server, event, bot):
        if server == self:
            for channel in self.initial_channels:
                self.join(channel)

    def update_channel(self, server, event, bot):
        if server == self:
            if event.type == "kick":
                nick = event.arguments[0]
                host = None
            elif event.type =="part" or event.type == "join" or event.type == "quit":
                nick = event.source.nick
                host = event.source.host

            if nick not in self.users:
                self.users[nick] = User(self, nick, host)
            user = self.users[nick]

            if event.type == "quit":
                for channel in user.channels:
                    channel.usercount -= 1
                    if channel.user_count == 0:
                        del self.channels[channel.name]
                del self.users[nick]
                return

            if event.target not in self.channels:
                self.channels[event.target] = Channel(self, event.target)
            channel = Channel(self, event.target)

            if event.type == "join":
                user.join(channel)
            elif event.type == "part" or event.type == "kick":
                # TODO: Need to take care of the part when _I_ part a channel, shouldn't track its state anymore
                user.part(channel)
                if len(user.channels) == 0:
                    del self.users[user.nick]
                if channel.user_count == 0:
                    del self.channels[channel.name]

    def on_namereply(self, server, event, bot):
        if server == self:
            # Hoping that the namreply is only additive! In theory, we shouldn't have missed any users leaving channels though
            if event.type == "namreply":
                channel_name = event.arguments[1]
                try:
                    channel = self.channels[channel_name]
                except KeyError:
                    return #ignore it, we're not in the channel

                nicks = event.arguments[2].strip()
                for nick in nicks.split(" "):
                    if nick not in self.users:
                        self.users[nick] = User(self, nick)
{'fraction_non_alphanumeric': 0.10574747322371399, 'fraction_numerical': 0.0027907678382863175, 'mean_word_length': 2.196480231436837, 'pattern_counts': {'":': 14, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7001219', 'n_tokens_mistral': 3749, 'n_tokens_neox': 3575, 'n_words': 1033}