<commit_before>#!/bin/python
# -*- coding: utf-8 -*-

import sys, os, os.path, types
import datetime, time
import lxml.html
import pymongo
from inspect import getdoc
from twisted.internet import reactor, defer, protocol
from twisted.python import log
from twisted.words.protocols import irc
from twisted.web.client import getPage
from twisted.application import internet
import config

def chanconf(channel):
    if channel:
        channel = channel.lstrip('#')
    return config.CHANNELS[channel]

class FileLogger:
    def __init__(self, channel=''):
        filename = config.BOTNAME
        if channel:
            filename += '_' + channel
        filename += '.log'
        if not os.path.isdir('log'):
            os.mkdir('log')
        self.file = open(os.path.join('log', filename), "a")

    def log(self, message):
        timestamp = time.strftime("[%H:%M:%S]", time.localtime(time.time()))
        self.file.write('%s %s\n' % (timestamp, message))
        self.file.flush()

    def close(self):
        self.file.close()

class IRCBot(irc.IRCClient):

    def __init__(self):
        #NickServ identification handled automatically
        self.nickname = config.BOTNAME
        self.username = config.BOTNAME
        self.password = config.BOTPASS
        self.nicks = {}
        self.sourceURL = 'https://github.com/RouxRC/gazouilleur'
        self.db = pymongo.Connection(config.MONGODB['HOST'], config.MONGODB['PORT'])[config.MONGODB['DATABASE']]
        self.db.authenticate(config.MONGODB['USER'], config.MONGODB['PSWD'])

    def log(self, message, user=None, channel=config.BOTNAME):
        if user:
            nick, _, host = user.partition('!')
            if channel not in self.nicks:
                self.nicks[channel] = {}
            if nick not in self.nicks[channel] or self.nicks[channel][nick] != host:
                self.nicks[channel][nick] = host
            else:
                user = nick
            host = self.nicks[channel][nick]
            self.db['logs'].insert({'timestamp': datetime.datetime.today(), 'channel': channel, 'user': nick, 'host': host, 'message': message})
            message = "%s: %s" % (user, message)
        if channel == "*" or channel == self.nickname or channel not in self.logger:
            channel = config.BOTNAME
        self.logger[channel].log(message)

    def connectionMade(self):
        irc.IRCClient.connectionMade(self)
        log.msg('Connection made')
        self.logger = {config.BOTNAME: FileLogger()}
        self.log("[connected at %s]" % time.asctime(time.localtime(time.time())))

    def connectionLost(self, reason):
        irc.IRCClient.connectionLost(self, reason)
        for channel in self.factory.channels:
            self.left(channel)
        log.msg('Connection lost because: %s.' % (reason,))
        self.log("[disconnected at %s]" % time.asctime(time.localtime(time.time())))
        self.logger[config.BOTNAME].close()

    def signedOn(self):
        log.msg("Signed on as %s." % (self.nickname,))
        for channel in self.factory.channels:
            self.join(channel)

    @defer.inlineCallbacks
    def _reclaimNick(self):
        if config.BOTPASS:
            yield self.msg("NickServ", 'regain %s %s' % (config.BOTNAME, config.BOTPASS,))
            yield self.msg("NickServ", 'identify %s %s' % (config.BOTNAME, config.BOTPASS,))
            log.msg("Reclaimed ident as %s." % (config.BOTNAME,))

    def nickChanged(self, nick):
        log.msg("Identified as %s." % (nick,))
        if nick != config.BOTNAME:
            self._reclaimNick()

    def noticed(self, user, channel, message):
        if 'is not a registered nickname' in message and 'NickServ' in user:
            self._reclaimNick()
        self.log(message, user, channel)

    def joined(self, channel):
        log.msg("Joined %s." % (channel,))
        self.logger[channel] = FileLogger(channel)
        self.log("[joined at %s]" % time.asctime(time.localtime(time.time())), None, channel)

    def left(self, channel):
        log.msg("Left %s." % (channel,))
        self.log("[left at %s]" % time.asctime(time.localtime(time.time())), None, channel)
        self.logger[channel].close()

    def _find_command_function(self, command):
        return getattr(self, 'command_' + command.lower(), None)

    def privmsg(self, user, channel, message):
        nick, _, _ = user.partition('!')
        message = message.strip()
        self.log(message, user, channel)
        d = None
        if not message.startswith('!'):
            if self.nickname.lower() in message.lower():
                d = defer.maybeDeferred(self.command_test)
            else:
                return
        log.msg("[%s] Received command from user %s: %s" % (channel, user, message))
        command, _, rest = message.lstrip('!').partition(' ')
        func = self._find_command_function(command)
        if func is None and d is None:
            d = defer.maybeDeferred(self.command_help, command, channel)
        if d is None:
            d = defer.maybeDeferred(func, rest, channel, nick)
        d.addErrback(self._show_error)
        if channel == self.nickname:
            d.addCallback(self._send_message, nick)
        else:
            d.addCallback(self._send_message, channel, nick)

#TODO apply
    def _check_user_rights(self, nick, channel):
        if nick not in config.GLOBAL_USERS and channel in self.factory.channels and ('USERS' not in chanconf(channel) or nick not in chanconf(channel)['USERS']):
            return 'Sorry only registered users are allowed to use me'
        return

    def _send_message(self, msg, target, nick=None):
        if isinstance(msg, types.ListType):
            msg = '\n'.join([m for (r, m) in msg])
        if nick:
            msg = '%s: %s' % (nick, msg)
        self.msg(target, msg)
        self.log(msg, self.nickname, target)

    def _show_error(self, failure):
        return failure.getErrorMessage()

    def command_help(self, rest, channel=None, *args):
        """!help [<command>]: Prints general help or help for specific <command>."""
        rest = rest.lstrip('!').lower()
        commands = [c.lstrip('command_') for c in dir(IRCBot) if c.startswith('command_')]
        if channel not in self.factory.channels or 'TWITTER' not in chanconf(channel):
            commands = [c for c in commands if not 'twitter' in self._find_command_function(c).__doc__.lower()]
        def_msg = 'My commands are:  !'+' ;  !'.join(commands)+'\nType "!help <command>" to get more details.'
        if rest == '':
            return def_msg
        elif rest in commands:
            return self._find_command_function(rest).__doc__
        return '!%s is not a valid command. %s' % (rest, def_msg)

    def command_ping(self, *args):
        """!ping : Ping test, should answer pong."""
        return 'Pong.'

    def command_test(self, *args):
        """!test : Simple test to check whether I'm present, TwiTTER similar as !ping."""
        return 'Hello? type "!help" to list my commands.'

    def command_source(self, *args):
        """!source : Gives the link to my sourcecode."""
        return 'My sourcecode is under free GPL 3.0 licence and available at the following address: %s' % self.sourceURL

    def command_last(self, rest, channel=None, *args):
        """!last [<N>] : Prints the last message or <N> last ones (maximum 5)."""
        return "TODO"
# TODO
    def command_lastseen(self, rest, channel=None, *args):
        """!lastseen <nickname> : Prints last time <nickname> was seen logging off and in."""<commit_msg>handle multiline outputs / command last to retrieve previous msgs<commit_after>#!/bin/python
# -*- coding: utf-8 -*-

import sys, os, os.path, types, re
import datetime, time
import lxml.html
import pymongo
from inspect import getdoc
from twisted.internet import reactor, defer, protocol
from twisted.python import log
from twisted.words.protocols import irc
from twisted.web.client import getPage
from twisted.application import internet
import config

def sint(s):
    try:
        return int(s.strip())
    except:
        return 0

def chanconf(channel):
    if channel:
        channel = channel.lstrip('#')
    return config.CHANNELS[channel]

class FileLogger:
    def __init__(self, channel=''):
        filename = config.BOTNAME
        if channel:
            filename += '_' + channel
        filename += '.log'
        if not os.path.isdir('log'):
            os.mkdir('log')
        self.file = open(os.path.join('log', filename), "a")

    def log(self, message):
        timestamp = time.strftime("[%H:%M:%S]", time.localtime(time.time()))
        self.file.write('%s %s\n' % (timestamp, message))
        self.file.flush()

    def close(self):
        self.file.close()

class IRCBot(irc.IRCClient):

    def __init__(self):
        #NickServ identification handled automatically
        self.nickname = config.BOTNAME
        self.username = config.BOTNAME
        self.password = config.BOTPASS
        self.nicks = {}
        self.sourceURL = 'https://github.com/RouxRC/gazouilleur'
        self.db = pymongo.Connection(config.MONGODB['HOST'], config.MONGODB['PORT'])[config.MONGODB['DATABASE']]
        self.db.authenticate(config.MONGODB['USER'], config.MONGODB['PSWD'])

    def log(self, message, user=None, channel=config.BOTNAME):
        if user:
            nick, _, host = user.partition('!')
            if channel not in self.nicks:
                self.nicks[channel] = {}
            if nick not in self.nicks[channel] or self.nicks[channel][nick] != host:
                self.nicks[channel][nick] = host
            else:
                user = nick
            host = self.nicks[channel][nick]
            self.db['logs'].insert({'timestamp': datetime.datetime.today(), 'channel': channel, 'user': nick, 'host': host, 'message': message})
            message = "%s: %s" % (user, message)
        if channel == "*" or channel == self.nickname or channel not in self.logger:
            channel = config.BOTNAME
        self.logger[channel].log(message)

    def connectionMade(self):
        irc.IRCClient.connectionMade(self)
        log.msg('Connection made')
        self.logger = {config.BOTNAME: FileLogger()}
        self.log("[connected at %s]" % time.asctime(time.localtime(time.time())))

    def connectionLost(self, reason):
        irc.IRCClient.connectionLost(self, reason)
        for channel in self.factory.channels:
            self.left(channel)
        log.msg('Connection lost because: %s.' % (reason,))
        self.log("[disconnected at %s]" % time.asctime(time.localtime(time.time())))
        self.logger[config.BOTNAME].close()

    def signedOn(self):
        log.msg("Signed on as %s." % (self.nickname,))
        for channel in self.factory.channels:
            self.join(channel)

    @defer.inlineCallbacks
    def _reclaimNick(self):
        if config.BOTPASS:
            yield self.msg("NickServ", 'regain %s %s' % (config.BOTNAME, config.BOTPASS,))
            yield self.msg("NickServ", 'identify %s %s' % (config.BOTNAME, config.BOTPASS,))
            log.msg("Reclaimed ident as %s." % (config.BOTNAME,))

    def nickChanged(self, nick):
        log.msg("Identified as %s." % (nick,))
        if nick != config.BOTNAME:
            self._reclaimNick()

    def noticed(self, user, channel, message):
        if 'is not a registered nickname' in message and 'NickServ' in user:
            self._reclaimNick()
        self.log(message, user, channel)

    def joined(self, channel):
        log.msg("Joined %s." % (channel,))
        self.logger[channel] = FileLogger(channel)
        self.log("[joined at %s]" % time.asctime(time.localtime(time.time())), None, channel)

    def left(self, channel):
        log.msg("Left %s." % (channel,))
        self.log("[left at %s]" % time.asctime(time.localtime(time.time())), None, channel)
        self.logger[channel].close()

    def _find_command_function(self, command):
        return getattr(self, 'command_' + command.lower(), None)

    def privmsg(self, user, channel, message):
        nick, _, _ = user.partition('!')
        message = message.strip()
        self.log(message, user, channel)
        d = None
        if not message.startswith('!'):
            if self.nickname.lower() in message.lower():
                d = defer.maybeDeferred(self.command_test)
            else:
                return
        log.msg("[%s] Received command from user %s: %s" % (channel, user, message))
        command, _, rest = message.lstrip('!').partition(' ')
        func = self._find_command_function(command)
        if func is None and d is None:
            d = defer.maybeDeferred(self.command_help, command, channel)
        if d is None:
            d = defer.maybeDeferred(func, rest, channel, nick)
        d.addErrback(self._show_error)
        if channel == self.nickname:
            d.addCallback(self._send_message, nick)
        else:
            d.addCallback(self._send_message, channel, nick)

#TODO apply
    def _check_user_rights(self, nick, channel):
        if nick not in config.GLOBAL_USERS and channel in self.factory.channels and ('USERS' not in chanconf(channel) or nick not in chanconf(channel)['USERS']):
            return 'Sorry only registered users are allowed to use me'
        return

    @defer.inlineCallbacks
    def _send_message(self, msg, target, nick=None):
        if not isinstance(msg, types.ListType):
            msgs = [(1, m) for m in msg.split('\n')]
        for m in [msg for (res, msg) in msgs if res]:
            if nick:
                m = '%s: %s' % (nick, m)
            yield self.msg(target, m)
            yield self.log(m, self.nickname, target)

    def _show_error(self, failure):
        return failure.getErrorMessage()

    def command_help(self, rest, channel=None, *args):
        """!help [<command>]: Prints general help or help for specific <command>."""
        rest = rest.lstrip('!').lower()
        commands = [c.lstrip('command_') for c in dir(IRCBot) if c.startswith('command_')]
        if channel not in self.factory.channels or 'TWITTER' not in chanconf(channel):
            commands = [c for c in commands if not 'twitter' in self._find_command_function(c).__doc__.lower()]
        def_msg = 'My commands are:  !'+' ;  !'.join(commands)+'\nType "!help <command>" to get more details.'
        if rest == '':
            return def_msg
        elif rest in commands:
            return self._find_command_function(rest).__doc__
        return '!%s is not a valid command. %s' % (rest, def_msg)

    def command_ping(self, *args):
        """!ping : Ping test, should answer pong."""
        return 'Pong.'

    def command_test(self, *args):
        """!test : Simple test to check whether I'm present, similar as !ping."""
        return 'Hello? type "!help" to list my commands.'

    def command_source(self, *args):
        """!source : Gives the link to my sourcecode."""
        return 'My sourcecode is under free GPL 3.0 licence and available at the following address: %s' % self.sourceURL

    re_shortdate = re.compile(r'^....-(..)-(.. ..:..).*$')
    def command_last(self, rest, channel=None, nick=None):
        """!last [<N>] : Prints the last message or <N> last ones (maximum 4)."""
        #TODO add --from --with
        nb, _, msg = rest.partition(' ')
        nb = max(1, min(sint(nb), 4))+1
        query = {'channel': channel, '$or': [{'user': {'$ne': self.nickname}}, {'message': {'$not': re.compile(r'^'+nick+': ', re.I)} }] }
        res = list(self.db['logs'].find(query, sort=[('timestamp', pymongo.DESCENDING)], limit=nb))
        if len(res) == 0:
            return "No match found in my history log."
        res.reverse()
        res.pop()
        res = "\n".join(['%s - %s: %s' % (self.re_shortdate.sub(r'\1/\2', str(l['timestamp'])), str(l['user']), str(l['message'])) for l in res])
        return res

# TODO
    def command_lastseen(self, rest, channel=None, *args):
        """!lastseen <nickname> : Prints last time <nickname> was seen logging off and in."""
{'fraction_non_alphanumeric': 0.1416941147225157, 'fraction_numerical': 0.0009943446647194083, 'mean_word_length': 2.5799777530589543, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1701808', 'n_tokens_mistral': 4982, 'n_tokens_neox': 4848, 'n_words': 1316}