<commit_before>#!/usr/bin/python

"""A modular python bot based on the twisted matrix irc library

@author Riku 'Shrike' Lindblad (<EMAIL>)
@copyright Copyright (c) 2006 <NAME>
@license BSD
"""

import sys
import os.path
import time
import urllib
import fnmatch

try:
    import psyco
    psyco.full()
except ImportError:
    print "Psyco not found, running unoptimized"

try:
    import yaml
except ImportError:
    print "PyYAML not found, please install from http://pyyaml.org/wiki/PyYAML"
    sys.exit(1)

# twisted imports
try:
    from twisted.protocols import irc
    from twisted.internet import reactor, protocol
    from twisted.python import log, rebuild
except ImportError:
    print "Twisted library not found, please install Twisted 1.3.0 from http://twistedmatrix.com/products/download"
    sys.exit(1)

from util import *
from util.BeautifulSoup import BeautifulSoup
import botcore

class URLCacheItem(object):
    """URL cache item object, fetches data only when needed"""
    
    def __init__(self, url):
        self.url = url
        self.content = None
        self.headers = None
        self.bs = None
        # maximum size in kB to download
        self.max_size = 2048
        self.fp = None

    def _open(self, url):
        """Returns the raw file pointer to the given URL"""
        if not self.fp:
            urllib._urlopener = BotURLOpener()
            try:
                self.fp = urllib.urlopen(self.url)
            except IOError, e:
                print "IOError when opening url %s" % url
                print e
        return self.fp

    def _checkstatus(self):
        """Check if all data has already been cached and close socket if so"""
        
        if self.content and \
           self.headers and \
           self.bs:
            self.fp.close()

    def getSize(self):
        """Get the content length of URL in kB

        @return None if the server doesn't return a content-length header"""
        if self.getHeaders().has_key('content-length'):
            length = int(self.getHeaders()['content-length'])/1024
            return length
        else:
            return None

    def getContent(self):
        """Get the actual file at the URL

        @return None if the file is too large (over 2MB)"""
        if not self.content:
            f = self._open(self.url)

            size = self.getSize()
            if size > self.max_size:
                print "CONTENT TOO LARGE, WILL NOT FETCH", size, self.url
                self.content = None
            else:
                self.content = f.read()

        self._checkstatus()
        return self.content

    def getHeaders(self):
        """Get headers for the URL"""
        
        if not self.headers:
            f = self._open(self.url)
            self.headers = f.info()
        
        self._checkstatus()
        return self.headers

    def getBS(self):
        """Get a beautifulsoup instance for the URL

        @return None if the url doesn't contain HTML
        """
        
        if not self.bs:
            # only attempt a bs parsing if the content is html or xhtml
            if self.getHeaders().has_key('content-type') and \
            self.getHeaders().getsubtype() == 'html' or \
            self.getHeaders().getsubtype() == 'xhtml+xml':
            
                bs = BeautifulSoup()
                bs.feed(self.getContent())
                self.bs = bs
            else:
                return None
            
        self._checkstatus()
        return self.bs

class BotURLOpener(urllib.FancyURLopener):
    """Url opener that fakes itself as Firefox and ignores all basic auth prompts"""
    
    def __init__(self, *args):
        # Firefox 1.0PR on w2k
        self.version = "Mozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) PyFiBot"
        urllib.FancyURLopener.__init__(self, *args)

    def prompt_user_passwd(self, host, realm):
        print "PASSWORD PROMPT:", host, realm
        return ('', '')

class Network:
    def __init__(self, root, alias, address, nickname, channels = None):
        self.alias = alias                         # network name
        self.address = address                     # server address
        self.nickname = nickname                   # nick to use
        self.channels = channels or {}             # channels to join

        # create network specific save directory
        p = os.path.join(root, alias)
        if not os.path.isdir(p):
            os.mkdir(p)

    def __repr__(self):
        return 'Network(%r, %r)' % (self.alias, self.address)

class InstantDisconnectProtocol(protocol.Protocol):
    def connectionMade(self):
        self.transport.loseConnection()

class ThrottledClientFactory(protocol.ClientFactory):
    """Client factory that inserts a slight delay to connecting and reconnecting"""
    
    lostDelay = 10
    failedDelay = 60
    
    def clientConnectionLost(self, connector, reason):
        print connector
        print "connection lost (%s): reconnecting in %d seconds" % (reason, self.lostDelay)
        reactor.callLater(self.lostDelay, connector.connect)
        
    def clientConnectionFailed(self, connector, reason):
        print connector
        print "connection failed (%s): reconnecting in %d seconds" % (reason, self.failedDelay)
        reactor.callLater(self.failedDelay, connector.connect)
                                                                        
class PyFiBotFactory(ThrottledClientFactory):
    """python.fi bot factory"""

    version = "$Revision$"

    protocol = botcore.PyFiBot
    allBots = None
    moduledir = os.path.join(sys.path[0], "modules/")
    startTime = None

    def __init__(self, config):
        """Initialize the factory"""

        self.config = config
        print config['admins']
        #self.admins = config['admins']
        self.data = {}
        self.data['networks'] = {}
        self.ns = {}

        # cache url contents for 10 minutes, check for old entries every minute
        self._urlcache = timeoutdict.TimeoutDict(timeout=600, pollinterval=60)<commit_msg>Minor change (useless?)<commit_after>#!/usr/bin/python

"""A modular python bot based on the twisted matrix irc library

@author Riku 'Shrike' Lindblad (<EMAIL>)
@copyright Copyright (c) 2006 <NAME>
@license BSD
"""

import sys
import os.path
import time
import urllib
import fnmatch

try:
    import psyco
    psyco.full()
except ImportError:
    print "Psyco not found, running unoptimized"

try:
    import yaml
except ImportError:
    print "PyYAML not found, please install from http://pyyaml.org/wiki/PyYAML"
    sys.exit(1)

# twisted imports
try:
    from twisted.protocols import irc
    from twisted.internet import reactor, protocol
    from twisted.python import log, rebuild
except ImportError:
    print "Twisted library not found, please install Twisted 1.3.0 from http://twistedmatrix.com/products/download"
    sys.exit(1)

from util import *
from util.BeautifulSoup import BeautifulSoup
import botcore

class URLCacheItem(object):
    """URL cache item object, fetches data only when needed"""
    
    def __init__(self, url):
        self.url = url
        self.content = None
        self.headers = None
        self.bs = None
        # maximum size in kB to download
        self.max_size = 2048
        self.fp = None

    def _open(self, url):
        """Returns the raw file pointer to the given URL"""
        if not self.fp:
            urllib._urlopener = BotURLOpener()
            try:
                self.fp = urllib.urlopen(self.url)
            except IOError, e:
                print "IOError when opening url %s" % url
                print e
        return self.fp

    def _checkstatus(self):
        """Check if all data has already been cached and close socket if so"""
        
        if self.content and \
           self.headers and \
           self.bs:
            self.fp.close()

    def getSize(self):
        """Get the content length of URL in kB

        @return None if the server doesn't return a content-length header"""
        if self.getHeaders().has_key('content-length'):
            length = int(self.getHeaders()['content-length'])/1024
            return length
        else:
            return None

    def getContent(self):
        """Get the actual file at the URL

        @return None if the file is too large (over 2MB)"""
        if not self.content:
            f = self._open(self.url)

            size = self.getSize()
            if size > self.max_size:
                print "CONTENT TOO LARGE, WILL NOT FETCH", size, self.url
                self.content = None
            else:
                self.content = f.read()

        self._checkstatus()
        return self.content

    def getHeaders(self):
        """Get headers for the URL"""
        
        if not self.headers:
            f = self._open(self.url)
            self.headers = f.info()
        
        self._checkstatus()
        return self.headers

    def getBS(self):
        """Get a beautifulsoup instance for the URL

        @return None if the url doesn't contain HTML
        """
        
        if not self.bs:
            # only attempt a bs parsing if the content is html or xhtml
            if self.getHeaders().has_key('content-type') and \
            self.getHeaders().getsubtype() == 'html' or \
            self.getHeaders().getsubtype() == 'xhtml+xml':
            
                bs = BeautifulSoup()
                bs.feed(self.getContent())
                self.bs = bs
            else:
                return None
            
        self._checkstatus()
        return self.bs

class BotURLOpener(urllib.FancyURLopener):
    """Url opener that fakes itself as Firefox and ignores all basic auth prompts"""
    
    def __init__(self, *args):
        # Firefox 1.0PR on w2k
        self.version = "Mozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) PyFiBot"
        urllib.FancyURLopener.__init__(self, *args)

    def prompt_user_passwd(self, host, realm):
        print "PASSWORD PROMPT:", host, realm
        return ('', '')

class Network:
    def __init__(self, root, alias, address, nickname, channels = None):
        self.alias = alias                         # network name
        self.address = address                     # server address
        self.nickname = nickname                   # nick to use
        self.channels = channels or {}             # channels to join

        # create network specific save directory
        p = os.path.join(root, alias)
        if not os.path.isdir(p):
            os.mkdir(p)

    def __repr__(self):
        return 'Network(%r, %r)' % (self.alias, self.address)

class InstantDisconnectProtocol(protocol.Protocol):
    def connectionMade(self):
        self.transport.loseConnection()

class ThrottledClientFactory(protocol.ClientFactory):
    """Client factory that inserts a slight delay to connecting and reconnecting"""
    
    lostDelay = 10
    failedDelay = 60
    
    def clientConnectionLost(self, connector, reason):
        print connector
        print "connection lost (%s): reconnecting in %d seconds" % (reason, self.lostDelay)
        reactor.callLater(self.lostDelay, connector.connect)
        
    def clientConnectionFailed(self, connector, reason):
        print connector
        print "connection failed (%s): reconnecting in %d seconds" % (reason, self.failedDelay)
        reactor.callLater(self.failedDelay, connector.connect)
                                                                        
class PyFiBotFactory(ThrottledClientFactory):
    """python.fi bot factory"""

    version = "$Revision$"

    protocol = botcore.PyFiBot
    allBots = None
    moduledir = os.path.join(sys.path[0], "modules/")
    startTime = None
    config = None

    def __init__(self, config):
        """Initialize the factory"""

        self.config = config
        print config['admins']
        #self.admins = config['admins']
        self.data = {}
        self.data['networks'] = {}
        self.ns = {}

        # cache url contents for 10 minutes, check for old entries every minute
        self._urlcache = timeoutdict.TimeoutDict(timeout=600, pollinterval=60)
{'fraction_non_alphanumeric': 0.09466098919095971, 'fraction_numerical': 0.006550933508024894, 'mean_word_length': 2.005167322834646, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7673780', 'n_tokens_mistral': 3510, 'n_tokens_neox': 3312, 'n_words': 1055}