<commit_before>from twisted.internet import reactor, protocol, task, defer
from twisted.enterprise import adbapi
from twisted.protocols import amp
from twisted.python import log, reflect
from twisted.web import xmlrpc
from infobat.redent import redent
from infobat.config import conf
from infobat import amp, chains, database, http
from datetime import datetime
from lxml import html, etree
from urllib import urlencode
from urlparse import urljoin
import operator
import random
import time
import sys
import re

_lol_regex = re.compile(r'\b([lo]{3,}|rofl+|lmao+)z*\b', re.I)
_lol_messages = [
    '#python is a no-LOL zone.', 
    'i mean it: no LOL in #python.', 
    'seriously, dude, no LOL in #python.']
_bad_pastebin_regex = re.compile(
    r'((https?://)?([a-z0-9-]+\.)*(pastebin\.(com|org|ca)|etherpad\.com)/)'
    r'([a-z0-9]+)/?', re.I)
_textarea_xpath = etree.XPath('//textarea[@name=$name][1]/text()')
_pastebin_textareas = {
    'pastebin.ca': 'content',
    'pastebin.com': 'code2',
    'pastebin.org': 'code2'}

class InfobatChild(amp.InfobatChildBase):
    db = dbpool = None
    
    def __init__(self):
        amp.InfobatChildBase.__init__(self)
        self.dbpool = database.InfobatDatabaseRunner()
        self.paste_proxy = xmlrpc.Proxy('http://paste.pocoo.org/xmlrpc/')
        self._load_database()
        self.looper = task.LoopingCall(self._sync_countdown)
        self.countdown = self.max_countdown
        self.looper.start(30)
    
    def connectionLost(self, reason):
        if self.db:
            self.db.sync()
        if self.dbpool:
            self.dbpool.close()
        amp.InfobatChildBase.connectionLost(self, reason)
    
    def _load_database(self):
        self.db = chains.Database(conf.get('database', 'db_file'))
    
    def _sync_countdown(self):
        if self.db is None: 
            return
        self.countdown -= 1
        if self.countdown == 0:
            self.db.sync()
            self.countdown = self.max_countdown
    
    def learn(self, string, action=False):
        if self.db is None: 
            return
        self.db.learn(string, action)
    
    def action(self, user, channel, message):
        if not user: 
            return
        if channel != self.nickname:
            self.learn(message, True)
    
    def privmsg(self, user, channel, message):
        if not user: return
        user = user.split('!', 1)[0]
        if user.lower() in ('nickserv', 'chanserv', 'memoserv'): return
        if channel == self.nickname:
            log.msg('privmsg from %s: %s' % (user, message))
            target = user
        else:
            target = channel
       
        if channel == '#python':
            if message.startswith('!') and message.lstrip('!'):
                self.notice(user, 'no triggers in %s.' % channel)
            elif _lol_regex.search(message):
                self.do_lol(user)
            else:
                m = _bad_pastebin_regex.search(message)
                if m:
                    base, _, _, which_bin, _, paste_id = m.groups()
                    full_url = m.group(0)
                    self.repaste(
                        target, user, base, which_bin, paste_id, full_url)
        
        if channel != self.nickname:
            self.learn(message)
        
        m = re.match(
            r'^s*%s\s*[,:> ]+(\S?.*?)[.!?]?\s*$' % self.nickname, message, re.I)
        if m:
            command, = m.groups()
        elif channel == self.nickname:
            command = message
        else:
            return
        s_command = command.split(' ')
        command_func = getattr(self, 'infobat_' + s_command[0], None)
        if command_func is not None:
            command_func(target, *s_command[1:])
        elif self.db and (channel in (
                self.nickname, '#python-offtopic', '#infobob')):
            action, result = self.db.splice()
            if action:
                self.me(target, result)
            else:
                if channel != self.nickname:
                    result = user + ', ' + result
                self.msg(target, result)
    
    @defer.inlineCallbacks
    def do_lol(self, nick):
        offenses = yield self.dbpool.add_lol(nick)
        message_idx = min(offenses, len(_lol_messages)) - 1
        self.notice(nick, _lol_messages[message_idx])
    
    @defer.inlineCallbacks
    def repaste(self, target, user, base, which_bin, paste_id, full_url):
        self.notice(user, 'in the future, please use a less awful pastebin '
            '(e.g. paste.pocoo.org) instead of %s.' % which_bin)
        repasted_url = yield self.dbpool.get_repaste(full_url)
        if repasted_url is None:
            if which_bin == 'etherpad.com':
                data, _ = yield http.get_page((
                    'http://etherpad.com/ep/pad/export/%s/latest?format=txt'
                ) % paste_id)
            else:
                page, _ = yield http.get_page(full_url)
                tree = html.document_fromstring(page)
                data, = _textarea_xpath(tree, 
                    name=_pastebin_textareas[which_bin])
            new_paste_id = yield self.paste_proxy.callRemote(
                'pastes.newPaste', 'python', str(data))
            repasted_url = 'http://paste.pocoo.org/show/%s/' % new_paste_id
            yield self.dbpool.add_repaste(full_url, repasted_url)
        self.msg(target, '%s (repasted for %s)' % (repasted_url, user))
    
    @defer.inlineCallbacks
    def infobat_redent(self, target, paste_target, *text):
        redented = (
            redent(' '.join(text).decode('utf8', 'replace')).encode('utf8'))
        try:
            paste_id = yield self.paste_proxy.callRemote(
                'pastes.newPaste', 'python', redented)
        except:
            self.msg(target, 'Error: %r' % sys.exc_info()[1])
            raise
        else:
            self.msg(target, '%s, http://paste.pocoo.org/show/%s/' % (
                paste_target, paste_id))
    
    @defer.inlineCallbacks
    def infobat_codepad(self, target, paste_target, *text):
        redented = (
            redent(' '.join(text).decode('utf8', 'replace')).encode('utf8'))
        post_data = urlencode(dict(
            code=redented, lang='Python', submit='Submit', run='True'))
        headers = {'Content-Type': 'application/x-www-form-urlencoded'}
        try:
            _, fac = yield http.get_page('http://codepad.org/', 
                method='POST', postdata=post_data, headers=headers)
        except:
            self.msg(target, 'Error: %r' % sys.exc_info()[1])
            raise
        else:
            paste_url = urljoin(
                'http://codepad.org/', fac.response_headers['location'][0])
            self.msg(target, '%s, %s' % (paste_target, paste_url))
    
    def infobat_sync(self, target):
        if self.db is not None: 
            self.db.sync()
            self.msg(target, 'Done.')<commit_msg>Falling back on bpaste.net for when paste.pocoo.org is down.<commit_after>from twisted.internet import reactor, protocol, task, defer
from twisted.enterprise import adbapi
from twisted.protocols import amp
from twisted.python import log, reflect
from twisted.web import xmlrpc
from infobat.redent import redent
from infobat.config import conf
from infobat import amp, chains, database, http
from datetime import datetime
from lxml import html, etree
from urllib import urlencode
from urlparse import urljoin
import operator
import random
import time
import sys
import re

_lol_regex = re.compile(r'\b([lo]{3,}|rofl+|lmao+)z*\b', re.I)
_lol_messages = [
    '#python is a no-LOL zone.', 
    'i mean it: no LOL in #python.', 
    'seriously, dude, no LOL in #python.']
_bad_pastebin_regex = re.compile(
    r'((https?://)?([a-z0-9-]+\.)*(pastebin\.(com|org|ca)|etherpad\.com)/)'
    r'([a-z0-9]+)/?', re.I)
_textarea_xpath = etree.XPath('//textarea[@name=$name][1]/text()')
_pastebin_textareas = {
    'pastebin.ca': 'content',
    'pastebin.com': 'code2',
    'pastebin.org': 'code2'}
good_pastebins = [
    'http://paste.pocoo.org',
    'http://bpaste.net',
]

class CouldNotPastebinError(Exception):
    pass

class InfobatChild(amp.InfobatChildBase):
    db = dbpool = None
    
    def __init__(self):
        amp.InfobatChildBase.__init__(self)
        self.dbpool = database.InfobatDatabaseRunner()
        self._load_database()
        self.looper = task.LoopingCall(self._sync_countdown)
        self.countdown = self.max_countdown
        self.looper.start(30)
    
    def connectionLost(self, reason):
        if self.db:
            self.db.sync()
        if self.dbpool:
            self.dbpool.close()
        amp.InfobatChildBase.connectionLost(self, reason)
    
    def _load_database(self):
        self.db = chains.Database(conf.get('database', 'db_file'))
    
    def _sync_countdown(self):
        if self.db is None: 
            return
        self.countdown -= 1
        if self.countdown == 0:
            self.db.sync()
            self.countdown = self.max_countdown
    
    def learn(self, string, action=False):
        if self.db is None: 
            return
        self.db.learn(string, action)
    
    def action(self, user, channel, message):
        if not user: 
            return
        if channel != self.nickname:
            self.learn(message, True)
    
    def privmsg(self, user, channel, message):
        if not user: return
        user = user.split('!', 1)[0]
        if user.lower() in ('nickserv', 'chanserv', 'memoserv'): return
        if channel == self.nickname:
            log.msg('privmsg from %s: %s' % (user, message))
            target = user
        else:
            target = channel
       
        if channel == '#python':
            if message.startswith('!') and message.lstrip('!'):
                self.notice(user, 'no triggers in %s.' % channel)
            elif _lol_regex.search(message):
                self.do_lol(user)
            else:
                m = _bad_pastebin_regex.search(message)
                if m:
                    base, _, _, which_bin, _, paste_id = m.groups()
                    full_url = m.group(0)
                    self.repaste(
                        target, user, base, which_bin, paste_id, full_url)
        
        if channel != self.nickname:
            self.learn(message)
        
        m = re.match(
            r'^s*%s\s*[,:> ]+(\S?.*?)[.!?]?\s*$' % self.nickname, message, re.I)
        if m:
            command, = m.groups()
        elif channel == self.nickname:
            command = message
        else:
            return
        s_command = command.split(' ')
        command_func = getattr(self, 'infobat_' + s_command[0], None)
        if command_func is not None:
            command_func(target, *s_command[1:])
        elif self.db and (channel in (
                self.nickname, '#python-offtopic', '#infobob')):
            action, result = self.db.splice()
            if action:
                self.me(target, result)
            else:
                if channel != self.nickname:
                    result = user + ', ' + result
                self.msg(target, result)
    
    @defer.inlineCallbacks
    def do_lol(self, nick):
        offenses = yield self.dbpool.add_lol(nick)
        message_idx = min(offenses, len(_lol_messages)) - 1
        self.notice(nick, _lol_messages[message_idx])
    
    @defer.inlineCallbacks
    def pastebin(self, language, data):
        for url in good_pastebins:
            proxy = xmlrpc.Proxy(url + '/xmlrpc/')
            try:
                new_paste_id = yield proxy.callRemote(
                    'pastes.newPaste', language, data)
            except:
                log.err()
                continue
            else:
                defer.returnValue('%s/show/%s/' % (url, new_paste_id))
        raise CouldNotPastebinError()
    
    @defer.inlineCallbacks
    def repaste(self, target, user, base, which_bin, paste_id, full_url):
        self.notice(user, 'in the future, please use a less awful pastebin '
            '(e.g. paste.pocoo.org) instead of %s.' % which_bin)
        repasted_url = yield self.dbpool.get_repaste(full_url)
        if repasted_url is None:
            if which_bin == 'etherpad.com':
                data, _ = yield http.get_page((
                    'http://etherpad.com/ep/pad/export/%s/latest?format=txt'
                ) % paste_id)
            else:
                page, _ = yield http.get_page(full_url)
                tree = html.document_fromstring(page)
                data, = _textarea_xpath(tree, 
                    name=_pastebin_textareas[which_bin])
            repasted_url = yield self.pastebin('python', data.encode('utf8'))
            yield self.dbpool.add_repaste(full_url, repasted_url)
        self.msg(target, '%s (repasted for %s)' % (repasted_url, user))
    
    @defer.inlineCallbacks
    def infobat_redent(self, target, paste_target, *text):
        redented = (
            redent(' '.join(text).decode('utf8', 'replace')).encode('utf8'))
        try:
            paste_url = yield self.pastebin('python', redented)
        except:
            self.msg(target, 'Error: %r' % sys.exc_info()[1])
            raise
        else:
            self.msg(target, '%s, %s' % (paste_target, paste_url))
    
    @defer.inlineCallbacks
    def infobat_codepad(self, target, paste_target, *text):
        redented = (
            redent(' '.join(text).decode('utf8', 'replace')).encode('utf8'))
        post_data = urlencode(dict(
            code=redented, lang='Python', submit='Submit', run='True'))
        headers = {'Content-Type': 'application/x-www-form-urlencoded'}
        try:
            _, fac = yield http.get_page('http://codepad.org/', 
                method='POST', postdata=post_data, headers=headers)
        except:
            self.msg(target, 'Error: %r' % sys.exc_info()[1])
            raise
        else:
            paste_url = urljoin(
                'http://codepad.org/', fac.response_headers['location'][0])
            self.msg(target, '%s, %s' % (paste_target, paste_url))
    
    def infobat_sync(self, target):
        if self.db is not None: 
            self.db.sync()
            self.msg(target, 'Done.')
{'fraction_non_alphanumeric': 0.12881236823612088, 'fraction_numerical': 0.003583977512297962, 'mean_word_length': 1.9997892074198989, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6916967', 'n_tokens_mistral': 4534, 'n_tokens_neox': 4393, 'n_words': 964}