<commit_before>""" Basic iembot/nwsbot implementation, upstream for this code is
    on github: https://github.com/akrherz/iembot
"""

from twisted.words.xish import domish
from twisted.words.xish import xpath
from twisted.words.protocols.jabber import jid, client
from twisted.words.xish.xmlstream import STREAM_END_EVENT
from twisted.internet import reactor
from twisted.application import internet
from twisted.python import log
from twisted.python.logfile import DailyLogFile
import twisted.web.error as weberror
from twisted.internet.task import LoopingCall
from twisted.mail import smtp
from twisted.web import client as webclient

from oauth import oauth

from xml.etree import ElementTree as ET

import datetime
import pytz
import json
import traceback
from email.MIMEText import MIMEText
import StringIO
import random
import os
import pwd
import urllib
import socket
import locale
import re
import glob

from twittytwister import twitter
locale.setlocale(locale.LC_ALL, 'en_US')

PRESENCE_MUC_ITEM = (
    "/presence/x[@xmlns='http://jabber.org/protocol/muc#user']/item")


def load_chatrooms_from_db(txn, bot, always_join):
    """ Load database configuration and do work

    Args:
      txn (dbtransaction): database cursor
      bot (basicbot): the running bot instance
      always_join (boolean): do we force joining each room, regardless
    """
    # Load up the channel keys
    txn.execute("SELECT id, channel_key from %s_channels" % (bot.name,))
    for row in txn:
        bot.channelkeys[row['channel_key']] = row['id']

    # Load up the routingtable for bot products
    rt = {}
    txn.execute("""
        SELECT roomname, channel from %s_room_subscriptions
    """ % (bot.name,))
    for row in txn:
        rm = row['roomname']
        channel = row['channel']
        if channel not in rt:
            rt[channel] = []
        rt[channel].append(rm)
    bot.routingtable = rt
    log.msg(("... loaded %s channel subscriptions for %s rooms"
             ) % (txn.rowcount, len(rt)))

    # Now we need to load up the syndication
    synd = {}
    txn.execute("""
        SELECT roomname, endpoint from %s_room_syndications
    """ % (bot.name,))
    for row in txn:
        rm = row['roomname']
        endpoint = row['endpoint']
        if rm not in synd:
            synd[rm] = []
        synd[rm].append(endpoint)
    bot.syndication = synd
    log.msg(("... loaded %s room syndications for %s rooms"
             ) % (txn.rowcount, len(synd)))

    # Load up a list of chatrooms
    txn.execute("""
        SELECT roomname, fbpage, twitter from %s_rooms ORDER by roomname ASC
    """ % (bot.name,))
    oldrooms = bot.rooms.keys()
    joined = 0
    for i, row in enumerate(txn):
        rm = row['roomname']
        # Setup Room Config Dictionary
        if rm not in bot.rooms:
            bot.rooms[rm] = {'fbpage': None, 'twitter': None,
                             'occupants': {}}
        bot.rooms[rm]['fbpage'] = row['fbpage']
        bot.rooms[rm]['twitter'] = row['twitter']

        if always_join or rm not in oldrooms:
            presence = domish.Element(('jabber:client', 'presence'))
            presence['to'] = "%s@%s/%s" % (rm, bot.conference,
                                           bot.myjid.user)
            reactor.callLater(i % 30, bot.xmlstream.send, presence)
            joined += 1
        if rm in oldrooms:
            oldrooms.remove(rm)

    # Check old rooms for any rooms we need to vacate!
    for rm in oldrooms:
        presence = domish.Element(('jabber:client', 'presence'))
        presence['to'] = "%s@%s/%s" % (rm, bot.conference, bot.myjid.user)
        presence['type'] = 'unavailable'
        bot.xmlstream.send(presence)

        del(bot.rooms[rm])
    log.msg(("... loaded %s chatrooms, joined %s of them, left %s of them"
             ) % (txn.rowcount, joined, len(oldrooms)))


def load_twitter_from_db(txn, bot):
    """ Load twitter config from database """
    txn.execute("""
        SELECT screen_name, channel from """+bot.name+"""_twitter_subs
        """)
    twrt = {}
    for row in txn:
        sn = row['screen_name']
        channel = row['channel']
        if sn == '' or channel == '':
            continue
        if channel not in twrt:
            twrt[channel] = []
        twrt[channel].append(sn)
    bot.tw_routingtable = twrt
    log.msg("load_twitter_from_db(): %s subs found" % (txn.rowcount,))

    txn.execute("""
        SELECT screen_name, access_token, access_token_secret
        from """+bot.name+"""_twitter_oauth
        """)
    for row in txn:
        sn = row['screen_name']
        at = row['access_token']
        ats = row['access_token_secret']
        bot.tw_access_tokens[sn] = oauth.OAuthToken(at, ats)
    log.msg("load_twitter_from_db(): %s oauth tokens found" % (txn.rowcount,))


def load_facebook_from_db(txn, bot):
    """ Load facebook config from database """
    txn.execute("""
        SELECT fbpid, channel from """+bot.name+"""_fb_subscriptions
        """)
    fbrt = {}
    for row in txn:
        page = row['fbpid']
        channel = row['channel']
        if channel not in fbrt:
            fbrt[channel] = []
        fbrt[channel].append(page)
    bot.fb_routingtable = fbrt

    txn.execute("""
        SELECT fbpid, access_token from """+bot.name+"""_fb_access_tokens
        """)

    for row in txn:
        page = row['fbpid']
        at = row['access_token']
        bot.fb_access_tokens[page] = at


def safe_twitter_text(text):
    """ Attempt to rip apart a message that is too long!
    To be safe, the URL is counted as 24 chars
    """
    # Convert two or more spaces into one
    text = ' '.join(text.split())
    # If we are already below 140, we don't have any more work to do...
    if len(text) < 140:
        return text
    chars = 0
    for word in text.split():
        if word.find('http') == 0:
            chars += 25
        else:
            chars += (len(word) + 1)
    if chars < 140:
        return text

    urls = re.findall('https?://[^\s]+', text)
    if len(urls) == 1:
        text2 = text.replace(urls[0], '')
        sections = re.findall('(.*) for (.*)( till [0-9A-Z].*)', text2)
        if len(sections) == 1:
            text = "%s%s%s" % (sections[0][0], sections[0][2], urls[0])
            if len(text) > 140:
                sz = 112 - len(sections[0][2])
                text = "%s%s%s" % (sections[0][0][:sz], sections[0][2],
                                   urls[0])
            return text
        if len(text) > 140:
            return "%s... %s" % (text2[:109], urls[0])
    return text[:140]


class basicbot:
    """ Here lies the Jabber Bot """

    def __init__(self, name, dbpool):
        """ Constructor """
        self.startup_time = datetime.datetime.utcnow().replace(
                                                tzinfo=pytz.timezone("UTC"))
        self.name = name
        self.dbpool = dbpool
        self.config = {}
        self.IQ = {}
        self.rooms = {}
        self.chatlog = {}
        self.seqnum = 0
        self.routingtable = {}
        self.tw_access_tokens = {}
        self.tw_routingtable = {}
        self.fb_access_tokens = {}
        self.fb_routingtable = {}
        self.has_football = True
        self.xmlstream = None
        self.firstlogin = False
        self.channelkeys = {}
        self.syndication = {}
        self.xmllog = DailyLogFile('xmllog', 'logs/')
        self.myjid = None
        self.ingestjid = None
        self.conference = None
        self.email_timestamps = []
        self.fortunes = open('startrek', 'r').read().split("\n%\n")
        self.twitter_oauth_consumer = None
        self.logins = 0

        lc2 = LoopingCall(self.purge_logs)
        lc2.start(60*60*24)

    def on_firstlogin(self):
        """ callbacked when we are first logged in """
        pass

    def authd(self, xmlstream):
        """ callback when we are logged into the server! """
        msg = "Logged into jabber server as %s" % (self.myjid,)
        self.email_error(None, msg)
        if not self.firstlogin:
            self.compute_daily_caller()
            self.on_firstlogin()
            self.firstlogin = True

        # Resets associated with the previous login session, perhaps
        self.rooms = {}
        self.IQ = {}

        # Assignment of xmlstream!
        self.xmlstream = xmlstream
        self.xmlstream.rawDataInFn = self.rawDataInFn
        self.xmlstream.rawDataOutFn = self.rawDataOutFn

        self.xmlstream.addObserver('/message',  self.message_processor)
        self.xmlstream.addObserver('/iq',  self.iq_processor)
        self.xmlstream.addObserver('/presence/x/item', self.presence_processor)

        self.load_twitter()
        self.send_presence()
        self.load_chatrooms(True)
        self.load_facebook()

        lc = LoopingCall(self.housekeeping)
        lc.start(60)
        self.xmlstream.addObserver(STREAM_END_EVENT, lambda _: lc.stop())

    def next_seqnum(self):
        """
        Simple tool to generate a sequence number for message logging
        """
        self.seqnum += 1
        return self.seqnum

    def load_chatrooms(self, always_join):
        """
        Load up the chatrooms and subscriptions from the database!, I also
        support getting called at a later date for any changes
        """
        log.msg("load_chatrooms() called...")
        df = self.dbpool.runInteraction(load_chatrooms_from_db, self, 
                                            always_join)
        df.addErrback( self.email_error, "load_chatrooms() failure" )
        
    def load_twitter(self):
        ''' Load the twitter subscriptions and access tokens '''
        log.msg("load_twitter() called...")
        df = self.dbpool.runInteraction(load_twitter_from_db, self)
        df.addErrback( self.email_error, "load_twitter() failure" )

    def load_facebook(self):
        """
        Load up the facebook configuration page/channel subscriptions
        """
        log.msg("load_facebook() called...")
        df = self.dbpool.runInteraction(load_facebook_from_db, self)
        df.addErrback( self.email_error )

    def tweet_cb(self, response, twttxt, room, myjid, twituser):
        """
        Called after success going to twitter
        """
        log.msg("tweet_cb() called...")
        if response is None:
            return
        url = "https://twitter.com/%s/status/%s" % (twituser, response)
        html = "Posted twitter message! View it <a href=\"%s\">here</a>." % (
                                                url,)
        plain = "Posted twitter message! %s" % (url,)
        if room is not None:
            self.send_groupchat(room, plain, html)

        # Log
        df = self.dbpool.runOperation("""
            INSERT into """ + self.name + """_social_log(medium, source,
            resource_uri, message, response, response_code)
            values (%s,%s,%s,%s,%s,%s)
            """, ('twitter', myjid, url, twttxt, response, 200))
        df.addErrback(log.err)
        return response

    def tweet_eb(self, err, twttxt, room, myjid, twituser):
        """
        Called after error going to twitter
        """
        log.msg("====== Twitter Error! ======")
        log.err(err)

        # Make sure we only are trapping API errors
        err.trap( weberror.Error )
        # Don't email duplication errors
        j = {}
        try:
            j = json.loads(err.value.response)
        except:
            log.msg("Unable to parse response |%s| as JSON" % (
                                                        err.value.response,))
        if (len(j.get('errors', [])) > 0 and 
            j['errors'][0].get('code', 0) !=  187):
            self.email_error(err, ("Room: %s\nmyjid: %s\ntwituser: %s\n"
                                   +"tweet: %s\nError:%s\n") % (
                                room, myjid, twituser, twttxt,
                                err.value.response))

        log.msg(err.getErrorMessage())
        log.msg(err.value.response)

        msg = "Sorry, an error was encountered with the tweet."
        htmlmsg = "Sorry, an error was encountered with the tweet."
        if err.value.status == "401":
            msg = "Post to twitter failed. Access token for %s " % (twituser,)
            msg += "is no longer valid."
            htmlmsg = msg + " Please refresh access tokens "
            htmlmsg += "<a href='https://nwschat.weather.gov/nws/twitter.php'>here</a>."
        if room is not None:
            self.send_groupchat(room, msg, htmlmsg)

        # Log this
        deffered = self.dbpool.runOperation("""
            INSERT into """+self.name+"""_social_log(medium, source, message,
            response, response_code, resource_uri) values (%s,%s,%s,%s,%s,%s)
            """, ('twitter', myjid, twttxt, err.value.response, 
                 err.value.status, "https://twitter.com/%s" % (twituser,) ))
        deffered.addErrback(log.err)

        # return err.value.response

    def email_error(self, exp, message=''):
        """
        Something to email errors when something fails
        """
        # Always log a message about our fun
        cstr = StringIO.StringIO()
        
        if exp is not None:
            traceback.print_exc(file=cstr)
            cstr.seek(0)
            if isinstance(exp, Exception):
                log.err( exp )
            else:
                log.msg( exp )
        log.msg( message )

        
        def should_email():
            utcnow = datetime.datetime.utcnow()
            self.email_timestamps.insert(0, utcnow )
            delta = self.email_timestamps[0] - self.email_timestamps[-1]
            if len(self.email_timestamps) < 10:
                return True
            while len(self.email_timestamps) > 10:
                self.email_timestamps.pop()

            return (delta > datetime.timedelta(hours=1))

        # Logic to prevent email bombs
        if not should_email():
            log.msg("Email threshold exceeded, so no email sent!")
            return False

        msg = MIMEText("""
System          : %s@%s [CWD: %s]
System UTC date : %s
process id      : %s
system load     : %s
Exception       :
%s
%s

Message:
%s""" % (pwd.getpwuid(os.getuid())[0], socket.gethostname(), os.getcwd(),
         datetime.datetime.utcnow(),
         os.getpid(), ' '.join(['%.2f' % (_,) for _ in os.getloadavg()]),
         cstr.read(), exp, message))

        # TODO: add his local name
        msg['subject'] = '[bot] Traceback -- %s' % (socket.gethostname(),)

        msg['From'] = self.config.get('bot.email_errors_from',
                                      'root@localhost')
        msg['To'] = self.config.get('bot.email_errors_to',
                                    'root@localhost')

        df = smtp.sendmail(self.config.get('bot.smtp_server', 'localhost'),
                           msg["From"], msg["To"], msg)
        df.addErrback(log.err)
        return True

    def check_for_football(self):
        """ Logic to check if we have the football or not, this should
        be over-ridden """
        self.has_football = True

    def fire_client_with_config(self, res, serviceCollection):
        """ Calledback once bot has loaded its database configuration """
        log.msg("fire_client_with_config() called ...")

        for row in res:
            self.config[ row['propname'] ] = row['propvalue']

        self.myjid = jid.JID("%s@%s/twisted_words" % (
                                                self.config["bot.username"],
                                                self.config["bot.xmppdomain"]))
        self.ingestjid = jid.JID("%s@%s" % (
                                            self.config["bot.ingest_username"],
                                            self.config["bot.xmppdomain"]))
        self.conference = self.config['bot.mucservice']

        self.twitter_oauth_consumer = oauth.OAuthConsumer(
                            self.config['bot.twitter.consumerkey'],
                            self.config['bot.twitter.consumersecret'])

        factory = client.basicClientFactory(self.myjid, 
                                            self.config['bot.password'])
        # Limit reconnection delay to 60 seconds
        factory.maxDelay = 60
        factory.addBootstrap('//event/stream/authd', self.authd)

        i = internet.TCPClient(self.config['bot.connecthost'], 5222, 
                               factory)
        i.setServiceParent(serviceCollection)

    def get_fortune(self):
        """ Get a random value from the array """
        offset = int((len(self.fortunes)-1) * random.random())
        return " ".join( self.fortunes[offset].replace("\n","").split() )

    def failure(self, f):
        log.err( f )

    def debug(self, elem):
        log.msg( elem )

    def rawDataInFn(self, data):
        self.xmllog.write("%s RECV %s\n" % (
                    datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), 
                                            data))

    def rawDataOutFn(self, data):
        self.xmllog.write("%s SEND %s\n" % (
                    datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
                                            data))

    def purge_logs(self):
        """ Remove chat logs on a 24 HR basis """
        log.msg("purge_logs() called...")
        basets = datetime.datetime.utcnow() - datetime.timedelta(
                days=int(self.config.get('bot.purge_xmllog_days', 7)))
        basets = basets.replace(tzinfo=pytz.timezone("UTC"))
        for fn in glob.glob("logs/xmllog.*"):
            ts = datetime.datetime.strptime(fn,'logs/xmllog.%Y_%m_%d')
            ts = ts.replace(tzinfo=pytz.timezone("UTC"))
            if ts < basets:
                log.msg("Purging logfile %s" % (fn,))
                os.remove(fn)

    def housekeeping(self):
        """ 
        This gets exec'd every minute to keep up after ourselves 
        1. XMPP Server Ping
        2. Check if we have the football
        3. Update presence
        """
        gmtnow = datetime.datetime.utcnow()
        self.check_for_football()

        if len( self.IQ.keys() ) > 0:
            log.msg("ERROR: missing IQs %s" % (self.IQ.keys(),))
        if len( self.IQ.keys() ) > 5:
            self.IQ = {}
            self.email_error("Logging out of Chat!", "IQ error limit reached...")
            if self.xmlstream is not None:
                self.xmlstream.sendFooter()
                self.xmlstream.connectionLost('BAD')
            return
        ping = domish.Element((None,'iq'))
        ping['to'] = self.myjid.host
        ping['type'] = 'get'
        pingid = "%s" % (gmtnow.strftime("%Y%m%d%H%M"), )
        ping['id'] = pingid
        ping.addChild( domish.Element(('urn:xmpp:ping', 'ping')) )
        if self.xmlstream is not None:
            self.IQ[ pingid ] = 1
            self.xmlstream.send( ping )
            if gmtnow.minute % 10 == 0:
                self.send_presence()
                # Reset our service guard
                self.logins = 1


    def send_privatechat(self, to, mess, htmlstr=None):
        """
        Helper method to send private messages

        @param to: String jabber ID to send the message too
        @param mess: String plain text version to send
        @param html: Optional String html version 
        """
        message = domish.Element(('jabber:client','message'))
        if to.find("@") == -1: # base username, add domain
            to = "%s@%s" % (to, self.config["bot.xmppdomain"])
        message['to'] = to
        message['type'] = "chat"
        message.addElement('body', None, mess)
        html = message.addElement('html', 'http://jabber.org/protocol/xhtml-im')
        body = html.addElement('body', 'http://www.w3.org/1999/xhtml')
        if htmlstr is not None:
            body.addRawXml(htmlstr)
        else:
            p = body.addElement('p')
            p.addContent(mess)
        self.xmlstream.send(message)

    def send_groupchat(self, room, plain, htmlstr=None, secondtrip=False):
        """Send a groupchat message to a given room

        Args:
          room (str): The roomname (which we should have already joined)
          plain (str): The message to send to the room, no escaping necessary
          htmlstr (str, optional): The HTML variant of the message
          secondtrip (bool, optional): Is this the second attempt to deliver

        Returns:
          twisted.words.xish.domish.Element: the element that was sent
        """
        message = domish.Element(('jabber:client', 'message'))
        message['to'] = "%s@%s" % (room, self.conference)
        message['type'] = "groupchat"
        message.addElement('body', None, plain)
        html = message.addElement('html',
                                  'http://jabber.org/protocol/xhtml-im')
        body = html.addElement('body',
                               'http://www.w3.org/1999/xhtml')
        if htmlstr is not None:
            body.addRawXml(htmlstr)
        else:
            # Careful here, we always want to have valid xhtml, so we should
            # wrap plain text in a paragraph tag
            p = body.addElement('p')
            p.addContent(plain)
        # Ensure that we have well formed XML before sending it
        try:
            _ = ET.fromstring(message.toXml())
        except Exception, exp:
            self.email_error(exp, message.toXml())
            return None
        else:
            self.send_groupchat_elem(message)
        return message

    def send_groupchat_elem(self, elem, to=None, secondtrip=False):
        """ Wrapper for sending groupchat elements """
        if to is not None:
            elem['to'] = to
        room = jid.JID(elem['to']).user
        if room not in self.rooms:
            self.email_error(("Attempted to send message to room [%s] "
                              "we have not joined...") % (room,), elem)
            return
        if self.myjid.user not in self.rooms[room]['occupants']:
            if secondtrip:
                log.msg("ABORT of send to room: %s, msg: %s, not in room" % (
                                                        room, elem))
                return
            log.msg("delaying send to room: %s, not in room yet" % (room,))
            # Need to prevent elem['to'] object overwriting
            reactor.callLater(300, self.send_groupchat_elem, elem, elem['to'],
                              True)
            return
        self.xmlstream.send(elem)

    def send_presence(self):
        """
        Set a presence for my login
        """
        presence = domish.Element(('jabber:client','presence'))
        msg = "Booted: %s Updated: %s UTC, Rooms: %s, Messages: %s" % (
                                        self.startup_time.strftime("%d %b"),
                                        datetime.datetime.utcnow().strftime("%H%M"),
                                        len(self.rooms), 
                                        locale.format("%d", self.seqnum, grouping=True) )
        presence.addElement('status').addContent(msg)
        if self.xmlstream is not None:
            self.xmlstream.send(presence)

    def tweet(self, twttxt, access_token, room=None, myjid=None, twituser=None,<commit_msg>remove un-nessary locale usage just to get a pretty number printed<commit_after>""" Basic iembot/nwsbot implementation, upstream for this code is
    on github: https://github.com/akrherz/iembot
"""

from twisted.words.xish import domish
from twisted.words.xish import xpath
from twisted.words.protocols.jabber import jid, client
from twisted.words.xish.xmlstream import STREAM_END_EVENT
from twisted.internet import reactor
from twisted.application import internet
from twisted.python import log
from twisted.python.logfile import DailyLogFile
import twisted.web.error as weberror
from twisted.internet.task import LoopingCall
from twisted.mail import smtp
from twisted.web import client as webclient

from oauth import oauth

from xml.etree import ElementTree as ET

import datetime
import pytz
import json
import traceback
from email.MIMEText import MIMEText
import StringIO
import random
import os
import pwd
import urllib
import socket
import re
import glob

from twittytwister import twitter

PRESENCE_MUC_ITEM = (
    "/presence/x[@xmlns='http://jabber.org/protocol/muc#user']/item")


def load_chatrooms_from_db(txn, bot, always_join):
    """ Load database configuration and do work

    Args:
      txn (dbtransaction): database cursor
      bot (basicbot): the running bot instance
      always_join (boolean): do we force joining each room, regardless
    """
    # Load up the channel keys
    txn.execute("SELECT id, channel_key from %s_channels" % (bot.name,))
    for row in txn:
        bot.channelkeys[row['channel_key']] = row['id']

    # Load up the routingtable for bot products
    rt = {}
    txn.execute("""
        SELECT roomname, channel from %s_room_subscriptions
    """ % (bot.name,))
    for row in txn:
        rm = row['roomname']
        channel = row['channel']
        if channel not in rt:
            rt[channel] = []
        rt[channel].append(rm)
    bot.routingtable = rt
    log.msg(("... loaded %s channel subscriptions for %s rooms"
             ) % (txn.rowcount, len(rt)))

    # Now we need to load up the syndication
    synd = {}
    txn.execute("""
        SELECT roomname, endpoint from %s_room_syndications
    """ % (bot.name,))
    for row in txn:
        rm = row['roomname']
        endpoint = row['endpoint']
        if rm not in synd:
            synd[rm] = []
        synd[rm].append(endpoint)
    bot.syndication = synd
    log.msg(("... loaded %s room syndications for %s rooms"
             ) % (txn.rowcount, len(synd)))

    # Load up a list of chatrooms
    txn.execute("""
        SELECT roomname, fbpage, twitter from %s_rooms ORDER by roomname ASC
    """ % (bot.name,))
    oldrooms = bot.rooms.keys()
    joined = 0
    for i, row in enumerate(txn):
        rm = row['roomname']
        # Setup Room Config Dictionary
        if rm not in bot.rooms:
            bot.rooms[rm] = {'fbpage': None, 'twitter': None,
                             'occupants': {}}
        bot.rooms[rm]['fbpage'] = row['fbpage']
        bot.rooms[rm]['twitter'] = row['twitter']

        if always_join or rm not in oldrooms:
            presence = domish.Element(('jabber:client', 'presence'))
            presence['to'] = "%s@%s/%s" % (rm, bot.conference,
                                           bot.myjid.user)
            reactor.callLater(i % 30, bot.xmlstream.send, presence)
            joined += 1
        if rm in oldrooms:
            oldrooms.remove(rm)

    # Check old rooms for any rooms we need to vacate!
    for rm in oldrooms:
        presence = domish.Element(('jabber:client', 'presence'))
        presence['to'] = "%s@%s/%s" % (rm, bot.conference, bot.myjid.user)
        presence['type'] = 'unavailable'
        bot.xmlstream.send(presence)

        del(bot.rooms[rm])
    log.msg(("... loaded %s chatrooms, joined %s of them, left %s of them"
             ) % (txn.rowcount, joined, len(oldrooms)))


def load_twitter_from_db(txn, bot):
    """ Load twitter config from database """
    txn.execute("""
        SELECT screen_name, channel from """+bot.name+"""_twitter_subs
        """)
    twrt = {}
    for row in txn:
        sn = row['screen_name']
        channel = row['channel']
        if sn == '' or channel == '':
            continue
        if channel not in twrt:
            twrt[channel] = []
        twrt[channel].append(sn)
    bot.tw_routingtable = twrt
    log.msg("load_twitter_from_db(): %s subs found" % (txn.rowcount,))

    txn.execute("""
        SELECT screen_name, access_token, access_token_secret
        from """+bot.name+"""_twitter_oauth
        """)
    for row in txn:
        sn = row['screen_name']
        at = row['access_token']
        ats = row['access_token_secret']
        bot.tw_access_tokens[sn] = oauth.OAuthToken(at, ats)
    log.msg("load_twitter_from_db(): %s oauth tokens found" % (txn.rowcount,))


def load_facebook_from_db(txn, bot):
    """ Load facebook config from database """
    txn.execute("""
        SELECT fbpid, channel from """+bot.name+"""_fb_subscriptions
        """)
    fbrt = {}
    for row in txn:
        page = row['fbpid']
        channel = row['channel']
        if channel not in fbrt:
            fbrt[channel] = []
        fbrt[channel].append(page)
    bot.fb_routingtable = fbrt

    txn.execute("""
        SELECT fbpid, access_token from """+bot.name+"""_fb_access_tokens
        """)

    for row in txn:
        page = row['fbpid']
        at = row['access_token']
        bot.fb_access_tokens[page] = at


def safe_twitter_text(text):
    """ Attempt to rip apart a message that is too long!
    To be safe, the URL is counted as 24 chars
    """
    # Convert two or more spaces into one
    text = ' '.join(text.split())
    # If we are already below 140, we don't have any more work to do...
    if len(text) < 140:
        return text
    chars = 0
    for word in text.split():
        if word.find('http') == 0:
            chars += 25
        else:
            chars += (len(word) + 1)
    if chars < 140:
        return text

    urls = re.findall('https?://[^\s]+', text)
    if len(urls) == 1:
        text2 = text.replace(urls[0], '')
        sections = re.findall('(.*) for (.*)( till [0-9A-Z].*)', text2)
        if len(sections) == 1:
            text = "%s%s%s" % (sections[0][0], sections[0][2], urls[0])
            if len(text) > 140:
                sz = 112 - len(sections[0][2])
                text = "%s%s%s" % (sections[0][0][:sz], sections[0][2],
                                   urls[0])
            return text
        if len(text) > 140:
            return "%s... %s" % (text2[:109], urls[0])
    return text[:140]


class basicbot:
    """ Here lies the Jabber Bot """

    def __init__(self, name, dbpool):
        """ Constructor """
        self.startup_time = datetime.datetime.utcnow().replace(
                                                tzinfo=pytz.timezone("UTC"))
        self.name = name
        self.dbpool = dbpool
        self.config = {}
        self.IQ = {}
        self.rooms = {}
        self.chatlog = {}
        self.seqnum = 0
        self.routingtable = {}
        self.tw_access_tokens = {}
        self.tw_routingtable = {}
        self.fb_access_tokens = {}
        self.fb_routingtable = {}
        self.has_football = True
        self.xmlstream = None
        self.firstlogin = False
        self.channelkeys = {}
        self.syndication = {}
        self.xmllog = DailyLogFile('xmllog', 'logs/')
        self.myjid = None
        self.ingestjid = None
        self.conference = None
        self.email_timestamps = []
        self.fortunes = open('startrek', 'r').read().split("\n%\n")
        self.twitter_oauth_consumer = None
        self.logins = 0

        lc2 = LoopingCall(self.purge_logs)
        lc2.start(60*60*24)

    def on_firstlogin(self):
        """ callbacked when we are first logged in """
        pass

    def authd(self, xmlstream):
        """ callback when we are logged into the server! """
        msg = "Logged into jabber server as %s" % (self.myjid,)
        self.email_error(None, msg)
        if not self.firstlogin:
            self.compute_daily_caller()
            self.on_firstlogin()
            self.firstlogin = True

        # Resets associated with the previous login session, perhaps
        self.rooms = {}
        self.IQ = {}

        # Assignment of xmlstream!
        self.xmlstream = xmlstream
        self.xmlstream.rawDataInFn = self.rawDataInFn
        self.xmlstream.rawDataOutFn = self.rawDataOutFn

        self.xmlstream.addObserver('/message',  self.message_processor)
        self.xmlstream.addObserver('/iq',  self.iq_processor)
        self.xmlstream.addObserver('/presence/x/item', self.presence_processor)

        self.load_twitter()
        self.send_presence()
        self.load_chatrooms(True)
        self.load_facebook()

        lc = LoopingCall(self.housekeeping)
        lc.start(60)
        self.xmlstream.addObserver(STREAM_END_EVENT, lambda _: lc.stop())

    def next_seqnum(self):
        """
        Simple tool to generate a sequence number for message logging
        """
        self.seqnum += 1
        return self.seqnum

    def load_chatrooms(self, always_join):
        """
        Load up the chatrooms and subscriptions from the database!, I also
        support getting called at a later date for any changes
        """
        log.msg("load_chatrooms() called...")
        df = self.dbpool.runInteraction(load_chatrooms_from_db, self, 
                                            always_join)
        df.addErrback( self.email_error, "load_chatrooms() failure" )
        
    def load_twitter(self):
        ''' Load the twitter subscriptions and access tokens '''
        log.msg("load_twitter() called...")
        df = self.dbpool.runInteraction(load_twitter_from_db, self)
        df.addErrback( self.email_error, "load_twitter() failure" )

    def load_facebook(self):
        """
        Load up the facebook configuration page/channel subscriptions
        """
        log.msg("load_facebook() called...")
        df = self.dbpool.runInteraction(load_facebook_from_db, self)
        df.addErrback( self.email_error )

    def tweet_cb(self, response, twttxt, room, myjid, twituser):
        """
        Called after success going to twitter
        """
        log.msg("tweet_cb() called...")
        if response is None:
            return
        url = "https://twitter.com/%s/status/%s" % (twituser, response)
        html = "Posted twitter message! View it <a href=\"%s\">here</a>." % (
                                                url,)
        plain = "Posted twitter message! %s" % (url,)
        if room is not None:
            self.send_groupchat(room, plain, html)

        # Log
        df = self.dbpool.runOperation("""
            INSERT into """ + self.name + """_social_log(medium, source,
            resource_uri, message, response, response_code)
            values (%s,%s,%s,%s,%s,%s)
            """, ('twitter', myjid, url, twttxt, response, 200))
        df.addErrback(log.err)
        return response

    def tweet_eb(self, err, twttxt, room, myjid, twituser):
        """
        Called after error going to twitter
        """
        log.msg("====== Twitter Error! ======")
        log.err(err)

        # Make sure we only are trapping API errors
        err.trap( weberror.Error )
        # Don't email duplication errors
        j = {}
        try:
            j = json.loads(err.value.response)
        except:
            log.msg("Unable to parse response |%s| as JSON" % (
                                                        err.value.response,))
        if (len(j.get('errors', [])) > 0 and 
            j['errors'][0].get('code', 0) !=  187):
            self.email_error(err, ("Room: %s\nmyjid: %s\ntwituser: %s\n"
                                   +"tweet: %s\nError:%s\n") % (
                                room, myjid, twituser, twttxt,
                                err.value.response))

        log.msg(err.getErrorMessage())
        log.msg(err.value.response)

        msg = "Sorry, an error was encountered with the tweet."
        htmlmsg = "Sorry, an error was encountered with the tweet."
        if err.value.status == "401":
            msg = "Post to twitter failed. Access token for %s " % (twituser,)
            msg += "is no longer valid."
            htmlmsg = msg + " Please refresh access tokens "
            htmlmsg += "<a href='https://nwschat.weather.gov/nws/twitter.php'>here</a>."
        if room is not None:
            self.send_groupchat(room, msg, htmlmsg)

        # Log this
        deffered = self.dbpool.runOperation("""
            INSERT into """+self.name+"""_social_log(medium, source, message,
            response, response_code, resource_uri) values (%s,%s,%s,%s,%s,%s)
            """, ('twitter', myjid, twttxt, err.value.response, 
                 err.value.status, "https://twitter.com/%s" % (twituser,) ))
        deffered.addErrback(log.err)

        # return err.value.response

    def email_error(self, exp, message=''):
        """
        Something to email errors when something fails
        """
        # Always log a message about our fun
        cstr = StringIO.StringIO()
        
        if exp is not None:
            traceback.print_exc(file=cstr)
            cstr.seek(0)
            if isinstance(exp, Exception):
                log.err( exp )
            else:
                log.msg( exp )
        log.msg( message )

        
        def should_email():
            utcnow = datetime.datetime.utcnow()
            self.email_timestamps.insert(0, utcnow )
            delta = self.email_timestamps[0] - self.email_timestamps[-1]
            if len(self.email_timestamps) < 10:
                return True
            while len(self.email_timestamps) > 10:
                self.email_timestamps.pop()

            return (delta > datetime.timedelta(hours=1))

        # Logic to prevent email bombs
        if not should_email():
            log.msg("Email threshold exceeded, so no email sent!")
            return False

        msg = MIMEText("""
System          : %s@%s [CWD: %s]
System UTC date : %s
process id      : %s
system load     : %s
Exception       :
%s
%s

Message:
%s""" % (pwd.getpwuid(os.getuid())[0], socket.gethostname(), os.getcwd(),
         datetime.datetime.utcnow(),
         os.getpid(), ' '.join(['%.2f' % (_,) for _ in os.getloadavg()]),
         cstr.read(), exp, message))

        # TODO: add his local name
        msg['subject'] = '[bot] Traceback -- %s' % (socket.gethostname(),)

        msg['From'] = self.config.get('bot.email_errors_from',
                                      'root@localhost')
        msg['To'] = self.config.get('bot.email_errors_to',
                                    'root@localhost')

        df = smtp.sendmail(self.config.get('bot.smtp_server', 'localhost'),
                           msg["From"], msg["To"], msg)
        df.addErrback(log.err)
        return True

    def check_for_football(self):
        """ Logic to check if we have the football or not, this should
        be over-ridden """
        self.has_football = True

    def fire_client_with_config(self, res, serviceCollection):
        """ Calledback once bot has loaded its database configuration """
        log.msg("fire_client_with_config() called ...")

        for row in res:
            self.config[ row['propname'] ] = row['propvalue']

        self.myjid = jid.JID("%s@%s/twisted_words" % (
                                                self.config["bot.username"],
                                                self.config["bot.xmppdomain"]))
        self.ingestjid = jid.JID("%s@%s" % (
                                            self.config["bot.ingest_username"],
                                            self.config["bot.xmppdomain"]))
        self.conference = self.config['bot.mucservice']

        self.twitter_oauth_consumer = oauth.OAuthConsumer(
                            self.config['bot.twitter.consumerkey'],
                            self.config['bot.twitter.consumersecret'])

        factory = client.basicClientFactory(self.myjid, 
                                            self.config['bot.password'])
        # Limit reconnection delay to 60 seconds
        factory.maxDelay = 60
        factory.addBootstrap('//event/stream/authd', self.authd)

        i = internet.TCPClient(self.config['bot.connecthost'], 5222, 
                               factory)
        i.setServiceParent(serviceCollection)

    def get_fortune(self):
        """ Get a random value from the array """
        offset = int((len(self.fortunes)-1) * random.random())
        return " ".join( self.fortunes[offset].replace("\n","").split() )

    def failure(self, f):
        log.err( f )

    def debug(self, elem):
        log.msg( elem )

    def rawDataInFn(self, data):
        self.xmllog.write("%s RECV %s\n" % (
                    datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), 
                                            data))

    def rawDataOutFn(self, data):
        self.xmllog.write("%s SEND %s\n" % (
                    datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
                                            data))

    def purge_logs(self):
        """ Remove chat logs on a 24 HR basis """
        log.msg("purge_logs() called...")
        basets = datetime.datetime.utcnow() - datetime.timedelta(
                days=int(self.config.get('bot.purge_xmllog_days', 7)))
        basets = basets.replace(tzinfo=pytz.timezone("UTC"))
        for fn in glob.glob("logs/xmllog.*"):
            ts = datetime.datetime.strptime(fn,'logs/xmllog.%Y_%m_%d')
            ts = ts.replace(tzinfo=pytz.timezone("UTC"))
            if ts < basets:
                log.msg("Purging logfile %s" % (fn,))
                os.remove(fn)

    def housekeeping(self):
        """ 
        This gets exec'd every minute to keep up after ourselves 
        1. XMPP Server Ping
        2. Check if we have the football
        3. Update presence
        """
        gmtnow = datetime.datetime.utcnow()
        self.check_for_football()

        if len( self.IQ.keys() ) > 0:
            log.msg("ERROR: missing IQs %s" % (self.IQ.keys(),))
        if len( self.IQ.keys() ) > 5:
            self.IQ = {}
            self.email_error("Logging out of Chat!", "IQ error limit reached...")
            if self.xmlstream is not None:
                self.xmlstream.sendFooter()
                self.xmlstream.connectionLost('BAD')
            return
        ping = domish.Element((None,'iq'))
        ping['to'] = self.myjid.host
        ping['type'] = 'get'
        pingid = "%s" % (gmtnow.strftime("%Y%m%d%H%M"), )
        ping['id'] = pingid
        ping.addChild( domish.Element(('urn:xmpp:ping', 'ping')) )
        if self.xmlstream is not None:
            self.IQ[ pingid ] = 1
            self.xmlstream.send( ping )
            if gmtnow.minute % 10 == 0:
                self.send_presence()
                # Reset our service guard
                self.logins = 1


    def send_privatechat(self, to, mess, htmlstr=None):
        """
        Helper method to send private messages

        @param to: String jabber ID to send the message too
        @param mess: String plain text version to send
        @param html: Optional String html version 
        """
        message = domish.Element(('jabber:client','message'))
        if to.find("@") == -1: # base username, add domain
            to = "%s@%s" % (to, self.config["bot.xmppdomain"])
        message['to'] = to
        message['type'] = "chat"
        message.addElement('body', None, mess)
        html = message.addElement('html', 'http://jabber.org/protocol/xhtml-im')
        body = html.addElement('body', 'http://www.w3.org/1999/xhtml')
        if htmlstr is not None:
            body.addRawXml(htmlstr)
        else:
            p = body.addElement('p')
            p.addContent(mess)
        self.xmlstream.send(message)

    def send_groupchat(self, room, plain, htmlstr=None, secondtrip=False):
        """Send a groupchat message to a given room

        Args:
          room (str): The roomname (which we should have already joined)
          plain (str): The message to send to the room, no escaping necessary
          htmlstr (str, optional): The HTML variant of the message
          secondtrip (bool, optional): Is this the second attempt to deliver

        Returns:
          twisted.words.xish.domish.Element: the element that was sent
        """
        message = domish.Element(('jabber:client', 'message'))
        message['to'] = "%s@%s" % (room, self.conference)
        message['type'] = "groupchat"
        message.addElement('body', None, plain)
        html = message.addElement('html',
                                  'http://jabber.org/protocol/xhtml-im')
        body = html.addElement('body',
                               'http://www.w3.org/1999/xhtml')
        if htmlstr is not None:
            body.addRawXml(htmlstr)
        else:
            # Careful here, we always want to have valid xhtml, so we should
            # wrap plain text in a paragraph tag
            p = body.addElement('p')
            p.addContent(plain)
        # Ensure that we have well formed XML before sending it
        try:
            _ = ET.fromstring(message.toXml())
        except Exception, exp:
            self.email_error(exp, message.toXml())
            return None
        else:
            self.send_groupchat_elem(message)
        return message

    def send_groupchat_elem(self, elem, to=None, secondtrip=False):
        """ Wrapper for sending groupchat elements """
        if to is not None:
            elem['to'] = to
        room = jid.JID(elem['to']).user
        if room not in self.rooms:
            self.email_error(("Attempted to send message to room [%s] "
                              "we have not joined...") % (room,), elem)
            return
        if self.myjid.user not in self.rooms[room]['occupants']:
            if secondtrip:
                log.msg("ABORT of send to room: %s, msg: %s, not in room" % (
                                                        room, elem))
                return
            log.msg("delaying send to room: %s, not in room yet" % (room,))
            # Need to prevent elem['to'] object overwriting
            reactor.callLater(300, self.send_groupchat_elem, elem, elem['to'],
                              True)
            return
        self.xmlstream.send(elem)

    def send_presence(self):
        """
        Set a presence for my login
        """
        presence = domish.Element(('jabber:client','presence'))
        msg = "Booted: %s Updated: %s UTC, Rooms: %s, Messages: %s" % (
                                        self.startup_time.strftime("%d %b"),
                                        datetime.datetime.utcnow().strftime("%H%M"),
                                        len(self.rooms), self.seqnum)
        presence.addElement('status').addContent(msg)
        if self.xmlstream is not None:
            self.xmlstream.send(presence)

    def tweet(self, twttxt, access_token, room=None, myjid=None, twituser=None,
{'fraction_non_alphanumeric': 0.12654772141014617, 'fraction_numerical': 0.005503009458297506, 'mean_word_length': 1.8925573587017348, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 8, 'lorem ipsum': 0, 'www.': 4, 'xml': 64}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 71, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3172938', 'n_tokens_mistral': 14138, 'n_tokens_neox': 13537, 'n_words': 3443}