<commit_before>import urllib2, re
from bs4 import BeautifulSoup

from socbot.pluginbase import Base, BadParams
# Regex found at http://mathiasbynens.be/demo/url-regex (@diegoperini), and modified.
patt = ur"(?P<protocol>(?:https?|ftp)://)?(?:\S+(?::\S*)?@)?(?:(?!10(?:\.\d{1,3}){3})(?!127(?:\.\d{1,3}){3})(?!169\.254(?:\.\d{1,3}){2})(?!192\.168(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:/[^\s]*)?"
urlpattern = re.compile(ur"^%s$"%patt, re.IGNORECASE)
titlepattern = re.compile(ur"(?P<url>%s)\w?"%patt, re.IGNORECASE)

class Plugin(Base):
    def initialize(self):
        self.bots = {} # bot: {channel: url, ...}, ...
        
    @Base.event("PRIVMSG")
    def on_privmsg(self, bot, command, prefix, params):
        if len(params) == 1:
            return
        
        said = params[1]
        channel = params[0].lower()
        
        match = titlepattern.search(said)
        
        if match:
            url = match.group('url')
            pro = match.group('protocol')
            
            if not pro:
                url = "http://" + url
                
            if not bot in self.bots:
                self.bots[bot] = {}
                
            self.bots[bot][channel] = url
        
    @Base.trigger("TIT", "TITLE")
    def on_title(self, bot, user, details):
        if not len(details['splitmsg']):
            if not bot in self.bots:
                return "No URL has been said recently."
            
            if not details["channel"].lower() in self.bots[bot]:
                return "No URL has been said recently in this channel."
            
            url = self.bots[bot][details['channel'].lower()]
        else:
            url = details['splitmsg'][0]
            match = urlpattern.match(url)
            
            if not match:
                return "Oops, try a valid url!"
            
            pro = match.group('protocol')
            if not pro:
                url = "http://" + url

        try:
            page = urllib2.urlopen(url)
            page = page.read()
        except urllib2.HTTPError as e:
            return "Looks like that page has an error on it! (%s: %i)" % (url, e.code)
        except urllib2.URLError, e:
            return "There was an error retrieving the page's data. (%s: %s)" % (url, e)
        
        title = BeautifulSoup(page).title.string
        
        return "%s (%s)" % (title, url)
        
    @Base.trigger("UP", "DOWN")
    def on_up(self, bot, user, details):
        if len(details['splitmsg']):
            url = details['splitmsg'][0]
            
            match = urlpattern.match(url)
            
            if not match:
                return "Oops, try a valid url!"
            
            pro = match.group('protocol')
            if not pro:
                url = "http://" + url
            
            try:
                _ = urllib2.urlopen(url, timeout=5)
            except urllib2.HTTPError as e:
                return "She's alive, but thats a %i! (%s)" % (e.code, url)
            except urllib2.URLError, e:
                return "She's dead, Jim! (%s)" % url
            
            return "Looks like she's alive! (%s)" % url
<commit_msg>Made ^TITLE and ^UP async. Made the title regex simpler to avoid long regex operations.<commit_after>import urllib2, re
from bs4 import BeautifulSoup

from twisted.internet import reactor
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from twisted.web.client import Agent, RedirectAgent

titlepattern = re.compile(r"\b(?P<pro>https?://)?(?P<url>.*?\.[-A-Z0-9+&@#/%=~_|$?!:,.]*[A-Z0-9+&@#/%=~_|$])", re.IGNORECASE)

from socbot.pluginbase import Base, BadParams

class DeferredPrinter(Protocol):
    def __init__(self, finished, url):
        self.finished = finished
        self.url = url
        self.data = ""

    def dataReceived(self, bytes):
        self.data += bytes

    def connectionLost(self, reason):
        title = BeautifulSoup(self.data).title
        
        if title:
            title = title.string
        else:
            title = "No title provided."
            
        self.finished.callback("%s (%s)" % (title.strip(), self.url))

class Plugin(Base):
    def initialize(self):
        self.bots = {} # bot: {channel: url, ...}, ...
        
    @Base.event("PRIVMSG")
    def on_privmsg(self, bot, command, prefix, params):
        if len(params) == 1:
            return
        
        said = params[1]
        channel = params[0].lower()
        
        match = titlepattern.search(said)
        
        if match:
            url = match.group('url')
            pro = match.group('pro')
            
            if not pro:
                url = "http://" + url
            else:
                url = pro + url
                
            if not bot in self.bots:
                self.bots[bot] = {}
                
            self.bots[bot][channel] = url
        
    @Base.trigger("TIT", "TITLE")
    def on_title(self, bot, user, details):
        if not len(details['splitmsg']):
            if not bot in self.bots:
                return "No URL has been said recently."
            
            if not details["channel"].lower() in self.bots[bot]:
                return "No URL has been said recently in this channel."
            
            url = self.bots[bot][details['channel'].lower()]
        else:
            url = details['splitmsg'][0]
            match = titlepattern.match(url)
            
            if not match:
                return "Oops, try a valid url!"
            
            pro = match.group('pro')
            if not pro:
                url = "http://" + url

        try:
            agent = RedirectAgent(Agent(reactor))
            d = agent.request('GET', url)
            
            def cbRequest(response):
                finished = Deferred()
                response.deliverBody(DeferredPrinter(finished, url))
                return finished
            
            d.addCallback(cbRequest)
            return d
        
        except urllib2.HTTPError as e:
            return "Looks like that page has an error on it! (%s: %i)" % (url, e.code)
        except urllib2.URLError, e:
            return "There was an error retrieving the page's data. (%s: %s)" % (url, e)
        
    @Base.trigger("UP", "DOWN", "ISUP", "ISDOWN")
    def on_up(self, bot, user, details):
        if len(details['splitmsg']):
            url = details['splitmsg'][0]
            
            match = titlepattern.match(url)
            
            if not match:
                return "Oops, try a valid url!"
            
            pro = match.group('pro')
            
            if not pro:
                url = "http://" + url
            else:
                url = pro + url
            
            d = agent = Agent(reactor)
            d = agent.request('GET', url)
            
            d.addCallback(lambda *x: "She's alive! (%s)" % url)
            d.addErrback(lambda *x: "She's dead, Jim. :( (%s)" % url)
            
            return d

{'fraction_non_alphanumeric': 0.15696649029982362, 'fraction_numerical': 0.017094017094017096, 'mean_word_length': 1.5641739130434782, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1948816', 'n_tokens_mistral': 2376, 'n_tokens_neox': 2301, 'n_words': 540}