<commit_before>import plugin
import urllib2
import logging
import re

from httplib import HTTPException
from htmllib import HTMLParser

class Plugin(plugin.baseplugin):
    def __init__(self, bot):
        self._regex = re.compile("(https?://.+)\s?")
        self._ua = "Mozilla/5.0 (X11; Linux x86_64; rv:2.0b8) Gecko/20100101 Firefox/4.0b8"
        bot.register_regex(self._regex, self.title)

    def unescape(self, s):
        p = HTMLParser(None)
        p.save_bgn()
        p.feed(s)
        return p.save_end()

    def title(self, bot, user, channel, text, match_obj):
        url = match_obj.groups()[0]
        req = urllib2.Request(url)
        req.add_header("User-Agent", self._ua)
        try:
            content = urllib2.urlopen(req).read()
        except (urllib2.URLError, HTTPException), e:
            logging.debug("%s - %s" % (e, url))
            return
        beg = content.find("<title>")
        if beg != -1:
            title = content[beg+7:content.find("</title>")].replace("\n","")
            title = self.unescape(title)
            bot.privmsg(channel, "Title: %s" % unicode(title, "utf-8"))
<commit_msg>Use the HTMLParser module instead of htmllib<commit_after>import plugin
import urllib2
import logging
import re

from httplib import HTTPException
import HTMLParser

class Plugin(plugin.baseplugin):
    def __init__(self, bot):
        self._regex = re.compile("(https?://.+)\s?")
        self._ua = "Mozilla/5.0 (X11; Linux x86_64; rv:2.0b8) Gecko/20100101 Firefox/4.0b8"
        bot.register_regex(self._regex, self.title)

    def unescape(self, s):
        p = HTMLParser.HTMLParser(None)
        p.save_bgn()
        p.feed(s)
        return p.save_end()

    def title(self, bot, user, channel, text, match_obj):
        url = match_obj.groups()[0]
        req = urllib2.Request(url)
        req.add_header("User-Agent", self._ua)
        try:
            content = urllib2.urlopen(req).read()
        except (urllib2.URLError, HTTPException), e:
            logging.debug("%s - %s" % (e, url))
            return
        beg = content.find("<title>")
        if beg != -1:
            title = content[beg+7:content.find("</title>")].replace("\n","")
            try:
                title = self.unescape(title)
            except HTMLParser.HTMLParseError, e:
                logging.info("%s -  %s" % (e.msg, url))
            bot.privmsg(channel, "Title: %s" % unicode(title, "utf-8"))

{'fraction_non_alphanumeric': 0.1469019286007386, 'fraction_numerical': 0.024620434961017644, 'mean_word_length': 2.3351573187414503, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7208133', 'n_tokens_mistral': 845, 'n_tokens_neox': 814, 'n_words': 174}