<commit_before># -*- coding: utf-8 -*-
"""
Created on Aug 31, 2015

@author: Tyranic-Moron
"""
import HTMLParser
import re

from IRCMessage import IRCMessage
from IRCResponse import IRCResponse, ResponseType
from CommandInterface import CommandInterface

from Utils import WebUtils

from bs4 import BeautifulSoup


class Slurp(CommandInterface):
    triggers = ['slurp']
    help = "slurp <attribute> <url> <css selector> - scrapes the given attribute from the tag selected at the given url"

    htmlParser = HTMLParser.HTMLParser()

    def execute(self, message):
        """
        @type message: IRCMessage
        """
        if len(message.ParameterList) < 3:
            return IRCResponse(ResponseType.Say, u"Not enough parameters, usage: {}".format(self.help), message.ReplyTo)

        prop, url, selector = (message.ParameterList[0], message.ParameterList[1], u" ".join(message.ParameterList[2:]))

        if not re.match(ur'^\w+://', url):
            url = u"http://{}".format(url)

        if 'slurp' in message.Metadata and url in message.Metadata['slurp']:
            page = message.Metadata['slurp'][url]
        else:
            page = WebUtils.fetchURL(url)

        if page is None:
            return IRCResponse(ResponseType.Say, u"Problem fetching {}".format(url), message.ReplyTo)

        soup = BeautifulSoup(page.body)
        tag = soup.select_one(selector)
<commit_msg>[Slurp] oops, wrong page variable<commit_after># -*- coding: utf-8 -*-
"""
Created on Aug 31, 2015

@author: Tyranic-Moron
"""
import HTMLParser
import re

from IRCMessage import IRCMessage
from IRCResponse import IRCResponse, ResponseType
from CommandInterface import CommandInterface

from Utils import WebUtils

from bs4 import BeautifulSoup


class Slurp(CommandInterface):
    triggers = ['slurp']
    help = "slurp <attribute> <url> <css selector> - scrapes the given attribute from the tag selected at the given url"
    runInThread = True

    htmlParser = HTMLParser.HTMLParser()

    def execute(self, message):
        """
        @type message: IRCMessage
        """
        if len(message.ParameterList) < 3:
            return IRCResponse(ResponseType.Say, u"Not enough parameters, usage: {}".format(self.help), message.ReplyTo)

        prop, url, selector = (message.ParameterList[0], message.ParameterList[1], u" ".join(message.ParameterList[2:]))

        if not re.match(ur'^\w+://', url):
            url = u"http://{}".format(url)

        if 'slurp' in message.Metadata and url in message.Metadata['slurp']:
            soup = message.Metadata['slurp'][url]
        else:
            page = WebUtils.fetchURL(url)
            if page is None:
                return IRCResponse(ResponseType.Say, u"Problem fetching {}".format(url), message.ReplyTo)
            soup = BeautifulSoup(page.body)

        tag = soup.select_one(selector)

{'fraction_non_alphanumeric': 0.11888111888111888, 'fraction_numerical': 0.008391608391608392, 'mean_word_length': 3.0352609308885756, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4578851', 'n_tokens_mistral': 872, 'n_tokens_neox': 841, 'n_words': 238}