<commit_before>from requests import get as req_get
from urlparse import urlsplit, urljoin
from bs4 import BeautifulSoup
from pika import BlockingConnection, ConnectionParameters
from re import search
from json import dumps as json_dumps
from time import time

RABBIT_QUEUE = 'daftpunk'

class DpSearcher(object):
    def __init__(self, config):
        self.config = config
        self.run()

    def reletive_url(self, href):
        return urljoin("http://%s" % self.host, href)

    def scrape_properties(self, soup):
        counters = soup.find_all(**{'class': "sr_counter"})
        hrefs = [counter.parent.a['href'] for counter in counters]

        return [self.reletive_url(href) for href in hrefs]

    def find_properties(self, search_url):
        self.host = urlsplit(search_url).netloc

        next_page = search_url
        properties = []
        while True:
            req = req_get(next_page)
            soup = BeautifulSoup(req.text)
            properties.extend(self.scrape_properties(soup))

            # Find next page of results
            try:
                href = soup.find(**{'class': "next_page"}).a['href']
            except:
                break
            next_page = self.reletive_url(href)

        return properties

    def rabbit_connect(self):
        conn = BlockingConnection(ConnectionParameters('localhost'))
        self.rabbit = conn.channel()
        self.rabbit.queue_declare(queue=RABBIT_QUEUE)

    def process_url(self, url):
        # Get prop_id
        url_match = search("/(\d*)/?$", url)
        prop_id = url_match.group(1)

        # Get prop_html
        resp = req_get(url)
        prop_html = resp.text

        timestamp = time()
        return json_dumps([url, prop_id, timestamp, prop_html])

    def run(self):
        self.rabbit_connect()

        for query in self.config['queries']:
            for url in self.find_properties(query):
                print url
                message = self.process_url(url)
                self.rabbit.basic_publish(
                        exchange='',
                        routing_key=RABBIT_QUEUE,
                        body=message,
                        )

        self.rabbit.close()

if __name__ == "__main__":
    config = {
            'queries': [
                'http://www.daft.ie/kildare/houses-for-sale/johnstownbridge/',
                ]
            }
    DpSearcher(config)<commit_msg>DpSearcher.process_url() -> DpSearcher.create_message().<commit_after>from requests import get as req_get
from urlparse import urlsplit, urljoin
from bs4 import BeautifulSoup
from pika import BlockingConnection, ConnectionParameters
from re import search
from json import dumps as json_dumps
from time import time

RABBIT_QUEUE = 'daftpunk'

class DpSearcher(object):
    def __init__(self, config):
        self.config = config
        self.run()

    def reletive_url(self, href):
        return urljoin("http://%s" % self.host, href)

    def scrape_properties(self, soup):
        counters = soup.find_all(**{'class': "sr_counter"})
        hrefs = [counter.parent.a['href'] for counter in counters]

        return [self.reletive_url(href) for href in hrefs]

    def find_properties(self, search_url):
        self.host = urlsplit(search_url).netloc

        next_page = search_url
        properties = []
        while True:
            req = req_get(next_page)
            soup = BeautifulSoup(req.text)
            properties.extend(self.scrape_properties(soup))

            # Find next page of results
            try:
                href = soup.find(**{'class': "next_page"}).a['href']
            except:
                break
            next_page = self.reletive_url(href)

        return properties

    def rabbit_connect(self):
        conn = BlockingConnection(ConnectionParameters('localhost'))
        self.rabbit = conn.channel()
        self.rabbit.queue_declare(queue=RABBIT_QUEUE)

    def create_message(self, url):
        # Get prop_id
        url_match = search("/(\d*)/?$", url)
        prop_id = url_match.group(1)

        # Get prop_html
        resp = req_get(url)
        prop_html = resp.text

        timestamp = time()
        return json_dumps([url, prop_id, timestamp, prop_html])

    def run(self):
        self.rabbit_connect()

        for query in self.config['queries']:
            for url in self.find_properties(query):
                print url
                message = self.create_message(url)
                self.rabbit.basic_publish(
                        exchange='',
                        routing_key=RABBIT_QUEUE,
                        body=message,
                        )

        self.rabbit.close()

if __name__ == "__main__":
    config = {
            'queries': [
                'http://www.daft.ie/kildare/houses-for-sale/johnstownbridge/',
                ]
            }
    DpSearcher(config)
{'fraction_non_alphanumeric': 0.10075618230124668, 'fraction_numerical': 0.0008174943797261394, 'mean_word_length': 2.0042971147943525, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5790432', 'n_tokens_mistral': 1467, 'n_tokens_neox': 1389, 'n_words': 312}