<commit_before>from scrapy.spiders import Spider
from hackernews_scrapy.items import HackernewsScrapyItem
from scrapy.selector import Selector

import datetime


class HackernewsSpider(Spider):
    name = 'pythonhackernews'
    allowed_hosts = ['news.ycombinator.com']
    start_urls = ['https://news.ycombinator.com']

    def parse(self, response):
        """Parse the HTML to get the information we need"""
        html_xpath_selector = Selector(response)
        titles = html_xpath_selector.xpath(
            '//td[@class="title"]/a/text()').extract()

        titles = filter(lambda t: 'python' in t.lower(), titles)

        for title in titles:
            item = HackernewsScrapyItem()
            item['title'] = title
            item['crawled_at'] = datetime.datetime.utcnow()

            yield item<commit_msg>Refactored Spider to be a Crawlspider<commit_after>from scrapy.spiders import CrawlSpider
from scrapy.spiders import Rule
from scrapy.linkextractors import LinkExtractor
from hackernews_scrapy.items import HackernewsScrapyItem
from scrapy.selector import Selector


class HackernewsCrawlSpider(CrawlSpider):
    name = 'pythonhackernews'
    allowed_hosts = ['news.ycombinator.com']
    start_urls = ['https://news.ycombinator.com/news?p=1']

    rules = (Rule(LinkExtractor(allow=r'news\?p=[0-2]'),
                  callback="parse_item",
                  follow=True),)

    def parse_item(self, response):
        """
            Parse the HTML to get the information we need
        """

        selector = Selector(response)
        links = selector.xpath('//td[@class="title"]')
        items = []

        for link in links:
            title = link.xpath("a/text()").extract()
            url = link.xpath("a/@href").extract()

            if title and url:
                title, url = title[0], url[0]

                if 'python' in title.lower():

                    item = HackernewsScrapyItem()
                    item['title'] = title
                    item['url'] = url

                    items.append(item)
        return items
{'fraction_non_alphanumeric': 0.11164581328200192, 'fraction_numerical': 0.002406159769008662, 'mean_word_length': 2.3915171288743884, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1451816', 'n_tokens_mistral': 626, 'n_tokens_neox': 588, 'n_words': 140}