<commit_before>import scrapy
import re
from datetime import datetime
from scrapy import Spider, Request, Item
from scraper.items import ArticleItem, AuthorItem


class TechChunchSpider(Spider):

    name = "tech_crunch"

    start_urls = [
        'https://techcrunch.com/',
    ]

    def parse(self, response):
        for block in response.css('.river .river-block'):
            url = block.xpath('@data-permalink').extract_first()
            title = block.xpath('@data-sharetitle').extract_first()
            byline = block.css('.byline')
            # raw data formatted as '2016-10-07 07:03:28'
            raw_publish_date = byline.css('time::attr(datetime)').extract_first()
            publish_date = datetime.strptime(raw_publish_date, '%Y-%m-%d %H:%M:%S')
            author_url = byline.css('a[rel=author]::attr(href)').extract_first()
            author_name = byline.css('a[rel=author]::text').extract_first()
            tags = block.css('.tags .tag > span::text').extract()
            content = block.css('p.excerpt::text').extract_first()
            yield ArticleItem(
                title = title,
                url = url,
                publish_date = publish_date,
                content = content,
                # TODO<commit_msg>Spider: Improvement on parsing datetime<commit_after>import scrapy
import re
from datetime import datetime
from scrapy import Spider, Request, Item
from scraper.items import ArticleItem, AuthorItem


class TechChunchSpider(Spider):

    name = "tech_crunch"

    start_urls = [
        'https://techcrunch.com/',
    ]

    def parse(self, response):
        for block in response.css('.river .river-block'):
            url = block.xpath('@data-permalink').extract_first()
            title = block.xpath('@data-sharetitle').extract_first()
            byline = block.css('.byline')
            # raw data formatted as '2016-10-07 07:03:28'
            raw_publish_date = byline.css('time::attr(datetime)').extract_first()
            try:
                publish_date = datetime.strptime(raw_publish_date, '%Y-%m-%d %H:%M:%S')
            except (TypeError, ValueError) as e:
                publish_date = None
            author_url = byline.css('a[rel=author]::attr(href)').extract_first()
            author_name = byline.css('a[rel=author]::text').extract_first()
            tags = block.css('.tags .tag > span::text').extract()
            content = block.css('p.excerpt::text').extract_first()
            yield ArticleItem(
                title = title,
                url = url,
                publish_date = publish_date,
                content = content,
                # TODO
{'fraction_non_alphanumeric': 0.12979683972911965, 'fraction_numerical': 0.010534236267870579, 'mean_word_length': 2.2908415841584158, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1669424', 'n_tokens_mistral': 796, 'n_tokens_neox': 752, 'n_words': 155}