<commit_before>import re
from datetime import datetime
from scrapy.contrib.spiders.crawl import Rule, CrawlSpider
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import Selector
from crawler.items import ArticleItem


# REGEX
DOT = '\.'
SLASH = '/'
YEAR = '201[1-4]'
MONTH = '[0-3]\d'
YEAR_MONTH = YEAR + SLASH + MONTH + SLASH
ANY_CHAR_DIGIT = '[\w\d]+'
ANY_CHAR_DIGIT_HYPHEN = '[\d\-\w]+'

# FOLHA SPIDER INFO
FOLHA_SPIDER_NAME = 'folha'
FOLHA_DOMAIN = 'www1.folha.uol.com.br'
FOLHA_START_URLS = [
                    'http://www1.folha.uol.com.br/poder',
                    'http://www1.folha.uol.com.br/poder/poderepolitica'
                    ]

FOLHA_URL_PATTERN = YEAR_MONTH + ANY_CHAR_DIGIT_HYPHEN + DOT + ANY_CHAR_DIGIT

FOLHA_PODER = re.compile('/poder/' + FOLHA_URL_PATTERN) 
FOLHA_PODEREPOLITICA = re.compile('/poder/poderepolitica/' + FOLHA_URL_PATTERN)


FOLHA_ARTICLE_TITLE = '//h1/text()'
FOLHA_ARTICLE_PUB_DATE = '//header/time/@datetime'
FOLHA_ARTICLE_CONTENT = '//article/div[@itemprop="articleBody"]/p [not(@class="wideVideoPlayer") and not(@class="star") and not(descendant::img)]'

class FolhaSpider(CrawlSpider):
    
    name = FOLHA_SPIDER_NAME
    allowed_domains = [ FOLHA_DOMAIN ]
    start_urls = FOLHA_START_URLS 

    rules = [
             Rule(SgmlLinkExtractor(
                                    allow=(FOLHA_PODER),
                                    ),
                                    callback='parse_item',
                                    follow=True,
                  ),
             Rule(SgmlLinkExtractor(
                                    allow=(FOLHA_PODEREPOLITICA),
                                    ),
                                    callback='parse_item',
                                    follow=True,
                  ),
             ]

    
    def parse_item(self, response):
        
        sel = Selector(response)
        
        article = ArticleItem()
        
        article['source'] = FOLHA_SPIDER_NAME
        
        article['url'] = response.url
        
        title = sel.xpath(FOLHA_ARTICLE_TITLE).extract()
        article['title'] = title[0] if title else None
        
        pub_date = sel.xpath(FOLHA_ARTICLE_PUB_DATE).extract()[0]
        article['pub_date'] = datetime.strptime(pub_date, "%Y-%m-%d %H:%M")
        
        content = ' '.join(sel.xpath(FOLHA_ARTICLE_CONTENT).extract())
        article['body'] = content if content else None 
         
        return article<commit_msg>Adicionando coleta de links na própria página. Falta remover links desnecessários para a referência<commit_after>import re
from datetime import datetime
from scrapy.contrib.spiders.crawl import Rule, CrawlSpider
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import Selector
from crawler.items import ArticleItem


# REGEX
DOT = '\.'
SLASH = '/'
YEAR = '201[1-4]'
MONTH = '[0-3]\d'
YEAR_MONTH = YEAR + SLASH + MONTH + SLASH
ANY_CHAR_DIGIT = '[\w\d]+'
ANY_CHAR_DIGIT_HYPHEN = '[\d\-\w]+'

# FOLHA SPIDER INFO
FOLHA_SPIDER_NAME = 'folha'
FOLHA_DOMAIN = 'www1.folha.uol.com.br'
FOLHA_START_URLS = [
                    'http://www1.folha.uol.com.br/poder',
                    'http://www1.folha.uol.com.br/poder/poderepolitica'
                    ]

FOLHA_URL_PATTERN = YEAR_MONTH + ANY_CHAR_DIGIT_HYPHEN + DOT + ANY_CHAR_DIGIT

FOLHA_PODER = re.compile('/poder/' + FOLHA_URL_PATTERN) 
FOLHA_PODEREPOLITICA = re.compile('/poder/poderepolitica/' + FOLHA_URL_PATTERN)


FOLHA_ARTICLE_TITLE = '//h1/text()'
FOLHA_ARTICLE_PUB_DATE = '//header/time/@datetime'
FOLHA_ARTICLE_CONTENT = '//article/div[@itemprop="articleBody"]/p [not(@class="wideVideoPlayer") and not(@class="star") and not(descendant::img)]'

class FolhaSpider(CrawlSpider):
    
    name = FOLHA_SPIDER_NAME
    allowed_domains = [ FOLHA_DOMAIN ]
    start_urls = FOLHA_START_URLS 

    rules = [
             Rule(SgmlLinkExtractor(
                                    allow=(FOLHA_PODER),
                                    ),
                                    callback='parse_item',
                                    follow=True,
                  ),
             Rule(SgmlLinkExtractor(
                                    allow=(FOLHA_PODEREPOLITICA),
                                    ),
                                    callback='parse_item',
                                    follow=True,
                  ),
             ]

    
    def parse_item(self, response):
        
        sel = Selector(response)
        
        article = ArticleItem()
        
        article['source'] = FOLHA_SPIDER_NAME
        
        article['url'] = response.url
        
        title = sel.xpath(FOLHA_ARTICLE_TITLE).extract()
        article['title'] = title[0] if title else None
        
        pub_date = sel.xpath(FOLHA_ARTICLE_PUB_DATE).extract()[0]
        article['pub_date'] = datetime.strptime(pub_date, "%Y-%m-%d %H:%M")
        
        content = ' '.join(sel.xpath(FOLHA_ARTICLE_CONTENT).extract())
        article['body'] = content if content else None 
        
        
        links = sel.xpath('//article//a/@href').extract()
        article['links'] = links
        
        return article
{'fraction_non_alphanumeric': 0.11308161708619374, 'fraction_numerical': 0.004958047292143402, 'mean_word_length': 1.912270960577457, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4882314', 'n_tokens_mistral': 1808, 'n_tokens_neox': 1734, 'n_words': 262}