<commit_before># -*- coding: utf-8 -*-
import re
import scrapy
from scrapy.spiders import CrawlSpider
from bs4 import BeautifulSoup
from ..items import *
import time
from datetime import datetime, timedelta


class WeiboForSomethingSpider(CrawlSpider):
    name = "weibo"
    allowed_domains = ["weibo.cn"]
    start_urls = ['http://weibo.cn/']

    def get_time(self, post_time):
        return datetime.strptime(post_time.decode('utf-8'), '%a, %d %b %Y %H:%M:%S %Z')

    def handle_time(self, now_time, post_time):
        if re.match(u'\d+分钟前', post_time):
            return now_time - timedelta(minutes=int(re.findall('\d+', post_time)[0]))
        elif re.match(u'今天', post_time):
            temp_time = re.findall('\d+', post_time)
            return datetime(
                year=now_time.date().year,
                month=now_time.date().month,
                day=now_time.date().day,
                hour=int(temp_time[0]),
                minute=int(temp_time[1])
            )
        else:
            temp_time = re.findall(r'\d+', post_time)
            # x-x-x x:x
            if int(temp_time[0]) > 12:
                return datetime(
                    year=int(temp_time[0]),
                    month=int(temp_time[1]),
                    day=int(temp_time[2]),
                    hour=int(temp_time[3]),
                    minute=int(temp_time[4])
                )
            # x月x日 x:x
            else:
                return datetime(
                    year=now_time.date().year,
                    month=int(temp_time[0]),
                    day=int(temp_time[1]),
                    hour=int(temp_time[2]),
                    minute=int(temp_time[3])
                )

    def start_requests(self):
        self.logger.info('start crawl for something ...')
        # keywords = self.settings.get("CRAWLED_WEIBO_KEYWORDS_LIST")
        keywords = [u"Yeah虚拟小号",u"郑爽", u"央视春节联欢晚会", u"美国大选", u"特朗普", u"希拉里", u"奥巴马", u"川普", u"春晚", u"鸡年春晚", u"央视春晚"]
        for keyword in keywords:
            keyword_url = u'http://weibo.cn/search/mblog?hideSearchFrame=&keyword={0}&page=1&vt=4'.format(keyword)
            self.logger.info(keyword)
            yield scrapy.Request(
                url=keyword_url,
                meta={'keyword': keyword},
                callback=self.parse_keyword
            )

    def parse_keyword(self, response):
        self.logger.info('parsekeyword..')
        soup = BeautifulSoup(response.body, "lxml")
        # self.logger.info(soup.prettify())
<commit_msg>删除话题，春晚<commit_after># -*- coding: utf-8 -*-
import re
import scrapy
from scrapy.spiders import CrawlSpider
from bs4 import BeautifulSoup
from ..items import *
import time
from datetime import datetime, timedelta


class WeiboForSomethingSpider(CrawlSpider):
    name = "weibo"
    allowed_domains = ["weibo.cn"]
    start_urls = ['http://weibo.cn/']

    def get_time(self, post_time):
        return datetime.strptime(post_time.decode('utf-8'), '%a, %d %b %Y %H:%M:%S %Z')

    def handle_time(self, now_time, post_time):
        if re.match(u'\d+分钟前', post_time):
            return now_time - timedelta(minutes=int(re.findall('\d+', post_time)[0]))
        elif re.match(u'今天', post_time):
            temp_time = re.findall('\d+', post_time)
            return datetime(
                year=now_time.date().year,
                month=now_time.date().month,
                day=now_time.date().day,
                hour=int(temp_time[0]),
                minute=int(temp_time[1])
            )
        else:
            temp_time = re.findall(r'\d+', post_time)
            # x-x-x x:x
            if int(temp_time[0]) > 12:
                return datetime(
                    year=int(temp_time[0]),
                    month=int(temp_time[1]),
                    day=int(temp_time[2]),
                    hour=int(temp_time[3]),
                    minute=int(temp_time[4])
                )
            # x月x日 x:x
            else:
                return datetime(
                    year=now_time.date().year,
                    month=int(temp_time[0]),
                    day=int(temp_time[1]),
                    hour=int(temp_time[2]),
                    minute=int(temp_time[3])
                )

    def start_requests(self):
        self.logger.info('start crawl for something ...')
        # keywords = self.settings.get("CRAWLED_WEIBO_KEYWORDS_LIST")
        keywords = [u"Yeah虚拟小号",u"郑爽",u"金正恩",u"朝鲜",u"金正男", u"美国大选", u"特朗普", u"希拉里", u"奥巴马", u"川普"]
        for keyword in keywords:
            keyword_url = u'http://weibo.cn/search/mblog?hideSearchFrame=&keyword={0}&page=1&vt=4'.format(keyword)
            self.logger.info(keyword)
            yield scrapy.Request(
                url=keyword_url,
                meta={'keyword': keyword},
                callback=self.parse_keyword
            )

    def parse_keyword(self, response):
        self.logger.info('parsekeyword..')
        soup = BeautifulSoup(response.body, "lxml")
        # self.logger.info(soup.prettify())

{'fraction_non_alphanumeric': 0.1387787470261697, 'fraction_numerical': 0.008326724821570182, 'mean_word_length': 1.8454596728708403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1047000', 'n_tokens_mistral': 1696, 'n_tokens_neox': 1664, 'n_words': 264}