<commit_before># -*- coding: utf-8 -*-

import json
import time
from datetime import datetime
import pymongo
import re
from amwatcher_spider.spiders.base import BaseSpider, KeywordEscape
from random import random
from scrapy import Spider, Request
from scrapy.http import HtmlResponse
from collections import defaultdict
import logging
import requests
from logging.handlers import RotatingFileHandler

logger = logging.getLogger(__name__)

BILIBILI_ACCOUNT_SET = 'amwatcher:spider:bilibili:accounts'
PROXY_KEY = 'amwatcher:spider:login_proxy:%s'


class BilibiliSpider(BaseSpider):
    name = 'bilibili'
    updrama_pattern = 'http://search.bilibili.com/ajax_api/video?keyword=%(keyword)s&page=1&order=pubdate&tids_1=11&tids_2=15'
    bangumi_pattern = 'http://search.bilibili.com/bangumi?keyword=%(keyword)s'
    bangumi_season_pattern = 'http://bangumi.bilibili.com/jsonp/seasoninfo/%(season_id)s.ver?callback=seasonListCallback&jsonp=jsonp'
    upbangumi_pattern = 'http://search.bilibili.com/ajax_api/video?keyword=%(keyword)s&page=1&order=pubdate&tids_1=13&tids_2=33'
    upvariety_pattern = 'http://search.bilibili.com/ajax_api/video?keyword=%(keyword)s&page=1&order=pubdate&tids_1=5&tids_2=71'
    download_delay = 0.5
    handle_httpstatus_list = [302]
    
    def __init__(self, mode='prod', *args, **kwargs):
        super(BilibiliSpider, self).__init__(mode)
    
    def start_requests(self):
        self.start_timestamp = datetime.now()
        
        logger.debug('检查并缓存帐号登陆信息')
        self.redis_db.delete(BILIBILI_ACCOUNT_SET) # 清空并重构帐号池
        pkeys = self.redis_db.keys(PROXY_KEY % '*')
        if pkeys:
            self.redis_db.delete(*pkeys) # 清空并重构帐号-代理映射
        accounts = self.mongo_accounts.find({'source': 'bilibili'})
        for account in accounts:
            logger.debug(account)
            login_info = { 
                'key': account['key'], 
                'cookies': account['cookies'] 
            }
            res = requests.get('http://space.bilibili.com', cookies=account['cookies'])
            regex = re.search('<title>(.+)的个人空间', res.text)
            if regex:
                username = regex.group(1)
                logging.info('用户%s登陆成功！' % username)
            else:
                username = account['username']
                logging.info('用户%s登陆失败！' % username)
                logging.error(res.text)
                
            self.redis_db.sadd(BILIBILI_ACCOUNT_SET, json.dumps(login_info))
        
        for kobj in self.mongo_keywords.find({'status': 'activated'}):
            # keyword = 
            search_words = [kobj['keyword']]
            if 'alias' in kobj:
                search_words.extend(kobj['alias'])
            for search_word in search_words:
                search_word_url = KeywordEscape(kobj['keyword'])
                if kobj['type'] == 'anime': <commit_msg>fix bilibili alias missing bug<commit_after># -*- coding: utf-8 -*-

import json
import time
from datetime import datetime
import pymongo
import re
from amwatcher_spider.spiders.base import BaseSpider, KeywordEscape
from random import random
from scrapy import Spider, Request
from scrapy.http import HtmlResponse
from collections import defaultdict
import logging
import requests
from logging.handlers import RotatingFileHandler

logger = logging.getLogger(__name__)

BILIBILI_ACCOUNT_SET = 'amwatcher:spider:bilibili:accounts'
PROXY_KEY = 'amwatcher:spider:login_proxy:%s'


class BilibiliSpider(BaseSpider):
    name = 'bilibili'
    updrama_pattern = 'http://search.bilibili.com/ajax_api/video?keyword=%(keyword)s&page=1&order=pubdate&tids_1=11&tids_2=15'
    bangumi_pattern = 'http://search.bilibili.com/bangumi?keyword=%(keyword)s'
    bangumi_season_pattern = 'http://bangumi.bilibili.com/jsonp/seasoninfo/%(season_id)s.ver?callback=seasonListCallback&jsonp=jsonp'
    upbangumi_pattern = 'http://search.bilibili.com/ajax_api/video?keyword=%(keyword)s&page=1&order=pubdate&tids_1=13&tids_2=33'
    upvariety_pattern = 'http://search.bilibili.com/ajax_api/video?keyword=%(keyword)s&page=1&order=pubdate&tids_1=5&tids_2=71'
    download_delay = 0.5
    handle_httpstatus_list = [302]
    
    def __init__(self, mode='prod', *args, **kwargs):
        super(BilibiliSpider, self).__init__(mode)
    
    def start_requests(self):
        self.start_timestamp = datetime.now()
        
        logger.debug('检查并缓存帐号登陆信息')
        self.redis_db.delete(BILIBILI_ACCOUNT_SET) # 清空并重构帐号池
        pkeys = self.redis_db.keys(PROXY_KEY % '*')
        if pkeys:
            self.redis_db.delete(*pkeys) # 清空并重构帐号-代理映射
        accounts = self.mongo_accounts.find({'source': 'bilibili'})
        for account in accounts:
            logger.debug(account)
            login_info = { 
                'key': account['key'], 
                'cookies': account['cookies'] 
            }
            res = requests.get('http://space.bilibili.com', cookies=account['cookies'])
            regex = re.search('<title>(.+)的个人空间', res.text)
            if regex:
                username = regex.group(1)
                logging.info('用户%s登陆成功！' % username)
            else:
                username = account['username']
                logging.info('用户%s登陆失败！' % username)
                logging.error(res.text)
                
            self.redis_db.sadd(BILIBILI_ACCOUNT_SET, json.dumps(login_info))
        
        for kobj in self.mongo_keywords.find({'status': 'activated'}):
            # keyword = 
            search_words = [kobj['keyword']]
            if 'alias' in kobj:
                search_words.extend(kobj['alias'])
            for search_word in search_words:
                search_word_url = KeywordEscape(search_word)
                if kobj['type'] == 'anime': 
{'fraction_non_alphanumeric': 0.12223576527947067, 'fraction_numerical': 0.009402751175343897, 'mean_word_length': 2.894237288135593, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6635758', 'n_tokens_mistral': 1963, 'n_tokens_neox': 1954, 'n_words': 292}