<commit_before># -*- coding: utf-8 -*-
import scrapy
import re
import pdb
import json
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from ..items import ZhihuItem,RelationItem
from scrapy.http import Request,FormRequest
from scrapy_redis.spiders import RedisSpider

# ------------------------------------------
#   版本：1.0
#   日期：2017-8-06
#   作者：AlexTan
#   <CSDN:   http://blog.csdn.net/alextan_>  
#   <e-mail: <EMAIL>>
# ------------------------------------------


#zhihuspider1是模拟浏览器爬（速度慢,不建议，仅供学习） zhihuspider0抓包爬（速度快）
class ZhihuspiderSpider(RedisSpider):
#class ZhihuspiderSpider(scrapy.Spider):
    name = "zhihuspider1"
    #allowed_domains = ["zhihu.com"]
    host = 'https://www.zhihu.com'
    redis_key = "zhihuspider:start_urls"
    #start_urls = ['https://www.zhihu.com/people/yun-he-shu-ju-8/answers']
    strat_user_id = ['yun-he-shu-ju-8']
    #pdb.set_trace()
    dcap = dict(DesiredCapabilities.PHANTOMJS)
    dcap["phantomjs.page.settings.userAgent"] = ("Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0")
    dcap["phantomjs.page.settings.loadImages"] = False
    obj = webdriver.PhantomJS(desired_capabilities=dcap)


    def start_requests(self):
        for one in self.strat_user_id:
            yield Request('https://www.zhihu.com/people/'+one+'/answers',callback=self.parse)
        #return [Request('https://www.zhihu.com/#signin',callback=self.start_login,meta={'cookiejar':1})] #这个登录已不可用，仅供学习

    def start_login(self,response):<commit_msg>Fix the bug about dupefilter<commit_after># -*- coding: utf-8 -*-
import scrapy
import re
import pdb
import json
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from ..items import ZhihuItem,RelationItem
from scrapy.http import Request,FormRequest
from scrapy_redis.spiders import RedisSpider

# ------------------------------------------
#   版本：1.0
#   日期：2017-8-06
#   作者：AlexTan
#   <CSDN:   http://blog.csdn.net/alextan_>  
#   <e-mail: <EMAIL>>
# ------------------------------------------


#zhihuspider1是模拟浏览器爬（速度慢,不建议，仅供学习） zhihuspider0抓包爬（速度快）
class ZhihuspiderSpider(RedisSpider):
#class ZhihuspiderSpider(scrapy.Spider):
    name = "zhihuspider1"
    #allowed_domains = ["zhihu.com"]
    host = 'https://www.zhihu.com'
    redis_key = "zhihuspider:start_urls"
    #start_urls = ['https://www.zhihu.com/people/yun-he-shu-ju-8/answers']
    strat_user_id = ['yun-he-shu-ju-8']
    #pdb.set_trace()
    dcap = dict(DesiredCapabilities.PHANTOMJS)
    dcap["phantomjs.page.settings.userAgent"] = ("Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0")
    dcap["phantomjs.page.settings.loadImages"] = False
    obj = webdriver.PhantomJS(desired_capabilities=dcap)


    def start_requests(self):
        for one in self.strat_user_id:
            yield Request('https://www.zhihu.com/people/'+one+'/answers',callback=self.parse,dont_filter=True)
        #return [Request('https://www.zhihu.com/#signin',callback=self.start_login,meta={'cookiejar':1})] #这个登录已不可用，仅供学习

    def start_login(self,response):
{'fraction_non_alphanumeric': 0.1879746835443038, 'fraction_numerical': 0.024050632911392405, 'mean_word_length': 3.3480055020632737, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 8, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2868864', 'n_tokens_mistral': 1282, 'n_tokens_neox': 1233, 'n_words': 173}