<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2017-02-07 16:02:55
# Project: douban_demo

from pyspider.libs.base_handler import *
import re
import time
import random as rd    # 随机数



def dt(i):
    return time.time() + int(round(rd.random()*10,0)) + i  # 延迟时间

class Handler(BaseHandler):
    crawl_config = {
        #"proxy":'172.16.17.32:80',
    }
    
    @every(minutes=24 * 60 *5)
    def on_start(self):
        self.crawl('https://movie.douban.com/tag/', callback=self.index_page)
    
    # 标签列表，10小时爬一个标签
    def index_page(self, response):
        for each in response.doc('a[href^="https"]').items():
            if re.match("https://movie.douban.com/tag/\w+",each.attr.href,re.U):
                self.crawl(each.attr.href, callback=self.list_page,exetime=time.time()+10*60*60)
                
    # 爬标签
    def list_page(self, response):
        for each in response.doc('a[href^="https"]').items():
            # 电影详情
            if re.match("https://movie.douban.com/subject/\w+",each.attr.href,re.U):
                self.crawl(each.attr.href, callback=self.detail_page,exetime=dt(15))
                
        for each in response.doc('a[href^="https"]').items():  
            # 翻页
            if each.text() == u'后页>':
                self.crawl(each.attr.href, callback=self.list_page,exetime=dt(60*1.5))
                
    # 抓取结果
    def detail_page(self, response):
        actors = [x.text() for x in response.doc('a[rel="v:starring"]').items()]
        return {
            "影片名称": response.doc('span[property="v:itemreviewed"]').text(),
            "导演": [x.text() for x in response.doc('a[rel="v:directedBy"]').items()],
            "主演":actors,
            "上映日期": [t.text() for t in  response.doc('span[property="v:initialReleaseDate"]').items()],
            "片长": response.doc('span[property="v:runtime"]').text(),
            "评分": response.doc('strong[property="v:average"]').text()
        }<commit_msg>增加随机代理和UA<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2017-02-07 16:02:55
# Project: douban_demo

from pyspider.libs.base_handler import *
import re
import time
import random as rd    # 随机数

def get_proxy():
    proxy=['192.168.127.12:9999',
           '192.168.127.12:8080','192.168.127.12:80','192.168.3.11:9999',
           '172.16.17.32:808',
           '192.168.127.12:9999',
           '172.16.58.3:3128',
           '172.16.17.32:80',
           '172.16.58.3:8909',
           '172.16.31.10:8088',
           '172.16.17.32:9999',
           '172.16.31.10:80',
           '192.168.127.12:3128',
           '172.16.17.32:9999',
           '192.168.3.11:80',
           '172.16.17.32:80',
           '172.16.17.32:808',
           '172.16.17.32:808',
           '172.16.58.3:3128',
           '172.16.31.10:808',
           '192.168.3.11:8080',
           '192.168.3.11:8118',
           '192.168.3.11:808',
           '192.168.127.12:80',
           '172.16.31.10:8080',
           '172.16.58.3:8118',
           '192.168.3.11:80',
           '172.16.17.32:80',
           '172.16.31.10:9999',
           '192.168.3.11:9999',
           '172.16.17.32:8000',
           '192.168.127.12:8998',
           '172.16.17.32:8080',
           '192.168.3.11:9999',
           '172.16.31.10:80',
           '192.168.127.12:80',
           '192.168.127.12:80',
           '192.168.127.12:808',
           '172.16.58.3:9999',
           '172.16.58.3:808',
           '172.16.17.32:808',
           '192.168.127.12:3128',
           '172.16.31.10:80',
           '172.16.58.3:9999',
           '192.168.127.12:8998',
           '172.16.58.3:9999',
           '192.168.127.12:8080',
           '192.168.127.12:9999',
           '192.168.3.11:80',
           '192.168.127.12:843',
           '192.168.127.12:80',
           '172.16.31.10:8998',
           '172.16.58.3:8998',
           '172.16.31.10:80',
           '172.16.17.32:8080',
           '172.16.31.10:9999',
           '192.168.127.12:80',
           '172.16.58.3:9999',
           '192.168.127.12:808',
           '172.16.31.10:3128',
           '192.168.127.12:8080',
           '172.16.17.32:80',
           '172.16.17.32:9999',
           '172.16.31.10:9999',
           '192.168.3.11:80',
           '172.16.58.3:9999',
           '172.16.31.10:3128',
           '172.16.31.10:8000',
           '192.168.127.12:9999',
           '172.16.58.3:9999',
           '172.16.31.10:9999',
           '192.168.127.12:8088',
           '172.16.58.3:80',
           '192.168.3.11:9999',
           '192.168.3.11:80',
           '172.16.58.3:3128',
           '172.16.17.32:9999',
           '172.16.17.32:80',
           '172.16.31.10:3128',
           '172.16.31.10:9999',
           '172.16.58.3:9999',
           '172.16.31.10:80',
           '192.168.127.12:808',
           '172.16.31.10:80',
           '172.16.31.10:8000',
           '192.168.3.11:80',
           '192.168.127.12:9999',
           '172.16.31.10:808',
           '172.16.31.10:80',
           '192.168.3.11:80',
           '192.168.127.12:8080',
           '192.168.127.12:8080',
           '192.168.127.12:80',
           '172.16.17.32:80',
           '172.16.58.3:3128',
           '192.168.127.12:8080',
           '192.168.127.12:9999',
           '192.168.127.12:808',
           '172.16.17.32:808',
           '192.168.3.11:8998']
    i = rd.randint(0,len(proxy)-1)
    ip = proxy[i]+''
    return ip

def get_header():
    UserAgent_List = [
	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
	"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
	"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
	"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
	"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
	"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
	"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
	"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
	"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
	"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"]
    i = rd.randint(0,len(UserAgent_List)-1)
    return UserAgent_List[i]

def dt(i):
    return time.time() + int(round(rd.random()*10,0)) + i  # 延迟时间

class Handler(BaseHandler):
    crawl_config = {
        'proxy': get_proxy(),
        'User-Agent': get_header()
    }
    
    @every(minutes=24 * 60 * 15)
    def on_start(self):
        print get_proxy(),get_header()
        self.crawl('https://movie.douban.com/tag/', callback=self.index_page)
    
    # 标签列表，1小时爬一个标签
    def index_page(self, response):
        print self.crawl_config['proxy']
        for each in response.doc('a[href^="https"]').items():
            if re.match("https://movie.douban.com/tag/\w+",each.attr.href,re.U):
                self.crawl(each.attr.href, callback=self.list_page,exetime=dt(60)+60*60,proxy=get_proxy(),headers= {'User-Agent': get_header()})
                
    # 爬标签
    def list_page(self, response):
        print self.crawl_config['proxy']
        for each in response.doc('a[href^="https"]').items():
            # 电影详情
            if re.match("https://movie.douban.com/subject/\w+",each.attr.href,re.U):
                self.crawl(each.attr.href, callback=self.detail_page,exetime=dt(30),proxy=get_proxy(),headers= {'User-Agent': get_header()})
                
        for each in response.doc('a[href^="https"]').items():  
            # 翻页
            if each.text() == u'后页>':
                self.crawl(each.attr.href, callback=self.list_page,exetime=dt(60*2),proxy=get_proxy(),headers= {'User-Agent': get_header()})
                
    # 抓取结果
    def detail_page(self, response):
        print self.crawl_config['proxy']
        actors = [x.text() for x in response.doc('a[rel="v:starring"]').items()]
        return {
            "影片名称": response.doc('span[property="v:itemreviewed"]').text(),
            "导演": [x.text() for x in response.doc('a[rel="v:directedBy"]').items()],
            "主演":actors,
            "上映日期": [t.text() for t in  response.doc('span[property="v:initialReleaseDate"]').items()],
            "片长": response.doc('span[property="v:runtime"]').text(),
            "评分": response.doc('strong[property="v:average"]').text()
        }
{'fraction_non_alphanumeric': 0.19580638431045275, 'fraction_numerical': 0.18380972251199665, 'mean_word_length': 2.06784, 'pattern_counts': {'":': 13, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 102, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1820471', 'n_tokens_mistral': 4880, 'n_tokens_neox': 3860, 'n_words': 531}