<commit_before># -*- coding: utf-8 -*-
import scrapy
import os
from scrapy.selector import Selector
from dmm_joke.items import DVDDetailItem


class DvdSpiderSpider(scrapy.Spider):
    name = "dvd_spider"
    allowed_domains = ["dmm.com"]
    start_url_template = 'http://www.dmm.com/rental/-/list/=/sort=ranking/page='
    # start_urls = ["http://www.dmm.com/rental/-/list/=/sort=ranking/page=2/"]
    # start_urls = ["http://www.dmm.com/en/rental/-/detail/=/cid=n_611vwdg6298r/"]
    start_urls = ["file:///tmp/index.html"]
    # def parse(self, response):
    #     filename = response.url.split("/")[-2]
    #     filepath = os.path.join("./tmp/",filename)
    #     with open(filepath, 'wb') as f:
    #         f.write(response.body)

    def get_item_list(self, sel, index, field_name, item):
        detail_xpath = "//div[@class='page-detail']/table/tr/td[1]/table/tr[%s]/td[2]/a"
        m_field_list = sel.xpath(detail_xpath%(index))
        item[field_name] = []
        for m_field in m_field_list:
            items = []
            items.append(m_field.xpath("text()").extract()[0])
            items.append(m_field.xpath("@href").extract()[0])
            item[field_name].append(items)


    def parse(self, response):
        sel = Selector(response)
        items = []
        item = DVDDetailItem()
        # item['publisher'] = self.publisher
        # item['id'] = self.id
        item['link'] = response.url
        item['img_url'] = sel.xpath("//div[@class='page-detail']/table/tr/td[1]/div/div[@id='sample-video']/a/img/@src").extract()
        item['title'] = sel.xpath("//div[@class='page-detail']/div[@class='area-headline group']/div[@class='hreview']/h1/span/text()").extract()
        detail_xpath = "//div[@class='page-detail']/table/tr/td[1]/table/tr[%s]/td[2]/text()"
        for index in xrange( len(DVDDetailItem.m_fields) ):
            item[item.m_fields[index]] = sel.xpath(detail_xpath%(index)).extract()
        self.get_item_list(sel, 5, 'performers', item)
        self.get_item_list(sel, 6, 'supervision', item)
        self.get_item_list(sel, 7, 'production', item)
        self.get_item_list(sel, 14, 'studios', item)
        self.get_item_list(sel, 15, 'genre', item)
        item['brief'] = sel.xpath("//div[@class='page-detail']/table[@class='mg-b12']/tr/td[1]/div/p/text()").extract()

        # item['rental_date'] = sel.xpath(detail_xpath%(1)).extract()
        # item['production_year'] = sel.xpath("//div[@class='page-detail']/table/tr/td[1]/table/tr[2]/td[2]/text()").extract()
        print item['brief'][0]

        items.append(item)<commit_msg>update with rules to auto get new webpage<commit_after># -*- coding: utf-8 -*-
import scrapy
import os
from scrapy.selector import Selector
from dmm_joke.items import DVDDetailItem
from scrapy.spiders import  CrawlSpider,Rule
from scrapy.linkextractors import LinkExtractor

class DvdSpiderSpider(CrawlSpider):
    name = "dvd_spider"
    allowed_domains = ["dmm.com"]
    start_url_template = 'http://www.dmm.com/rental/-/list/=/sort=ranking/page='
    start_urls = ["http://www.dmm.com/en/rental/-/list/=/article=maker/id=60011/limit=30/view=text/page=2/"]
    # start_urls = ["http://www.dmm.com/en/rental/-/detail/=/cid=n_611vwdg6298r/"]
    # start_urls = ["file:///tmp/index.html"]
    # start_urls = ["file:///tmp/index.html.1"]
    # 再一次爬去中, scrapy在rule中是自动去重的,额，省了很多事
    # Rule(LinkExtractor(allow=('/rental/\-/list/=/.*page', ))),
    rules = (
        Rule(LinkExtractor(allow=('/rental/\-/list/=/.*page=3', ))),
        Rule(LinkExtractor(allow=('/rental/\-/detail/=/cid=\w+/', )), callback='parse_item'),

    )

    def get_item_list(self, sel, index, field_name, item):
        detail_xpath = "//div[@class='page-detail']/table/tr/td[1]/table/tr[%s]/td[2]/a"
        m_field_list = sel.xpath(detail_xpath%(index))
        item[field_name] = []
        for m_field in m_field_list:
            items = []
            items.append(m_field.xpath("text()").extract()[0])
            items.append(m_field.xpath("@href").extract()[0])
            item[field_name].append(items)

    def parse_item(self, response):
        sel = Selector(response)
        items = []
        item = DVDDetailItem()
        item['link'] = response.url
        item['img_url'] = sel.xpath("//div[@class='page-detail']/table/tr/td[1]/div/div[@id='sample-video']/a/img/@src").extract()
        item['title'] = sel.xpath("//div[@class='page-detail']/div[@class='area-headline group']/div[@class='hreview']/h1/span/text()").extract()
        detail_xpath = "//div[@class='page-detail']/table/tr/td[1]/table/tr[%s]/td[2]/text()"
        for index in xrange( len(DVDDetailItem.m_fields) ):
            item[item.m_fields[index]] = sel.xpath(detail_xpath%(index)).extract()
        self.get_item_list(sel, 5, 'performers', item)
        self.get_item_list(sel, 6, 'supervision', item)
        self.get_item_list(sel, 7, 'production', item)
        self.get_item_list(sel, 14, 'studios', item)
        self.get_item_list(sel, 15, 'genre', item)
        item['brief'] = sel.xpath("//div[@class='page-detail']/table[@class='mg-b12']/tr/td[1]/div/p/text()").extract()
        items.append(item)
{'fraction_non_alphanumeric': 0.20131375579598146, 'fraction_numerical': 0.013330757341576507, 'mean_word_length': 3.1615755627009645, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4895028', 'n_tokens_mistral': 1936, 'n_tokens_neox': 1870, 'n_words': 236}