# Path: scrapywork/items.py
# class articleItem(scrapy.Item):
#     #文章的链接
#     url = scrapy.Field()
#     #站点
#     site=scrapy.Field()
#     #文章标题
#     title=scrapy.Field()
#     #文章发布时间
#     time=scrapy.Field()
#     #文章类型
#     type=scrapy.Field()
#     #文章发布方
#     publish=scrapy.Field()
#     html =scrapy.Field()
#     text=scrapy.Field()
#     xml=scrapy.Field()
# 
#     #文件下载
#     file_urls=scrapy.Field()
#     files=scrapy.Field()
#     #图片下载
#     image_urls=scrapy.Field()
#     images=scrapy.Field()


import scrapy
import re
import logging
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapywork.items import articleItem
from scrapy.http import Request
        # 添加`下一页`的规则
        if rule.next_page:
            rule_list.append(Rule(LinkExtractor(restrict_xpaths=rule.next_page,unique=True),follow=True))
        #添加 '列表'的规则
        if rule.list_page:
            rule_list.append(Rule(LinkExtractor(allow=rule.list_page,unique=True),callback='parse_url',follow=True))
        #添加抽取文章链接的规则
        rule_list.append(Rule(LinkExtractor(
            allow=rule.allow_url,unique=True),
            callback='parse_item',follow=True)
        )
        self.rules=tuple(rule_list)
        super(ArticleSpider,self).__init__()


    def parse_url(self,response):
        yield response
        self.log('this is a list page %s'% response.url)
        pages=response.xpath(self.rule.list_page_total).extract_first()
        totalpage=re.findall(r'\d+',pages)
        if not totalpage:totalpage='1'
        self.log('this is a pagetoal %s' % totalpage)
        for i in range(2,int(totalpage[0])+1):
           url=response.url.split('.html')[0]+'_{}.html'.format(i)
           self.log('this ia list page %s' % url)
           yield Request(url)

    def parse_item(self, response):
        self.log('this is article page %s' % response.url)


# Path: scrapywork/items.py
# class articleItem(scrapy.Item):
#     #文章的链接
#     url = scrapy.Field()
#     #站点
#     site=scrapy.Field()
#     #文章标题
#     title=scrapy.Field()
#     #文章发布时间
#     time=scrapy.Field()
#     #文章类型
#     type=scrapy.Field()
#     #文章发布方
#     publish=scrapy.Field()
#     html =scrapy.Field()
#     text=scrapy.Field()
#     xml=scrapy.Field()
# 
#     #文件下载
#     file_urls=scrapy.Field()
#     files=scrapy.Field()
#     #图片下载
#     image_urls=scrapy.Field()
#     images=scrapy.Field()


# Path: scrapywork/spiders/articleSpider.py
import scrapy
import re
import logging
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapywork.items import articleItem
from scrapy.http import Request

        # 添加`下一页`的规则
        if rule.next_page:
            rule_list.append(Rule(LinkExtractor(restrict_xpaths=rule.next_page,unique=True),follow=True))
        #添加 '列表'的规则
        if rule.list_page:
            rule_list.append(Rule(LinkExtractor(allow=rule.list_page,unique=True),callback='parse_url',follow=True))
        #添加抽取文章链接的规则
        rule_list.append(Rule(LinkExtractor(
            allow=rule.allow_url,unique=True),
            callback='parse_item',follow=True)
        )
        self.rules=tuple(rule_list)
        super(ArticleSpider,self).__init__()


    def parse_url(self,response):
        yield response
        self.log('this is a list page %s'% response.url)
        pages=response.xpath(self.rule.list_page_total).extract_first()
        totalpage=re.findall(r'\d+',pages)
        if not totalpage:totalpage='1'
        self.log('this is a pagetoal %s' % totalpage)
        for i in range(2,int(totalpage[0])+1):
           url=response.url.split('.html')[0]+'_{}.html'.format(i)
           self.log('this ia list page %s' % url)
           yield Request(url)

    def parse_item(self, response):
        self.log('this is article page %s' % response.url)


        articleInfo=articleItem()