# Path: scrapywork/items.py
# class articleItem(scrapy.Item):
#     #文章的链接
#     url = scrapy.Field()
#     #站点
#     site=scrapy.Field()
#     #文章标题
#     title=scrapy.Field()
#     #文章发布时间
#     time=scrapy.Field()
#     #文章类型
#     type=scrapy.Field()
#     #文章发布方
#     publish=scrapy.Field()
#     html =scrapy.Field()
#     text=scrapy.Field()
#     xml=scrapy.Field()
# 
#     #文件下载
#     file_urls=scrapy.Field()
#     files=scrapy.Field()
#     #图片下载
#     image_urls=scrapy.Field()
#     images=scrapy.Field()


import scrapy
import re
from scrapywork.items import articleItem
        self.urllist=[]
        super(AutoSpider,self).__init__()

    def start_requests(self):
        try:
            for i in range(1,int(self.rule.pages),int(self.rule.pagestep)):
                url=self.rule.next_page_url.format(str(i))
                yield scrapy.Request(url,callback=self.parse_url)
        except:
            print('autospider request failed')

    def parse_url(self,response):
        try:
            print(response.text)
            host_url=self.rule.host_url
            urls=re.findall(self.rule.url_regex,response.text)
            for url in urls:
                self.urllist.append(host_url+url)
        except:
            print('auto spider urllist error.')
        finally:
            for url in self.urllist:
                try:
                    url=url.replace('\\','')
                    yield scrapy.Request(url,callback=self.parse_item)
                except:
                    continue

    def parse_item(self,response):
        self.log('this is iaaf article page %s' % response.url)

# Path: scrapywork/items.py
# class articleItem(scrapy.Item):
#     #文章的链接
#     url = scrapy.Field()
#     #站点
#     site=scrapy.Field()
#     #文章标题
#     title=scrapy.Field()
#     #文章发布时间
#     time=scrapy.Field()
#     #文章类型
#     type=scrapy.Field()
#     #文章发布方
#     publish=scrapy.Field()
#     html =scrapy.Field()
#     text=scrapy.Field()
#     xml=scrapy.Field()
# 
#     #文件下载
#     file_urls=scrapy.Field()
#     files=scrapy.Field()
#     #图片下载
#     image_urls=scrapy.Field()
#     images=scrapy.Field()


# Path: scrapywork/spiders/autoSpider.py
import scrapy
import re
from scrapywork.items import articleItem

        self.urllist=[]
        super(AutoSpider,self).__init__()

    def start_requests(self):
        try:
            for i in range(1,int(self.rule.pages),int(self.rule.pagestep)):
                url=self.rule.next_page_url.format(str(i))
                yield scrapy.Request(url,callback=self.parse_url)
        except:
            print('autospider request failed')

    def parse_url(self,response):
        try:
            print(response.text)
            host_url=self.rule.host_url
            urls=re.findall(self.rule.url_regex,response.text)
            for url in urls:
                self.urllist.append(host_url+url)
        except:
            print('auto spider urllist error.')
        finally:
            for url in self.urllist:
                try:
                    url=url.replace('\\','')
                    yield scrapy.Request(url,callback=self.parse_item)
                except:
                    continue

    def parse_item(self,response):
        self.log('this is iaaf article page %s' % response.url)

        articleInfo = articleItem()