<commit_before># -*- coding: utf-8 -*-
import scrapy
from lesson01.items import Article
from scrapy.exceptions import CloseSpider


class MovienewsSpider(scrapy.Spider):
    name = "movieNews"
    allowed_domains = ["www.1905.com"]
    start_urls = ['http://www.1905.com/list-p-catid-%s.html']

    def start_requests(self):
        # catid = 220
        catid = getattr(self, 'catid', None)
        if catid is None:
            raise CloseSpider("catid param is not set")

        for url in self.start_urls:
            url = url % catid
            yield scrapy.Request(url, self.parse)

    def parse(self, response):
        """抽取新闻列表信息"""
        print("*"*30, "parse")

        # 特别注意li的class名字后面有一个空格，这是一个坑
        li_list = response.xpath("//ul[@class='pic-event-over']/li[contains(@class, 'pic-pack-out')]/div[@class='pic-pack-inner']")

        for li in li_list:
            # 循环获取title，url，date等信息
            title = li.xpath("./h3/a/text()").extract()
            url = li.xpath("./h3/a/@href").extract()
            date = li.xpath(".//span[@class='timer fl']/text()").extract()

            # 注意：extract方法的返回值是一个列表，取第一个元素即可
            # yield是python中的生成器
            item = Article(title=title[0], url=url[0], date=date[0])

            # meta参数可以将已经获取的item数据传给详情解释函数
            # callback参数指定详情页面的解释函数
            yield scrapy.Request(url[0],
                                 meta={'item': item},
                                 callback=self.parse_desc)<commit_msg>lesson08<commit_after># -*- coding: utf-8 -*-
import scrapy
from lesson01.items import Article
from scrapy.exceptions import CloseSpider
from scrapy.shell import inspect_response

class MovienewsSpider(scrapy.Spider):
    name = "movieNews"
    allowed_domains = ["www.1905.com"]
    start_urls = ['http://www.1905.com/list-p-catid-%s.html']

    def start_requests(self):
        # catid = 220
        catid = getattr(self, 'catid', None)
        if catid is None:
            raise CloseSpider("catid param is not set")

        for url in self.start_urls:
            url = url % catid
            yield scrapy.Request(url, self.parse)

    def parse(self, response):
        """抽取新闻列表信息"""
        print("*"*30, "parse")

        # 特别注意li的class名字后面有一个空格，这是一个坑
        li_list = response.xpath("//ul[@class='pic-event-over']/li[contains(@class, 'pic-pack-out')]/div[@class='pic-pack-inner']")

        # 进入debug命令行
        # 这里可以加if判断，例如：if len(li_list) == 0
        # 这时才调用shell进行测试调试
        inspect_response(response, self)

        for li in li_list:
            # 循环获取title，url，date等信息
            title = li.xpath("./h3/a/text()").extract()
            url = li.xpath("./h3/a/@href").extract()
            date = li.xpath(".//span[@class='timer fl']/text()").extract()

            # 注意：extract方法的返回值是一个列表，取第一个元素即可
            # yield是python中的生成器
            item = Article(title=title[0], url=url[0], date=date[0])

            # meta参数可以将已经获取的item数据传给详情解释函数
            # callback参数指定详情页面的解释函数
            yield scrapy.Request(url[0],
                                 meta={'item': item},
                                 callback=self.parse_desc)
{'fraction_non_alphanumeric': 0.1414494596312778, 'fraction_numerical': 0.014939605848696759, 'mean_word_length': 2.118929633300297, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3824136', 'n_tokens_mistral': 1150, 'n_tokens_neox': 1152, 'n_words': 178}