<commit_before># -*- coding: utf-8 -*-
import os

import scrapy

from sc_spider.items import SongCiItem
from sc_spider.spiders.spider_utils import page_name_from_url


class SongSanSpider(scrapy.Spider):
    name = "songsan"
    allowed_domains = ["gushiwen.org"]
    start_urls = (
        'http://www.gushiwen.org/gushi/songsan.aspx',
    )

    STORAGE_PATH = './out/'

    def __init__(self, name=None, **kwargs):
        super().__init__(name, **kwargs)
        if not os.path.exists(self.STORAGE_PATH):
            os.makedirs(self.STORAGE_PATH)

    def parse(self, response):
        filename = self.STORAGE_PATH + page_name_from_url(response.url)
        with open(filename, 'wb') as f:
            self.logger.debug('saving page: %s', filename)
            f.write(response.body)
            self.logger.info('page save: %s', filename)

        cont2s = response.xpath('//div[@class="guwencont2"]')
        items = []
        for cont2 in cont2s:
            links = cont2.xpath('a')
            for link in links:
                items.append(self._parse_item(link))

        return items

    @staticmethod
    def _parse_item(link):
        item = SongCiItem()
        item['url'] = link.xpath('@href').extract_first()
        title_author = link.xpath('text()').re('(.+)\((.+)\)')
        if title_author:
            item['title'] = title_author[0]
            item['author'] = title_author[1]
        else:
            item['title'] = link.xpath('text()').extract_first()

        return item<commit_msg>use yield instead of return.<commit_after># -*- coding: utf-8 -*-
import os

import scrapy

from sc_spider.items import SongCiItem
from sc_spider.spiders.spider_utils import page_name_from_url


class SongSanSpider(scrapy.Spider):
    name = "songsan"
    allowed_domains = ["gushiwen.org"]
    start_urls = (
        'http://www.gushiwen.org/gushi/songsan.aspx',
    )

    STORAGE_PATH = './out/'

    def __init__(self, name=None, **kwargs):
        super().__init__(name, **kwargs)
        if not os.path.exists(self.STORAGE_PATH):
            os.makedirs(self.STORAGE_PATH)

    def parse(self, response):
        filename = self.STORAGE_PATH + page_name_from_url(response.url)
        with open(filename, 'wb') as f:
            self.logger.debug('saving page: %s', filename)
            f.write(response.body)
            self.logger.info('page save: %s', filename)

        cont2s = response.xpath('//div[@class="guwencont2"]')
        for cont2 in cont2s:
            links = cont2.xpath('a')
            for link in links:
                yield self._parse_item(link)

    @staticmethod
    def _parse_item(link):
        item = SongCiItem()
        item['url'] = link.xpath('@href').extract_first()
        title_author = link.xpath('text()').re('(.+)\((.+)\)')
        if title_author:
            item['title'] = title_author[0]
            item['author'] = title_author[1]
        else:
            item['title'] = link.xpath('text()').extract_first()

        return item
{'fraction_non_alphanumeric': 0.14471057884231536, 'fraction_numerical': 0.0053226879574184965, 'mean_word_length': 2.393905191873589, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6932467', 'n_tokens_mistral': 1006, 'n_tokens_neox': 968, 'n_words': 183}