<commit_before># -*- coding: utf-8 -*-

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from gotcha.items import GotchaItem

class HumorUnivSpider(CrawlSpider):
    name = "HumorUnivSpider"
    allowed_domains = ["web.humoruniv.com"]
    start_urls = [
        'http://web.humoruniv.com/board/humor/list.html?table=pds',
    ]

    rules = (
        Rule(LinkExtractor(allow=('/board/humor/read\.html\?.*table=pds.*', )), callback='parse_pds'),
    )

    def parse_pds(self, response):
        # self.logger.info('Hi, this is an item page! %s', response.url)
        item = GotchaItem()
        item['title'] = response.css('#ai_cm_title::text').extract()
        item['content'] = response.css('#ai_cm_content::text').extract()
        item['writer'] = response.css('#if_wrt .hu_nick_txt::text').extract()
        item['created_at'] = response.css('#if_date:first-child::text').extract()
        item['url'] = response.url
        item['category'] = 'humor'

        self.logger.info(str(item))
        return item<commit_msg>페이지 파싱 셀렉터 css -> xpath 로 교체<commit_after># -*- coding: utf-8 -*-

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from gotcha.items import GotchaItem

class HumorUnivSpider(CrawlSpider):
    name = "HumorUnivSpider"
    allowed_domains = ["web.humoruniv.com"]
    start_urls = [
        'http://web.humoruniv.com/board/humor/list.html?table=pds',
    ]

    rules = (
        Rule(LinkExtractor(allow=('/board/humor/read\.html\?.*table=pds.*', )), callback='parse_pds'),
    )

    def parse_pds(self, response):
        # self.logger.info('Hi, this is an item page! %s', response.url)
        item = GotchaItem()
        item['title'] = u"".join(response.xpath('//span[@id="ai_cm_title"]//text()').extract()).strip()
        item['content'] = u"".join(response.xpath('//span[@id="ai_cm_content"]//text()').extract()).strip()
        item['writer'] = u"".join(response.xpath('//div[@id="if_wrt"]//span[contains(@class, "hu_nick_txt")]//text()').extract()).strip()
        item['created_at'] = u"".join(response.xpath('//div[@id="if_date"]/span[1]//text()').extract()).strip()
        item['url'] = response.url
        item['category'] = 'humor'
        return item
{'fraction_non_alphanumeric': 0.18858131487889274, 'fraction_numerical': 0.0012975778546712802, 'mean_word_length': 3.4825581395348837, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1665621', 'n_tokens_mistral': 813, 'n_tokens_neox': 807, 'n_words': 120}