<commit_before><commit_msg>2jipink.py: add<commit_after>#!/usr/bin/env python

"""
No description.
"""

import sys
from scrapy import cmdline, Request
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

TARGET_DOMAIN = '2ji.pink'

class Crawler(CrawlSpider):
    """No descrition"""

    name = TARGET_DOMAIN
    allowed_domains = [TARGET_DOMAIN]
    custom_settings = {
        'DOWNLOAD_DELAY': 1,
    }

    rules = (
        Rule(LinkExtractor(restrict_xpaths='//section/h2[@class="entry-title"]/a'), callback='parse_entry'),
        Rule(LinkExtractor(restrict_xpaths='//ul[@class="pagination"]/li/a')),
    )

    def start_requests(self):
        """No descrition"""

        url = self.tag
        yield Request(url, dont_filter=True)

    def parse_entry(self, response):
        """No descrition"""

        if images := response.xpath('//a[@target="_blank"]/img[contains(@src, "img.2ji.pink")]/@src').getall():
            yield {
                'title': response.xpath('//title/text()').get(),
                'url': response.url,
                'images': images}


if __name__ == '__main__':
    #cmdline.execute(f"scrapy runspider {sys.argv[0]} -a tag={sys.argv[1]} -O images.csv".split())
    command_line = ["scrapy", "runspider"]
    command_line.extend(sys.argv)
    cmdline.execute(command_line)
{'fraction_non_alphanumeric': 0.16126656848306334, 'fraction_numerical': 0.005154639175257732, 'mean_word_length': 3.1944444444444446, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7072110', 'n_tokens_mistral': 464, 'n_tokens_neox': 444, 'n_words': 78}