<commit_before>.. _Development:

Writing a custom spider
=======================
Feeds already supports a number of websites (see :ref:`Supported Websites`) but
adding support for a new website doesn't take too much time.

A quick example
---------------
Writing a spider is easy! Consider the slightly simplified spider for
:ref:`spider_indiehackers.com`:

.. code-block:: python

    import scrapy

    from feeds.loaders import FeedEntryItemLoader
    from feeds.spiders import FeedsSpider


    class IndieHackersComSpider(FeedsSpider):
        name = "indiehackers.com"
        allowed_domains = [name]
        start_urls = ["https://www.indiehackers.com/interviews/page/1"]
        _title = "Indie Hackers"

        def parse(self, response):
            interviews = response.css(
                ".interview__link::attr(href), .interview__date::text"
            ).extract()
            for link, date in zip(interviews[::2], interviews[1::2]):
                yield scrapy.Request(
                    response.urljoin(link),
                    self._parse_interview,
                    meta={"updated": date.strip()},
                )

        def _parse_interview(self, response):
            remove_elems = [
                ".shareable-quote",
                ".share-bar",
                # Remove the last two h2s and all paragraphs below.
                ".interview-body > h2:last-of-type ~ p",
                ".interview-body > h2:last-of-type",
                ".interview-body > h2:last-of-type ~ p",
                ".interview-body > h2:last-of-type",<commit_msg>Further simplify indiehackers.com spider in the docs.<commit_after>.. _Development:

Writing a custom spider
=======================
Feeds already supports a number of websites (see :ref:`Supported Websites`) but
adding support for a new website doesn't take too much time.

A quick example
---------------
Writing a spider is easy! Consider the slightly simplified spider for
:ref:`spider_indiehackers.com`:

.. code-block:: python

    import scrapy

    from feeds.loaders import FeedEntryItemLoader
    from feeds.spiders import FeedsSpider


    class IndieHackersComSpider(FeedsSpider):
        name = "indiehackers.com"
        allowed_domains = [name]
        start_urls = ["https://www.indiehackers.com/interviews/page/1"]
        _title = "Indie Hackers"

        def parse(self, response):
            interview_links = response.css(".interview__link::attr(href)").extract()
            interview_dates = response.css(".interview__date::text").extract()
            for link, date in zip(interview_links, interview_dates):
                yield scrapy.Request(
                    response.urljoin(link),
                    self._parse_interview,
                    meta={"updated": date.strip()},
                )

        def _parse_interview(self, response):
            remove_elems = [
                ".shareable-quote",
                ".share-bar",
{'fraction_non_alphanumeric': 0.12369043595809395, 'fraction_numerical': 0.0033795201081446434, 'mean_word_length': 2.1759656652360513, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5773912', 'n_tokens_mistral': 864, 'n_tokens_neox': 792, 'n_words': 207}