<commit_before>import scrapy

from feeds.loaders import FeedEntryItemLoader
from feeds.spiders import FeedsSpider


class BibliowebAtSpider(FeedsSpider):
    name = "biblioweb.at"

    _days = 60

    def start_requests(self):
        self._library = self.settings.get(
            "FEEDS_SPIDER_BIBLIOWEB_AT_LOCATION", ""
        ).lower()
        if self._library:
            self.feed_path = self._library
            self.feed_title = "Bibliothek {}".format(self._library.title())
            self.feed_subtitle = "Neue Titel in der {}".format(self.feed_title)
            self.feed_link = "https://www.biblioweb.at/{}/".format(self._library)
            yield scrapy.Request(
                "https://www.biblioweb.at/{}/start.asp".format(self._library),
                callback=self.parse,
                meta={"dont_cache": True},
            )
        else:
            # Key location or section biblioweb.at not found in feeds.cfg.
            self.logger.error(
                "A location is required for spider '{name}'. Please add a "
                "configuration block for '{name}' with the 'location' you "
                "want to scrape.".format(name=self.name)
            )

    def parse(self, response):
        # Ignore the initial response to start.asp as it is required to get the
        # ASP cookie. Without this cookie the requests to webopac123 (!) are
        # ignored and will be redirected to the "login" page.
        return scrapy.Request(
            "https://www.biblioweb.at/webopac123/webopac.asp"
            "?kat=1&content=show_new&seit={}&order_by=Sachtitel".format(self._days),
            callback=self.parse_overview_page,
            meta={"dont_cache": True},
        )

    def parse_overview_page(self, response):
        # Find other pages
        for href in response.xpath(
            '//div[@id="p_main"][1]/div/a/div[@id!="p_aktuell"]/../@href'
        ):
            url = response.urljoin(href.extract())
            yield scrapy.Request(
                url, self.parse_overview_page, meta={"dont_cache": True}
            )

        # Find content
        for href in response.xpath('//a[contains(@href, "mnr")]/@href'):
            url = response.urljoin(href.extract())
            yield scrapy.Request(url, self.parse_content)

    def parse_content(self, response):
        parts = self._extract_parts(response)
        il = FeedEntryItemLoader(<commit_msg>biblioweb.at: Enable dupe filter.<commit_after>import scrapy

from feeds.loaders import FeedEntryItemLoader
from feeds.spiders import FeedsSpider


class BibliowebAtSpider(FeedsSpider):
    name = "biblioweb.at"
    custom_settings = {"DUPEFILTER_CLASS": "scrapy.dupefilters.RFPDupeFilter"}

    _days = 60

    def start_requests(self):
        self._library = self.settings.get(
            "FEEDS_SPIDER_BIBLIOWEB_AT_LOCATION", ""
        ).lower()
        if self._library:
            self.feed_path = self._library
            self.feed_title = "Bibliothek {}".format(self._library.title())
            self.feed_subtitle = "Neue Titel in der {}".format(self.feed_title)
            self.feed_link = "https://www.biblioweb.at/{}/".format(self._library)
            yield scrapy.Request(
                "https://www.biblioweb.at/{}/start.asp".format(self._library),
                callback=self.parse,
                meta={"dont_cache": True},
            )
        else:
            # Key location or section biblioweb.at not found in feeds.cfg.
            self.logger.error(
                "A location is required for spider '{name}'. Please add a "
                "configuration block for '{name}' with the 'location' you "
                "want to scrape.".format(name=self.name)
            )

    def parse(self, response):
        # Ignore the initial response to start.asp as it is required to get the
        # ASP cookie. Without this cookie the requests to webopac123 (!) are
        # ignored and will be redirected to the "login" page.
        return scrapy.Request(
            "https://www.biblioweb.at/webopac123/webopac.asp"
            "?kat=1&content=show_new&seit={}&order_by=Sachtitel".format(self._days),
            callback=self.parse_overview_page,
            meta={"dont_cache": True},
        )

    def parse_overview_page(self, response):
        # Find other pages
        # Note that the dupefilter has to be enabled, otherwise already
        # parsed pages will be parsed again.
        for href in response.xpath(
            '//div[@id="p_main"][1]/div/a/div[@id!="p_aktuell"]/../@href'
        ):
            url = response.urljoin(href.extract())
            yield scrapy.Request(
                url, self.parse_overview_page, meta={"dont_cache": True}
            )

        # Find content
        for href in response.xpath('//a[contains(@href, "mnr")]/@href'):
            url = response.urljoin(href.extract())
            yield scrapy.Request(url, self.parse_content)

    def parse_content(self, response):
        parts = self._extract_parts(response)
        il = FeedEntryItemLoader(
{'fraction_non_alphanumeric': 0.117008681925809, 'fraction_numerical': 0.003946329913180742, 'mean_word_length': 2.1621958827199004, 'pattern_counts': {'":': 7, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7460635', 'n_tokens_mistral': 1545, 'n_tokens_neox': 1477, 'n_words': 331}