<commit_before>import json
import re
from urllib.parse import urlparse

import scrapy
from inline_requests import inline_requests

from feeds.loaders import FeedEntryItemLoader
from feeds.spiders import FeedsXMLFeedSpider
from feeds.utils import generate_feed_header


class OrfAtSpider(FeedsXMLFeedSpider):
    name = "orf.at"
    namespaces = [
        ("dc", "http://purl.org/dc/elements/1.1/"),
        ("orfon", "http://rss.orf.at/1.0/"),
        ("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"),
        # Default (empty) namespaces are not supported so we just come up with one.
        ("rss", "http://purl.org/rss/1.0/"),
    ]
    itertag = "rss:item"
    # Use XML iterator instead of regex magic which would fail due to the
    # introduced rss namespace prefix.
    iterator = "xml"

    def start_requests(self):
        channels = self.settings.get("FEEDS_SPIDER_ORF_AT_CHANNELS")
        if channels:
            channels = set(channels.split())
        else:
            channels = {"news"}
        available_channels = {
            "burgenland",
            "fm4",
            "help",
            "kaernten",
            "news",
            "noe",
            "oe3",
            "oesterreich",
            "ooe",
            "religion",
            "salzburg",
            "science",
            "sport",
            "steiermark",
            "tirol",
            "vorarlberg",
            "wien",
        }
        unknown_channels = channels - available_channels
        if unknown_channels:
            self.logger.warning(
                "Unknown channel(s) in config file: {}".format(
                    ", ".join(unknown_channels)
                )
            )

        for channel in channels:
            yield scrapy.Request(
                "https://rss.orf.at/{}.xml".format(channel),
                meta={"path": channel, "dont_cache": True},
            )

        self._channels = channels

        self._authors = [
            author
            for author in (
                self.settings.get("FEEDS_SPIDER_ORF_AT_AUTHORS", "").split("\n")
            )
            if author
        ]

    def feed_headers(self):
        for channel in self._channels:
            channel_url = "{}.ORF.at".format(channel)
            yield generate_feed_header(
                title=channel_url,
                link="https://{}".format(channel_url.lower()),
                path=channel,
                logo=self._get_logo(channel),
            )

        for author in self._authors:
            yield generate_feed_header(title="ORF.at: {}".format(author), path=author)

    def parse_node(self, response, node):
        categories = [
            node.xpath("orfon:storyType/@rdf:resource").re_first("urn:orfon:type:(.*)"),
            node.xpath("dc:subject/text()").extract_first(),
        ]
        substories = node.xpath(
            "orfon:substories/rdf:Bag/rdf:li/@rdf:resource"
        ).extract()
        updated = node.xpath("dc:date/text()").extract_first()
        meta = {
            "path": response.meta["path"],
            "categories": categories,
            "updated": updated,
        }
        if substories:
            links = substories
        else:
            links = [node.xpath("rss:link/text()").extract_first()]
        for link in links:
            fixed_link = self._extract_link(link)
            if fixed_link is None or any(
                fixed_link.startswith(url)
                for url in ["https://debatte.orf.at", "https://iptv.orf.at"]
            ):
                self.logger.debug(
                    "Ignoring link to '{}' ('{}')".format(link, fixed_link)
                )
            else:
                yield scrapy.Request(fixed_link, self._parse_article, meta=meta)

    @staticmethod
    def _extract_link(link):
        """Extract a working link from a possibly broken link."""
        if link is None:
            return None
        match = re.search(r"https?://(?:[^\.]+\.)?orf\.at/(?:news/)?stories/\d*", link)
        return match.group(0) + "/" if match else None

    @inline_requests
    def _parse_article(self, response):
        # Heuristic for news.ORF.at to to detect teaser articles.
        more = self._extract_link(
            response.css(
                ".story-story p > strong:contains('Mehr') + a::attr(href), "
                + ".story-story p > a:contains('Lesen Sie mehr')::attr(href)"
            ).extract_first()
        )
        if more and more != response.url:<commit_msg>ORF.at: Add support for two kinds of XML documents.<commit_after>import json
import re
from urllib.parse import urlparse

import lxml
import scrapy
from scrapy.selector import Selector
from inline_requests import inline_requests

from feeds.loaders import FeedEntryItemLoader
from feeds.spiders import FeedsXMLFeedSpider
from feeds.utils import generate_feed_header


class OrfAtSpider(FeedsXMLFeedSpider):
    name = "orf.at"
    namespaces = [
        ("dc", "http://purl.org/dc/elements/1.1/"),
        ("orfon", "http://rss.orf.at/1.0/"),
        ("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"),
        # Default (empty) namespaces are not supported so we just come up with one.
        ("rss", "http://purl.org/rss/1.0/"),
    ]
    itertag = "rss:item"
    # Use XML iterator instead of regex magic which would fail due to the
    # introduced rss namespace prefix.
    iterator = "xml"

    def start_requests(self):
        channels = self.settings.get("FEEDS_SPIDER_ORF_AT_CHANNELS")
        if channels:
            channels = set(channels.split())
        else:
            channels = {"news"}
        available_channels = {
            "burgenland",
            "fm4",
            "help",
            "kaernten",
            "news",
            "noe",
            "oe3",
            "oesterreich",
            "ooe",
            "religion",
            "salzburg",
            "science",
            "sport",
            "steiermark",
            "tirol",
            "vorarlberg",
            "wien",
        }
        unknown_channels = channels - available_channels
        if unknown_channels:
            self.logger.warning(
                "Unknown channel(s) in config file: {}".format(
                    ", ".join(unknown_channels)
                )
            )

        for channel in channels:
            yield scrapy.Request(
                "https://rss.orf.at/{}.xml".format(channel),
                meta={"path": channel, "dont_cache": True},
            )

        self._channels = channels

        self._authors = [
            author
            for author in (
                self.settings.get("FEEDS_SPIDER_ORF_AT_AUTHORS", "").split("\n")
            )
            if author
        ]

    def feed_headers(self):
        for channel in self._channels:
            channel_url = "{}.ORF.at".format(channel)
            yield generate_feed_header(
                title=channel_url,
                link="https://{}".format(channel_url.lower()),
                path=channel,
                logo=self._get_logo(channel),
            )

        for author in self._authors:
            yield generate_feed_header(title="ORF.at: {}".format(author), path=author)

    def parse(self, response):
        selector = Selector(response, type="xml")

        doc = lxml.etree.ElementTree(lxml.etree.fromstring(response.body))
        if doc.getroot().nsmap:
            self._register_namespaces(selector)
            nodes = selector.xpath("//%s" % self.itertag)
        else:
            nodes = selector.xpath("//item")

        return self.parse_nodes(response, nodes)

    def parse_node(self, response, node):
        if "orfon" in node.namespaces:
            return self._parse_extended_node(response, node)
        else:
            return self._parse_simple_node(response, node)

    def _parse_extended_node(self, response, node):
        categories = [
            node.xpath("orfon:storyType/@rdf:resource").re_first("urn:orfon:type:(.*)"),
            node.xpath("dc:subject/text()").extract_first(),
        ]
        substories = node.xpath(
            "orfon:substories/rdf:Bag/rdf:li/@rdf:resource"
        ).extract()
        updated = node.xpath("dc:date/text()").extract_first()
        meta = {
            "path": response.meta["path"],
            "categories": categories,
            "updated": updated,
        }
        if substories:
            links = substories
        else:
            links = [node.xpath("rss:link/text()").extract_first()]
        for link in links:
            fixed_link = self._extract_link(link)
            if fixed_link is None or any(
                fixed_link.startswith(url)
                for url in ["https://debatte.orf.at", "https://iptv.orf.at"]
            ):
                self.logger.debug(
                    "Ignoring link to '{}' ('{}')".format(link, fixed_link)
                )
            else:
                yield scrapy.Request(fixed_link, self._parse_article, meta=meta)

    def _parse_simple_node(self, response, node):
        meta = {
            "path": response.meta["path"],
            "categories": node.xpath("category/text()").extract(),
            "updated": node.xpath("pubDate/text()").extract_first(),
        }
        link = node.xpath("link/text()").extract_first()
        return scrapy.Request(link, self._parse_article, meta=meta)

    @staticmethod
    def _extract_link(link):
        """Extract a working link from a possibly broken link."""
        if link is None:
            return None
        match = re.search(r"https?://(?:[^\.]+\.)?orf\.at/(?:news/)?stories/\d*", link)
        return match.group(0) + "/" if match else None

    @inline_requests
    def _parse_article(self, response):
        # Heuristic for news.ORF.at to to detect teaser articles.
        more = self._extract_link(
            response.css(
                ".story-story p > strong:contains('Mehr') + a::attr(href), "
                + ".story-story p > a:contains('Lesen Sie mehr')::attr(href)"
            ).extract_first()
        )
        if more and more != response.url:
{'fraction_non_alphanumeric': 0.12966620580683388, 'fraction_numerical': 0.003555204424254395, 'mean_word_length': 1.7813787421038176, 'pattern_counts': {'":': 13, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5488752', 'n_tokens_mistral': 2977, 'n_tokens_neox': 2889, 'n_words': 609}