<commit_before>import json

import scrapy

from feeds.loaders import FeedEntryItemLoader
from feeds.spiders import FeedsSpider


class Oe1OrfAtSpider(FeedsSpider):
    name = "oe1.orf.at"
    allowed_domains = ["audioapi.orf.at", name]
    start_urls = ["https://audioapi.orf.at/oe1/api/json/current/broadcasts"]

    _title = "oe1.ORF.at"
    _subtitle = "Ö1 Webradio"
    _link = "https://oe1.orf.at"
    _timezone = "Europe/Vienna"
    _logo = "https://{}/static/img/logo_oe1.png".format(name)

    def parse(self, response):
        for day in json.loads(response.text)[-2:]:
            for broadcast in day["broadcasts"]:
                # Only parse if already recorded (i.e. not live/in the future).
                if broadcast["state"] == "C":
                    yield scrapy.Request(broadcast["href"], self._parse_broadcast)

    def _parse_broadcast(self, response):
        broadcast = json.loads(response.text)
        il = FeedEntryItemLoader(
            response=response, timezone=self._timezone, dayfirst=False
        )
        link = "https://{}/player/{}/{}".format(
            self.name, broadcast["broadcastDay"], broadcast["programKey"]
        )
        il.add_value("link", link)
        il.add_value("title", broadcast["programTitle"])
        il.add_value("title", broadcast["title"])
        if broadcast.get("streams"):
            stream = "https://loopstream01.apa.at/?channel=oe1&id={}".format(
                broadcast["streams"][0]["loopStreamId"]
            )
            il.add_value("enclosure_iri", stream)
            il.add_value("enclosure_type", "audio/mpeg")
        il.add_value("updated", broadcast["niceTimeISO"])
        if broadcast["subtitle"]:
            il.add_value(
                "content_html", "<strong>{}</strong>".format(broadcast["subtitle"])
            )
        image = broadcast["images"][0]["versions"][0]
        il.add_value(
            "content_html",
            '<img src="{image[path]}" width="{image[width]}">'.format(image=image),
        )
        for item in broadcast["items"]:
            if "title" in item:
                il.add_value("content_html", "<h3>{}</h3>".format(item["title"]))
            il.add_value("content_html", item.get("description"))
        il.add_value("content_html", broadcast["description"])
        il.add_value("category", broadcast["tags"])
        if "no_canonical_url" not in broadcast["url"]:
            yield scrapy.Request(<commit_msg>oe1.ORF.at: Include broadcast URL in content for filtering.<commit_after>import json

import scrapy

from feeds.loaders import FeedEntryItemLoader
from feeds.spiders import FeedsSpider


class Oe1OrfAtSpider(FeedsSpider):
    name = "oe1.orf.at"
    allowed_domains = ["audioapi.orf.at", name]
    start_urls = ["https://audioapi.orf.at/oe1/api/json/current/broadcasts"]

    _title = "oe1.ORF.at"
    _subtitle = "Ö1 Webradio"
    _link = "https://oe1.orf.at"
    _timezone = "Europe/Vienna"
    _logo = "https://{}/static/img/logo_oe1.png".format(name)

    def parse(self, response):
        for day in json.loads(response.text)[-2:]:
            for broadcast in day["broadcasts"]:
                # Only parse if already recorded (i.e. not live/in the future).
                if broadcast["state"] == "C":
                    yield scrapy.Request(broadcast["href"], self._parse_broadcast)

    def _parse_broadcast(self, response):
        broadcast = json.loads(response.text)
        il = FeedEntryItemLoader(
            response=response, timezone=self._timezone, dayfirst=False
        )
        link = "https://{}/player/{}/{}".format(
            self.name, broadcast["broadcastDay"], broadcast["programKey"]
        )
        il.add_value("link", link)
        il.add_value("title", broadcast["programTitle"])
        il.add_value("title", broadcast["title"])
        if broadcast.get("streams"):
            stream = "https://loopstream01.apa.at/?channel=oe1&id={}".format(
                broadcast["streams"][0]["loopStreamId"]
            )
            il.add_value("enclosure_iri", stream)
            il.add_value("enclosure_type", "audio/mpeg")
        il.add_value("updated", broadcast["niceTimeISO"])
        if broadcast["subtitle"]:
            il.add_value(
                "content_html", "<strong>{}</strong>".format(broadcast["subtitle"])
            )
        image = broadcast["images"][0]["versions"][0]
        il.add_value(
            "content_html",
            '<img src="{image[path]}" width="{image[width]}">'.format(image=image),
        )
        for item in broadcast["items"]:
            if "title" in item:
                il.add_value("content_html", "<h3>{}</h3>".format(item["title"]))
            il.add_value("content_html", item.get("description"))
        il.add_value("content_html", broadcast["description"])
        il.add_value(
            "content_html",
            '<a href="{broadcast[url]}">{broadcast[urlText]}</a>'.format(
                broadcast=broadcast
            ),
        )
        il.add_value("category", broadcast["tags"])
        if "no_canonical_url" not in broadcast["url"]:
            yield scrapy.Request(
{'fraction_non_alphanumeric': 0.15607385811467445, 'fraction_numerical': 0.00641399416909621, 'mean_word_length': 2.5174299384825702, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1524276', 'n_tokens_mistral': 1589, 'n_tokens_neox': 1559, 'n_words': 263}