<commit_before>import json

import scrapy

from feeds.loaders import FeedEntryItemLoader
from feeds.spiders import FeedsSpider


class Oe1OrfAtSpider(FeedsSpider):
    name = 'oe1.orf.at'
    allowed_domains = ['audioapi.orf.at']
    start_urls = ['https://audioapi.orf.at/oe1/api/json/current/broadcasts']

    _title = 'oe1.ORF.at'
    _subtitle = 'Ö1 Webradio'
    _link = 'http://oe1.orf.at'
    _timezone = 'Europe/Vienna'

    def parse(self, response):
        for day in json.loads(response.text)[-2:]:
            for broadcast in day['broadcasts']:
                # Only parse if already recorded (i.e. not live/in the future).
                if broadcast['state'] == 'C':
                    yield scrapy.Request(broadcast['href'],
                                         self.parse_broadcast,
                                         meta={'oe1_day': day['day']})

    def parse_broadcast(self, response):
        broadcast = json.loads(response.text)
        il = FeedEntryItemLoader(response=response, timezone=self._timezone,
                                 dayfirst=False)
        link = 'https://{}/programm/{}/{}'.format(
            self.name, response.meta['oe1_day'], broadcast['programKey'])
        il.add_value('link', link)
        il.add_value('title', broadcast['title'])
        stream = 'http://loopstream01.apa.at/?channel=oe1&id={}'.format(
            broadcast['streams'][0]['loopStreamId'])
        il.add_value('enclosure_iri', stream)
        il.add_value('enclosure_type', 'audio/mpeg')
        il.add_value('updated', broadcast['niceTimeISO'])
        if broadcast['subtitle']:
            il.add_value('content_html', '<strong>{}</strong>'.format(
                broadcast['subtitle']))
        for item in broadcast['items']:
            il.add_value('content_html', '<h3>{}</h3>'.format(item['title']))
            il.add_value('content_html', item['description'])
        il.add_value('content_html', broadcast['description'])
        yield il.load_item()<commit_msg>oe1.ORF.at: Fix title extraction for items.<commit_after>import json

import scrapy

from feeds.loaders import FeedEntryItemLoader
from feeds.spiders import FeedsSpider


class Oe1OrfAtSpider(FeedsSpider):
    name = 'oe1.orf.at'
    allowed_domains = ['audioapi.orf.at']
    start_urls = ['https://audioapi.orf.at/oe1/api/json/current/broadcasts']

    _title = 'oe1.ORF.at'
    _subtitle = 'Ö1 Webradio'
    _link = 'http://oe1.orf.at'
    _timezone = 'Europe/Vienna'

    def parse(self, response):
        for day in json.loads(response.text)[-2:]:
            for broadcast in day['broadcasts']:
                # Only parse if already recorded (i.e. not live/in the future).
                if broadcast['state'] == 'C':
                    yield scrapy.Request(broadcast['href'],
                                         self.parse_broadcast,
                                         meta={'oe1_day': day['day']})

    def parse_broadcast(self, response):
        broadcast = json.loads(response.text)
        il = FeedEntryItemLoader(response=response, timezone=self._timezone,
                                 dayfirst=False)
        link = 'https://{}/programm/{}/{}'.format(
            self.name, response.meta['oe1_day'], broadcast['programKey'])
        il.add_value('link', link)
        il.add_value('title', broadcast['title'])
        stream = 'http://loopstream01.apa.at/?channel=oe1&id={}'.format(
            broadcast['streams'][0]['loopStreamId'])
        il.add_value('enclosure_iri', stream)
        il.add_value('enclosure_type', 'audio/mpeg')
        il.add_value('updated', broadcast['niceTimeISO'])
        if broadcast['subtitle']:
            il.add_value('content_html', '<strong>{}</strong>'.format(
                broadcast['subtitle']))
        for item in broadcast['items']:
            if 'title' in item:
                il.add_value('content_html',
                             '<h3>{}</h3>'.format(item['title']))
            il.add_value('content_html', item.get('description'))
        il.add_value('content_html', broadcast['description'])
        yield il.load_item()
{'fraction_non_alphanumeric': 0.148776350860189, 'fraction_numerical': 0.007511509571117034, 'mean_word_length': 2.3479318734793186, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6785714', 'n_tokens_mistral': 1239, 'n_tokens_neox': 1219, 'n_words': 212}