<commit_before>#!/usr/bin/python3

import datetime

import scrapy

from feeds.items import FeedEntryItem
from feeds.items import FeedItem


class CbirdAtSpider(scrapy.Spider):
    name = 'cbird.at'
    allowed_domains = ['cbird.at']
    start_urls = ['http://cbird.at/hilfe/neu/']

    def parse(self, response):
        yield FeedItem(
            title='Neue cbird Versionen',
            subtitle='The neuesten Versionen von cbird.',
            link=self.start_urls[0],
            author={'name': self.name})

        for href in response.xpath(
                '//table[@class="act"]//*[string(@href)]/@href'):
            url = response.urljoin(href.extract())
            yield scrapy.Request(url, callback=self.parse_announcement)

    def parse_announcement(self, response):
        sel = response.xpath('//div[@class="main"]')
        item = FeedEntryItem(author={'name': self.name})
        item['title'] = sel.xpath('h1/text()').extract()[0]
        item['link'] = response.url
        item['updated'] = datetime.datetime.strptime(
            response.url.split('/')[-1], '%Y%m%d')
        item['content_html'] = ''.join(
            sel.xpath('h1/following-sibling::*').extract())
        yield item

# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 smartindent autoindent<commit_msg>cbird.at: Typo<commit_after>#!/usr/bin/python3

import datetime

import scrapy

from feeds.items import FeedEntryItem
from feeds.items import FeedItem


class CbirdAtSpider(scrapy.Spider):
    name = 'cbird.at'
    allowed_domains = ['cbird.at']
    start_urls = ['http://cbird.at/hilfe/neu/']

    def parse(self, response):
        yield FeedItem(
            title='Neue cbird Versionen',
            subtitle='Die neuesten Versionen von cbird.',
            link=self.start_urls[0],
            author={'name': self.name})

        for href in response.xpath(
                '//table[@class="act"]//*[string(@href)]/@href'):
            url = response.urljoin(href.extract())
            yield scrapy.Request(url, callback=self.parse_announcement)

    def parse_announcement(self, response):
        sel = response.xpath('//div[@class="main"]')
        item = FeedEntryItem(author={'name': self.name})
        item['title'] = sel.xpath('h1/text()').extract()[0]
        item['link'] = response.url
        item['updated'] = datetime.datetime.strptime(
            response.url.split('/')[-1], '%Y%m%d')
        item['content_html'] = ''.join(
            sel.xpath('h1/following-sibling::*').extract())
        yield item

# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 smartindent autoindent
{'fraction_non_alphanumeric': 0.15791490992717516, 'fraction_numerical': 0.006899195093905711, 'mean_word_length': 2.777134587554269, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '781586', 'n_tokens_mistral': 841, 'n_tokens_neox': 818, 'n_words': 150}