<commit_before># -*- coding: utf-8 -*-
"""A spider to crawl the Berlinovo website."""
import re

from scrapy import Request, Spider
from scrapy.linkextractors import LinkExtractor
from scrapy.loader import ItemLoader
from w3lib.url import url_query_cleaner

from tegenaria.items import ApartmentItem
from tegenaria.spiders import SpiderMixin


class BerlinovoSpider(Spider, SpiderMixin):
    """A spider to crawl the Berlinovo website."""

    name = 'berlinovo'
    allowed_domains = ['berlinovo.de']
    start_urls = (
        'https://www.berlinovo.de/en/suche-apartments',
    )
    searched_pages = set()

    def parse(self, response):
        """Parse a search result HTML page.

        @url https://www.berlinovo.de/en/suche-apartments
        @returns items 0 0
        @returns requests 15 20
        """
        for link in LinkExtractor(allow=r'/en/suche-apartments.+page=').extract_links(response):
            if link.url not in (self.searched_pages, self.start_urls):
                self.searched_pages.add(link.url)
                yield Request(link.url, callback=self.parse)

        for link in LinkExtractor(allow=r'/en/apartment/', process_value=url_query_cleaner).extract_links(response):
            yield Request(link.url, callback=self.parse_item)

    def parse_item(self, response):
        """Parse an ad page, with an apartment.

        @url https://www.berlinovo.de/en/apartment/2-room-suite-house-heinrich-heine-stra-e-18-24-berlin-mitte
        @returns items 1 1
        @scrapes url title description location address other neighborhood rooms
        """
        self.shutdown_on_error()
        item = ItemLoader(ApartmentItem(), response=response)
        item.add_value('url', response.url)
        item.add_css('title', 'h1.title::text')
        item.add_xpath('description', '//div[contains(@class, field-name-body)]/div/div[4]/div/div/p/text()')
        item.add_xpath('location', '//div[contains(@class, field-name-field-position)]/div/div[5]/div[2]/div/text()')

        zipcode = response.xpath(
            '//*[@id="block-views-aktuelle-wohnung-block-3"]/div/div/div/div/div[3]/div/span/text()[1]') \
            .extract()[0].strip()
        street = response.xpath(
            '//*[@id="block-views-aktuelle-wohnung-block-3"]/div/div/div/div/div[3]/div/span/text()[2]') \
            .extract()[0].strip()
        item.add_value('address', u'{}, {}'.format(street, zipcode))

        item.add_xpath(
            'equipment',
            '//*[@id="block-views-aktuelle-wohnung-block-3"]/div/div/div/div/div[18]/div/div/ul/li/span/text()')
        item.add_xpath('warm_rent',
                       '//*[@id="block-views-aktuelle-wohnung-block-3"]/div/div/div/div/div[5]/span[2]/text()')

        item.add_xpath('other', '//*[@id="block-views-aktuelle-wohnung-block-3"]/div/div/div/div/div/span/text()')

        item.add_value('neighborhood', response.css('#page-title::text').extract()[0].strip().split('Berlin-')[-1])

        room_list = response.xpath('//*[@id="block-views-aktuelle-wohnung-block-3"]/div/div/div/div/div'
                                   '[contains(@class, views-field-field-rooms-description)]/div/text()').extract()
        item.add_value('rooms', re.findall(r'([0-9]+)', ' '.join(room_list))[0])

        yield item.load_item()<commit_msg>Crawl Berlinovo housings (#29)<commit_after># -*- coding: utf-8 -*-
"""Furnished and regular apartments from Berlinovo."""
import re
from typing import Any, Dict

from scrapy.linkextractors import LinkExtractor
from scrapy.loader import ItemLoader
from scrapy.spiders import CrawlSpider, Rule
from w3lib.url import url_query_cleaner

from tegenaria.items import ApartmentItem
from tegenaria.spiders import SpiderMixin


class BerlinovoSpider(CrawlSpider, SpiderMixin):
    """Furnished and regular apartments from Berlinovo."""

    name = 'berlinovo'
    allowed_domains = ['berlinovo.de']
    start_urls = [
        # Furnished apartments
        'https://www.berlinovo.de/en/suche-apartments',
        # Regular housing
        'https://www.berlinovo.de/en/suche-wohnungen',
    ]
    rules = (
        Rule(LinkExtractor(allow=r'/en/suche-(apartments|wohnungen).+page=')),
        Rule(LinkExtractor(allow=r'/en/apartment/', process_value=url_query_cleaner),
             callback='parse_furnished', follow=True),
        Rule(LinkExtractor(allow=r'/en/wohnung/', process_value=url_query_cleaner),
             callback='parse_regular', follow=True),
    )

    def parse_common(self, response):
        """Parse common fields for both."""
        self.shutdown_on_error()
        item = ItemLoader(ApartmentItem(), response=response)
        item.add_value('url', response.url)
        item.add_css('title', 'h1.title::text')
        return item

    def parse_furnished(self, response):
        """Parse an ad page, with an apartment.

        @url https://www.berlinovo.de/en/apartment/2-room-suite-house-heinrich-heine-stra-e-18-24-berlin-mitte
        @returns items 1 1
        @scrapes url title description location address other neighborhood rooms
        """
        item = self.parse_common(response)
        item.add_xpath('description', '//div[contains(@class, field-name-body)]/div/div[4]/div/div/p/text()')
        item.add_xpath('location', '//div[contains(@class, field-name-field-position)]/div/div[5]/div[2]/div/text()')

        zipcode = response.xpath(
            '//*[@id="block-views-aktuelle-wohnung-block-3"]/div/div/div/div/div[3]/div/span/text()[1]') \
            .extract()[0].strip()
        street = response.xpath(
            '//*[@id="block-views-aktuelle-wohnung-block-3"]/div/div/div/div/div[3]/div/span/text()[2]') \
            .extract()[0].strip()
        item.add_value('address', u'{}, {}'.format(street, zipcode))

        item.add_xpath(
            'equipment',
            '//*[@id="block-views-aktuelle-wohnung-block-3"]/div/div/div/div/div[18]/div/div/ul/li/span/text()')
        item.add_xpath('warm_rent',
                       '//*[@id="block-views-aktuelle-wohnung-block-3"]/div/div/div/div/div[5]/span[2]/text()')

        item.add_xpath('other', '//*[@id="block-views-aktuelle-wohnung-block-3"]/div/div/div/div/div/span/text()')

        item.add_value('neighborhood', response.css('#page-title::text').extract()[0].strip().split('Berlin-')[-1])

        room_list = response.xpath('//*[@id="block-views-aktuelle-wohnung-block-3"]/div/div/div/div/div'
                                   '[contains(@class, views-field-field-rooms-description)]/div/text()').extract()
        item.add_value('rooms', re.findall(r'([0-9]+)', ' '.join(room_list))[0])

        yield item.load_item()

    def parse_regular(self, response):
        """Parse a regular housing apartment ad.

        @url https://www.berlinovo.de/en/wohnung/single-wohnung-hellersdorf-zu-vermieten
        @returns items 1 1
        @scrapes url title address rooms size cold_rent warm_rent description equipment
        """
        item = self.parse_common(response)
        item.add_xpath('address', '//span[@class="address"]/text()')

        for field, class_ in {'rooms': 'views-label-field-rooms', 'size': 'views-label-field-net-area-1',
                              'cold_rent': 'views-label-field-net-rent'}.items():
            item.add_xpath(field, '//span[contains(@class, "{}")]/following-sibling::span/text()'.format(class_))

        item.add_xpath('warm_rent', '//div[contains(@class, "views-field-field-total-rent")]'
                                    '/child::span[@class="field-content"]/text()')
        item.add_xpath('description', '//div[contains(@class, field-name-field-description)]/div/div/p/text()')
        item.add_xpath('equipment', '//div[starts-with(normalize-space(.), "Ausstattung")]//div/text()')
        yield item.load_item()

    def clean_item(self, data: Dict[str, Any]) -> Dict[str, Any]:
        """Clean the item before loading."""
        self.clean_number(data, 'rooms')
        self.clean_number(data, 'size')
        return data
{'fraction_non_alphanumeric': 0.17267623206487837, 'fraction_numerical': 0.009856519026824703, 'mean_word_length': 2.8818401937046003, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 7, 'lorem ipsum': 0, 'www.': 7, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5371642', 'n_tokens_mistral': 2696, 'n_tokens_neox': 2614, 'n_words': 391}