<commit_before><commit_msg>Add files via upload<commit_after># -*- coding: utf-8 -*-
import scrapy
import re

from locations.items import GeojsonPointItem


class HiHostelsSpider(scrapy.Spider):
    name = "hihostels"
    allowed_domains = ['hihostels.com']
    start_urls = (
        'https://www.hihostels.com/sitemap.xml',
    )

    def parse(self, response):
        response.selector.remove_namespaces()
        city_urls = response.xpath('//url/loc/text()').extract()
        for path in city_urls:
            regex = re.compile(r'http\S+hihostels.com/\S+/hostels/\S+')
            if not re.search(regex,path):
                pass
            else:
                yield scrapy.Request(
                    path.strip(),
                    callback=self.parse_store,
                )

    def parse_store(self, response):

        properties = {
            'name': " ".join(response.xpath('/html/body/div[1]/div[6]/div[2]/div[1]/h1/span/text()').extract()[0].split()),
            'website': response.xpath('//head/link[@rel="canonical"]/@href').extract_first(),
            'addr_full': " ".join(response.xpath('/html/body/div[1]/div[6]/div[2]/div[1]/div[2]/p[1]/text()').extract()[0].split(',')[0].split()),
            'city': " ".join(response.xpath('/html/body/div[1]/div[6]/div[2]/div[1]/div[2]/p[1]/text()').extract()[0].split(',')[1].split()),
            'postcode': " ".join(response.xpath('/html/body/div[1]/div[6]/div[2]/div[1]/div[2]/p[1]/text()').extract()[0].split(',')[-2].split()),
            'country': " ".join(response.xpath('/html/body/div[1]/div[6]/div[2]/div[1]/div[2]/p[1]/text()').extract()[0].split(',')[-1].split()),
            'lon': float(response.xpath('//*[@id ="lat"]/@value').extract()[0]),
            'lat': float(response.xpath('//*[@id ="lon"]/@value').extract()[0]),
        }
        open('/tmp/tmp.txt', 'w').write(str(properties))


        yield GeojsonPointItem(**properties)
{'fraction_non_alphanumeric': 0.23590274185204346, 'fraction_numerical': 0.021210553543714435, 'mean_word_length': 2.662878787878788, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3842699', 'n_tokens_mistral': 668, 'n_tokens_neox': 648, 'n_words': 69}