<commit_before># -*- coding: utf8 -*-

import re
from urlparse import urljoin
from functools import partial

from scrapy.selector import HtmlXPathSelector
from scrapy.http import Request
from scrapy.spider import BaseSpider

from mscrap.loaders import LegisladorItemLoader


_RE_RESOURCE_ID = re.compile(r'http://webappl.hcdn.gov.ar/diputados/([-\w]+)')

class DiputadosSpider(BaseSpider):
    name = 'diputados'
    allowed_domains = ['webappl.hcdn.gov.ar']
    start_urls = 'http://webappl.hcdn.gov.ar/diputados/listadodiputados.html',

    def parse(self, response):
        hxs = HtmlXPathSelector(response)

        dipus = hxs.select('//div[@id="tablaPpal"]//table//tr/td[2]/a')
        for dipu in dipus:
            apellido, nombre = dipu.select('text()').extract()[0].split(',')
            resource_id = dipu.select('@href').extract()[0]
            resource_url = urljoin(response.url, resource_id)
            item_data = {
                'apellido': apellido,
                'nombre': nombre,
                'resource_id': resource_id,
                'resource_url': resource_url }
            yield Request(resource_url, callback=partial(self.parse_diputado, item_data=item_data))

    def parse_diputado(self, response, item_data):
        hxs = HtmlXPathSelector(response)
        #uname =  urlsplit(response.url).path.split('/')[-1]

        l = LegisladorItemLoader(selector=hxs.select('//div[@id="page"]//div[@class="primera"]'))

        l.add_value('id', item_data['resource_url']) # unique enough :)

        l.add_value('resource_source', u'webappl.hcdn.gov.ar')
        l.add_value('resource_id', item_data['resource_id'])
        l.add_value('resource_url', item_data['resource_url'])

        l.add_value('camara', 'D')
        l.add_value('nombre', item_data['nombre'])
        l.add_value('apellido', item_data['apellido'])

        l.add_xpath('foto_url', './/img[@alt="Foto del legislador"]/@src')
        l.add_xpath('bloque_nombre', './/*[@class="quinto1"][3]/p/text()')
        l.add_xpath('distrito_nombre', './/div[@class="cuarto1"]/p[contains(., "Distrito:")]/text()',
                                       re='Distrito:\xa0 ([a-zA-Z0-9 ]+)')
        l.add_xpath('mandato_inicio', './/div[@class="quinto1"]/p[contains(., "Mandato:")]/text()',
                                      re=r'.*(\d\d/\d\d/\d\d\d\d)\xa0-.*')
        l.add_xpath('mandato_fin', './/div[@class="quinto1"]/p[contains(., "Mandato:")]/text()',
                                   re=r'.*-\xa0(\d\d/\d\d/\d\d\d\d)$')
        l.add_xpath('email', './/div[@class="quinto1"]/p[contains(., "E-Mail:")]/a/text()')
        l.add_xpath('telefono', './/div[@class="cuarto1"]/p[contains(., "Teléfono:")]/text()', re='([-\d]+)')

        yield l.load_item()


SPIDER = DiputadosSpider()<commit_msg>Updates to diputados spider<commit_after># -*- coding: utf8 -*-

import re
from urlparse import urljoin
from functools import partial

from scrapy.selector import HtmlXPathSelector
from scrapy.http import Request
from scrapy.spider import BaseSpider

from mscrap.loaders import LegisladorItemLoader


_RE_RESOURCE_ID = re.compile(r'http://webappl.hcdn.gov.ar/diputados/([-\w]+)')

class DiputadosSpider(BaseSpider):
    name = 'diputados'
    allowed_domains = ['www.hcdn.gov.ar']
    start_urls = 'http://www.hcdn.gov.ar/diputados/listadip.html',

    def parse(self, response):
        hxs = HtmlXPathSelector(response)

        dipus = hxs.select('//div[@id="tablaPpal"]//table//a[starts-with(@href, "/diputados/")]')
        for dipu in dipus:
            apellido, nombre = dipu.select('text()').extract()[0].split(',')
            resource_id = dipu.select('@href').extract()[0]
            resource_url = urljoin(response.url, resource_id)
            item_data = {
                'apellido': apellido,
                'nombre': nombre,
                'resource_id': resource_id,
                'resource_url': resource_url }
            yield Request(resource_url, callback=partial(self.parse_diputado, item_data=item_data))

    def parse_diputado(self, response, item_data):
        hxs = HtmlXPathSelector(response)
        #uname =  urlsplit(response.url).path.split('/')[-1]

        l = LegisladorItemLoader(selector=hxs.select('//div[@class="diputados-principal"]'))

        l.add_value('id', item_data['resource_url']) # unique enough :)

        l.add_value('resource_source', u'www.hcdn.gov.ar')
        l.add_value('resource_id', item_data['resource_id'])
        l.add_value('resource_url', item_data['resource_url'])

        l.add_value('camara', 'D')
        l.add_value('nombre', item_data['nombre'])
        l.add_value('apellido', item_data['apellido'])

        l.add_xpath('foto_url', './/div[@class="foto-diputados-principal"]/img/@src')
        l.add_xpath('distrito_nombre', './/div[@class="info-diputados-principal1"]//text()[2]',
                                       re='Distrito:\xa0 ([a-zA-Z0-9 ]+)')
        l.add_xpath('telefono', './/div[@class="info-diputados-principal1"]//text()[3]', re='([-\d]+)')
        l.add_xpath('bloque_nombre', './/div[@class="info-diputados-principal2"]//h3/text()')
        l.add_xpath('mandato_inicio', './/div[@class="info-diputados-principal2"]//text()[2]',
                                      re=r'.*(\d\d/\d\d/\d\d\d\d)\xa0-.*')
        l.add_xpath('mandato_fin', './/div[@class="info-diputados-principal2"]//text()[2]',
                                   re=r'.*-\xa0(\d\d/\d\d/\d\d\d\d).*')
        l.add_xpath('email', './/div[@class="info-diputados-principal2"]//a/text()')

        yield l.load_item()


SPIDER = DiputadosSpider()
{'fraction_non_alphanumeric': 0.19020480854853072, 'fraction_numerical': 0.0065894924309884235, 'mean_word_length': 2.7920324105334235, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4104025', 'n_tokens_mistral': 2040, 'n_tokens_neox': 1966, 'n_words': 250}