<commit_before>from __future__ import unicode_literals
from utils import CanadianScraper, CanadianPerson as Person

import re

COUNCIL_PAGE = 'http://www.gatineau.ca/page.asp?p=la_ville/conseil_municipal'
MAYOR_CONTACT_PAGE = 'http://www.gatineau.ca/portail/default.aspx?p=la_ville/conseil_municipal/maire'


class GatineauPersonScraper(CanadianScraper):

    def scrape(self):
        page = self.lxmlize(COUNCIL_PAGE)

        # it's all javascript rendered on the client... wow.
        js = page.xpath('string(//div[@class="inner_container"]/div/script[2])')  # allow string()
        districts = re.findall(r'arrayDistricts\[a.+"(.+)"', js)
        names = re.findall(r'arrayMembres\[a.+"(.+)"', js)
        urls = re.findall(r'arrayLiens\[a.+"(.+)"', js)
        # first item in list is mayor
        p = Person(primary_org='legislature', name=names[0], district='Gatineau', role='Maire')
        p.add_source(COUNCIL_PAGE)
        p.add_source(MAYOR_CONTACT_PAGE)
        email = '<EMAIL>'  # hardcoded
        p.add_contact('email', email)
        yield p

        for raw_district, name, url in list(zip(districts, names, urls))[1:]:
            if name == 'Vacant':
                continue

            profile_url = COUNCIL_PAGE + '/' + url.split('/')[-1]
            profile_page = self.lxmlize(profile_url)
            photo_url = profile_page.xpath('//img/@src')[0]
            district = 'District ' + re.search('\d+', raw_district).group(0)
            email = self.get_email(profile_page)
            p = Person(primary_org='legislature', name=name, district=district, role='Conseiller')
            p.add_source(COUNCIL_PAGE)
            p.add_source(profile_url)
            p.image = photo_url
            p.add_contact('email', email)
            yield p<commit_msg>ca_qc_gatineau: Fix image<commit_after>from __future__ import unicode_literals
from utils import CanadianScraper, CanadianPerson as Person

import re

COUNCIL_PAGE = 'http://www.gatineau.ca/page.asp?p=la_ville/conseil_municipal'
MAYOR_CONTACT_PAGE = 'http://www.gatineau.ca/portail/default.aspx?p=la_ville/conseil_municipal/maire'


class GatineauPersonScraper(CanadianScraper):

    def scrape(self):
        page = self.lxmlize(COUNCIL_PAGE)

        # it's all javascript rendered on the client... wow.
        js = page.xpath('string(//div[@class="inner_container"]/div/script[2])')  # allow string()
        districts = re.findall(r'arrayDistricts\[a.+"(.+)"', js)
        names = re.findall(r'arrayMembres\[a.+"(.+)"', js)
        urls = re.findall(r'arrayLiens\[a.+"(.+)"', js)
        # first item in list is mayor
        p = Person(primary_org='legislature', name=names[0], district='Gatineau', role='Maire')
        p.add_source(COUNCIL_PAGE)
        p.add_source(MAYOR_CONTACT_PAGE)
        email = '<EMAIL>'  # hardcoded
        p.add_contact('email', email)
        yield p

        for raw_district, name, url in list(zip(districts, names, urls))[1:]:
            if name == 'Vacant':
                continue

            profile_url = COUNCIL_PAGE + '/' + url.split('/')[-1]
            profile_page = self.lxmlize(profile_url)
            photo_url = profile_page.xpath('//table//img/@src')[0]
            district = 'District ' + re.search('\d+', raw_district).group(0)
            email = self.get_email(profile_page)
            p = Person(primary_org='legislature', name=name, district=district, role='Conseiller')
            p.add_source(COUNCIL_PAGE)
            p.add_source(profile_url)
            p.image = photo_url
            p.add_contact('email', email)
            yield p
{'fraction_non_alphanumeric': 0.14214186369958276, 'fraction_numerical': 0.003337969401947149, 'mean_word_length': 2.7972544878563887, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '887877', 'n_tokens_mistral': 1268, 'n_tokens_neox': 1220, 'n_words': 205}