<commit_before># coding: utf-8
from pupa.scrape import Scraper

from utils import lxmlize, CanadianLegislator as Legislator

import re
from urlparse import urljoin
from itertools import takewhile

COUNCIL_PAGE = 'http://www.regionofwaterloo.ca/en/regionalgovernment/regionalcouncil.asp'
CHAIR_URL = 'http://www.regionofwaterloo.ca/en/regionalGovernment/regionalchairandsupportstaff.asp'


class WaterlooPersonScraper(Scraper):

  def get_people(self):
    page = lxmlize(COUNCIL_PAGE)

    regions = page.xpath('//*[@id="contentIntleft"]//h3')[1:]
    for region in regions:
      #the links in all <p> tags immediately following each <h3>
      councillors = [elem[0] for elem in 
        takewhile(lambda elem: elem.tag == 'p', 
          region.xpath('./following-sibling::*'))]
      for councillor in councillors:
        post = re.search('of (.*)', region.text).group(1)
        p = Legislator(name=councillor.text, post_id=post, role='Councillor')
        p.add_source(COUNCIL_PAGE)
        councillor_url = councillor.attrib['href']
        p.add_source(councillor_url)
        email, phone, address, photo_url = councillor_data(councillor_url)
        p.add_contact('email', email, None)
        p.add_contact('voice', phone, 'legislature')
        p.add_contact('address', address, 'legislature')
        p.image = photo_url
        yield p

    chairpage = lxmlize(CHAIR_URL)
    name = re.search('Chair (.*) -', 
        chairpage.xpath('string(//title)')).group(1)
    email = chairpage.xpath('string(//a[contains(text(), "E-mail")]/@href)')
    phone = chairpage.xpath('string((//span[@class="labelTag"][contains(text(), "Phone")]/parent::*/text())[1])').strip(':')
    address = '\n'.join(
        chairpage.xpath('//div[@class="contactBody"]//p[1]/text()'))
    photo_url_src = chairpage.xpath(
        'string(//div[id="contentIntleft"]/img[1]/@src)')
    photo_url = urljoin(CHAIR_URL, photo_url_src)
    p = Legislator(name=name, post_id='Waterloo', role='Regional Chair')
    p.add_source(CHAIR_URL)
    p.add_contact('email', email, None)
    p.add_contact('voice', phone, 'legislature')
    p.add_contact('address', address, 'legislature')
    p.image = photo_url
    yield p

def councillor_data(url):<commit_msg>fix waterloo region chair image<commit_after># coding: utf-8
from pupa.scrape import Scraper

from utils import lxmlize, CanadianLegislator as Legislator

import re
from urlparse import urljoin
from itertools import takewhile

COUNCIL_PAGE = 'http://www.regionofwaterloo.ca/en/regionalgovernment/regionalcouncil.asp'
CHAIR_URL = 'http://www.regionofwaterloo.ca/en/regionalGovernment/regionalchairandsupportstaff.asp'


class WaterlooPersonScraper(Scraper):

  def get_people(self):
    page = lxmlize(COUNCIL_PAGE)

    regions = page.xpath('//*[@id="contentIntleft"]//h3')[1:]
    for region in regions:
      #the links in all <p> tags immediately following each <h3>
      councillors = [elem[0] for elem in 
        takewhile(lambda elem: elem.tag == 'p', 
          region.xpath('./following-sibling::*'))]
      for councillor in councillors:
        post = re.search('of (.*)', region.text).group(1)
        p = Legislator(name=councillor.text, post_id=post, role='Councillor')
        p.add_source(COUNCIL_PAGE)
        councillor_url = councillor.attrib['href']
        p.add_source(councillor_url)
        email, phone, address, photo_url = councillor_data(councillor_url)
        p.add_contact('email', email, None)
        p.add_contact('voice', phone, 'legislature')
        p.add_contact('address', address, 'legislature')
        p.image = photo_url
        yield p

    chairpage = lxmlize(CHAIR_URL)
    name = re.search('Chair (.*) -', 
        chairpage.xpath('string(//title)')).group(1)
    email = chairpage.xpath('string(//a[contains(text(), "E-mail")]/@href)')
    phone = chairpage.xpath('string((//span[@class="labelTag"][contains(text(), "Phone")]/parent::*/text())[1])').strip(':')
    address = '\n'.join(
        chairpage.xpath('//div[@class="contactBody"]//p[1]/text()'))
    photo_url_src = chairpage.xpath(
        'string(//div[@id="contentIntleft"]//img[1]/@src)')
    photo_url = urljoin(CHAIR_URL, photo_url_src)
    p = Legislator(name=name, post_id='Waterloo', role='Regional Chair')
    p.add_source(CHAIR_URL)
    p.add_contact('email', email, None)
    p.add_contact('voice', phone, 'legislature')
    p.add_contact('address', address, 'legislature')
    p.image = photo_url
    yield p

def councillor_data(url):
{'fraction_non_alphanumeric': 0.15672306322350846, 'fraction_numerical': 0.004452359750667854, 'mean_word_length': 3.8311827956989246, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1340012', 'n_tokens_mistral': 1611, 'n_tokens_neox': 1543, 'n_words': 271}