<commit_before># coding: utf-8
from pupa.scrape import Scraper

from utils import lxmlize, CanadianLegislator as Legislator

import re
from urlparse import ulrjoin

COUNCIL_PAGE = 'http://www.calgary.ca/General/Pages/Calgary-City-Council.aspx'


class CalgaryPersonScraper(Scraper):

  def get_people(self):
    page = lxmlize(COUNCIL_PAGE)
    councillor_urls = page.xpath('//div[@class="cocis-image"]/a/@href')
    for url in councillor_urls:
      yield councillor_data(urljoin(COUNCIL_PAGE, url))

def councillor_data(url):
  page = lxmlize(url)
  name_and_ward = page.xpath('string(//h1[@id="cocis-content-page-title"])')
  ward, name = re.match(r'(Ward [0-9]+) Councillor (.+)').groups()
  photo_url_rel = page.xpath('div[@id="contactInfo"]//img[1]/@src')
  photo_url = urljoin(url, photo_url_rel)
  # no email, there's a contact form!
  phone = page.xpath('//strong[contains(., "Phone")]/next-sibling::text()')

  p = Legislator(name=name, post_id=ward, role='Councillor')
  p.add_source(COUNCIL_PAGE)
  p.add_contact('voice', phone, 'legislature')
  p.image = photo

  return p
<commit_msg>fix calgary<commit_after># coding: utf-8
from pupa.scrape import Scraper

from utils import lxmlize, CanadianLegislator as Legislator

import re
from urlparse import urljoin

COUNCIL_PAGE = 'http://www.calgary.ca/General/Pages/Calgary-City-Council.aspx'
MAYOR_PAGE = 'http://calgarymayor.ca/forms_all.php'


class CalgaryPersonScraper(Scraper):

  def get_people(self):
    page = lxmlize(COUNCIL_PAGE)
    nodes = page.xpath('//div[contains(@class,"cocis-has-caption")]')[1:]
    for node in nodes:
      url = urljoin(COUNCIL_PAGE, node.xpath('string(.//a[1]/href)'))
      name = node.xpath('string(.//a[2])')
      ward = ' '.join(node.xpath('string(.//strong)').split()[:-1])
      yield councillor_data(url, name, ward)

    mayor_node = page.xpath('//div[contains(@class, "cocis-image-panel")]')[0]
    photo_url = urljoin(COUNCIL_PAGE, mayor_node.xpath('string(.//img/@src)'))
    name = mayor_node.xpath('string(.//a[2])')
    mayor_page = lxmlize(MAYOR_PAGE)
    email = mayor_page.xpath('string(//a[contains(., "@")])')
    phone = mayor_page.xpath('string(//strong[contains(., "Phone")]/'
                             'following-sibling::text())')
    m = Legislator(name=name, post_id='Calgary', role='Mayor')
    m.add_source(COUNCIL_PAGE)
    m.add_source(MAYOR_PAGE)
    m.add_contact('email', email, None)
    m.add_contact('voice', phone, 'legislature')
    m.image = photo_url
    yield m

def councillor_data(url, name, ward):
  page = lxmlize(url)
  photo_url_rel = page.xpath('div[@id="contactInfo"]//img[1]/@src')
  photo_url = urljoin(url, photo_url_rel)
  # no email, there's a contact form!
  phone = page.xpath('//strong[contains(., "Phone")]/following-sibling::text()')

  p = Legislator(name=name, post_id=ward, role='Councillor')
  p.add_source(COUNCIL_PAGE)
  p.add_contact('voice', phone, 'legislature')
  p.image = photo_url

  return p

{'fraction_non_alphanumeric': 0.16981132075471697, 'fraction_numerical': 0.004380053908355795, 'mean_word_length': 4.427787934186472, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7355286', 'n_tokens_mistral': 1165, 'n_tokens_neox': 1117, 'n_words': 187}