<commit_before>from pupa.scrape import Scraper

from utils import lxmlize, CanadianLegislator as Legislator

import re

COUNCIL_PAGE = 'http://www.newmarket.ca/en/townhall/contactinformationmayorandtowncouncil.asp'


class NewmarketPersonScraper(Scraper):

  def get_people(self):
    page = lxmlize(COUNCIL_PAGE)

    councillors = page.xpath('//div[@id="printArea"]//table//tr//td')[4:-1]
    for councillor in councillors:
      if councillor == councillors[0]:
        yield self.scrape_mayor(councillor)
        continue
      name = councillor.xpath('.//a/text()')[0]
      district = councillor.xpath('.//strong/text()')[1].replace('Councillor- ', '')
      district = district if not 'Regional' in district else 'Newmarket'
      role = councillor.xpath('.//strong/text()')[1].split('-')[0]  # @todo Mayor is not being set
      url = councillor.xpath('.//a/@href')[0]

      p = Legislator(name=name, post_id=district, role=role)
      p.add_source(COUNCIL_PAGE)
      p.add_source(url)

      p.image = councillor.xpath('.//img/@src')[0]

      page = lxmlize(url)
      info = page.xpath('//div[@id="printArea"]')[0]
      info = info.xpath('.//p[@class="heading"][2]/following-sibling::p')
      address = info.pop(0).text_content().strip()
      if not address:
        address = info.pop(0).text_content().strip()

      if 'Ward' in info[0].text_content():
        info.pop(0)<commit_msg>ca_on_newmarket fixes<commit_after>from pupa.scrape import Scraper

from utils import lxmlize, CanadianLegislator as Legislator

import re

COUNCIL_PAGE = 'http://www.newmarket.ca/en/townhall/contactinformationmayorandtowncouncil.asp'


class NewmarketPersonScraper(Scraper):

  def get_people(self):
    page = lxmlize(COUNCIL_PAGE)

    councillors = page.xpath('//div[@id="printArea"]//table//tr//td')[4:-1]
    yield self.scrape_mayor(councillors[0])
    for councillor in councillors[1:]:
      name = ' '.join(councillor.xpath('string(.//strong/a[last()])').split())
      infostr = councillor.xpath('string(.//strong)')
      try:
        district = infostr.split('-')[1]
        role = 'Councillor'
      except IndexError:
        district = 'Newmarket'
        role = 'Regional Councillor'
      url = councillor.xpath('.//a/@href')[0]

      p = Legislator(name=name, post_id=district, role=role)
      p.add_source(COUNCIL_PAGE)
      p.add_source(url)

      p.image = councillor.xpath('.//img/@src')[0]

      page = lxmlize(url)
      info = page.xpath('//div[@id="printArea"]')[0]
      info = info.xpath('.//p[@class="heading"][2]/following-sibling::p')
      address = info.pop(0).text_content().strip()
      if not address:
        address = info.pop(0).text_content().strip()

      if 'Ward' in info[0].text_content():
        info.pop(0)
{'fraction_non_alphanumeric': 0.16829533116178066, 'fraction_numerical': 0.010133912414042706, 'mean_word_length': 3.2852713178294572, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6241662', 'n_tokens_mistral': 1035, 'n_tokens_neox': 973, 'n_words': 155}