<commit_before>import urllib
import urllib2
from BeautifulSoup import BeautifulSoup
import re
import time
import datetime
import mechanize
from collections import defaultdict
import slate
import cStringIO

class LegistarScraper (object):
  """
  A programatic interface onto a hosted Legistar website.
  """

  def __init__(self, config):
    """
    Construct a new Legistar scraper.  Pass in the host name of the Legistar
    instance, e.g. 'phila.legistar.com'.
    """
    self.config = config
    self.host = 'http://%s/' % self.config['hostname']
    self.fulltext = self.config['fulltext']

    # Assume that the legislation and calendar URLs are constructed regularly.
    self._legislation_uri = (
      self.host + self.config.get('legislation_path', 'Legislation.aspx'))
    self._calendar_uri = (
      self.host + self.config.get('calendar_path', 'Calendar.aspx'))
    self._people_uri = (
      self.host + self.config.get('people_path', 'People.aspx'))
    
  def searchLegislation(self, search_text, created_before=None, created_after=None, num_pages = None):
    """
    Submit a search query on the legislation search page, and return a list
    of summary results.
    """
    br = self._get_new_browser()
    br.open(self._legislation_uri)

    try:
      # Check for the link to the advanced search form
      br.find_link(text_regex='Advanced.*')

    except mechanize.LinkNotFoundError:
      # If it's not there, then we're already on the advanced search form.
      pass

    else:
      # If it is there, the navigate to the advanced search form.
      br.select_form('aspnetForm')
      data = self._data(br.form, None)
      data['ctl00$ContentPlaceHolder1$btnSwitch'] = ''
      data = urllib.urlencode(data)

      br.open(self._legislation_uri, data)

    br.select_form('aspnetForm')
    br.form.set_all_readonly(False)

    # Enter the search parameters
    # TODO: Each of the possible form fields should be represented as keyword
    # arguments to this function. The default query string should be for the
    # the default 'Legislative text' field.
    br.form['ctl00$ContentPlaceHolder1$txtText'] = search_text


    if created_before or created_after:
      if created_before :
        creation_date = created_before
        relation = '<'
      else:
        creation_date = created_after
        relation = '>'

      br.form['ctl00$ContentPlaceHolder1$radFileCreated'] = [relation]
      br.form['ctl00_ContentPlaceHolder1_txtFileCreated1_dateInput_ClientState'] = '{"enabled":true,"emptyMessage":"","validationText":"%s-00-00-00","valueAsString":"%s-00-00-00","minDateStr":"1980-01-01-00-00-00","maxDateStr":"2099-12-31-00-00-00"}' % (creation_date, creation_date)


    # Submit the form
    data = self._data(br.form, None)

    # Return up to one million search results
    data['ctl00_ContentPlaceHolder1_lstMax_ClientState'] = '{"value":"1000000"}'

    data['ctl00$ContentPlaceHolder1$btnSearch'] = 'Search Legislation'
    data['ctl00_ContentPlaceHolder1_lstYearsAdvanced_ClientState'] = '{"logEntries":[],"value":"All","text":"All Years","enabled":true,"checkedIndices":[],"checkedItemsTextOverflows":false}'

    data = urllib.urlencode(data)

    response = _try_connect(br, self._legislation_uri, data)

    # Loop through the pages, yielding each of the results
    all_results = False
    while all_results is False :
      soup = BeautifulSoup(response.read())

      legislation_iter = self.parseSearchResults(soup)
      for legislation in legislation_iter:
        yield legislation


      current_page = soup.fetch('a', {'class': 'rgCurrentPage'})
      if current_page :
        current_page = current_page[0]
        print 'page', current_page.text
        print
        next_page = current_page.findNextSibling('a')
      else :
        next_page = None

      if next_page :
        event_target = next_page['href'].split("'")[1]

        br.select_form('aspnetForm')
        data = self._data(br.form, event_target)
        data = urllib.urlencode(data)
        response = _try_connect(br, self._legislation_uri, data)

      else :
        all_results = True

    raise StopIteration

  def parseSearchResults(self, soup) :
    """Take a page of search results and return a sequence of data
    of tuples about the legislation, of the form

    ('Document ID', 'Document URL', 'Type', 'Status', 'Introduction Date'
     'Passed Date', 'Main Sponsor', 'Title')
    """
    table = soup.find('table', id='ctl00_ContentPlaceHolder1_gridMain_ctl00')
    for legislation, headers, row in self.parseDataTable(table):
      # Do legislation search-specific stuff
      # ------------------------------------
      # First column should be the ID of the record.
      id_key = headers[0]
      legislation_id = legislation[id_key]['label']
      legislation_url = legislation[id_key]['url']
      legislation[id_key] = legislation_id<commit_msg>some debugging code around<commit_after>import urllib
import urllib2
from BeautifulSoup import BeautifulSoup
import re
import time
import datetime
import mechanize
from collections import defaultdict
import slate
import cStringIO

class LegistarScraper (object):
  """
  A programatic interface onto a hosted Legistar website.
  """

  def __init__(self, config):
    """
    Construct a new Legistar scraper.  Pass in the host name of the Legistar
    instance, e.g. 'phila.legistar.com'.
    """
    self.config = config
    self.host = 'http://%s/' % self.config['hostname']
    self.fulltext = self.config['fulltext']

    # Assume that the legislation and calendar URLs are constructed regularly.
    self._legislation_uri = (
      self.host + self.config.get('legislation_path', 'Legislation.aspx'))
    self._calendar_uri = (
      self.host + self.config.get('calendar_path', 'Calendar.aspx'))
    self._people_uri = (
      self.host + self.config.get('people_path', 'People.aspx'))
    
  def searchLegislation(self, search_text, created_before=None, created_after=None, num_pages = None):
    """
    Submit a search query on the legislation search page, and return a list
    of summary results.
    """
    br = self._get_new_browser()
    br.open(self._legislation_uri)

    try:
      # Check for the link to the advanced search form
      br.find_link(text_regex='Advanced.*')

    except mechanize.LinkNotFoundError:
      # If it's not there, then we're already on the advanced search form.
      pass

    else:
      # If it is there, the navigate to the advanced search form.
      br.select_form('aspnetForm')
      data = self._data(br.form, None)
      data['ctl00$ContentPlaceHolder1$btnSwitch'] = ''
      data = urllib.urlencode(data)

      br.open(self._legislation_uri, data)

    br.select_form('aspnetForm')
    br.form.set_all_readonly(False)

    # Enter the search parameters
    # TODO: Each of the possible form fields should be represented as keyword
    # arguments to this function. The default query string should be for the
    # the default 'Legislative text' field.
    br.form['ctl00$ContentPlaceHolder1$txtText'] = search_text


    if created_before or created_after:
      if created_before :
        creation_date = created_before
        relation = '<'
      else:
        creation_date = created_after
        relation = '>'

      br.form['ctl00$ContentPlaceHolder1$radFileCreated'] = [relation]
      br.form['ctl00_ContentPlaceHolder1_txtFileCreated1_dateInput_ClientState'] = '{"enabled":true,"emptyMessage":"","validationText":"%s-00-00-00","valueAsString":"%s-00-00-00","minDateStr":"1980-01-01-00-00-00","maxDateStr":"2099-12-31-00-00-00"}' % (creation_date, creation_date)


    # Submit the form
    data = self._data(br.form, None)

    # Return up to one million search results
    data['ctl00_ContentPlaceHolder1_lstMax_ClientState'] = '{"value":"1000000"}'

    data['ctl00$ContentPlaceHolder1$btnSearch'] = 'Search Legislation'
    data['ctl00_ContentPlaceHolder1_lstYearsAdvanced_ClientState'] = '{"logEntries":[],"value":"All","text":"All Years","enabled":true,"checkedIndices":[],"checkedItemsTextOverflows":false}'

    data = urllib.urlencode(data)

    response = _try_connect(br, self._legislation_uri, data)

    # Loop through the pages, yielding each of the results
    all_results = False
    while all_results is False :
      soup = BeautifulSoup(response.read())

      legislation_iter = self.parseSearchResults(soup)
      for legislation in legislation_iter:
        yield legislation


      current_page = soup.fetch('a', {'class': 'rgCurrentPage'})
      if current_page :
        current_page = current_page[0]
        print 'page', current_page.text
        print
        next_page = current_page.findNextSibling('a')
      else :
        next_page = None

      if next_page :
        event_target = next_page['href'].split("'")[1]

        br.select_form('aspnetForm')
        data = self._data(br.form, event_target)
        data = urllib.urlencode(data)
        response = _try_connect(br, self._legislation_uri, data)

      else :
        all_results = True

    raise StopIteration

  def parseSearchResults(self, soup) :
    """Take a page of search results and return a sequence of data
    of tuples about the legislation, of the form

    ('Document ID', 'Document URL', 'Type', 'Status', 'Introduction Date'
     'Passed Date', 'Main Sponsor', 'Title')
    """
    table = soup.find('table', id='ctl00_ContentPlaceHolder1_gridMain_ctl00')
    for legislation, headers, row in self.parseDataTable(table):
      # Do legislation search-specific stuff
      # ------------------------------------
      # First column should be the ID of the record.
      id_key = headers[0]
      try:
        legislation_id = legislation[id_key]['label']
      except:
        print legislation
        raise
      legislation_url = legislation[id_key]['url']
      legislation[id_key] = legislation_id
{'fraction_non_alphanumeric': 0.11567239635995956, 'fraction_numerical': 0.01577350859453994, 'mean_word_length': 3.3173286774334354, 'pattern_counts': {'":': 26, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1734213', 'n_tokens_mistral': 3042, 'n_tokens_neox': 2898, 'n_words': 739}