<commit_before>import re, logging, datetime, time, json
from lxml import etree

import utils

from bill_info import sponsor_for, actions_for


# downloads amendment information from THOMAS.gov, 
# parses out basic information, writes JSON to disk

def fetch_amendment(amendment_id, options):
  logging.info("\n[%s] Fetching..." % amendment_id)
  
  body = utils.download(
    amendment_url_for(amendment_id), 
    amendment_cache_for(amendment_id, "information.html"),
    options)

  if not body:
    return {'saved': False, 'ok': False, 'reason': "failed to download"}

  if options.get("download_only", False):
    return {'saved': False, 'ok': True, 'reason': "requested download only"}
    
  amendment_type, number, congress = utils.split_bill_id(amendment_id)
  
  actions = actions_for(body, amendment_id, is_amendment=True)
  if actions is None: actions = []
  parse_amendment_actions(actions)

  amdt = {
    'amendment_id': amendment_id,
    'amendment_type': amendment_type,
    'chamber': amendment_type[0],
    'number': number,
    'congress': congress,
    
    'amends': amends_for(body, grab_bill=False),
    'amends_bill': amends_for(body, grab_bill=True),

    'offered_at': offered_at_for(body, 'offered'),
    'submitted_at': offered_at_for(body, 'submitted'),
    'proposed_at': offered_at_for(body, 'proposed'),
    'sponsor': sponsor_for(body),

    'title': amendment_simple_text_for(body, "title"),
    'description': amendment_simple_text_for(body, "description"),
    'purpose': amendment_simple_text_for(body, "purpose"),
    
    'actions': actions,
    
    'updated_at': datetime.datetime.fromtimestamp(time.time()),
  }

  amdt['status'], amdt['status_at'] = amendment_status_for(amdt)

  # only set a house_number if it's a House bill -
  # this lets us choke if it's not found.
  if amdt['chamber'] == 'h':
    amdt['house_number'] = house_number_for(body)
  
  output_amendment(amdt, options)

  return {'ok': True, 'saved': True}

def output_amendment(amdt, options):
  logging.info("[%s] Writing to disk..." % amdt['amendment_id'])

  # output JSON - so easy!
  utils.write(
    json.dumps(amdt, sort_keys=True, indent=2, default=utils.format_datetime), 
    output_for_amdt(amdt['amendment_id'], "json")
  )

  # output XML
  govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' }
  root = etree.Element("amendment")
  root.set("session", amdt['congress'])
  root.set("chamber", amdt['amendment_type'][0])
  root.set("number", amdt['number'])
  root.set("updated", utils.format_datetime(amdt['updated_at']))
  
  make_node = utils.make_node
  
  make_node(root, "amends", None,
    type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]],
    number=str(amdt["amends_bill"]["number"]),
    sequence=str(int(amdt["house_number"][1:])) if amdt.get("house_number", None) else "") # chop off A from the house_number
  
  make_node(root, "status", amdt['status'], datetime=amdt['status_at'])

  if amdt['sponsor'] and amdt['sponsor']['type'] == 'person':
    v = amdt['sponsor']['thomas_id']
    if not options.get("govtrack", False):
      make_node(root, "sponsor", None, thomas_id=v)
    else:
      v = str(utils.get_govtrack_person_id('thomas', v))
      make_node(root, "sponsor", None, id=v)
  elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee':
    make_node(root, "sponsor", None, committee=amdt['sponsor']['name'])
  else:
    make_node(root, "sponsor", None)

  make_node(root, "offered", None, datetime=amdt['offered_at'] if amdt['offered_at'] else amdt['submitted_at'])
      
  if amdt["title"]: make_node(root, "title", amdt["title"])
  make_node(root, "description", amdt["description"] if amdt["description"] else amdt["purpose"])
  make_node(root, "purpose", amdt["purpose"])
      
  actions = make_node(root, "actions", None)
  for action in amdt['actions']:
      a = make_node(actions,
        action['type'] if action['type'] in ("vote",) else "action",
        None,
        datetime=action['acted_at'])
      if action['type'] == 'vote':
        a.set("how", action["how"])
        a.set("result", action["result"])
        if action.get("roll") != None: a.set("roll", str(action["roll"]))
      if action.get('text'): make_node(a, "text", action['text'])
      if action.get('in_committee'): make_node(a, "committee", None, name=action['in_committee'])
      for cr in action['references']:
          make_node(a, "reference", None, ref=cr['reference'], label=cr['type'])
          
  utils.write(
    etree.tostring(root, pretty_print=True),
    output_for_amdt(amdt['amendment_id'], "xml")
  )


# assumes this is a House amendment, and it should choke if it doesn't find a number
def house_number_for(body):
  match = re.search(r"H.AMDT.\d+</b>\n \((A\d+)\)", body, re.I)
  if match:
    return match.group(1)
  else:
    raise Exception("Choked finding House amendment number.")
    
def amends_for(body, grab_bill):
  # When an amendment amends an amendment, the bill is listed first, followed by a comma
  # and newline. Skip the bill when it exists and just parse the amendment.
  match = re.search(r"Amends: "
      + ("(?:.*\n, )?" if not grab_bill else "")
      + "<a href=\"/cgi-bin/bdquery/z\?d(\d+):([A-Z]+)(\d+):",
      body)
  if match:
    congress = int(match.group(1))<commit_msg>Lose the A on house amendment numbers and store as an integer, store regular number as an integer<commit_after>import re, logging, datetime, time, json
from lxml import etree

import utils

from bill_info import sponsor_for, actions_for


# downloads amendment information from THOMAS.gov, 
# parses out basic information, writes JSON to disk

def fetch_amendment(amendment_id, options):
  logging.info("\n[%s] Fetching..." % amendment_id)
  
  body = utils.download(
    amendment_url_for(amendment_id), 
    amendment_cache_for(amendment_id, "information.html"),
    options)

  if not body:
    return {'saved': False, 'ok': False, 'reason': "failed to download"}

  if options.get("download_only", False):
    return {'saved': False, 'ok': True, 'reason': "requested download only"}
    
  amendment_type, number, congress = utils.split_bill_id(amendment_id)
  
  actions = actions_for(body, amendment_id, is_amendment=True)
  if actions is None: actions = []
  parse_amendment_actions(actions)

  amdt = {
    'amendment_id': amendment_id,
    'amendment_type': amendment_type,
    'chamber': amendment_type[0],
    'number': int(number),
    'congress': congress,
    
    'amends': amends_for(body, grab_bill=False),
    'amends_bill': amends_for(body, grab_bill=True),

    'offered_at': offered_at_for(body, 'offered'),
    'submitted_at': offered_at_for(body, 'submitted'),
    'proposed_at': offered_at_for(body, 'proposed'),
    'sponsor': sponsor_for(body),

    'title': amendment_simple_text_for(body, "title"),
    'description': amendment_simple_text_for(body, "description"),
    'purpose': amendment_simple_text_for(body, "purpose"),
    
    'actions': actions,
    
    'updated_at': datetime.datetime.fromtimestamp(time.time()),
  }

  amdt['status'], amdt['status_at'] = amendment_status_for(amdt)

  # only set a house_number if it's a House bill -
  # this lets us choke if it's not found.
  if amdt['chamber'] == 'h':
    amdt['house_number'] = house_number_for(body)
  
  output_amendment(amdt, options)

  return {'ok': True, 'saved': True}

def output_amendment(amdt, options):
  logging.info("[%s] Writing to disk..." % amdt['amendment_id'])

  # output JSON - so easy!
  utils.write(
    json.dumps(amdt, sort_keys=True, indent=2, default=utils.format_datetime), 
    output_for_amdt(amdt['amendment_id'], "json")
  )

  # output XML
  govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' }
  root = etree.Element("amendment")
  root.set("session", amdt['congress'])
  root.set("chamber", amdt['amendment_type'][0])
  root.set("number", str(amdt['number']))
  root.set("updated", utils.format_datetime(amdt['updated_at']))
  
  make_node = utils.make_node
  
  make_node(root, "amends", None,
    type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]],
    number=str(amdt["amends_bill"]["number"]),
    sequence=str(amdt["house_number"]) if amdt.get("house_number", None) else "")
  
  make_node(root, "status", amdt['status'], datetime=amdt['status_at'])

  if amdt['sponsor'] and amdt['sponsor']['type'] == 'person':
    v = amdt['sponsor']['thomas_id']
    if not options.get("govtrack", False):
      make_node(root, "sponsor", None, thomas_id=v)
    else:
      v = str(utils.get_govtrack_person_id('thomas', v))
      make_node(root, "sponsor", None, id=v)
  elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee':
    make_node(root, "sponsor", None, committee=amdt['sponsor']['name'])
  else:
    make_node(root, "sponsor", None)

  make_node(root, "offered", None, datetime=amdt['offered_at'] if amdt['offered_at'] else amdt['submitted_at'])
      
  if amdt["title"]: make_node(root, "title", amdt["title"])
  make_node(root, "description", amdt["description"] if amdt["description"] else amdt["purpose"])
  make_node(root, "purpose", amdt["purpose"])
      
  actions = make_node(root, "actions", None)
  for action in amdt['actions']:
      a = make_node(actions,
        action['type'] if action['type'] in ("vote",) else "action",
        None,
        datetime=action['acted_at'])
      if action['type'] == 'vote':
        a.set("how", action["how"])
        a.set("result", action["result"])
        if action.get("roll") != None: a.set("roll", str(action["roll"]))
      if action.get('text'): make_node(a, "text", action['text'])
      if action.get('in_committee'): make_node(a, "committee", None, name=action['in_committee'])
      for cr in action['references']:
          make_node(a, "reference", None, ref=cr['reference'], label=cr['type'])
          
  utils.write(
    etree.tostring(root, pretty_print=True),
    output_for_amdt(amdt['amendment_id'], "xml")
  )


# assumes this is a House amendment, and it should choke if it doesn't find a number
def house_number_for(body):
  match = re.search(r"H.AMDT.\d+</b>\n \(A(\d+)\)", body, re.I)
  if match:
    return int(match.group(1))
  else:
    raise Exception("Choked finding House amendment number.")
    
def amends_for(body, grab_bill):
  # When an amendment amends an amendment, the bill is listed first, followed by a comma
  # and newline. Skip the bill when it exists and just parse the amendment.
  match = re.search(r"Amends: "
      + ("(?:.*\n, )?" if not grab_bill else "")
      + "<a href=\"/cgi-bin/bdquery/z\?d(\d+):([A-Z]+)(\d+):",
      body)
  if match:
    congress = int(match.group(1))
{'fraction_non_alphanumeric': 0.16993039443155453, 'fraction_numerical': 0.0010208816705336427, 'mean_word_length': 4.401503759398496, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4857729', 'n_tokens_mistral': 3841, 'n_tokens_neox': 3721, 'n_words': 844}