<commit_before>import utils
from utils import log
import re
import json
from lxml import etree
import time, datetime
from lxml.html import fromstring

# can be run on its own, just require a bill_id
def run(options):
  bill_id = options.get('bill_id', None)
  
  if bill_id:
    result = fetch_bill(bill_id, options)
    log("\n%s" % result)
  else:
    log("To run this task directly, supply a bill_id.")


# download and cache landing page for bill
# can raise an exception under various conditions
def fetch_bill(bill_id, options):
  log("\n[%s] Fetching..." % bill_id)


  body = utils.download(
    bill_url_for(bill_id), 
    bill_cache_for(bill_id, "information.html"),
    options.get('force', False))

  body = utils.unescape(body)


  if options.get("download_only", False):
    return {'saved': False, 'ok': True, 'reason': "requested download only"}

  # two conditions where we want to parse the bill from multiple pages instead of one:
  # 1) the all info page is truncated (~5-10 bills a congress)
  #     e.g. s1867-112, hr2112-112, s3240-112
  # 2) there are > 150 amendments, use undocumented amendments list (~5-10 bills a congress)
  #     e.g. hr3590-111, sconres13-111, s3240-112
  if "</html>" not in body:
    log("[%s] Main page truncated, fetching many pages..." % bill_id)
    bill = parse_bill_split(bill_id, body, options)
  elif too_many_amendments(body):
    log("[%s] Too many amendments, fetching many pages..." % bill_id)
    bill = parse_bill_split(bill_id, body, options)
  elif options.get('force_split', False):
    log("[%s] Forcing a split, fetching many pages..." % bill_id)<commit_msg>Skip un-downloaded bills and mark them as such<commit_after>import utils
from utils import log
import re
import json
from lxml import etree
import time, datetime
from lxml.html import fromstring

# can be run on its own, just require a bill_id
def run(options):
  bill_id = options.get('bill_id', None)
  
  if bill_id:
    result = fetch_bill(bill_id, options)
    log("\n%s" % result)
  else:
    log("To run this task directly, supply a bill_id.")


# download and cache landing page for bill
# can raise an exception under various conditions
def fetch_bill(bill_id, options):
  log("\n[%s] Fetching..." % bill_id)


  body = utils.download(
    bill_url_for(bill_id), 
    bill_cache_for(bill_id, "information.html"),
    options.get('force', False))

  if not body:
    return {'saved': False, 'ok': False, 'reason': "failed to download"}

  body = utils.unescape(body)


  if options.get("download_only", False):
    return {'saved': False, 'ok': True, 'reason': "requested download only"}

  # two conditions where we want to parse the bill from multiple pages instead of one:
  # 1) the all info page is truncated (~5-10 bills a congress)
  #     e.g. s1867-112, hr2112-112, s3240-112
  # 2) there are > 150 amendments, use undocumented amendments list (~5-10 bills a congress)
  #     e.g. hr3590-111, sconres13-111, s3240-112
  if "</html>" not in body:
    log("[%s] Main page truncated, fetching many pages..." % bill_id)
    bill = parse_bill_split(bill_id, body, options)
  elif too_many_amendments(body):
    log("[%s] Too many amendments, fetching many pages..." % bill_id)
    bill = parse_bill_split(bill_id, body, options)
  elif options.get('force_split', False):
    log("[%s] Forcing a split, fetching many pages..." % bill_id)
{'fraction_non_alphanumeric': 0.11996445497630331, 'fraction_numerical': 0.030213270142180094, 'mean_word_length': 3.8242857142857143, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4509766', 'n_tokens_mistral': 1209, 'n_tokens_neox': 1104, 'n_words': 382}