<commit_before>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin

from utils import utils, inspector

# http://www.fmc.gov/bureaus_offices/office_of_inspector_general.aspx
archive = 2005

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
# - The link to http://www.fmc.gov/userfiles/pages/file/OR10-03_UserFeeCalculations.pdf
# is incorrect. See `REPORT_URL_MAPPING`
# - Fix all links in `BLACKLIST_REPORT_URLS`

AUDIT_REPORTS_URL = "http://www.fmc.gov/about/oig_audits_evaluations_and_reviews.aspx"
SEMIANNUAL_REPORTS_URL = "http://www.fmc.gov/about/oig_semiannual_reports.aspx"

REPORT_URL_MAPPING = {
  "http://www.fmc.gov/userfiles/pages/file/OR10-03_UserFeeCalculations.pdf":
  "http://www.fmc.gov/assets/1/Page/OR10-03_UserFeeCalculations.pdf",
}

BLACKLIST_REPORT_URLS = [
  "http://www.fmc.gov/UserFiles/pages/File/FY_08_Audited_Financial_Statements_09-01.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_09-01A.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Report_OR09-01.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-02.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-03.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-04.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Report_A09-05.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-06.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-07.pdf",
]

def run(options):
  year_range = inspector.year_range(options, archive)

  # Pull the audit reports
  doc = utils.beautifulsoup_from_url(AUDIT_REPORTS_URL)
  results = doc.select("table tr")
  if not results:
    raise inspector.NoReportsFoundError("Federal Maritime Commission (audits)")
  for index, result in enumerate(results):
    if not index:
      # Skip the header row
      continue
    report = report_from(result, AUDIT_REPORTS_URL, report_type='audit', year_range=year_range)
    if report:
      inspector.save_report(report)

  # Pull historical audits
  audit_year_links = doc.select("div.col-2-3 ul li a")
  for year_link in audit_year_links:
    audit_year_url = urljoin(AUDIT_REPORTS_URL, year_link.get('href'))
    doc = utils.beautifulsoup_from_url(audit_year_url)
    results = doc.select("table tr")
    if not results:
      # Grab results other than first and last (header and extra links)
      results = doc.select("div.col-2-2 ul")[1:-1]
    if not results:
      raise inspector.NoReportsFoundError("Federal Maritime Commission (%s)" % audit_year_url)
    for index, result in enumerate(results):
      if not index:
        # Skip the header row
        continue
      report = report_from(result, AUDIT_REPORTS_URL, report_type='audit', year_range=year_range)
      if report:
        inspector.save_report(report)

  # Pull the semiannual reports
  doc = utils.beautifulsoup_from_url(SEMIANNUAL_REPORTS_URL)
  results = doc.select("div.col-2-2 p a") + doc.select("div.col-2-2 li a")
  if not results:
    raise inspector.NoReportsFoundError("Federal Maritime Commission (semiannual reports)")
  for result in results:
    report = report_from(result.parent, AUDIT_REPORTS_URL, report_type='semiannual_report', year_range=year_range)
    if report:
      inspector.save_report(report)

def report_from(result, landing_url, report_type, year_range):
  link = result.find("a")
  report_url = urljoin(landing_url, link.get('href'))
  title = link.text
<commit_msg>[fmc] Handle header rows for second tables<commit_after>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin

from utils import utils, inspector

# http://www.fmc.gov/bureaus_offices/office_of_inspector_general.aspx
archive = 2005

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
# - The link to http://www.fmc.gov/userfiles/pages/file/OR10-03_UserFeeCalculations.pdf
# is incorrect. See `REPORT_URL_MAPPING`
# - Fix all links in `BLACKLIST_REPORT_URLS`

AUDIT_REPORTS_URL = "http://www.fmc.gov/about/oig_audits_evaluations_and_reviews.aspx"
SEMIANNUAL_REPORTS_URL = "http://www.fmc.gov/about/oig_semiannual_reports.aspx"

REPORT_URL_MAPPING = {
  "http://www.fmc.gov/userfiles/pages/file/OR10-03_UserFeeCalculations.pdf":
  "http://www.fmc.gov/assets/1/Page/OR10-03_UserFeeCalculations.pdf",
}

BLACKLIST_REPORT_URLS = [
  "http://www.fmc.gov/UserFiles/pages/File/FY_08_Audited_Financial_Statements_09-01.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_09-01A.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Report_OR09-01.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-02.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-03.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-04.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Report_A09-05.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-06.pdf",
  "http://www.fmc.gov/UserFiles/pages/File/OIG_Final_Report_A09-07.pdf",
]

def run(options):
  year_range = inspector.year_range(options, archive)

  # Pull the audit reports
  doc = utils.beautifulsoup_from_url(AUDIT_REPORTS_URL)
  results = doc.select("table tr")
  if not results:
    raise inspector.NoReportsFoundError("Federal Maritime Commission (audits)")
  for result in results:
    if result.th:
      # Skip the header row
      continue
    report = report_from(result, AUDIT_REPORTS_URL, report_type='audit', year_range=year_range)
    if report:
      inspector.save_report(report)

  # Pull historical audits
  audit_year_links = doc.select("div.col-2-3 ul li a")
  for year_link in audit_year_links:
    audit_year_url = urljoin(AUDIT_REPORTS_URL, year_link.get('href'))
    doc = utils.beautifulsoup_from_url(audit_year_url)
    results = doc.select("table tr")
    if not results:
      # Grab results other than first and last (header and extra links)
      results = doc.select("div.col-2-2 ul")[1:-1]
    if not results:
      raise inspector.NoReportsFoundError("Federal Maritime Commission (%s)" % audit_year_url)
    for result in results:
      if result.th:
        # Skip the header row
        continue
      report = report_from(result, AUDIT_REPORTS_URL, report_type='audit', year_range=year_range)
      if report:
        inspector.save_report(report)

  # Pull the semiannual reports
  doc = utils.beautifulsoup_from_url(SEMIANNUAL_REPORTS_URL)
  results = doc.select("div.col-2-2 p a") + doc.select("div.col-2-2 li a")
  if not results:
    raise inspector.NoReportsFoundError("Federal Maritime Commission (semiannual reports)")
  for result in results:
    report = report_from(result.parent, AUDIT_REPORTS_URL, report_type='semiannual_report', year_range=year_range)
    if report:
      inspector.save_report(report)

def report_from(result, landing_url, report_type, year_range):
  link = result.find("a")
  report_url = urljoin(landing_url, link.get('href'))
  title = link.text

{'fraction_non_alphanumeric': 0.11987025807361444, 'fraction_numerical': 0.01833309829361162, 'mean_word_length': 4.376800606520091, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 30, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1154095', 'n_tokens_mistral': 2712, 'n_tokens_neox': 2604, 'n_words': 450}