#!/usr/bin/env python

import datetime
import logging
import os

from bs4 import BeautifulSoup
from utils import utils, inspector

# http://www.peacecorps.gov/about/inspgen/
# Oldest report: 1989

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
#

REPORTS_URL = "http://www.peacecorps.gov/about/inspgen/reports/"

SECTION_TITLES_TO_SKIP = [
  "Management Advisory Reports",
  "Peer Reviews",
  "Special Reports/Reviews",
]

def run(options):
  year_range = inspector.year_range(options)

  # Pull the reports
  doc = BeautifulSoup(utils.download(REPORTS_URL))
  results = doc.select("li div li")
  for result in results:
    report = report_from(result, year_range)
    if report:
      inspector.save_report(report)

def report_from(result, year_range):
  link = result.find("a")
  report_url = link.get('href')
  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)
  title = link.text

  section_title = result.find_previous("h3").text.strip()
  if section_title in SECTION_TITLES_TO_SKIP:
    return

  estimated_date = False
  try:
    published_on_text = title.split("–")[-1].strip()
    published_on = datetime.datetime.strptime(published_on_text, '%B %d, %Y')
  except ValueError:
    # For reports where we can only find the year, set them to Nov 1st of that year
    published_on_year =int(section_title.lstrip("FY "))
    published_on = datetime.datetime(published_on_year, 11, 1)
    estimated_date = True

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'peacecorps',
    'inspector_url': 'http://www.peacecorps.gov/about/inspgen/',
    'agency': 'peacecorps',
    'agency_name': 'Peace Corps',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  if estimated_date:
    report['estimated_date'] = estimated_date
  return report

utils.run(run) if (__name__ == "__main__") else None

#!/usr/bin/env python

import datetime
import logging
import os

from bs4 import BeautifulSoup
from utils import utils, inspector

# http://www.peacecorps.gov/about/inspgen/
# Oldest report: 1989

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
#

REPORTS_URL = "http://www.peacecorps.gov/about/inspgen/reports/"

REPORT_PUBLISHED_MAPPING = {
  "Burkina_Faso_Medical_Supply_Management_Advisory_Report": datetime.datetime(2013, 3, 14),
  "PCIG_Final_MAR_Certification_of_Volunteer_Payments": datetime.datetime(2013, 9, 24),
  "MAR_Cost_Savings_Opportunity_on_Value_Added_Tax": datetime.datetime(2013, 2, 13),
  "Management_Advisory_Report-Peace_Corps_Drug_Free_Workplace_Program": datetime.datetime(2012, 8, 16),
  "PCIG_2014_Peace_Corps_OIG_Peer_Review_Final": datetime.datetime(2014, 3, 27),
  "MAR_Sierra_Leone": datetime.datetime(2013, 3, 14),
  "PC_Morocco_Assessment_of_Medical_Care": datetime.datetime(2010, 2, 1),
  "PC_Recurring_Issues_OIG_Post_Audits_Evaluations_FYs_2009-2011": datetime.datetime(2012, 4, 1),
  "PC_Vanuatu_SR_Advice_and_Assistance": datetime.datetime(2010, 5, 1),
  "PC_Gambia_SR_Grant_Activities": datetime.datetime(2010, 5, 14),
  "PC_Ecuador_Special_Review_IG1005SR": datetime.datetime(2010, 9, 1),
}

def run(options):
  year_range = inspector.year_range(options)

  # Pull the reports
  doc = BeautifulSoup(utils.download(REPORTS_URL))
  results = doc.select("li div li")
  for result in results:
    report = report_from(result, year_range)
    if report:
      inspector.save_report(report)

def report_from(result, year_range):
  link = result.find("a")
  report_url = link.get('href')
  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)
  title = link.text

  section_title = result.find_previous("h3").text.strip()

  estimated_date = False
  if report_id in REPORT_PUBLISHED_MAPPING:
    published_on = REPORT_PUBLISHED_MAPPING[report_id]
  else:
    try:
      published_on_text = title.split("–")[-1].strip()
      published_on = datetime.datetime.strptime(published_on_text, '%B %d, %Y')
    except ValueError:
      # For reports where we can only find the year, set them to Nov 1st of that year
      published_on_year =int(section_title.lstrip("FY "))
      published_on = datetime.datetime(published_on_year, 11, 1)
      estimated_date = True

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'peacecorps',
    'inspector_url': 'http://www.peacecorps.gov/about/inspgen/',
    'agency': 'peacecorps',
    'agency_name': 'Peace Corps',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  if estimated_date:
    report['estimated_date'] = estimated_date
  return report

utils.run(run) if (__name__ == "__main__") else None

Add additional reports.