<commit_before>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin

from utils import utils, inspector

# https://www.fca.gov/home/inspector.html
archive = 1989

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
#

REPORTS_URL = "https://www.fca.gov/home/inspector/ig_reports.html"
REPORT_ARCHIVE_URL = "https://www.fca.gov/home/inspector/oigarchive.html"
SEMIANNUAL_REPORTS_URL = "https://www.fca.gov/home/inspector/semiannual.html"

REPORT_PUBLISHED_MAPPING = {
  "RegulatoryDevelopmentSurvey2002": datetime.datetime(2002, 10, 22),
}

def run(options):
  year_range = inspector.year_range(options, archive)

  # Pull the general reports
  doc = utils.beautifulsoup_from_url(REPORTS_URL)
  results = doc.select("div#mainContent li.mainContenttext a")
  if not results:
    raise inspector.NoReportsFoundError("Farm Credit Administration (reports)")
  for result in results:
    report = report_from(result, REPORTS_URL, year_range)
    if report:
      inspector.save_report(report)

  # Pull the archive reports
  doc = utils.beautifulsoup_from_url(REPORT_ARCHIVE_URL)
  results = doc.select("div#mainContent li.mainContenttext a") + doc.select("div#mainContent span.mainContenttext a")
  if not results:
    raise inspector.NoReportsFoundError("Farm Credit Administration (archive)")
  for result in results:
    if not result.text:
      continue
    report = report_from(result, REPORT_ARCHIVE_URL, year_range)
    if report:
      inspector.save_report(report)

  # Pull the semiannual reports
  doc = utils.beautifulsoup_from_url(SEMIANNUAL_REPORTS_URL)
  results = doc.select("div#mainContent li.mainContenttext a")
  if not results:
    raise inspector.NoReportsFoundError("Farm Credit Administration (semiannual reports)")
  for result in results:
    report = semiannual_report_from(result, year_range)
    if report:
      inspector.save_report(report)

def clean_text(text):
  return text.replace("Â\xa0", " ").strip()

def type_for_report(text):
  if "Audit" in text:
    return "audit"
  elif "Semiannual Report" in text:
    return "semiannual_report"
  elif "Inspection" in text:
    return "inspection"
  elif "FISMA" in text:
    return "fisma"
  elif "Peer Review" in text:
    return "peer_review"
  elif "Survey" in text:
    return "research"
  elif "Performance" in text:
    return "performance"
  else:
    return "other"

def report_from(result, landing_url, year_range):
  report_url = urljoin(landing_url, result.get('href'))

  if landing_url + '#' in report_url:
    # These are just anchor links, skip them.
    return

  if result.find_parent("ul") and result.find_parent("ul").get('type') == 'disc':
    # These are just anchor links, skip them.
    return

  title = clean_text(result.text)

  if title == 'Inspector General Reports':
    # Just a return link to the main IG page
    return

  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)

  estimated_date = False
  if report_id in REPORT_PUBLISHED_MAPPING:
    published_on = REPORT_PUBLISHED_MAPPING[report_id]
  else:
    try:
      published_on_text = result.parent.contents[1].lstrip(",").split("(")[0].strip()
    except (IndexError, TypeError):
      published_on_text = result.text.strip()
    published_on_text = clean_text(published_on_text)
<commit_msg>[fca] Fix finding date text<commit_after>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin

from utils import utils, inspector

# https://www.fca.gov/home/inspector.html
archive = 1989

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
#

REPORTS_URL = "https://www.fca.gov/home/inspector/ig_reports.html"
REPORT_ARCHIVE_URL = "https://www.fca.gov/home/inspector/oigarchive.html"
SEMIANNUAL_REPORTS_URL = "https://www.fca.gov/home/inspector/semiannual.html"

REPORT_PUBLISHED_MAPPING = {
  "RegulatoryDevelopmentSurvey2002": datetime.datetime(2002, 10, 22),
}

def run(options):
  year_range = inspector.year_range(options, archive)

  # Pull the general reports
  doc = utils.beautifulsoup_from_url(REPORTS_URL)
  results = doc.select("div#mainContent li.mainContenttext a")
  if not results:
    raise inspector.NoReportsFoundError("Farm Credit Administration (reports)")
  for result in results:
    report = report_from(result, REPORTS_URL, year_range)
    if report:
      inspector.save_report(report)

  # Pull the archive reports
  doc = utils.beautifulsoup_from_url(REPORT_ARCHIVE_URL)
  results = doc.select("div#mainContent li.mainContenttext a") + doc.select("div#mainContent span.mainContenttext a")
  if not results:
    raise inspector.NoReportsFoundError("Farm Credit Administration (archive)")
  for result in results:
    if not result.text:
      continue
    report = report_from(result, REPORT_ARCHIVE_URL, year_range)
    if report:
      inspector.save_report(report)

  # Pull the semiannual reports
  doc = utils.beautifulsoup_from_url(SEMIANNUAL_REPORTS_URL)
  results = doc.select("div#mainContent li.mainContenttext a")
  if not results:
    raise inspector.NoReportsFoundError("Farm Credit Administration (semiannual reports)")
  for result in results:
    report = semiannual_report_from(result, year_range)
    if report:
      inspector.save_report(report)

def clean_text(text):
  return text.replace("Â\xa0", " ").strip()

def type_for_report(text):
  if "Audit" in text:
    return "audit"
  elif "Semiannual Report" in text:
    return "semiannual_report"
  elif "Inspection" in text:
    return "inspection"
  elif "FISMA" in text:
    return "fisma"
  elif "Peer Review" in text:
    return "peer_review"
  elif "Survey" in text:
    return "research"
  elif "Performance" in text:
    return "performance"
  else:
    return "other"

def report_from(result, landing_url, year_range):
  report_url = urljoin(landing_url, result.get('href'))

  if landing_url + '#' in report_url:
    # These are just anchor links, skip them.
    return

  if result.find_parent("ul") and result.find_parent("ul").get('type') == 'disc':
    # These are just anchor links, skip them.
    return

  title = clean_text(result.text)

  if title == 'Inspector General Reports':
    # Just a return link to the main IG page
    return

  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)

  estimated_date = False
  if report_id in REPORT_PUBLISHED_MAPPING:
    published_on = REPORT_PUBLISHED_MAPPING[report_id]
  else:
    try:
      li = result.parent
      if li.name == "u":
        li = li.parent
      published_on_text = li.contents[1].lstrip(",").split("(")[0].strip()
    except (IndexError, TypeError):
      published_on_text = result.text.strip()
    published_on_text = clean_text(published_on_text)

{'fraction_non_alphanumeric': 0.10487380330722368, 'fraction_numerical': 0.005802146794313896, 'mean_word_length': 4.243346007604563, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7103682', 'n_tokens_mistral': 2394, 'n_tokens_neox': 2280, 'n_words': 559}