<commit_before>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin

from bs4 import BeautifulSoup
from utils import utils, inspector

# http://arts.gov/oig
archive = 2005

# options:
#   standard since/year options for a year range to fetch from.
#   report_id: only bother to process a single report
#
# Notes for IG's web team:
#

AUDIT_REPORTS_URL = "http://arts.gov/oig/reports/audits"
SPECIAL_REVIEWS_URL = "http://arts.gov/oig/reports/specials"
SEMIANNUAL_REPORTS_URL = "http://arts.gov/oig/reports/semi-annual"
PEER_REVIEWS_URL = "http://arts.gov/oig/reports/external-peer-reviews"
FISMA_REPORTS_URL = "http://arts.gov/oig/reports/fisma"

REPORT_URLS = {
  "audit": AUDIT_REPORTS_URL,
  "evaluation": SPECIAL_REVIEWS_URL,
  "semiannual_report": SEMIANNUAL_REPORTS_URL,
  "peer_review": PEER_REVIEWS_URL,
  "fisma": FISMA_REPORTS_URL,
}

def run(options):
  year_range = inspector.year_range(options, archive)

  only_report_id = options.get('report_id')

  # Pull the reports
  for report_type, url in REPORT_URLS.items():
    doc = BeautifulSoup(utils.download(url))
    results = doc.select("div.field-item li")
    for result in results:
      report = report_from(result, url, report_type, year_range)

      if report:
        # debugging convenience: can limit to single report
        if only_report_id and (report['report_id'] != only_report_id):
          continue

        inspector.save_report(report)

def report_from(result, landing_url, report_type, year_range):
  link = result.find("a")
  if not link:
    return
<commit_msg>[nea] remove fisma page scraping for now, see #186<commit_after>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin

from bs4 import BeautifulSoup
from utils import utils, inspector

# http://arts.gov/oig
archive = 2005

# options:
#   standard since/year options for a year range to fetch from.
#   report_id: only bother to process a single report
#
# Notes for IG's web team:
#

AUDIT_REPORTS_URL = "http://arts.gov/oig/reports/audits"
SPECIAL_REVIEWS_URL = "http://arts.gov/oig/reports/specials"
SEMIANNUAL_REPORTS_URL = "http://arts.gov/oig/reports/semi-annual"
PEER_REVIEWS_URL = "http://arts.gov/oig/reports/external-peer-reviews"

# NEA took this section down sometime in
# FISMA_REPORTS_URL = "http://arts.gov/oig/reports/fisma"

REPORT_URLS = {
  "audit": AUDIT_REPORTS_URL,
  "evaluation": SPECIAL_REVIEWS_URL,
  "semiannual_report": SEMIANNUAL_REPORTS_URL,
  "peer_review": PEER_REVIEWS_URL,
  # "fisma": FISMA_REPORTS_URL,
}

def run(options):
  year_range = inspector.year_range(options, archive)

  only_report_id = options.get('report_id')

  # Pull the reports
  for report_type, url in REPORT_URLS.items():
    doc = BeautifulSoup(utils.download(url))
    results = doc.select("div.field-item li")
    for result in results:
      report = report_from(result, url, report_type, year_range)

      if report:
        # debugging convenience: can limit to single report
        if only_report_id and (report['report_id'] != only_report_id):
          continue

        inspector.save_report(report)

def report_from(result, landing_url, report_type, year_range):
  link = result.find("a")
  if not link:
    return

{'fraction_non_alphanumeric': 0.10793554271815141, 'fraction_numerical': 0.003952569169960474, 'mean_word_length': 4.238853503184713, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '609152', 'n_tokens_mistral': 1210, 'n_tokens_neox': 1159, 'n_words': 269}