<commit_before>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin, unquote

from utils import utils, inspector, admin

# https://www.eac.gov/inspector-general/
archive = 2005

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
#

CONGRESSIONAL_TESTIMONY_URL = "https://www.eac.gov/inspector-general/congressional-testimony/"
HAVA_AUDITS_URL = "https://www.eac.gov/inspector-general/inspector-general/hava-fund-audits/"
EAC_AUDITS_URL = "https://www.eac.gov/inspector-general/audits-evaluations-of-eac/"
CONGRESSIONAL_REPORTS_URL = "https://www.eac.gov/inspector-general/congressional-reports/"
INVESTIGATIONS_URL = "https://www.eac.gov/inspector-general/investigation-reports/"
PEER_REVIEWS_URL = "https://www.eac.gov/inspector-general/peer-review-reports/"

REPORT_URLS = [
  ("testimony", CONGRESSIONAL_TESTIMONY_URL),
  ("audit", HAVA_AUDITS_URL),
  ("audit", EAC_AUDITS_URL),
  ("congress", CONGRESSIONAL_REPORTS_URL),
  ("investigation", INVESTIGATIONS_URL),
  ("peer_review", PEER_REVIEWS_URL),
]

REPORT_PUBLISHED_MAP = {
  "US-EAC-OIG-2016-Peer-Review-Report": datetime.datetime(2016, 7, 20),
  "USEAC-2016-Final-Financial-Audit-Report-I-PA-EAC-01-16": datetime.datetime(2016, 11, 10),
  "USEAC-2016-Final-FISMA-Report-I-PA-EAC-02-16": datetime.datetime(2016, 11, 10),
  "U.S.-Election-Assistance-Commissions-Financial-Statements-for-Fiscal-Years-2015-and-20141": datetime.datetime(2015, 11, 12),
  "Final-Report---EAC-Compliance-with-FISMA-20151": datetime.datetime(2015, 11, 13),
  "2014-financial-audit-web": datetime.datetime(2014, 11, 12),
  "2014-FISMA-issued-report": datetime.datetime(2014, 11, 10),<commit_msg>[eac] Add date<commit_after>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin, unquote

from utils import utils, inspector, admin

# https://www.eac.gov/inspector-general/
archive = 2005

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
#

CONGRESSIONAL_TESTIMONY_URL = "https://www.eac.gov/inspector-general/congressional-testimony/"
HAVA_AUDITS_URL = "https://www.eac.gov/inspector-general/inspector-general/hava-fund-audits/"
EAC_AUDITS_URL = "https://www.eac.gov/inspector-general/audits-evaluations-of-eac/"
CONGRESSIONAL_REPORTS_URL = "https://www.eac.gov/inspector-general/congressional-reports/"
INVESTIGATIONS_URL = "https://www.eac.gov/inspector-general/investigation-reports/"
PEER_REVIEWS_URL = "https://www.eac.gov/inspector-general/peer-review-reports/"

REPORT_URLS = [
  ("testimony", CONGRESSIONAL_TESTIMONY_URL),
  ("audit", HAVA_AUDITS_URL),
  ("audit", EAC_AUDITS_URL),
  ("congress", CONGRESSIONAL_REPORTS_URL),
  ("investigation", INVESTIGATIONS_URL),
  ("peer_review", PEER_REVIEWS_URL),
]

REPORT_PUBLISHED_MAP = {
  "OIG-Semi-Annual-Report-Letter-5-26-2017-MM-signed-ocr": datetime.datetime(2017, 5, 26),
  "US-EAC-OIG-2016-Peer-Review-Report": datetime.datetime(2016, 7, 20),
  "USEAC-2016-Final-Financial-Audit-Report-I-PA-EAC-01-16": datetime.datetime(2016, 11, 10),
  "USEAC-2016-Final-FISMA-Report-I-PA-EAC-02-16": datetime.datetime(2016, 11, 10),
  "U.S.-Election-Assistance-Commissions-Financial-Statements-for-Fiscal-Years-2015-and-20141": datetime.datetime(2015, 11, 12),
  "Final-Report---EAC-Compliance-with-FISMA-20151": datetime.datetime(2015, 11, 13),
  "2014-financial-audit-web": datetime.datetime(2014, 11, 12),
  "2014-FISMA-issued-report": datetime.datetime(2014, 11, 10),
{'fraction_non_alphanumeric': 0.1585297418630752, 'fraction_numerical': 0.06060606060606061, 'mean_word_length': 5.189236111111111, 'pattern_counts': {'":': 15, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 14, 'lorem ipsum': 0, 'www.': 14, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3662404', 'n_tokens_mistral': 1599, 'n_tokens_neox': 1350, 'n_words': 193}