<commit_before>import os, os.path, errno, sys, traceback, subprocess
import re, html.entities
import json
import logging
import yaml
from bs4 import BeautifulSoup
from datetime import datetime
import requests
import urllib.parse
import io
import gzip
import certifi
import docx
import zipfile
from urllib.parse import urljoin
import inspect
import pdfrw

from . import admin

logging.getLogger("pdfrw").setLevel(logging.CRITICAL)

# scraper should be instantiated at class-load time, so that it can rate limit appropriately
import scrapelib
scraper = scrapelib.Scraper(requests_per_minute=120, retry_attempts=3)
scraper.user_agent = "unitedstates/inspectors-general (https://github.com/unitedstates/inspectors-general)"
scraper.timeout = 60

class Soft404HttpAdapter(requests.adapters.HTTPAdapter):
  """Transport adapter that checks all responses against a blacklist of "file
  not found" pages that are served with 200 status codes."""

  SOFT_404_URLS_RE = re.compile(r"^(http://www\.dodig\.mil/errorpages/index\.html|http://www\.fec\.gov/404error\.shtml|http://www\.gpo\.gov/maintenance/error\.htm)$")
  SOFT_404_BODY_SIGNATURES = {
    "cftc.gov": b"<title>404 Page Not Found - CFTC</title>",
    "cpb.org": b"<title>CPB: Page Not Found</title>",
    "ncua.gov": b"Redirect.aspx?404",
    "si.edu": b"<title>Page Not Found Smithsonian</title>",
  }

  def build_response(self, req, resp):
    domain = urllib.parse.urlparse(req.url)[1].split(':')[0]
    base_domain = ".".join(domain.split(".")[-2:])
    if base_domain in self.SOFT_404_BODY_SIGNATURES:
      if resp.getheader("Content-Type") in ["text/html; charset=utf-8",
                                            "text/html"]:
        data = resp.data
        headers = resp.headers
        if resp.getheader("Content-Encoding") == "gzip":
          decompressed_data = gzip.decompress(data)
        else:
          decompressed_data = data
        if resp.getheader("Transfer-Encoding") == "chunked":
          headers.pop("Transfer-Encoding")
        body = io.BytesIO(data)
        resp = requests.packages.urllib3.response.HTTPResponse(
                body=body,
                headers=headers,
                status=resp.status,
                version=resp.version,
                reason=resp.reason,
                strict=resp.strict,
                preload_content=False,
        )
        if decompressed_data.find(self.SOFT_404_BODY_SIGNATURES[base_domain], 0, 10240) != -1:
          result = super(Soft404HttpAdapter, self).build_response(req, resp)
          result.status_code = 404 # tells scrapelib to not retry
          return result

    redirect = resp.get_redirect_location()
    result = super(Soft404HttpAdapter, self).build_response(req, resp)
    if redirect and self.SOFT_404_URLS_RE.match(redirect):
      result.status_code = 404 # tells scrapelib to not retry

    return result

scraper.mount("http://www.cftc.gov/", Soft404HttpAdapter())
scraper.mount("http://cftc.gov/", Soft404HttpAdapter())
scraper.mount("http://www.dodig.mil/", Soft404HttpAdapter())
scraper.mount("http://dodig.mil/", Soft404HttpAdapter())
scraper.mount("http://www.fec.gov/", Soft404HttpAdapter())
scraper.mount("http://fec.gov/", Soft404HttpAdapter())
scraper.mount("http://www.gpo.gov/", Soft404HttpAdapter())
scraper.mount("http://gpo.gov/", Soft404HttpAdapter())
scraper.mount("http://www.cpb.org/", Soft404HttpAdapter())
scraper.mount("http://cpb.org/", Soft404HttpAdapter())
scraper.mount("http://www.ncua.gov/", Soft404HttpAdapter())
scraper.mount("http://ncua.gov/", Soft404HttpAdapter())
scraper.mount("http://www.si.edu/", Soft404HttpAdapter())
scraper.mount("http://si.edu/", Soft404HttpAdapter())

class CipherListAdapter(requests.adapters.HTTPAdapter):
  def __init__(self, ciphers):
    self.ciphers = ciphers
    super(CipherListAdapter, self).__init__()

  def init_poolmanager(self, num_pools, maxsize, block=False, *args, **kwargs):
    context = requests.packages.urllib3.util.ssl_.create_urllib3_context(
        ciphers=self.ciphers
    )
    kwargs["ssl_context"] = context
    self.poolmanager = requests.packages.urllib3.poolmanager.PoolManager(
        num_pools=num_pools,
        maxsize=maxsize,
        block=block,
        *args,
        **kwargs
    )

# The ARC server or middlebox only supports one cipher suite,
# DES-CBC3-SHA, and it also needs it to be sufficiently far forward in the
# cipher suite list.
# (as of 9/8/2016)
scraper.mount("https://www.arc.gov/", CipherListAdapter("DES-CBC3-SHA"))

WHITELIST_INSECURE_DOMAINS = (
  "https://www.ignet.gov/",  # incomplete chain as of 1/25/2015
  "https://www.va.gov/",  # incomplete chain as of 12/6/2015
  "https://origin.www.fhfaoig.gov/",  # incomplete chain as of 1/5/2016
  "https://www.archives.gov/",  # incomplete chain as of 2/19/2016
  "https://www.arc.gov/",  # incomplete chain as of 8/24/2016
  "https://www.ncua.gov/",  # incomplete chain as of 12/10/2016
  "https://www.oig.dot.gov/",  # incomplete chain as of 4/12/2017
  "https://www.oig.dhs.gov",  # incomplete chain as of 5/6/2017
  "https://oig.eeoc.gov/",  # incomplete chain as of 6/3/2017
  "https://www.cpb.org/",  # incomplete chain as of 10/1/2017

  # The following domains will 301/302 redirect to the above domains, so
  # validate=False is needed for these cases as well
  "http://www.ignet.gov/",
  "http://ignet.gov/",
  "http://www.arc.gov/",
  "http://www.cpb.org/",
)
WHITELIST_SHA1_DOMAINS = (
  "https://www.sba.gov/",
  "http://www.sba.gov/",
  "http://sba.gov/",
  "https://www.sigar.mil/",
  "http://www.sigar.mil/",<commit_msg>[arc] Remove cipher workaround, no longer needed<commit_after>import os, os.path, errno, sys, traceback, subprocess
import re, html.entities
import json
import logging
import yaml
from bs4 import BeautifulSoup
from datetime import datetime
import requests
import urllib.parse
import io
import gzip
import certifi
import docx
import zipfile
from urllib.parse import urljoin
import inspect
import pdfrw

from . import admin

logging.getLogger("pdfrw").setLevel(logging.CRITICAL)

# scraper should be instantiated at class-load time, so that it can rate limit appropriately
import scrapelib
scraper = scrapelib.Scraper(requests_per_minute=120, retry_attempts=3)
scraper.user_agent = "unitedstates/inspectors-general (https://github.com/unitedstates/inspectors-general)"
scraper.timeout = 60

class Soft404HttpAdapter(requests.adapters.HTTPAdapter):
  """Transport adapter that checks all responses against a blacklist of "file
  not found" pages that are served with 200 status codes."""

  SOFT_404_URLS_RE = re.compile(r"^(http://www\.dodig\.mil/errorpages/index\.html|http://www\.fec\.gov/404error\.shtml|http://www\.gpo\.gov/maintenance/error\.htm)$")
  SOFT_404_BODY_SIGNATURES = {
    "cftc.gov": b"<title>404 Page Not Found - CFTC</title>",
    "cpb.org": b"<title>CPB: Page Not Found</title>",
    "ncua.gov": b"Redirect.aspx?404",
    "si.edu": b"<title>Page Not Found Smithsonian</title>",
  }

  def build_response(self, req, resp):
    domain = urllib.parse.urlparse(req.url)[1].split(':')[0]
    base_domain = ".".join(domain.split(".")[-2:])
    if base_domain in self.SOFT_404_BODY_SIGNATURES:
      if resp.getheader("Content-Type") in ["text/html; charset=utf-8",
                                            "text/html"]:
        data = resp.data
        headers = resp.headers
        if resp.getheader("Content-Encoding") == "gzip":
          decompressed_data = gzip.decompress(data)
        else:
          decompressed_data = data
        if resp.getheader("Transfer-Encoding") == "chunked":
          headers.pop("Transfer-Encoding")
        body = io.BytesIO(data)
        resp = requests.packages.urllib3.response.HTTPResponse(
                body=body,
                headers=headers,
                status=resp.status,
                version=resp.version,
                reason=resp.reason,
                strict=resp.strict,
                preload_content=False,
        )
        if decompressed_data.find(self.SOFT_404_BODY_SIGNATURES[base_domain], 0, 10240) != -1:
          result = super(Soft404HttpAdapter, self).build_response(req, resp)
          result.status_code = 404 # tells scrapelib to not retry
          return result

    redirect = resp.get_redirect_location()
    result = super(Soft404HttpAdapter, self).build_response(req, resp)
    if redirect and self.SOFT_404_URLS_RE.match(redirect):
      result.status_code = 404 # tells scrapelib to not retry

    return result

scraper.mount("http://www.cftc.gov/", Soft404HttpAdapter())
scraper.mount("http://cftc.gov/", Soft404HttpAdapter())
scraper.mount("http://www.dodig.mil/", Soft404HttpAdapter())
scraper.mount("http://dodig.mil/", Soft404HttpAdapter())
scraper.mount("http://www.fec.gov/", Soft404HttpAdapter())
scraper.mount("http://fec.gov/", Soft404HttpAdapter())
scraper.mount("http://www.gpo.gov/", Soft404HttpAdapter())
scraper.mount("http://gpo.gov/", Soft404HttpAdapter())
scraper.mount("http://www.cpb.org/", Soft404HttpAdapter())
scraper.mount("http://cpb.org/", Soft404HttpAdapter())
scraper.mount("http://www.ncua.gov/", Soft404HttpAdapter())
scraper.mount("http://ncua.gov/", Soft404HttpAdapter())
scraper.mount("http://www.si.edu/", Soft404HttpAdapter())
scraper.mount("http://si.edu/", Soft404HttpAdapter())

WHITELIST_INSECURE_DOMAINS = (
  "https://www.ignet.gov/",  # incomplete chain as of 1/25/2015
  "https://www.va.gov/",  # incomplete chain as of 12/6/2015
  "https://origin.www.fhfaoig.gov/",  # incomplete chain as of 1/5/2016
  "https://www.archives.gov/",  # incomplete chain as of 2/19/2016
  "https://www.arc.gov/",  # incomplete chain as of 8/24/2016
  "https://www.ncua.gov/",  # incomplete chain as of 12/10/2016
  "https://www.oig.dot.gov/",  # incomplete chain as of 4/12/2017
  "https://www.oig.dhs.gov",  # incomplete chain as of 5/6/2017
  "https://oig.eeoc.gov/",  # incomplete chain as of 6/3/2017
  "https://www.cpb.org/",  # incomplete chain as of 10/1/2017

  # The following domains will 301/302 redirect to the above domains, so
  # validate=False is needed for these cases as well
  "http://www.ignet.gov/",
  "http://ignet.gov/",
  "http://www.arc.gov/",
  "http://www.cpb.org/",
)
WHITELIST_SHA1_DOMAINS = (
  "https://www.sba.gov/",
  "http://www.sba.gov/",
  "http://sba.gov/",
  "https://www.sigar.mil/",
  "http://www.sigar.mil/",
{'fraction_non_alphanumeric': 0.14958502219648717, 'fraction_numerical': 0.03541787299749083, 'mean_word_length': 3.415423945462292, 'pattern_counts': {'":': 12, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 27, 'lorem ipsum': 0, 'www.': 53, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2851480', 'n_tokens_mistral': 3819, 'n_tokens_neox': 3479, 'n_words': 700}