<commit_before>from __future__ import unicode_literals, division, absolute_import
import urllib2
import time
import logging
from datetime import timedelta, datetime
from urlparse import urlparse
import requests
# Allow some request objects to be imported from here instead of requests
from requests import RequestException
from flexget.utils.tools import parse_timedelta

log = logging.getLogger('requests')

# Don't emit info level urllib3 log messages or below
logging.getLogger('requests.packages.urllib3').setLevel(logging.WARNING)

# Remembers sites that have timed out
unresponsive_hosts = {}
# Time to wait before trying an unresponsive site again
WAIT_TIME = timedelta(seconds=60)


def is_unresponsive(url):
    """
    Checks if host of given url has timed out within WAIT_TIME

    :param url: The url to check
    :return: True if the host has timed out within WAIT_TIME
    :rtype: bool
    """
    host = urlparse(url).hostname
    if host in unresponsive_hosts and unresponsive_hosts[host] + WAIT_TIME < datetime.now():
        return True
    return False


def set_unresponsive(url):
    """
    Marks the host of a given url as unresponsive

    :param url: The url that timed out
    """
    host = urlparse(url).hostname
    unresponsive_hosts[host] = datetime.now()


class FileAdapter(requests.adapters.BaseAdapter):
    """Handles file:// URIs by passing them to urllib2"""
    def send(self, request):
        url = request.url
        try:
            raw = urllib2.urlopen(url)
        except IOError as e:
            msg = 'IOError openening file %s: %s' % (url, e)
            log.error(msg)
            raise RequestException(msg)
        resp = requests.Response()
        resp.raw = raw
        resp.status_code = 200
        resp.headers = requests.structures.CaseInsensitiveDict(raw.headers)
        return resp

    def close(self):
        pass

class Session(requests.Session):
    """Subclass of requests Session class which defines some of our own defaults, records unresponsive sites,
    and raises errors by default."""

    def __init__(self, timeout=None, max_retries=None):<commit_msg>Make sure file:// uris do not crash requests utility. refs #1898<commit_after>from __future__ import unicode_literals, division, absolute_import
import urllib2
import time
import logging
from datetime import timedelta, datetime
from urlparse import urlparse
import requests
# Allow some request objects to be imported from here instead of requests
from requests import RequestException
from flexget.utils.tools import parse_timedelta

log = logging.getLogger('requests')

# Don't emit info level urllib3 log messages or below
logging.getLogger('requests.packages.urllib3').setLevel(logging.WARNING)

# Remembers sites that have timed out
unresponsive_hosts = {}
# Time to wait before trying an unresponsive site again
WAIT_TIME = timedelta(seconds=60)


def is_unresponsive(url):
    """
    Checks if host of given url has timed out within WAIT_TIME

    :param url: The url to check
    :return: True if the host has timed out within WAIT_TIME
    :rtype: bool
    """
    host = urlparse(url).hostname
    if host in unresponsive_hosts and unresponsive_hosts[host] + WAIT_TIME < datetime.now():
        return True
    return False


def set_unresponsive(url):
    """
    Marks the host of a given url as unresponsive

    :param url: The url that timed out
    """
    host = urlparse(url).hostname
    unresponsive_hosts[host] = datetime.now()


class FileAdapter(requests.adapters.BaseAdapter):
    """Handles file:// URIs by passing them to urllib2"""
    def send(self, request, **kwargs):
        url = request.url
        try:
            raw = urllib2.urlopen(url)
        except IOError as e:
            msg = 'IOError openening file %s: %s' % (url, e)
            log.error(msg)
            raise RequestException(msg)
        resp = requests.Response()
        resp.raw = raw
        resp.status_code = 200
        resp.headers = requests.structures.CaseInsensitiveDict(raw.headers)
        return resp

    def close(self):
        pass

class Session(requests.Session):
    """Subclass of requests Session class which defines some of our own defaults, records unresponsive sites,
    and raises errors by default."""

    def __init__(self, timeout=None, max_retries=None):
{'fraction_non_alphanumeric': 0.0777597778292062, 'fraction_numerical': 0.0055542698449433, 'mean_word_length': 3.3437185929648243, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5279839', 'n_tokens_mistral': 1254, 'n_tokens_neox': 1185, 'n_words': 434}