<commit_before>from __future__ import absolute_import
from django.db import transaction
from django.conf import settings
from django_atomic_celery import task

import requests
from requests.exceptions import InvalidURL
import redis
from cachecontrol import CacheControl
from cachecontrol.caches import RedisCache

from .utils import ResultDict, logger, response_to_dict
from thezombies.models import URLInspection

try:
    from urllib.parse import urlparse, urlunparse
except ImportError:
    from urlparse import urlparse, urlunparse


REQUEST_TIMEOUT = getattr(settings, 'REQUEST_TIMEOUT', 60)

pool = redis.ConnectionPool.from_url(settings.REDIS_URL)
r = redis.Redis(connection_pool=pool)
session = CacheControl(requests.Session(), cache=RedisCache(r), cache_etags=False)


@task
def check_and_correct_url(url, method='GET'):
    """Check a url for issues, record exceptions, and attempt to correct the url.

    :param url: URL to check and correct
    :param method: http method to use, as a string. Default is 'GET'
    """
    returnval = ResultDict({'initial_url': url})
    try:
        logger.info('Checking URL: {0}'.format(url))
        scheme, netloc, path, params, query, fragments = urlparse(str(url))
        if scheme is '':
            # Maybe it is an http url without the scheme?
            scheme, netloc, path, params, query, fragments = urlparse("http://{0}".format(str(url)))
        elif not (scheme.startswith('http') or scheme.startswith('sftp') or scheme.startswith('ftp')):
            # Not a typical 'web' scheme
            raise InvalidURL('Invalid scheme (not http(s) or (s)ftp)')
<commit_msg>Add simple function for opening a URL stream rather than downloading whole URL.<commit_after>from __future__ import absolute_import
from django.db import transaction
from django.conf import settings
from django_atomic_celery import task

import requests
from requests.exceptions import InvalidURL

from .utils import ResultDict, logger, response_to_dict
from thezombies.models import URLInspection

try:
    from urllib.parse import urlparse, urlunparse
except ImportError:
    from urlparse import urlparse, urlunparse


REQUEST_TIMEOUT = getattr(settings, 'REQUEST_TIMEOUT', 60)

session = requests.Session()


def open_url_stream(method, url):
    """Open a URL for streaming. The file-like object will be available under resp.raw"""
    resp = session.request(method.upper(), url, stream=True,
                           allow_redirects=True, timeout=REQUEST_TIMEOUT, verify=False)
    return resp


@task
def check_and_correct_url(url, method='GET'):
    """Check a url for issues, record exceptions, and attempt to correct the url.

    :param url: URL to check and correct
    :param method: http method to use, as a string. Default is 'GET'
    """
    returnval = ResultDict({'initial_url': url})
    try:
        logger.info('Checking URL: {0}'.format(url))
        scheme, netloc, path, params, query, fragments = urlparse(str(url))
        if scheme is '':
            # Maybe it is an http url without the scheme?
            scheme, netloc, path, params, query, fragments = urlparse("http://{0}".format(str(url)))
        elif not (scheme.startswith('http') or scheme.startswith('sftp') or scheme.startswith('ftp')):
            # Not a typical 'web' scheme
            raise InvalidURL('Invalid scheme (not http(s) or (s)ftp)')

{'fraction_non_alphanumeric': 0.09608540925266904, 'fraction_numerical': 0.002372479240806643, 'mean_word_length': 3.9530102790014685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '629681', 'n_tokens_mistral': 965, 'n_tokens_neox': 951, 'n_words': 331}