<commit_before>'''
(C) 2015
Author: <NAME> <<EMAIL>>
Author: <NAME> <<EMAIL>>
License: MIT License
Fetches canonical (or open graph) version of URL
Failing that, returns redirect url if redirect happens.

'''

from __future__ import print_function
import logging
import re
import requests
from bs4 import BeautifulSoup, UnicodeDammit, FeatureNotFound
import urlparse
import rfc3987
from canonicalencoding import get_declared_encodings 
from canonicalencoding import detect_encoding, try_encoding, try_encoding_force



REQ_TIMEOUT = 30
MAX_READ = 2 * 1024 * 1024  # 2MB - Probably Enough for The Verge's HTML


def url_encode_non_ascii(b):
    return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)


def ensure_url(iri):
    '''If IRI, convert to URL
    If fragments (#), remove
    http://stackoverflow.com/posts/4391299/revisions
    '''
    parts = urlparse.urlparse(iri)
    url_parts = []

    for index, part in enumerate(parts):
        if index == 1:
            url_parts.append(part.lower().encode('idna'))
        else:
            url_parts.append(url_encode_non_ascii(part.encode('utf-8')))

    url = urlparse.urlunparse(url_parts)
    url = urlparse.urldefrag(url)[0]

    return url


def validate_url(url):
    '''
    Validates URL (actually, IRIs).
    '''
    try:
        rfc3987.parse(url, rule='IRI')
    except:
        return False

    return True


def get_web_page(url):
    ''' Fetches content at a given URL.
    Args:
        url - unicode string

    Returns: (html, headers, final_url) or (None, None, None) on error.
    '''

    # if it's not unicode, it must be utf8, otherwise fail
    if not isinstance(url, unicode):
        try:
            s = url.decode('utf8')
        except Exception as e:
            logging.exception(e)
            return (url, None, None)


    # Convert URI to URL if necessary
    try:
        url = ensure_url(url)
    except Exception as e:
        logging.exception(e)
        return (url, None, None)

    # Validate URL
    if not validate_url(url):
        logging.info('bad url: %s ' % url)
        return (url, None, None)

    # 
    # Download and Processs
    #
    try:
        # Not currently using Custom Headers
        headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 5.1; rv:10.0.1) Gecko/20100101 Firefox/10.0.1',
        }
        r = requests.get(url, timeout=REQ_TIMEOUT, allow_redirects=True,
                         stream=True)
        r.raise_for_status()<commit_msg>removed old unused headers.<commit_after>'''
(C) 2015
Author: <NAME> <<EMAIL>>
Author: <NAME> <<EMAIL>>
License: MIT License
Fetches canonical (or open graph) version of URL
Failing that, returns redirect url if redirect happens.

'''

from __future__ import print_function
import logging
import re
import requests
from bs4 import BeautifulSoup, UnicodeDammit, FeatureNotFound
import urlparse
import rfc3987
from canonicalencoding import get_declared_encodings 
from canonicalencoding import detect_encoding, try_encoding, try_encoding_force



REQ_TIMEOUT = 30
MAX_READ = 2 * 1024 * 1024  # 2MB - Probably Enough for The Verge's HTML


def url_encode_non_ascii(b):
    return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)


def ensure_url(iri):
    '''If IRI, convert to URL
    If fragments (#), remove
    http://stackoverflow.com/posts/4391299/revisions
    '''
    parts = urlparse.urlparse(iri)
    url_parts = []

    for index, part in enumerate(parts):
        if index == 1:
            url_parts.append(part.lower().encode('idna'))
        else:
            url_parts.append(url_encode_non_ascii(part.encode('utf-8')))

    url = urlparse.urlunparse(url_parts)
    url = urlparse.urldefrag(url)[0]

    return url


def validate_url(url):
    '''
    Validates URL (actually, IRIs).
    '''
    try:
        rfc3987.parse(url, rule='IRI')
    except:
        return False

    return True


def get_web_page(url):
    ''' Fetches content at a given URL.
    Args:
        url - unicode string

    Returns: (html, headers, final_url) or (None, None, None) on error.
    '''

    # if it's not unicode, it must be utf8, otherwise fail
    if not isinstance(url, unicode):
        try:
            s = url.decode('utf8')
        except Exception as e:
            logging.exception(e)
            return (url, None, None)


    # Convert URI to URL if necessary
    try:
        url = ensure_url(url)
    except Exception as e:
        logging.exception(e)
        return (url, None, None)

    # Validate URL
    if not validate_url(url):
        logging.info('bad url: %s ' % url)
        return (url, None, None)

    # 
    # Download and Processs
    #
    try:
        r = requests.get(url, timeout=REQ_TIMEOUT, allow_redirects=True,
                         stream=True)
        r.raise_for_status()
{'fraction_non_alphanumeric': 0.10832642916321458, 'fraction_numerical': 0.021541010770505385, 'mean_word_length': 2.628099173553719, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7008458', 'n_tokens_mistral': 1620, 'n_tokens_neox': 1499, 'n_words': 444}