<commit_before>import justext
import requests


class ParserError(Exception):
    '''Base class for exceptions in the parser module'''
    pass


class URLRetrievalError(ParserError):
    '''Exception raised for errors in the URL input.

    Attributes:
        expr -- input expression in which the error occurred
        msg  -- explanation of the error
        e    -- the original exception that was raised
    '''

    def __init__(self, expr, msg, e=None):
        self.expr = expr
        self.msg = msg
        self.e = e


class DocumentParsingError(ParserError):
    '''Exception raised for errors in the parsing of the URL.

    Attributes:
        expr -- input expression in which the error occurred
    '''

    def __init__(self, msg):
        self.msg = msg


def get_document(url):
    ''' This function will check if the url is valid and then
    proceed to parse it to produce a clean text (no html) which
    can be used as input to a recommendation engine.

    Arguments:
        url  -- input url that needs to be checked and parsed
    '''
    try:
        r = requests.head(url)
    except requests.exceptions.ConnectionError as e:
        raise URLRetrievalError(url, 'Could not connect', e)
    if r.status_code != requests.codes.ok:
        raise URLRetrievalError(url, 'Invalid response code from remote server: {}'
                                .format(r.status_code))
    if r.headers["content-type"].split(';')[0] not in ["text/html",
                                                       "text/plain"]:
        raise URLRetrievalError(url, 'Document has invalid MIME type: {}'
                                .format(r.headers["content-type"]))
    raw = requests.get(url)<commit_msg>Parser handles 303 now.<commit_after>import justext
import requests


class ParserError(Exception):
    '''Base class for exceptions in the parser module'''
    pass


class URLRetrievalError(ParserError):
    '''Exception raised for errors in the URL input.

    Attributes:
        expr -- input expression in which the error occurred
        msg  -- explanation of the error
        e    -- the original exception that was raised
    '''

    def __init__(self, expr, msg, e=None):
        self.expr = expr
        self.msg = msg
        self.e = e


class DocumentParsingError(ParserError):
    '''Exception raised for errors in the parsing of the URL.

    Attributes:
        expr -- input expression in which the error occurred
    '''

    def __init__(self, msg):
        self.msg = msg


def get_document(url):
    ''' This function will check if the url is valid and then
    proceed to parse it to produce a clean text (no html) which
    can be used as input to a recommendation engine.

    Arguments:
        url  -- input url that needs to be checked and parsed
    '''
    try:
        r = requests.head(url, allow_redirects = True)
    except requests.exceptions.ConnectionError as e:
        raise URLRetrievalError(url, 'Could not connect', e)
    if r.status_code != requests.codes.ok:
        raise URLRetrievalError(url, 'Invalid response code from remote server: {}'
                                .format(r.status_code))
    if r.headers["content-type"].split(';')[0] not in ["text/html",
                                                       "text/plain"]:
        raise URLRetrievalError(url, 'Document has invalid MIME type: {}'
                                .format(r.headers["content-type"]))
    raw = requests.get(url)
{'fraction_non_alphanumeric': 0.08894645941278066, 'fraction_numerical': 0.0014392630972941854, 'mean_word_length': 2.188073394495413, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2585243', 'n_tokens_mistral': 924, 'n_tokens_neox': 891, 'n_words': 342}