<commit_before>from .exceptions import *

import httplib2
import socket


def get_request(url, headers=None, timeout=30):
    """
    Retrieve contents of a page, passing any exceptions. Does not follow redirects.

    :param url: URL to retrieve
    :param headers: request headers
    :param timeout: timeout of request in seconds
    :return: page data
    """

    try:
        h = httplib2.Http(timeout=timeout)
        h.follow_redirects = False
        resp, content = h.request(url, 'GET', '', headers=headers)

    except (httplib2.ServerNotFoundError, ConnectionRefusedError) as e:
        raise HttpError("Could not retrieve URL %s. Additional info: %s" % (url, e))
    except socket.timeout:
        raise HttpError("Request timed out after specified timeout period of %s seconds" % timeout)

    return [resp, content]


def post_request(url, data, headers=None, timeout=30):
    """
    Retrieve contents of a page with a POST request and one payload data object, passing any exceptions.  Does not
    follow redirects.

    :param url: URL to retrieve
    :param data: Payload data object
    :param headers: request headers
    :param timeout: timeout of request in seconds
    :return: page data
    """

    # Prepare headers even if there are none, as we're doing a post request
    if headers is None:
        headers = {}
    headers['Content-type'] = 'application/x-www-form-urlencoded'

    try:
        h = httplib2.Http(timeout=timeout)
        h.follow_redirects = False
        resp, content = h.request(url, 'POST', data, headers=headers)

    except (httplib2.ServerNotFoundError, ConnectionRefusedError) as e:
        raise HttpError("Could not retrieve URL %s. Additional info: %s" % (url, e))
    except socket.timeout:
        raise HttpError("Request timed out after specified timeout period of %s seconds" % timeout)

    return [resp, content]


def delete_request(url, headers=None, timeout=30):
    """
    Retrieve contents of a page, passing any exceptions. Does not follow redirects.

    :param url: URL to retrieve
    :param headers: request headers
    :param timeout: timeout of request in seconds
    :return: page data
    """

    try:
        h = httplib2.Http(timeout=timeout)
        h.follow_redirects = False
        resp, content = h.request(url, 'DELETE', '', headers=headers)

    except (httplib2.ServerNotFoundError, ConnectionRefusedError) as e:
        raise HttpError("Could not retrieve URL %s. Additional info: %s" % (url, e))
    except socket.timeout:
        raise HttpError("Request timed out after specified timeout period of %s seconds" % timeout)

    return [resp, content]


def robust_get_file(url, file_handle, block_size=1048576, timeout=15):
    """
    Download an object in a robust way using HTTP partial downloading

    :param url: URL to download
    :param file: open pointer to file to store download in, data is appended
    :param block_size: size of individual download chunks during partial downloading
    :param timeout: timeout in seconds till individual block downloads are failed
    :return: None
    """

    # Retrieve header first in order to determine file size
    try:
        h = httplib2.Http(timeout=timeout)
        resp, content = h.request(url, 'HEAD', '', headers={})

    except httplib2.ServerNotFoundError as e:<commit_msg>Include additional verifications in robust transfer method<commit_after>from .exceptions import *

import httplib2
import socket


def get_request(url, headers=None, timeout=30):
    """
    Retrieve contents of a page, passing any exceptions. Does not follow redirects.

    :param url: URL to retrieve
    :param headers: request headers
    :param timeout: timeout of request in seconds
    :return: page data
    """

    try:
        h = httplib2.Http(timeout=timeout)
        h.follow_redirects = False
        resp, content = h.request(url, 'GET', '', headers=headers)

    except (httplib2.ServerNotFoundError, ConnectionRefusedError) as e:
        raise HttpError("Could not retrieve URL %s. Additional info: %s" % (url, e))
    except socket.timeout:
        raise HttpError("Request timed out after specified timeout period of %s seconds" % timeout)

    return [resp, content]


def post_request(url, data, headers=None, timeout=30):
    """
    Retrieve contents of a page with a POST request and one payload data object, passing any exceptions.  Does not
    follow redirects.

    :param url: URL to retrieve
    :param data: Payload data object
    :param headers: request headers
    :param timeout: timeout of request in seconds
    :return: page data
    """

    # Prepare headers even if there are none, as we're doing a post request
    if headers is None:
        headers = {}
    headers['Content-type'] = 'application/x-www-form-urlencoded'

    try:
        h = httplib2.Http(timeout=timeout)
        h.follow_redirects = False
        resp, content = h.request(url, 'POST', data, headers=headers)

    except (httplib2.ServerNotFoundError, ConnectionRefusedError) as e:
        raise HttpError("Could not retrieve URL %s. Additional info: %s" % (url, e))
    except socket.timeout:
        raise HttpError("Request timed out after specified timeout period of %s seconds" % timeout)

    return [resp, content]


def delete_request(url, headers=None, timeout=30):
    """
    Retrieve contents of a page, passing any exceptions. Does not follow redirects.

    :param url: URL to retrieve
    :param headers: request headers
    :param timeout: timeout of request in seconds
    :return: page data
    """

    try:
        h = httplib2.Http(timeout=timeout)
        h.follow_redirects = False
        resp, content = h.request(url, 'DELETE', '', headers=headers)

    except (httplib2.ServerNotFoundError, ConnectionRefusedError) as e:
        raise HttpError("Could not retrieve URL %s. Additional info: %s" % (url, e))
    except socket.timeout:
        raise HttpError("Request timed out after specified timeout period of %s seconds" % timeout)

    return [resp, content]


def robust_get_file(url, file_handle, block_size=1048576, timeout=15):
    """
    Download an object in a robust way using HTTP partial downloading

    :param url: URL to download
    :param file: open pointer to file to store download in, data is appended
    :param block_size: size of individual download chunks during partial downloading
    :param timeout: timeout in seconds till individual block downloads are failed
    :return: None
    """

    # Verify block size parameter
    if not isinstance(block_size, int):
        raise HttpError("The block size should be an integer")
    elif block_size < 512:
        raise HttpError("The block size should be at least 512 bytes")
    elif block_size > 268435456:
        raise HttpError("The block size can not be more than 256 megabytes")

    # Verify timeout parameter
    if not isinstance(timeout, int):
        raise HttpError("The timeout should be an integer")
    elif timeout < 1:
        raise HttpError("The timeout should be at least 1 second")
    elif timeout > 86400:
        raise HttpError("The timeout can not be more than 86400 seconds")

    # Retrieve header first in order to determine file size
    try:
        h = httplib2.Http(timeout=timeout)
        resp, content = h.request(url, 'HEAD', '', headers={})

    except httplib2.ServerNotFoundError as e:
{'fraction_non_alphanumeric': 0.08512945641859834, 'fraction_numerical': 0.01057340382269215, 'mean_word_length': 3.2135922330097086, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7018992', 'n_tokens_mistral': 2074, 'n_tokens_neox': 1976, 'n_words': 797}