<commit_before>import bs4
import requests
import requests.exceptions


class InfiksiError(Exception):
    pass


class UnreachableError(InfiksiError):
    pass


class TemporaryError(InfiksiError):
    pass


class OEmbedResponse(object):
    def __init__(self, title=None, description=None, canonical_url=None, author_name=None,
                 og_title=None, og_description=None, og_image=None, og_image_width=None, og_image_height=None,
                 thumbnail_url=None, thumbnail_width=None, thumbnail_height=None,
                 ):
        self.canonical_url = canonical_url
        self.og_image_height = og_image_height
        self.og_image_width = og_image_width
        self.og_image = og_image
        self.thumbnail_height = thumbnail_height
        self.thumbnail_width = thumbnail_width
        self.thumbnail_url = thumbnail_url
        self.author_name = author_name
        self.description = description
        self.og_title = og_title
        self.og_description = og_description
        self.title = title
        self.version = '1.0'
        self.type = 'article'

        if og_image and not thumbnail_url:
            self.thumbnail_url = og_image

            if og_image_width and not thumbnail_width:
                self.thumbnail_width = og_image_width
            if og_image_height and not thumbnail_height:
                self.thumbnail_height = og_image_height


def retrieve_html(url, timeout=1000):
    try:
        result = requests.get(url, timeout=(1.0 / timeout))
    except requests.exceptions.ConnectionError:
        raise UnreachableError("Could not reach {}".format(url))
    except requests.exceptions.TooManyRedirects:
        raise UnreachableError("Too many redirects while attempting to resolve {}".format(url))
    except requests.exceptions.RequestException:
        raise UnreachableError("Unknown error while trying to download {}".format(url))

    if result.status_code // 100 == 4:  # 4xx
        raise UnreachableError("Could not find page {}, got status code {}".format(url, result.status_code))

    if result.status_code // 100 == 5:  # 5xx
        raise TemporaryError("Could not find page {}, got status code {}".format(url, result.status_code))

    return result.text, result.url


def get_metadata(url, timeout=1000):<commit_msg>Correct timeout bug<commit_after>import bs4
import requests
import requests.exceptions


class InfiksiError(Exception):
    pass


class UnreachableError(InfiksiError):
    pass


class TemporaryError(InfiksiError):
    pass


class OEmbedResponse(object):
    def __init__(self, title=None, description=None, canonical_url=None, author_name=None,
                 og_title=None, og_description=None, og_image=None, og_image_width=None, og_image_height=None,
                 thumbnail_url=None, thumbnail_width=None, thumbnail_height=None,
                 ):
        self.canonical_url = canonical_url
        self.og_image_height = og_image_height
        self.og_image_width = og_image_width
        self.og_image = og_image
        self.thumbnail_height = thumbnail_height
        self.thumbnail_width = thumbnail_width
        self.thumbnail_url = thumbnail_url
        self.author_name = author_name
        self.description = description
        self.og_title = og_title
        self.og_description = og_description
        self.title = title
        self.version = '1.0'
        self.type = 'article'

        if og_image and not thumbnail_url:
            self.thumbnail_url = og_image

            if og_image_width and not thumbnail_width:
                self.thumbnail_width = og_image_width
            if og_image_height and not thumbnail_height:
                self.thumbnail_height = og_image_height


def retrieve_html(url, timeout=2000):
    try:
        result = requests.get(url, timeout=(timeout/1000.0))
    except requests.exceptions.ConnectionError:
        raise UnreachableError("Could not reach {}".format(url))
    except requests.exceptions.TooManyRedirects:
        raise UnreachableError("Too many redirects while attempting to resolve {}".format(url))
    except requests.exceptions.RequestException:
        raise UnreachableError("Unknown error while trying to download {}".format(url))

    if result.status_code // 100 == 4:  # 4xx
        raise UnreachableError("Could not find page {}, got status code {}".format(url, result.status_code))

    if result.status_code // 100 == 5:  # 5xx
        raise TemporaryError("Could not find page {}, got status code {}".format(url, result.status_code))

    return result.text, result.url


def get_metadata(url, timeout=2000):
{'fraction_non_alphanumeric': 0.08157951833369495, 'fraction_numerical': 0.010631373399869819, 'mean_word_length': 3.0262008733624453, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3175147', 'n_tokens_mistral': 1361, 'n_tokens_neox': 1292, 'n_words': 304}