<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Utilities relating to online services"""
from __future__ import (print_function, unicode_literals)

import time

import requests


def url_check(url=None, method='get', timeout=None, expected_codes=200, interval=None, healthy_threshold=None):
    """Check if a request for a URL gives an expected response"""
    if expected_codes:
        if not isinstance(expected_codes, list):
            expected_codes = expected_codes,
        response = None
        wait_timeout = time.time() + timeout
        healthy_count = 0
        while wait_timeout > time.time():
            try:
                if method:
                    if method.lower() == 'get':
                        response = requests.get(url=url, timeout=10, verify=False)
                    if method.lower() == 'head':
                        response = requests.head(url=url, timeout=10, verify=False)
                if response.status_code in expected_codes:
                    healthy_count += 1
                if healthy_count == healthy_threshold:
                    return True
            except:
                healthy_count = 0
                time.sleep(5)<commit_msg>add options to url_check and provide reasonable defaults.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Utilities relating to online services"""
from __future__ import (print_function, unicode_literals)

import time

import requests
from requests.exceptions import BaseHTTPError


def url_check(url=None, method='get', timeout=None, response_timeout=2, expected_codes=200, interval=2,
              healthy_threshold=1, ssl_verify=False):
    """Check if a request for a URL gives an expected response"""
    if not isinstance(expected_codes, list):
        expected_codes = expected_codes,
    response = None
    wait_timeout = time.time() + timeout
    healthy_count = 0
    while wait_timeout > time.time():
        try:
            if method.lower() == 'get':
                response = requests.get(url=url, timeout=response_timeout, verify=ssl_verify)
            elif method.lower() == 'head':
                response = requests.head(url=url, timeout=response_timeout, verify=ssl_verify)
            if response.status_code in expected_codes:
                healthy_count += 1
            if healthy_count == healthy_threshold:
                return True
        except BaseHTTPError:
            healthy_count = 0
            time.sleep(interval)
{'fraction_non_alphanumeric': 0.09401366010445962, 'fraction_numerical': 0.008838891120932101, 'mean_word_length': 2.3422818791946307, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5272215', 'n_tokens_mistral': 659, 'n_tokens_neox': 630, 'n_words': 181}