<commit_before># -*- coding: utf-8 -*-

'''
This module contains functions responsible for
validating arguments for other functions and
methods provided by the library
'''

import functools
import re
from urlparse import urlparse

import validators

from .exceptions import InvalidURLError

def is_valid_host(value):
    ''' Check if given value is valid host string
    
    :param value: a value to test
    :returns: True if the value is valid host string
    '''
    host_validators = validators.ipv4, validators.ipv6, validators.domain
    return any(f(value) for f in host_validators)
    

def accepts_valid_urls(f):
    @functools.wraps(f)
    def wrapper(obj, urls, *args, **kwargs):
        '''Run the function and return its return value
         if all given urls are valid - otherwise raise InvalidURLError
        :param obj: an object in whose class f is defined
        :param urls: an iterable containing urls
        :returns: a return value of the function f
        :raises InvalidURLError: if the iterable contains invalid urls
        '''
        invalid_urls = filter(lambda u: not is_valid_url(u), urls)
        if invalid_urls:
            msg = 'The values: {} are not valid urls'.format(','.join(invalid_urls))
            raise InvalidURLError, msg
        
        return f(obj, urls, *args, **kwargs)
    
    return wrapper

url_regex = re.compile(r'^[a-z0-9\.\-\+]*://' #scheme
                       r'(?:\S+(?::\S*)?@)?' #authentication
                       r'(?:[^/:]+|\[[0-9a-f:\.]+\])' # host
                       r'(?::\d{2,5})?' # port
                       r'(?:[/?#][^\s]*)?' # path, query or fragment
                       r'$', re.IGNORECASE)

def is_valid_url(value):
    ''' Check if given value is valid url string
    
    :param value: a value to test
    :returns: True if the value is valid url string
    '''
    host_validators = validators.ipv4, validators.ipv6, validators.domain
    
    match = url_regex.match(value)
    
    host = urlparse(value).hostname
    
    return (match and any(f(host) for f in host_validators))<commit_msg>Refactor is_valid_url to use is_valid_host<commit_after># -*- coding: utf-8 -*-

'''
This module contains functions responsible for
validating arguments for other functions and
methods provided by the library
'''

import functools
import re
from urlparse import urlparse

import validators

from .exceptions import InvalidURLError

def is_valid_host(value):
    ''' Check if given value is valid host string
    
    :param value: a value to test
    :returns: True if the value is valid host string
    '''
    host_validators = validators.ipv4, validators.ipv6, validators.domain
    return any(f(value) for f in host_validators)
    

def accepts_valid_urls(f):
    @functools.wraps(f)
    def wrapper(obj, urls, *args, **kwargs):
        '''Run the function and return its return value
         if all given urls are valid - otherwise raise InvalidURLError
        :param obj: an object in whose class f is defined
        :param urls: an iterable containing urls
        :returns: a return value of the function f
        :raises InvalidURLError: if the iterable contains invalid urls
        '''
        invalid_urls = filter(lambda u: not is_valid_url(u), urls)
        if invalid_urls:
            msg = 'The values: {} are not valid urls'.format(','.join(invalid_urls))
            raise InvalidURLError, msg
        
        return f(obj, urls, *args, **kwargs)
    
    return wrapper

url_regex = re.compile(r'^[a-z0-9\.\-\+]*://' #scheme
                       r'(?:\S+(?::\S*)?@)?' #authentication
                       r'(?:[^/:]+|\[[0-9a-f:\.]+\])' # host
                       r'(?::\d{2,5})?' # port
                       r'(?:[/?#][^\s]*)?' # path, query or fragment
                       r'$', re.IGNORECASE)

def is_valid_url(value):
    ''' Check if given value is valid url string
    
    :param value: a value to test
    :returns: True if the value is valid url string
    '''
    
    match = url_regex.match(value)
    host_str = urlparse(value).hostname
    
    return (match and is_valid_host(host_str))
{'fraction_non_alphanumeric': 0.11508704061895551, 'fraction_numerical': 0.004835589941972921, 'mean_word_length': 2.4417637271214643, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3794746', 'n_tokens_mistral': 1251, 'n_tokens_neox': 1248, 'n_words': 411}