<commit_before># coding: utf-8
'''
utility functions.
'''
from wsgiref.util import is_hop_by_hop
try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse
import six


def format_prefix(prefix):
    '''
    Format url path prefix, ensure '/' in the begining.
    '''
    prefix = '' if prefix is None else prefix
    prefix = prefix if prefix.startswith('/') else '/%s' % prefix
    return prefix


def filter_request_headers(req):
    ret = [(k, v) for (k, v) in req.headers.items()\
           if not is_hop_by_hop(k)]
    remove_headers = ['content-length', 'host']
    ret = dict(filter(lambda x: x[0].lower() not in remove_headers,
                      ret))
    return ret


def replace_location_host(location, upstream, request_host):
    parsed_location = urlparse(location)
    parsed_upstream = urlparse(upstream)
    if parsed_location.netloc and parsed_upstream.netloc\
            and parsed_location.scheme == parsed_upstream.scheme\
            and parsed_upstream.netloc.lower() in parsed_location.netloc.lower():
        return location.lower().replace(parsed_upstream.netloc.lower(),
                                        request_host)
    else:
        return location


def concat_path(*parts):
    ret = ''
    for i in parts:
        if ret.endswith('/') and i.startswith('/'):
            ret = ret[:-1] + i
        else:<commit_msg>allow host header to be proxied at default stage<commit_after># coding: utf-8
'''
utility functions.
'''
from wsgiref.util import is_hop_by_hop
try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse
import six


def format_prefix(prefix):
    '''
    Format url path prefix, ensure '/' in the begining.
    '''
    prefix = '' if prefix is None else prefix
    prefix = prefix if prefix.startswith('/') else '/%s' % prefix
    return prefix


def filter_request_headers(req):
    ret = [(k, v) for (k, v) in req.headers.items()\
           if not is_hop_by_hop(k)]
    remove_headers = ['content-length']
    ret = dict(filter(lambda x: x[0].lower() not in remove_headers,
                      ret))
    return ret


def replace_location_host(location, upstream, request_host):
    parsed_location = urlparse(location)
    parsed_upstream = urlparse(upstream)
    if parsed_location.netloc and parsed_upstream.netloc\
            and parsed_location.scheme == parsed_upstream.scheme\
            and parsed_upstream.netloc.lower() in parsed_location.netloc.lower():
        return location.lower().replace(parsed_upstream.netloc.lower(),
                                        request_host)
    else:
        return location


def concat_path(*parts):
    ret = ''
    for i in parts:
        if ret.endswith('/') and i.startswith('/'):
            ret = ret[:-1] + i
        else:
{'fraction_non_alphanumeric': 0.10804109103790294, 'fraction_numerical': 0.0021253985122210413, 'mean_word_length': 2.6205128205128205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1401680', 'n_tokens_mistral': 846, 'n_tokens_neox': 834, 'n_words': 234}