<commit_before># Copyright (c) 2010 <NAME>.
# See LICENSE for details.

from urlparse import urlsplit, urljoin
from twisted.python.failure import Failure
from twisted.internet.protocol import Protocol, ClientFactory, ClientCreator
from twisted.internet.defer import fail, Deferred

__all__ = [
    'Headers',
    'Request',
    'Response',
    'Agent',
    'TunnelCreator',
]

_canonicalHeaderParts = { 'www' : 'WWW' }
def _canonicalHeaderName(name):
    def canonical(part):
        return _canonicalHeaderParts.get(part) or part.capitalize()
    return '-'.join(canonical(part.lower()) for part in name.split('-'))

def _parseProxy(proxy):
    try:
        if isinstance(proxy, basestring):
            proxyhost, proxyport = proxy.split(':', 1)
        else:
            proxyhost, proxyport = proxy<commit_msg>_parseProxy shouldn't fail if there is no proxy<commit_after># Copyright (c) 2010 <NAME>.
# See LICENSE for details.

from urlparse import urlsplit, urljoin
from twisted.python.failure import Failure
from twisted.internet.protocol import Protocol, ClientFactory, ClientCreator
from twisted.internet.defer import fail, Deferred

__all__ = [
    'Headers',
    'Request',
    'Response',
    'Agent',
    'TunnelCreator',
]

_canonicalHeaderParts = { 'www' : 'WWW' }
def _canonicalHeaderName(name):
    def canonical(part):
        return _canonicalHeaderParts.get(part) or part.capitalize()
    return '-'.join(canonical(part.lower()) for part in name.split('-'))

def _parseProxy(proxy):
    if not proxy:
        return proxy
    try:
        if isinstance(proxy, basestring):
            proxyhost, proxyport = proxy.split(':', 1)
        else:
            proxyhost, proxyport = proxy
{'fraction_non_alphanumeric': 0.11405055849500294, 'fraction_numerical': 0.005878894767783657, 'mean_word_length': 3.538666666666667, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4082945', 'n_tokens_mistral': 518, 'n_tokens_neox': 496, 'n_words': 143}