<commit_before>#!/usr/bin/env python
from __future__ import with_statement

from cStringIO import StringIO
from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError
from tornado.httputil import HTTPHeaders
from tornado.ioloop import IOLoop
from tornado.iostream import IOStream, SSLIOStream
from tornado import stack_context

import contextlib
import errno
import functools
import logging
import re
import socket
import urlparse

try:
    import ssl # python 2.6+
except ImportError:
    ssl = None

class SimpleAsyncHTTPClient(object):
    """Non-blocking HTTP client with no external dependencies.

    WARNING:  This class is still in development and not yet recommended
    for production use.

    This class implements an HTTP 1.1 client on top of Tornado's IOStreams.
    It does not currently implement all applicable parts of the HTTP
    specification, but it does enough to work with major web service APIs
    (mostly tested against the Twitter API so far).

    Many features found in the curl-based AsyncHTTPClient are not yet
    implemented.  The currently-supported set of parameters to HTTPRequest
    are url, method, headers, body, streaming_callback, and header_callback.
    Connections are not reused, and no attempt is made to limit the number
    of outstanding requests.

    Python 2.6 or higher is required for HTTPS support.  Users of Python 2.5
    should use the curl-based AsyncHTTPClient if HTTPS support is required.
    """
    # TODO: singleton magic?
    def __init__(self, io_loop=None):
        self.io_loop = io_loop or IOLoop.instance()

    def close(self):
        pass

    def fetch(self, request, callback, **kwargs):
        if not isinstance(request, HTTPRequest):
            request = HTTPRequest(url=request, **kwargs)
        if not isinstance(request.headers, HTTPHeaders):
            request.headers = HTTPHeaders(request.headers)
        callback = stack_context.wrap(callback)
        _HTTPConnection(self.io_loop, request, callback)



class _HTTPConnection(object):
    def __init__(self, io_loop, request, callback):
        self.io_loop = io_loop
        self.request = request
        self.callback = callback
        self.code = None
        self.headers = None
        self.chunks = None
        with stack_context.StackContext(self.cleanup):
            parsed = urlparse.urlsplit(self.request.url)
            if ":" in parsed.netloc:
                host, _, port = parsed.netloc.partition(":")
                port = int(port)
            else:
                host = parsed.netloc
                port = 443 if parsed.scheme == "https" else 80

            if parsed.scheme == "https":
                # TODO: cert verification, etc
                self.stream = SSLIOStream(socket.socket(),
                                          io_loop=self.io_loop)
            else:
                self.stream = IOStream(socket.socket(),
                                       io_loop=self.io_loop)
            self.stream.connect((host, port),
                                functools.partial(self._on_connect, parsed))

    def _on_connect(self, parsed):
        if "Host" not in self.request.headers:
            self.request.headers["Host"] = parsed.netloc
        if self.request.auth_username:
            auth = "%s:%s" % (self.request.auth_username,
                              self.request.auth_password)
            self.request.headers["Authorization"] = ("Basic %s" %
                                                     auth.encode("base64"))<commit_msg>Support minor HTTP request attributes in SimpleAsyncHTTPClient.<commit_after>#!/usr/bin/env python
from __future__ import with_statement

from cStringIO import StringIO
from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError
from tornado.httputil import HTTPHeaders
from tornado.ioloop import IOLoop
from tornado.iostream import IOStream, SSLIOStream
from tornado import stack_context

import contextlib
import errno
import functools
import logging
import re
import socket
import urlparse

try:
    import ssl # python 2.6+
except ImportError:
    ssl = None

class SimpleAsyncHTTPClient(object):
    """Non-blocking HTTP client with no external dependencies.

    WARNING:  This class is still in development and not yet recommended
    for production use.

    This class implements an HTTP 1.1 client on top of Tornado's IOStreams.
    It does not currently implement all applicable parts of the HTTP
    specification, but it does enough to work with major web service APIs
    (mostly tested against the Twitter API so far).

    Many features found in the curl-based AsyncHTTPClient are not yet
    implemented.  The currently-supported set of parameters to HTTPRequest
    are url, method, headers, body, streaming_callback, and header_callback.
    Connections are not reused, and no attempt is made to limit the number
    of outstanding requests.

    Python 2.6 or higher is required for HTTPS support.  Users of Python 2.5
    should use the curl-based AsyncHTTPClient if HTTPS support is required.
    """
    # TODO: singleton magic?
    def __init__(self, io_loop=None):
        self.io_loop = io_loop or IOLoop.instance()

    def close(self):
        pass

    def fetch(self, request, callback, **kwargs):
        if not isinstance(request, HTTPRequest):
            request = HTTPRequest(url=request, **kwargs)
        if not isinstance(request.headers, HTTPHeaders):
            request.headers = HTTPHeaders(request.headers)
        callback = stack_context.wrap(callback)
        _HTTPConnection(self.io_loop, request, callback)



class _HTTPConnection(object):
    _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE"])

    def __init__(self, io_loop, request, callback):
        self.io_loop = io_loop
        self.request = request
        self.callback = callback
        self.code = None
        self.headers = None
        self.chunks = None
        with stack_context.StackContext(self.cleanup):
            parsed = urlparse.urlsplit(self.request.url)
            if ":" in parsed.netloc:
                host, _, port = parsed.netloc.partition(":")
                port = int(port)
            else:
                host = parsed.netloc
                port = 443 if parsed.scheme == "https" else 80

            if parsed.scheme == "https":
                # TODO: cert verification, etc
                self.stream = SSLIOStream(socket.socket(),
                                          io_loop=self.io_loop)
            else:
                self.stream = IOStream(socket.socket(),
                                       io_loop=self.io_loop)
            self.stream.connect((host, port),
                                functools.partial(self._on_connect, parsed))

    def _on_connect(self, parsed):
        if (self.request.method not in self._SUPPORTED_METHODS and
            not self.request.allow_nonstandard_methods):
            raise KeyError("unknown method %s" % self.request.method)
        if self.request.network_interface:
            raise NotImplementedError(
                "network interface selection not supported")
        if "Host" not in self.request.headers:
            self.request.headers["Host"] = parsed.netloc
        if self.request.auth_username:
            auth = "%s:%s" % (self.request.auth_username,
                              self.request.auth_password)
            self.request.headers["Authorization"] = ("Basic %s" %
                                                     auth.encode("base64"))
        if self.request.user_agent:
            self.request.headers["User-Agent"] = self.request.user_agent
{'fraction_non_alphanumeric': 0.07660130718954249, 'fraction_numerical': 0.00392156862745098, 'mean_word_length': 2.3265217391304347, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '492445', 'n_tokens_mistral': 2000, 'n_tokens_neox': 1921, 'n_words': 641}