<commit_before># Module:   http
# Date:     13th September 2007
# Author:   <NAME>, prologic at shortcircuit dot net dot au

"""Hyper Text Transfer Protocol

This module implements the Hyper Text Transfer Protocol
or commonly known as HTTP.
"""


from urllib import unquote
from urlparse import urlparse

from circuits.tools import tryimport

StringIO = tryimport(("cStringIO", "StringIO",))

from circuits.net.sockets import Close, Write
from circuits.core import handler, BaseComponent, Value

import wrappers
from utils import quoted_slash
from headers import parseHeaders
from exceptions import HTTPException
from errors import HTTPError, NotFound
from events import Request, Response, Stream
from errors import Redirect as RedirectError
from exceptions import Redirect as RedirectException


class HTTP(BaseComponent):
    """HTTP Protocol Component

    Implements the HTTP server protocol and parses and processes incoming
    HTTP messages creating and sending an appropriate response.
    """

    def __init__(self, *args, **kwargs):
        super(HTTP, self).__init__(*args, **kwargs)

        self._clients = {}
        self._buffers = {}

    @handler("stream")
    def _on_stream(self, response, data):
        if data is not None:
            if data:
                if response.chunked:
                    buf = [hex(len(data))[2:], "\r\n", data, "\r\n"]
                    data = "".join(buf)
                self.push(Write(response.request.sock, data))
            if response.body and not response.done:
                try:
                    data = response.body.next()
                except StopIteration:
                    data = None
                self.push(Stream(response, data))
        else:
            if response.body:
                response.body.close()
            if response.chunked:
                self.push(Write(response.request.sock, "0\r\n\r\n"))
            if response.close:
                self.push(Close(response.request.sock))
            response.done = True

    @handler("response")
    def _on_response(self, response):
        self.push(Write(response.request.sock, str(response)))

        if response.stream and response.body:
            try:
                data = response.body.next()
            except StopIteration:
                data = None
            self.push(Stream(response, data))
        else:
            body = "".join(response.body)

            if body:
                if response.chunked:
                    buf = [hex(len(body))[2:], "\r\n", body, "\r\n"]
                    body = "".join(buf)

                self.push(Write(response.request.sock, body))

                if response.chunked:
                    self.push(Write(response.request.sock, "0\r\n\r\n"))

            if not response.stream:
                if response.close:
                    self.push(Close(response.request.sock))
                response.done = True

    @handler("disconnect")
    def _on_disconnect(self, sock):
        if sock in self._clients:
            del self._clients[sock]

    @handler("read")
    def _on_read(self, sock, data):
        """Read Event Handler

        Process any incoming data appending it to an internal buffer.
        Split the buffer by the standard HTTP delimiter CRLF and create
        Raw Event per line. Any unfinished lines of text, leave in the buffer.
        """

        if sock in self._clients:
            request, response = self._clients[sock]
            if response.done:
                del self._clients[sock]

        if sock in self._clients:
            request, response = self._clients[sock]
            request.body.write(data)
            contentLength = int(request.headers.get("Content-Length", "0"))
            if not request.body.tell() == contentLength:
                return
        else:
            if data.find('\r\n\r\n') == -1:
                self._buffers.setdefault(sock, []).append(data)
                return
            if sock in self._buffers:
                self._buffers[sock].append(data)
                data = ''.join(self._buffers[sock])
                del self._buffers[sock]

            requestline, data = data.split("\n", 1)
            requestline = requestline.strip()
            method, path, protocol = requestline.split(" ", 2)
            scheme, location, path, params, qs, frag = urlparse(path)

            protocol = tuple(map(int, protocol[5:].split(".")))
            request = wrappers.Request(sock, method, scheme, path,
                    protocol, qs)
            response = wrappers.Response(request)
            self._clients[sock] = request, response

            if frag:
                return self.push(HTTPError(request, response, 400))

            if params:
                path = "%s;%s" % (path, params)

            # Unquote the path+params (e.g. "/this%20path" -> "this path").
            # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2
            #
            # But note that "...a URI must be separated into its components<commit_msg>limit the number of header fragments (prevent oom exploit)<commit_after># Module:   http
# Date:     13th September 2007
# Author:   <NAME>, prologic at shortcircuit dot net dot au

"""Hyper Text Transfer Protocol

This module implements the Hyper Text Transfer Protocol
or commonly known as HTTP.
"""


from urllib import unquote
from urlparse import urlparse

from circuits.tools import tryimport

StringIO = tryimport(("cStringIO", "StringIO",))

from circuits.net.sockets import Close, Write
from circuits.core import handler, BaseComponent, Value

import wrappers
from utils import quoted_slash
from headers import parseHeaders
from exceptions import HTTPException
from errors import HTTPError, NotFound
from events import Request, Response, Stream
from errors import Redirect as RedirectError
from exceptions import Redirect as RedirectException

MAX_HEADER_FRAGENTS = 20


class HTTP(BaseComponent):
    """HTTP Protocol Component

    Implements the HTTP server protocol and parses and processes incoming
    HTTP messages creating and sending an appropriate response.
    """

    def __init__(self, *args, **kwargs):
        super(HTTP, self).__init__(*args, **kwargs)

        self._clients = {}
        self._buffers = {}

    @handler("stream")
    def _on_stream(self, response, data):
        if data is not None:
            if data:
                if response.chunked:
                    buf = [hex(len(data))[2:], "\r\n", data, "\r\n"]
                    data = "".join(buf)
                self.push(Write(response.request.sock, data))
            if response.body and not response.done:
                try:
                    data = response.body.next()
                except StopIteration:
                    data = None
                self.push(Stream(response, data))
        else:
            if response.body:
                response.body.close()
            if response.chunked:
                self.push(Write(response.request.sock, "0\r\n\r\n"))
            if response.close:
                self.push(Close(response.request.sock))
            response.done = True

    @handler("response")
    def _on_response(self, response):
        self.push(Write(response.request.sock, str(response)))

        if response.stream and response.body:
            try:
                data = response.body.next()
            except StopIteration:
                data = None
            self.push(Stream(response, data))
        else:
            body = "".join(response.body)

            if body:
                if response.chunked:
                    buf = [hex(len(body))[2:], "\r\n", body, "\r\n"]
                    body = "".join(buf)

                self.push(Write(response.request.sock, body))

                if response.chunked:
                    self.push(Write(response.request.sock, "0\r\n\r\n"))

            if not response.stream:
                if response.close:
                    self.push(Close(response.request.sock))
                response.done = True

    @handler("disconnect")
    def _on_disconnect(self, sock):
        if sock in self._clients:
            del self._clients[sock]

    @handler("read")
    def _on_read(self, sock, data):
        """Read Event Handler

        Process any incoming data appending it to an internal buffer.
        Split the buffer by the standard HTTP delimiter CRLF and create
        Raw Event per line. Any unfinished lines of text, leave in the buffer.
        """

        if sock in self._clients:
            request, response = self._clients[sock]
            if response.done:
                del self._clients[sock]

        if sock in self._clients:
            request, response = self._clients[sock]
            request.body.write(data)
            contentLength = int(request.headers.get("Content-Length", "0"))
            if not request.body.tell() == contentLength:
                return
        else:
            if data.find('\r\n\r\n') == -1:
                buf = self._buffers.setdefault(sock, [])
                buf.append(data)
                if len(buf) > MAX_HEADER_FRAGENTS:
                    return self.push(HTTPError(request, response, 400))
                return
            if sock in self._buffers:
                self._buffers[sock].append(data)
                data = ''.join(self._buffers[sock])
                del self._buffers[sock]

            requestline, data = data.split("\n", 1)
            requestline = requestline.strip()
            method, path, protocol = requestline.split(" ", 2)
            scheme, location, path, params, qs, frag = urlparse(path)

            protocol = tuple(map(int, protocol[5:].split(".")))
            request = wrappers.Request(sock, method, scheme, path,
                    protocol, qs)
            response = wrappers.Response(request)
            self._clients[sock] = request, response

            if frag:
                return self.push(HTTPError(request, response, 400))

            if params:
                path = "%s;%s" % (path, params)

            # Unquote the path+params (e.g. "/this%20path" -> "this path").
            # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2
            #
            # But note that "...a URI must be separated into its components
{'fraction_non_alphanumeric': 0.10291843834557403, 'fraction_numerical': 0.006861229223038268, 'mean_word_length': 1.861210948299696, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6571196', 'n_tokens_mistral': 2868, 'n_tokens_neox': 2703, 'n_words': 785}