<commit_before>from ._config import PluginConfiguration
from ._parser import EndpointParser
from treq.client import HTTPClient
from twisted.internet import reactor, defer
from twisted.internet.interfaces import IHalfCloseableProtocol
from twisted.python import log
from twisted.python.failure import Failure
from twisted.web import server, proxy
from twisted.web.client import Agent
from twisted.web.resource import Resource
from twisted.web.server import NOT_DONE_YET
from urllib import quote as urlquote
from zope.interface import directlyProvides
import StringIO
import json
import treq
import urlparse

class NoPostHooks(Exception):
    """
    Do not run any post-hooks, because of an incompatible Docker response type
    (streaming/hijacked or chunked).
    """

class DockerProxyClient(proxy.ProxyClient):
    """
    An HTTP proxy which knows how to break HTTP just right so that Docker
    stream (attach/events) API calls work.

    self.http: A boolean which reflects whether the connection is in HTTP mode
        (True) or "hijack" mode (False). See
        https://docs.docker.com/reference/api/docker_remote_api_v1.14/#32-hijacking
    """

    http = True
    _streaming = False
    _listener = None
    _responsePartBuffer = b""

    def _fireListener(self, result):
        if self._listener is not None:
            d = self._listener
            self._listener = None
            d.callback(result)

    def setStreamingMode(self, streamingMode):
        """
        Allow anyone with a reference to us to toggle on/off streaming mode.
        Useful when we have no post-hooks (and no indication from Docker that
        it's sending packets of JSON e.g. with build) and we want to avoid
        buffering slow responses in memory.
        """
        self._streaming = streamingMode
        if streamingMode:
            self._fireListener(Failure(NoPostHooks()))

    def registerListener(self, d):
        """
        Register a one shot listener, which can fire either with:
           * Failure(NoPostHooks()) if the proxy is handling comms back to
             the client (streaming/chunked modes), or
           * A tuple containing the (response, code, content-type).
        """
        self._listener = d

    def _handleRawStream(self):
        """
        Switch the current connection to be a "hijacked" aka raw stream: one
        where bytes just get naively proxied back and forth.
        """
        def loseWriteConnectionReason(reason):
            # discard the reason, for compatibility with readConnectionLost
            self.transport.loseWriteConnection()
        self.father.transport.readConnectionLost = loseWriteConnectionReason
        directlyProvides(self.father.transport, IHalfCloseableProtocol)
        self.http = False
        self.father.transport.write(
            "HTTP/1.1 200 OK\r\n"
            "Content-Type: application/vnd.docker.raw-stream\r\n"
            "\r\n")
        def stdinHandler(data):
            self.transport.write(data)
        self.father.transport.protocol.dataReceived = stdinHandler
        self.setStreamingMode(True)

    def handleHeader(self, key, value):
        if (key.lower() == "content-type" and
                value == "application/vnd.docker.raw-stream"):
            self._handleRawStream()
        # XXX Turns out, the build endpoint doesn't actually used chunked
        # encoding. It just sends some JSON documents which maybe happen to
        # line up with packet boundaries. So the following if statement is both
        # untested and potentially never triggered in practice. :(
        if key.lower() == "transfer-encoding" and value == "chunked":
            self.setStreamingMode(True)
        return proxy.ProxyClient.handleHeader(self, key, value)


    def handleResponsePart(self, buffer):
        """
        If we're not in streaming mode, buffer the response part(s).
        """
        if self._streaming:
            proxy.ProxyClient.handleResponsePart(self, buffer)
        else:
            self._responsePartBuffer += buffer


    def handleResponseEnd(self):
        """
        If we're completing a chunked response, up-call to handle it like a
        regular reverse proxy.

        If we're completing a non-chunked response, fire the post-hooks.

        If we're completing a hijacked response, pass through the connection
        close.
        """
        if self.http:
            if self._streaming:<commit_msg>address review comment - remove untested code<commit_after>from ._config import PluginConfiguration
from ._parser import EndpointParser
from treq.client import HTTPClient
from twisted.internet import reactor, defer
from twisted.internet.interfaces import IHalfCloseableProtocol
from twisted.python import log
from twisted.python.failure import Failure
from twisted.web import server, proxy
from twisted.web.client import Agent
from twisted.web.resource import Resource
from twisted.web.server import NOT_DONE_YET
from urllib import quote as urlquote
from zope.interface import directlyProvides
import StringIO
import json
import treq
import urlparse

class NoPostHooks(Exception):
    """
    Do not run any post-hooks, because of an incompatible Docker response type
    (streaming/hijacked or chunked).
    """

class DockerProxyClient(proxy.ProxyClient):
    """
    An HTTP proxy which knows how to break HTTP just right so that Docker
    stream (attach/events) API calls work.

    self.http: A boolean which reflects whether the connection is in HTTP mode
        (True) or "hijack" mode (False). See
        https://docs.docker.com/reference/api/docker_remote_api_v1.14/#32-hijacking
    """

    http = True
    _streaming = False
    _listener = None
    _responsePartBuffer = b""

    def _fireListener(self, result):
        if self._listener is not None:
            d = self._listener
            self._listener = None
            d.callback(result)

    def setStreamingMode(self, streamingMode):
        """
        Allow anyone with a reference to us to toggle on/off streaming mode.
        Useful when we have no post-hooks (and no indication from Docker that
        it's sending packets of JSON e.g. with build) and we want to avoid
        buffering slow responses in memory.
        """
        self._streaming = streamingMode
        if streamingMode:
            self._fireListener(Failure(NoPostHooks()))

    def registerListener(self, d):
        """
        Register a one shot listener, which can fire either with:
           * Failure(NoPostHooks()) if the proxy is handling comms back to
             the client (streaming/chunked modes), or
           * A tuple containing the (response, code, content-type).
        """
        self._listener = d

    def _handleRawStream(self):
        """
        Switch the current connection to be a "hijacked" aka raw stream: one
        where bytes just get naively proxied back and forth.
        """
        def loseWriteConnectionReason(reason):
            # discard the reason, for compatibility with readConnectionLost
            self.transport.loseWriteConnection()
        self.father.transport.readConnectionLost = loseWriteConnectionReason
        directlyProvides(self.father.transport, IHalfCloseableProtocol)
        self.http = False
        self.father.transport.write(
            "HTTP/1.1 200 OK\r\n"
            "Content-Type: application/vnd.docker.raw-stream\r\n"
            "\r\n")
        def stdinHandler(data):
            self.transport.write(data)
        self.father.transport.protocol.dataReceived = stdinHandler
        self.setStreamingMode(True)

    def handleHeader(self, key, value):
        if (key.lower() == "content-type" and
                value == "application/vnd.docker.raw-stream"):
            self._handleRawStream()
        return proxy.ProxyClient.handleHeader(self, key, value)


    def handleResponsePart(self, buffer):
        """
        If we're not in streaming mode, buffer the response part(s).
        """
        if self._streaming:
            proxy.ProxyClient.handleResponsePart(self, buffer)
        else:
            self._responsePartBuffer += buffer


    def handleResponseEnd(self):
        """
        If we're completing a chunked response, up-call to handle it like a
        regular reverse proxy.

        If we're completing a non-chunked response, fire the post-hooks.

        If we're completing a hijacked response, pass through the connection
        close.
        """
        if self.http:
            if self._streaming:
{'fraction_non_alphanumeric': 0.07523731395757646, 'fraction_numerical': 0.0023438415563107936, 'mean_word_length': 2.552872606161532, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5685695', 'n_tokens_mistral': 2248, 'n_tokens_neox': 2161, 'n_words': 763}