<commit_before>import socket
import re

from time import sleep
from select import select
from threading import Thread


class CollectAllProxy(Thread):
    into = []
    outof = []
    inchannel = {}
    outchannel = {}
    is_shutdown = False

    def __init__(self,
                 inhost, inport,
                 outhost, outport):
        self.inhost = inhost
        self.inport = inport
        self.outhost = outhost
        self.outport = outport

        super(CollectAllProxy, self).__init__()

        self.incon = socket.socket(
            socket.AF_INET,
            socket.SOCK_STREAM)

        self.incon.setsockopt(
            socket.SOL_SOCKET,
            socket.SO_REUSEADDR,
            1)

        # bind the socket
        self.incon.bind((self.inhost, self.inport))
        self.incon.listen(10)

    def shutdown(self):
        print "shutting down"
        self.is_shutdown = True

    def run(self):
        self.into.append(self.incon)
        while not self.is_shutdown:
            sleep(.1)
            print "selecting"
            rready, wready, xready = select(self.into, [], [], 1)
            print "done selecting"
            for self.inc in rready:
                print "running rready"

                if self.incon == self.inc:
                    print "begin accept"
                    (clientsocket, addr) = self.inc.accept()
                    self.handle_accept(clientsocket, addr)
        self.incon.close()

    def handle_accept(self, clientsocket, addr):
        # set low so we have to iter over recv
        bufsize = 50
        eof_re = re.compile("\r\n\r\n")

        message = ""
        recved = clientsocket.recv(bufsize)
        while recved:
            print "*%s*" % recved
            message += recved
            if eof_re.search(message):
                break
            recved = clientsocket.recv(bufsize)
            print "---\nCNT:", message, ":TNC\n----\n"

        reply = self.forward_message(clientsocket, message)

        self.send_response(clientsocket, reply)

        clientsocket.close()

    def forward_message(self, clientsocket, message):
        out = socket.socket(
            socket.AF_INET,
            socket.SOCK_STREAM)

        out.connect((self.outhost, self.outport))

        out.send(message)

        reply = ""
        newContent = True
        buffsize = 50
        while newContent:
            sleep(.01)
            newContent = out.recv(buffsize)
            if not newContent:
                break
            reply += newContent

        print "\nThe Reply:\n"
        print reply
        print "\n---end reply---\n"

        reply = reply.replace("Content-Length: 130", "Content-Length: 141")
        reply += "\nINJECTED!\n"

        return reply

    def adjust_content_length(self, reply):
        pass

    def send_response(self, clientsocket, http):
        print "\n--- ready to send ---\n"
        print http
        print "\n--- done sending ---\n"
        clientsocket.send(http)


if __name__ == '__main__':
    cap = CollectAllProxy(
        '127.0.0.1', 8877,
        '127.0.0.1', 8000)
    cap.run()<commit_msg>dynamically set new content-length<commit_after>import socket
import re

from time import sleep
from select import select
from threading import Thread


class CollectAllProxy(Thread):
    into = []
    outof = []
    inchannel = {}
    outchannel = {}
    is_shutdown = False

    def __init__(self,
                 inhost, inport,
                 outhost, outport):
        self.inhost = inhost
        self.inport = inport
        self.outhost = outhost
        self.outport = outport

        super(CollectAllProxy, self).__init__()

        self.incon = socket.socket(
            socket.AF_INET,
            socket.SOCK_STREAM)

        self.incon.setsockopt(
            socket.SOL_SOCKET,
            socket.SO_REUSEADDR,
            1)

        # bind the socket
        self.incon.bind((self.inhost, self.inport))
        self.incon.listen(10)

    def shutdown(self):
        print "shutting down"
        self.is_shutdown = True

    def run(self):
        self.into.append(self.incon)
        while not self.is_shutdown:
            sleep(.1)
            print "selecting"
            rready, wready, xready = select(self.into, [], [], 1)
            print "done selecting"
            for self.inc in rready:
                print "running rready"

                if self.incon == self.inc:
                    print "begin accept"
                    (clientsocket, addr) = self.inc.accept()
                    self.handle_accept(clientsocket, addr)
        self.incon.close()

    def handle_accept(self, clientsocket, addr):
        # set low so we have to iter over recv
        bufsize = 50
        eof_re = re.compile("\r\n\r\n")

        message = ""
        recved = clientsocket.recv(bufsize)
        while recved:
            print "*%s*" % recved
            message += recved
            if eof_re.search(message):
                break
            recved = clientsocket.recv(bufsize)
            print "---\nCNT:", message, ":TNC\n----\n"

        reply = self.forward_message(clientsocket, message)

        self.send_response(clientsocket, reply)

        clientsocket.close()

    def forward_message(self, clientsocket, message):
        out = socket.socket(
            socket.AF_INET,
            socket.SOCK_STREAM)

        out.connect((self.outhost, self.outport))

        out.send(message)

        reply = ""
        newContent = True
        buffsize = 50
        while newContent:
            sleep(.01)
            newContent = out.recv(buffsize)
            if not newContent:
                break
            reply += newContent

        reply += "\nINJECTED!\n"
        reply = self.adjust_content_length(reply)

        print "\nThe Reply:\n"
        print reply
        print "\n---end reply---\n"

        return reply

    def adjust_content_length(self, reply):

        bits = reply.split("\r\n\r\n")

        if len(bits) == 1:
            return reply

        new_content_length = "Content-Length: " + str(len(bits[1]))

        clre = re.compile(r"Content-Length: .*", re.I)
        bits[0] = clre.sub(new_content_length, bits[0])

        return "\r\n\r\n".join(bits)

    def send_response(self, clientsocket, http):
        print "\n--- ready to send ---\n"
        print http
        print "\n--- done sending ---\n"
        clientsocket.send(http)


if __name__ == '__main__':
    cap = CollectAllProxy(
        '127.0.0.1', 8877,
        '127.0.0.1', 8000)
    cap.run()
{'fraction_non_alphanumeric': 0.10933698296836983, 'fraction_numerical': 0.010948905109489052, 'mean_word_length': 1.6245011971268954, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1829015', 'n_tokens_mistral': 2033, 'n_tokens_neox': 1906, 'n_words': 446}