<commit_before>from twisted.python import log
from twisted.web import http, proxy
from twisted.internet import reactor
from twisted.internet.protocol import Protocol
from twisted.internet.endpoints import serverFromString
from twisted.names import client
from twisted.web.client import Agent, readBody
from urlparse import urlparse, urlunparse
from twisted.internet.defer import inlineCallbacks, succeed, Deferred

# blacklist of disallowed domains (change this to ips later, and move)

DEFAULT_BLACKLIST = ["192.168.3.11"]

#  Set up proxy


class ProxyFactory(http.HTTPFactory):

    def __init__(self, blacklist, resolver, http_client):
        http.HTTPFactory.__init__(self)
        self.blacklist = blacklist
        self.resolver = resolver
        self.http_client = http_client

    def buildProtocol(self, addr):
        return Proxy(self.blacklist, self.resolver, self.http_client)


class CheckProxyRequest(proxy.ProxyRequest):
    def process(self):
        host, _, port = self.getAllHeaders()['host'].partition(':')
        d = self.channel.resolver.getHostByName(host)
        d.addCallback(self.setIP, host)
        d.addErrback(self.handleError)
        return d

    def handleError(self, failure):
        log.err(failure)
        self.setResponseCode(400)
        self.write("<html>Denied</html>")
        self.finish()
        return

    def setIP(self, ip_addr, host):
        if not ip_addr:
            self.setResponseCode(400)
            self.write("<html>ERROR: No IP adresses found for name %r" %
                       host + " </html>")
            self.finish()
            return
        if ip_addr in DEFAULT_BLACKLIST:
            self.setResponseCode(400)
            self.write("<html>Denied</html>")
            self.finish()
            return
        else:
            uri = self.replaceHostWithIP(self.uri, ip_addr)
            headers = self.requestHeaders
            d = self.channel.http_client.request(
                self.method, uri, headers, StringProducer(self.content.read()))
            print 'HERE'
            d.addCallback(self.sendResponseBack)

    def replaceHostWithIP(self, uri, ip_addr):
        scheme, netloc, path, params, query, fragment = urlparse(uri)
        _, _, port = netloc.partition(':')
        if port:
            ip_addr = "%s:%s" % (ip_addr, port)
        return urlunparse((scheme, ip_addr, path, params, query, fragment))

    @inlineCallbacks
    def sendResponseBack(self, r):
        self.setResponseCode(r.code)
        for key, value in r.headers.getAllRawHeaders():
            self.responseHeaders.addRawHeader(key, value)
        print "ASDf"
        body = yield readBody(r)
        print "Last"
        self.write(body)
        self.finish()


class StringProducer(object):
    def __init__(self, body):
        self.body = body
        self.length = len(body)

    def startProducing(self, consumer):
        consumer.write(self.body)
        return succeed(None)

    def pauseProducing(self):
        pass

    def stopProducing(self):
        pass


class Proxy(proxy.Proxy):

    requestFactory = CheckProxyRequest

    def __init__(self, blacklist, resolver, http_client):
        proxy.Proxy.__init__(self)<commit_msg>Remove debugging prints! Whee.<commit_after>from twisted.python import log
from twisted.web import http, proxy
from twisted.internet import reactor
from twisted.internet.protocol import Protocol
from twisted.internet.endpoints import serverFromString
from twisted.names import client
from twisted.web.client import Agent, readBody
from urlparse import urlparse, urlunparse
from twisted.internet.defer import inlineCallbacks, succeed, Deferred

# blacklist of disallowed domains (change this to ips later, and move)

DEFAULT_BLACKLIST = ["192.168.3.11"]

#  Set up proxy


class ProxyFactory(http.HTTPFactory):

    def __init__(self, blacklist, resolver, http_client):
        http.HTTPFactory.__init__(self)
        self.blacklist = blacklist
        self.resolver = resolver
        self.http_client = http_client

    def buildProtocol(self, addr):
        return Proxy(self.blacklist, self.resolver, self.http_client)


class CheckProxyRequest(proxy.ProxyRequest):
    def process(self):
        host, _, port = self.getAllHeaders()['host'].partition(':')
        d = self.channel.resolver.getHostByName(host)
        d.addCallback(self.setIP, host)
        d.addErrback(self.handleError)
        return d

    def handleError(self, failure):
        log.err(failure)
        self.setResponseCode(400)
        self.write("<html>Denied</html>")
        self.finish()
        return

    def setIP(self, ip_addr, host):
        if not ip_addr:
            self.setResponseCode(400)
            self.write("<html>ERROR: No IP adresses found for name %r" %
                       host + " </html>")
            self.finish()
            return
        if ip_addr in DEFAULT_BLACKLIST:
            self.setResponseCode(400)
            self.write("<html>Denied</html>")
            self.finish()
            return
        else:
            uri = self.replaceHostWithIP(self.uri, ip_addr)
            headers = self.requestHeaders
            d = self.channel.http_client.request(
                self.method, uri, headers, StringProducer(self.content.read()))
            d.addCallback(self.sendResponseBack)

    def replaceHostWithIP(self, uri, ip_addr):
        scheme, netloc, path, params, query, fragment = urlparse(uri)
        _, _, port = netloc.partition(':')
        if port:
            ip_addr = "%s:%s" % (ip_addr, port)
        return urlunparse((scheme, ip_addr, path, params, query, fragment))

    @inlineCallbacks
    def sendResponseBack(self, r):
        self.setResponseCode(r.code)
        for key, value in r.headers.getAllRawHeaders():
            self.responseHeaders.addRawHeader(key, value)
        body = yield readBody(r)
        self.write(body)
        self.finish()


class StringProducer(object):
    def __init__(self, body):
        self.body = body
        self.length = len(body)

    def startProducing(self, consumer):
        consumer.write(self.body)
        return succeed(None)

    def pauseProducing(self):
        pass

    def stopProducing(self):
        pass


class Proxy(proxy.Proxy):

    requestFactory = CheckProxyRequest

    def __init__(self, blacklist, resolver, http_client):
        proxy.Proxy.__init__(self)
{'fraction_non_alphanumeric': 0.09953051643192488, 'fraction_numerical': 0.005633802816901409, 'mean_word_length': 2.588433464345873, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1529656', 'n_tokens_mistral': 1865, 'n_tokens_neox': 1813, 'n_words': 438}