<commit_before>"""

:mod:`Tor2Web`
=====================================================

.. automodule:: Tor2Web
   :synopsis: Main Tor2web Server Implementation

.. moduleauthor:: <NAME>' <<EMAIL>>
.. moduleauthor:: <NAME> <<EMAIL>>

"""

# -*- coding: utf-8 -*-
from __future__ import print_function

import os
import re
import sys
import mimetypes
import random
import signal
import socket
import zlib
import hashlib
from StringIO import StringIO
from random import choice
from functools import partial
from urlparse import urlsplit
from cgi import parse_header

from OpenSSL._util import ffi as _ffi, lib as _lib

from zope.interface import implements
from twisted.spread import pb
from twisted.internet import reactor, protocol, defer, address
from twisted.internet.abstract import isIPAddress, isIPv6Address
from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
from twisted.protocols.policies import WrappingFactory
from twisted.web import http, client, _newclient
from twisted.web.error import SchemeNotSupported
from twisted.web.http import datetimeToString, StringTransport, \
    _IdentityTransferDecoder, _ChunkedTransferDecoder, parse_qs
from twisted.web.http_headers import Headers
from twisted.web.server import NOT_DONE_YET
from twisted.web.template import flattenString, XMLString
from twisted.web.iweb import IBodyProducer
from twisted.python import log, logfile
from twisted.python.compat import networkString, intToBytes
from twisted.python.failure import Failure
from twisted.python.filepath import FilePath
from twisted.internet.task import LoopingCall
from tor2web import __version__
from tor2web.utils.config import Config
from tor2web.utils.daemon import Daemon, set_pdeathsig, set_proctitle
from tor2web.utils.hostsmap import HostsMap
from tor2web.utils.lists import LimitedSizeDict, List, TorExitNodeList
from tor2web.utils.mail import sendmail, MailException
from tor2web.utils.misc import listenTCPonExistingFD, listenSSLonExistingFD, re_sub, verify_onion
from tor2web.utils.socks import SOCKSError, SOCKS5ClientEndpoint, TLSWrapClientEndpoint
from tor2web.utils.ssl import T2WSSLContextFactory, HTTPSVerifyingContextFactory
from tor2web.utils.stats import T2WStats
from tor2web.utils.storage import Storage
from tor2web.utils.templating import PageTemplate
from tor2web.utils.gettor import getRedirectURL, getOSandLC, processGetTorRequest, getTorTask
from tor2web.utils.urls import normalize_url, parent_urls

SOCKS_errors = {
    0x00: "error_sock_generic.tpl",
    0x23: "error_sock_hs_not_found.tpl",
    0x24: "error_sock_hs_not_reachable.tpl"
}


class T2WRPCServer(pb.Root):
    def __init__(self, config):
        self.config = config
        self.stats = T2WStats()
        self.block_list = []
        self.block_regexps = []
        self.blocked_ua = []
        self.hosts_map = {}
        self.TorExitNodes = []
        self.certs_tofu = LimitedSizeDict(size_limit=config.ssl_tofu_cache_size)

        self.load_lists()

    def load_lists(self):
        if config.mode == "BLOCKLIST":
            self.block_list = List(config.t2w_file_path('lists/blocklist_hashed.txt'),
                                   config.automatic_blocklist_updates_source,
                                   config.automatic_blocklist_updates_mode,
                                   config.automatic_blocklist_updates_refresh)

            # clear local cleartext list
            # (load -> hash -> clear feature; for security reasons)
            self.blocklist_cleartext = List(config.t2w_file_path('lists/blocklist_cleartext.txt'))
            for i in self.blocklist_cleartext:
                self.block_list.add(hashlib.md5(normalize_url(i)).hexdigest())

            self.block_list.dump()

            self.blocklist_cleartext.clear()
            self.blocklist_cleartext.dump()

            self.block_regexps = List(config.t2w_file_path('lists/blocklist_regexp.txt'))
            self.block_regexps = [re.compile(regexp_pattern) for regexp_pattern in self.block_regexps]

        if config.blockcrawl:
            self.blocked_ua = [ua.lower() for ua in List(config.t2w_file_path('lists/blocked_ua.txt'))]

        # Load Exit Nodes list with the refresh rate configured  in config file
        self.TorExitNodes = TorExitNodeList(config.t2w_file_path('lists/exitnodelist.txt'),
                                            'https://check.torproject.org/exit-addresses',
                                            'REPLACE',
                                            config.exit_node_list_refresh)

        self.hosts_map = HostsMap(config.t2w_file_path('lists/hosts_map.txt')).hosts
        if config.mode == "TRANSLATION":
            self.hosts_map[config.basehost] = config.onion

    def remote_get_config(self):
        return self.config.__dict__

    def remote_get_blocked_ua_list(self):
        return list(self.blocked_ua)

    def remote_get_block_list(self):
        return list(self.block_list)

    def remote_get_block_regexps(self):
        return list(self.block_regexps)

    def remote_get_tor_exits_list(self):
        return list(self.TorExitNodes)

    def remote_get_hosts_map(self):
        return dict(self.hosts_map)

    def remote_is_https(self, hostname):
        return hostname in self.certs_tofu

    def remote_verify_tls_tofu(self, hostname, cert):
        h = hashlib.sha512(cert).hexdigest()
        if hostname not in self.certs_tofu:
            self.certs_tofu[hostname] = h
            return True

        return self.certs_tofu[hostname] == h

    def remote_update_stats(self, onion):
        self.stats.update(onion)

    def remote_get_yesterday_stats(self):
        return self.stats.yesterday_stats

    def remote_log_access(self, line):
        t2w_daemon.logfile_access.write(line)

    def remote_log_debug(self, line):
        date = datetimeToString()
        t2w_daemon.logfile_debug.write(date + " " + str(line) + "\n")

    def remote_shutdown(self):
        reactor.stop()


@defer.inlineCallbacks
def rpc(f, *args, **kwargs):
    d = rpc_factory.getRootObject()
    d.addCallback(lambda obj: obj.callRemote(f, *args, **kwargs))
    ret = yield d
    defer.returnValue(ret)


def rpc_log(msg):
    rpc("log_debug", str(msg))


def rpc_shutdown():
    rpc("shutdown")


class T2WPP(protocol.ProcessProtocol):
    def __init__(self, father, childFDs, fds_https, fds_http):
        self.father = father
        self.childFDs = childFDs
        self.fds_https = fds_https
        self.fds_http = fds_http

    def connectionMade(self):
        self.pid = self.transport.pid

    def processExited(self, reason):
        for i, subprocess in enumerate(self.father.subprocesses):
            if subprocess == self.pid:
                del self.father.subprocesses[i]
                break

        if not self.father.quitting:
            subprocess = spawnT2W(self.father, self.childFDs, self.fds_https, self.fds_http)
            self.father.subprocesses.append(subprocess.pid)

        if len(self.father.subprocesses) == 0:
            try:
                reactor.stop()
            except Exception:
                pass


def spawnT2W(father, childFDs, fds_https, fds_http):
    child_env = os.environ.copy()
    child_env['T2W_FDS_HTTPS'] = fds_https
    child_env['T2W_FDS_HTTP'] = fds_http

    return reactor.spawnProcess(T2WPP(father, childFDs, fds_https, fds_http),
                                sys.executable,
                                [sys.executable, __file__] + sys.argv[1:],
                                env=child_env,
                                childFDs=childFDs)


class Tor2webObj(object):
    def __init__(self):
        # The destination hidden service identifier
        self.onion = None

        # The path portion of the URI
        self.path = None

        # The full address (hostname + uri) that must be requested
        self.address = None

        # The headers to be sent
        self.headers = None

        # The requested uri
        self.uri = None

        # A boolean that keeps track of client gzip support
        self.client_supports_gzip = False

        # A boolean that keeps track of server gzip support
        self.server_response_is_gzip = False

        # A variably that keep tracks of special contents to be parsed
        self.special_content = None


class BodyReceiver(protocol.Protocol):
    def __init__(self, finished):
        self._finished = finished
        self._data = []

    def dataReceived(self, chunk):
        self._data.append(chunk)

    def write(self, chunk):
        self._data.append(chunk)

    def connectionLost(self, reason):
        self._finished.callback(''.join(self._data))


class BodyStreamer(protocol.Protocol):
    def __init__(self, streamfunction, finished):
        self._finished = finished
        self._streamfunction = streamfunction

    def dataReceived(self, data):
        self._streamfunction(data)

    def connectionLost(self, reason):
        self._finished.callback('')


class BodyProducer(object):
    implements(IBodyProducer)

    def __init__(self):
        self.length = _newclient.UNKNOWN_LENGTH
        self.finished = defer.Deferred()
        self.consumer = None
        self.can_stream = False
        self.can_stream_d = defer.Deferred()

    def startProducing(self, consumer):
        self.consumer = consumer
        self.can_stream = True
        self.can_stream_d.callback(True)
        return self.finished

    @defer.inlineCallbacks
    def dataReceived(self, data):
        if not self.can_stream:
            yield self.can_stream_d
        self.consumer.write(data)

    def allDataReceived(self):
        self.finished.callback(None)

    def resumeProducing(self):
        pass

    def pauseProducing(self):
        pass

    def stopProducing(self):
        pass

class Agent(client.Agent):
    def __init__(self, reactor,
                 contextFactory=client.WebClientContextFactory(),
                 connectTimeout=None, bindAddress=None,
                 pool=None, sockhost=None, sockport=None):
        self._sockhost = sockhost
        self._sockport = sockport
        client.Agent.__init__(self, reactor, contextFactory,
                               connectTimeout, bindAddress, pool)

    def _getEndpoint(self, scheme, host, port):
        def verify_tofu(hostname, cert):
            return rpc("verify_tls_tofu", hostname, cert)

        kwargs = {}
        if self._connectTimeout is not None:
            kwargs['timeout'] = self._connectTimeout
        kwargs['bindAddress'] = self._bindAddress

        if scheme == 'http':
            return SOCKS5ClientEndpoint(self._reactor, self._sockhost, self._sockport,
                                        host, port, config.socksoptimisticdata, **kwargs)
        elif scheme == 'https':
            torSockEndpoint = SOCKS5ClientEndpoint(self._reactor, self._sockhost,
                                                   self._sockport, host, port, config.socksoptimisticdata, **kwargs)
            return TLSWrapClientEndpoint(HTTPSVerifyingContextFactory(host, verify_tofu), torSockEndpoint)
        elif scheme == 'normal-http':
             return TCP4ClientEndpoint(self._reactor, host, port, **kwargs)
        elif scheme == 'normal-https':
            return SSL4ClientEndpoint(self._reactor, host, port,
                                      self._wrapContextFactory(host, port),
                                      **kwargs)
        else:
            raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,))

    @defer.inlineCallbacks
    def request(self, method, uri, headers, bodyProducer=None):
        """
        Edited version of twisted Agent.request in order to make it asyncronous!
        """
        parsedURI = client._URI.fromBytes(uri)

        is_https = yield rpc("is_https", parsedURI.host)

        scheme = 'https' if is_https else 'http'

        for key, values in headers.getAllRawHeaders():
            fixed_values = [re_sub(rexp['w2t'], r'' + scheme + r'://\2.onion', value) for value in values]
            headers.setRawHeaders(key, fixed_values)

        try:
            endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host,
                                         parsedURI.port)
        except SchemeNotSupported:
            defer.returnValue(Failure())

        key = (parsedURI.scheme, parsedURI.host, parsedURI.port)
        ret = yield self._requestWithEndpoint(key, endpoint, method, parsedURI,
                                              headers, bodyProducer,
                                              parsedURI.originForm)
        defer.returnValue(ret)

class RedirectAgent(client.RedirectAgent):
    """
    Overridden client.RedirectAgent version where we evaluate and handle automatically only HTTPS redirects
    """
    def _handleResponse(self, response, method, uri, headers, redirectCount):
        locationHeaders = response.headers.getRawHeaders('location', [])
        if locationHeaders:
            location = self._resolveLocation(uri, locationHeaders[0])
            parsed = client._URI.fromBytes(location)
            if parsed.scheme == 'https':
                return client.RedirectAgent._handleResponse(self, response, method, uri, headers, redirectCount)

        return response

class T2WRequest(http.Request):
    """
    Used by Tor2webProxy to implement a simple web proxy.
    """

    def __init__(self, channel, queued, reactor=reactor):
        """
        Method overridden to change some part of proxy.Request and of the base http.Request
        """
        self.reactor = reactor
        self.notifications = []
        self.channel = channel
        self.queued = queued
        self.requestHeaders = Headers()
        self.received_cookies = {}
        self.responseHeaders = Headers()
        self.cookies = []  # outgoing cookies
        self.bodyProducer = BodyProducer()
        self.proxy_d = None
        self.proxy_response = None

        self.stream = ''

        self.header_injected = False
        # If we should disable the banner,
        # say that we have already injected it.
        if config.disable_banner:
            self.header_injected = True


        self.transport = StringTransport() if queued else self.channel.transport

        self.obj = Tor2webObj()
        self.var = Storage()
        self.var['version'] = __version__
        self.var['basehost'] = config.basehost
        self.var['errorcode'] = None

        self.decoderGzip = None
        self.encoderGzip = None

        self.pool = pool

    def getRequestHostname(self):
        """
            Function overload to fix ipv6 bug:
                http://twistedmatrix.com/trac/ticket/6014
        """
        host = self.getHeader(b'host')
        if not host:
            return networkString(self.getHost().host)
        if host[0] == '[':
            return host.split(']', 1)[0] + "]"

        # return everything before the ':'
        return networkString(host.split(':', 1)[0])


    def forwardData(self, data, end=False):
        if not self.startedWriting:
            if self.obj.client_supports_gzip:
                self.setHeader(b'content-encoding', b'gzip')

            if config.extra_http_response_headers:
                for header, value in config.extra_http_response_headers.iteritems():
                    self.setHeader(header, value)

            if data and end:
                self.setHeader(b'content-length', intToBytes(len(data)))

        if data:
            self.write(data)

    def getForwarders(self):
        forwarders = []

        try:
            port = self.channel.transport.getPeer().port

            xForwardedFor = self.requestHeaders.getRawHeaders("X-Forwarded-For")
            for forwardHeader in xForwardedFor or []:
                forwardList = forwardHeader.replace(" ", "").split(",")

                for ip in forwardList:
                    if isIPAddress(ip):
                        forwarders.append(address.IPv4Address("TCP",
                                                              ip.strip(),
                                                              port))
                    elif isIPv6Address(ip):
                        forwarders.append(address.IPv6Address("TCP",
                                                              ip.strip(),
                                                              port))
                    else:
                        raise Exception
        except Exception:
            return []

        return forwarders

    def requestReceived(self, command, path, version):
        """
        Method overridden to reduce the function actions
        """
        self.method, self.uri = command, path
        self.clientproto = version

        # we get the ip address of the user that can be:
        #  - written in the x-forwared-for header if a proxy is in between
        #  - the ip address of the transport endpoint
        forwarders = self.getForwarders()
        if forwarders:
            self.client = forwarders[0]
        else:
            self.client = self.channel.transport.getPeer()

        self.host = self.channel.transport.getHost()

        self.proto = 'http://' if config.transport == 'HTTP' else 'https://'

        port = self.channel.transport.getHost().port
        self.port = '' if port in [80,443] else ':%d' % port

        self.process()

    def add_banner(self, banner, data):
        """
        Inject tor2web banner inside the returned page
        """
        return str(data.group(1)) + str(banner)

    @defer.inlineCallbacks
    def handleFixPart(self, data):
        if self.obj.server_response_is_gzip:
            data = self.unzip(data)

        data = self.stream + data

        if len(data) >= config.bufsize * 2:
            if self.obj.special_content == 'HTML':
                if not self.header_injected and data.find("<body") != -1:
                    banner = yield flattenString(self, templates['banner.tpl'])
                    data = re.sub(rexp['body'], partial(self.add_banner, banner), data)
                    self.header_injected = True

            if config.avoid_rewriting_visible_content and self.obj.special_content == 'HTML':
                data = re_sub(rexp['html_t2w'], r'\1\2' + self.proto + r'\3.' + config.basehost + self.port + r'\4', data)
            else:
                data = re_sub(rexp['t2w'], self.proto + r'\2.' + config.basehost + self.port, data)

            forward = data[:-config.bufsize]

            self.forwardData(self.handleCleartextForwardPart(forward))

            self.stream = data[-config.bufsize:]

        else:
            self.stream = data

    @defer.inlineCallbacks
    def handleFixEnd(self, data):
        if self.obj.server_response_is_gzip:
            data = self.unzip(data, True)

        data = self.stream + data

        if self.obj.special_content == 'HTML':
            if not self.header_injected and data.find("<body") != -1:
                banner = yield flattenString(self, templates['banner.tpl'])
                data = re.sub(rexp['body'], partial(self.add_banner, banner), data)
                self.header_injected = True

        if config.avoid_rewriting_visible_content and self.obj.special_content == 'HTML':
            data = re_sub(rexp['html_t2w'], r'\1\2' + self.proto + r'\3.' + config.basehost + self.port + r'\4', data)
        else:
            data = re_sub(rexp['t2w'], self.proto + r'\2.' + config.basehost + self.port, data)

        data = self.handleCleartextForwardPart(data, True)
        self.forwardData(data, True)

        self.stream = ''

        self.finish()

    def handleGzippedForwardPart(self, data, end=False):
        if not self.obj.client_supports_gzip:
            data = self.unzip(data, end)

        return data

    def handleCleartextForwardPart(self, data, end=False):
        if self.obj.client_supports_gzip:
            data = self.zip(data, end)

        return data

    def handleForwardPart(self, data):
        if self.obj.server_response_is_gzip:
            data = self.handleGzippedForwardPart(data)
        else:
            data = self.handleCleartextForwardPart(data)

        self.forwardData(data)

    def handleForwardEnd(self, data):
        if self.obj.server_response_is_gzip:
            data = self.handleGzippedForwardPart(data, True)
        else:
            data = self.handleCleartextForwardPart(data, True)

        self.forwardData(data, True)
        self.finish()

    def contentFinish(self, data):
        if self.obj.client_supports_gzip:
            self.setHeader(b'content-encoding', b'gzip')
            data = self.zip(data, True)

        self.setHeader(b'content-length', intToBytes(len(data)))

        if self.isSecure():
            self.setHeader(b'strict-transport-security', b'max-age=31536000')

        if config.mode == 'TRANSLATION':
            # no additional headers are injected when in translation mode
            return

        if config.deny_caching:
            self.setHeader(b'cache-control', b'no-cache no-store, must-revalidate')
            self.setHeader(b'pragma', 'no-cache')
            self.setHeader(b'expires', b'-1')
<commit_msg>Fixed mixed-content warnings on HTTPS<commit_after>"""

:mod:`Tor2Web`
=====================================================

.. automodule:: Tor2Web
   :synopsis: Main Tor2web Server Implementation

.. moduleauthor:: <NAME>' <<EMAIL>>
.. moduleauthor:: <NAME> <<EMAIL>>

"""

# -*- coding: utf-8 -*-
from __future__ import print_function

import os
import re
import sys
import mimetypes
import random
import signal
import socket
import zlib
import hashlib
from StringIO import StringIO
from random import choice
from functools import partial
from urlparse import urlsplit
from cgi import parse_header

from OpenSSL._util import ffi as _ffi, lib as _lib

from zope.interface import implements
from twisted.spread import pb
from twisted.internet import reactor, protocol, defer, address
from twisted.internet.abstract import isIPAddress, isIPv6Address
from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
from twisted.protocols.policies import WrappingFactory
from twisted.web import http, client, _newclient
from twisted.web.error import SchemeNotSupported
from twisted.web.http import datetimeToString, StringTransport, \
    _IdentityTransferDecoder, _ChunkedTransferDecoder, parse_qs
from twisted.web.http_headers import Headers
from twisted.web.server import NOT_DONE_YET
from twisted.web.template import flattenString, XMLString
from twisted.web.iweb import IBodyProducer
from twisted.python import log, logfile
from twisted.python.compat import networkString, intToBytes
from twisted.python.failure import Failure
from twisted.python.filepath import FilePath
from twisted.internet.task import LoopingCall
from tor2web import __version__
from tor2web.utils.config import Config
from tor2web.utils.daemon import Daemon, set_pdeathsig, set_proctitle
from tor2web.utils.hostsmap import HostsMap
from tor2web.utils.lists import LimitedSizeDict, List, TorExitNodeList
from tor2web.utils.mail import sendmail, MailException
from tor2web.utils.misc import listenTCPonExistingFD, listenSSLonExistingFD, re_sub, verify_onion
from tor2web.utils.socks import SOCKSError, SOCKS5ClientEndpoint, TLSWrapClientEndpoint
from tor2web.utils.ssl import T2WSSLContextFactory, HTTPSVerifyingContextFactory
from tor2web.utils.stats import T2WStats
from tor2web.utils.storage import Storage
from tor2web.utils.templating import PageTemplate
from tor2web.utils.gettor import getRedirectURL, getOSandLC, processGetTorRequest, getTorTask
from tor2web.utils.urls import normalize_url, parent_urls

SOCKS_errors = {
    0x00: "error_sock_generic.tpl",
    0x23: "error_sock_hs_not_found.tpl",
    0x24: "error_sock_hs_not_reachable.tpl"
}


class T2WRPCServer(pb.Root):
    def __init__(self, config):
        self.config = config
        self.stats = T2WStats()
        self.block_list = []
        self.block_regexps = []
        self.blocked_ua = []
        self.hosts_map = {}
        self.TorExitNodes = []
        self.certs_tofu = LimitedSizeDict(size_limit=config.ssl_tofu_cache_size)

        self.load_lists()

    def load_lists(self):
        if config.mode == "BLOCKLIST":
            self.block_list = List(config.t2w_file_path('lists/blocklist_hashed.txt'),
                                   config.automatic_blocklist_updates_source,
                                   config.automatic_blocklist_updates_mode,
                                   config.automatic_blocklist_updates_refresh)

            # clear local cleartext list
            # (load -> hash -> clear feature; for security reasons)
            self.blocklist_cleartext = List(config.t2w_file_path('lists/blocklist_cleartext.txt'))
            for i in self.blocklist_cleartext:
                self.block_list.add(hashlib.md5(normalize_url(i)).hexdigest())

            self.block_list.dump()

            self.blocklist_cleartext.clear()
            self.blocklist_cleartext.dump()

            self.block_regexps = List(config.t2w_file_path('lists/blocklist_regexp.txt'))
            self.block_regexps = [re.compile(regexp_pattern) for regexp_pattern in self.block_regexps]

        if config.blockcrawl:
            self.blocked_ua = [ua.lower() for ua in List(config.t2w_file_path('lists/blocked_ua.txt'))]

        # Load Exit Nodes list with the refresh rate configured  in config file
        self.TorExitNodes = TorExitNodeList(config.t2w_file_path('lists/exitnodelist.txt'),
                                            'https://check.torproject.org/exit-addresses',
                                            'REPLACE',
                                            config.exit_node_list_refresh)

        self.hosts_map = HostsMap(config.t2w_file_path('lists/hosts_map.txt')).hosts
        if config.mode == "TRANSLATION":
            self.hosts_map[config.basehost] = config.onion

    def remote_get_config(self):
        return self.config.__dict__

    def remote_get_blocked_ua_list(self):
        return list(self.blocked_ua)

    def remote_get_block_list(self):
        return list(self.block_list)

    def remote_get_block_regexps(self):
        return list(self.block_regexps)

    def remote_get_tor_exits_list(self):
        return list(self.TorExitNodes)

    def remote_get_hosts_map(self):
        return dict(self.hosts_map)

    def remote_is_https(self, hostname):
        return hostname in self.certs_tofu

    def remote_verify_tls_tofu(self, hostname, cert):
        h = hashlib.sha512(cert).hexdigest()
        if hostname not in self.certs_tofu:
            self.certs_tofu[hostname] = h
            return True

        return self.certs_tofu[hostname] == h

    def remote_update_stats(self, onion):
        self.stats.update(onion)

    def remote_get_yesterday_stats(self):
        return self.stats.yesterday_stats

    def remote_log_access(self, line):
        t2w_daemon.logfile_access.write(line)

    def remote_log_debug(self, line):
        date = datetimeToString()
        t2w_daemon.logfile_debug.write(date + " " + str(line) + "\n")

    def remote_shutdown(self):
        reactor.stop()


@defer.inlineCallbacks
def rpc(f, *args, **kwargs):
    d = rpc_factory.getRootObject()
    d.addCallback(lambda obj: obj.callRemote(f, *args, **kwargs))
    ret = yield d
    defer.returnValue(ret)


def rpc_log(msg):
    rpc("log_debug", str(msg))


def rpc_shutdown():
    rpc("shutdown")


class T2WPP(protocol.ProcessProtocol):
    def __init__(self, father, childFDs, fds_https, fds_http):
        self.father = father
        self.childFDs = childFDs
        self.fds_https = fds_https
        self.fds_http = fds_http

    def connectionMade(self):
        self.pid = self.transport.pid

    def processExited(self, reason):
        for i, subprocess in enumerate(self.father.subprocesses):
            if subprocess == self.pid:
                del self.father.subprocesses[i]
                break

        if not self.father.quitting:
            subprocess = spawnT2W(self.father, self.childFDs, self.fds_https, self.fds_http)
            self.father.subprocesses.append(subprocess.pid)

        if len(self.father.subprocesses) == 0:
            try:
                reactor.stop()
            except Exception:
                pass


def spawnT2W(father, childFDs, fds_https, fds_http):
    child_env = os.environ.copy()
    child_env['T2W_FDS_HTTPS'] = fds_https
    child_env['T2W_FDS_HTTP'] = fds_http

    return reactor.spawnProcess(T2WPP(father, childFDs, fds_https, fds_http),
                                sys.executable,
                                [sys.executable, __file__] + sys.argv[1:],
                                env=child_env,
                                childFDs=childFDs)


class Tor2webObj(object):
    def __init__(self):
        # The destination hidden service identifier
        self.onion = None

        # The path portion of the URI
        self.path = None

        # The full address (hostname + uri) that must be requested
        self.address = None

        # The headers to be sent
        self.headers = None

        # The requested uri
        self.uri = None

        # A boolean that keeps track of client gzip support
        self.client_supports_gzip = False

        # A boolean that keeps track of server gzip support
        self.server_response_is_gzip = False

        # A variably that keep tracks of special contents to be parsed
        self.special_content = None


class BodyReceiver(protocol.Protocol):
    def __init__(self, finished):
        self._finished = finished
        self._data = []

    def dataReceived(self, chunk):
        self._data.append(chunk)

    def write(self, chunk):
        self._data.append(chunk)

    def connectionLost(self, reason):
        self._finished.callback(''.join(self._data))


class BodyStreamer(protocol.Protocol):
    def __init__(self, streamfunction, finished):
        self._finished = finished
        self._streamfunction = streamfunction

    def dataReceived(self, data):
        self._streamfunction(data)

    def connectionLost(self, reason):
        self._finished.callback('')


class BodyProducer(object):
    implements(IBodyProducer)

    def __init__(self):
        self.length = _newclient.UNKNOWN_LENGTH
        self.finished = defer.Deferred()
        self.consumer = None
        self.can_stream = False
        self.can_stream_d = defer.Deferred()

    def startProducing(self, consumer):
        self.consumer = consumer
        self.can_stream = True
        self.can_stream_d.callback(True)
        return self.finished

    @defer.inlineCallbacks
    def dataReceived(self, data):
        if not self.can_stream:
            yield self.can_stream_d
        self.consumer.write(data)

    def allDataReceived(self):
        self.finished.callback(None)

    def resumeProducing(self):
        pass

    def pauseProducing(self):
        pass

    def stopProducing(self):
        pass

class Agent(client.Agent):
    def __init__(self, reactor,
                 contextFactory=client.WebClientContextFactory(),
                 connectTimeout=None, bindAddress=None,
                 pool=None, sockhost=None, sockport=None):
        self._sockhost = sockhost
        self._sockport = sockport
        client.Agent.__init__(self, reactor, contextFactory,
                               connectTimeout, bindAddress, pool)

    def _getEndpoint(self, scheme, host, port):
        def verify_tofu(hostname, cert):
            return rpc("verify_tls_tofu", hostname, cert)

        kwargs = {}
        if self._connectTimeout is not None:
            kwargs['timeout'] = self._connectTimeout
        kwargs['bindAddress'] = self._bindAddress

        if scheme == 'http':
            return SOCKS5ClientEndpoint(self._reactor, self._sockhost, self._sockport,
                                        host, port, config.socksoptimisticdata, **kwargs)
        elif scheme == 'https':
            torSockEndpoint = SOCKS5ClientEndpoint(self._reactor, self._sockhost,
                                                   self._sockport, host, port, config.socksoptimisticdata, **kwargs)
            return TLSWrapClientEndpoint(HTTPSVerifyingContextFactory(host, verify_tofu), torSockEndpoint)
        elif scheme == 'normal-http':
             return TCP4ClientEndpoint(self._reactor, host, port, **kwargs)
        elif scheme == 'normal-https':
            return SSL4ClientEndpoint(self._reactor, host, port,
                                      self._wrapContextFactory(host, port),
                                      **kwargs)
        else:
            raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,))

    @defer.inlineCallbacks
    def request(self, method, uri, headers, bodyProducer=None):
        """
        Edited version of twisted Agent.request in order to make it asyncronous!
        """
        parsedURI = client._URI.fromBytes(uri)

        is_https = yield rpc("is_https", parsedURI.host)

        scheme = 'https' if is_https else 'http'

        for key, values in headers.getAllRawHeaders():
            fixed_values = [re_sub(rexp['w2t'], r'' + scheme + r'://\2.onion', value) for value in values]
            headers.setRawHeaders(key, fixed_values)

        try:
            endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host,
                                         parsedURI.port)
        except SchemeNotSupported:
            defer.returnValue(Failure())

        key = (parsedURI.scheme, parsedURI.host, parsedURI.port)
        ret = yield self._requestWithEndpoint(key, endpoint, method, parsedURI,
                                              headers, bodyProducer,
                                              parsedURI.originForm)
        defer.returnValue(ret)

class RedirectAgent(client.RedirectAgent):
    """
    Overridden client.RedirectAgent version where we evaluate and handle automatically only HTTPS redirects
    """
    def _handleResponse(self, response, method, uri, headers, redirectCount):
        locationHeaders = response.headers.getRawHeaders('location', [])
        if locationHeaders:
            location = self._resolveLocation(uri, locationHeaders[0])
            parsed = client._URI.fromBytes(location)
            if parsed.scheme == 'https':
                return client.RedirectAgent._handleResponse(self, response, method, uri, headers, redirectCount)

        return response

class T2WRequest(http.Request):
    """
    Used by Tor2webProxy to implement a simple web proxy.
    """

    def __init__(self, channel, queued, reactor=reactor):
        """
        Method overridden to change some part of proxy.Request and of the base http.Request
        """
        self.reactor = reactor
        self.notifications = []
        self.channel = channel
        self.queued = queued
        self.requestHeaders = Headers()
        self.received_cookies = {}
        self.responseHeaders = Headers()
        self.cookies = []  # outgoing cookies
        self.bodyProducer = BodyProducer()
        self.proxy_d = None
        self.proxy_response = None

        self.stream = ''

        self.header_injected = False
        # If we should disable the banner,
        # say that we have already injected it.
        if config.disable_banner:
            self.header_injected = True


        self.transport = StringTransport() if queued else self.channel.transport

        self.obj = Tor2webObj()
        self.var = Storage()
        self.var['version'] = __version__
        self.var['basehost'] = config.basehost
        self.var['errorcode'] = None

        self.decoderGzip = None
        self.encoderGzip = None

        self.pool = pool

    def getRequestHostname(self):
        """
            Function overload to fix ipv6 bug:
                http://twistedmatrix.com/trac/ticket/6014
        """
        host = self.getHeader(b'host')
        if not host:
            return networkString(self.getHost().host)
        if host[0] == '[':
            return host.split(']', 1)[0] + "]"

        # return everything before the ':'
        return networkString(host.split(':', 1)[0])


    def forwardData(self, data, end=False):
        if not self.startedWriting:
            if self.obj.client_supports_gzip:
                self.setHeader(b'content-encoding', b'gzip')

            if config.extra_http_response_headers:
                for header, value in config.extra_http_response_headers.iteritems():
                    self.setHeader(header, value)

            if data and end:
                self.setHeader(b'content-length', intToBytes(len(data)))

        if data:
            self.write(data)

    def getForwarders(self):
        forwarders = []

        try:
            port = self.channel.transport.getPeer().port

            xForwardedFor = self.requestHeaders.getRawHeaders("X-Forwarded-For")
            for forwardHeader in xForwardedFor or []:
                forwardList = forwardHeader.replace(" ", "").split(",")

                for ip in forwardList:
                    if isIPAddress(ip):
                        forwarders.append(address.IPv4Address("TCP",
                                                              ip.strip(),
                                                              port))
                    elif isIPv6Address(ip):
                        forwarders.append(address.IPv6Address("TCP",
                                                              ip.strip(),
                                                              port))
                    else:
                        raise Exception
        except Exception:
            return []

        return forwarders

    def requestReceived(self, command, path, version):
        """
        Method overridden to reduce the function actions
        """
        self.method, self.uri = command, path
        self.clientproto = version

        # we get the ip address of the user that can be:
        #  - written in the x-forwared-for header if a proxy is in between
        #  - the ip address of the transport endpoint
        forwarders = self.getForwarders()
        if forwarders:
            self.client = forwarders[0]
        else:
            self.client = self.channel.transport.getPeer()

        self.host = self.channel.transport.getHost()

        self.proto = 'http://' if config.transport == 'HTTP' else 'https://'

        port = self.channel.transport.getHost().port
        self.port = '' if port in [80,443] else ':%d' % port

        self.process()

    def add_banner(self, banner, data):
        """
        Inject tor2web banner inside the returned page
        """
        return str(data.group(1)) + str(banner)

    @defer.inlineCallbacks
    def handleFixPart(self, data):
        if self.obj.server_response_is_gzip:
            data = self.unzip(data)

        data = self.stream + data

        if len(data) >= config.bufsize * 2:
            if self.obj.special_content == 'HTML':
                if not self.header_injected and data.find("<body") != -1:
                    banner = yield flattenString(self, templates['banner.tpl'])
                    data = re.sub(rexp['body'], partial(self.add_banner, banner), data)
                    self.header_injected = True

            if config.avoid_rewriting_visible_content and self.obj.special_content == 'HTML':
                data = re_sub(rexp['html_t2w'], r'\1\2' + self.proto + r'\3.' + config.basehost + self.port + r'\4', data)
            else:
                data = re_sub(rexp['t2w'], self.proto + r'\2.' + config.basehost + self.port, data)

            forward = data[:-config.bufsize]

            self.forwardData(self.handleCleartextForwardPart(forward))

            self.stream = data[-config.bufsize:]

        else:
            self.stream = data

    @defer.inlineCallbacks
    def handleFixEnd(self, data):
        if self.obj.server_response_is_gzip:
            data = self.unzip(data, True)

        data = self.stream + data

        if self.obj.special_content == 'HTML':
            if not self.header_injected and data.find("<body") != -1:
                banner = yield flattenString(self, templates['banner.tpl'])
                data = re.sub(rexp['body'], partial(self.add_banner, banner), data)
                self.header_injected = True

        if config.avoid_rewriting_visible_content and self.obj.special_content == 'HTML':
            data = re_sub(rexp['html_t2w'], r'\1\2' + self.proto + r'\3.' + config.basehost + self.port + r'\4', data)
        else:
            data = re_sub(rexp['t2w'], self.proto + r'\2.' + config.basehost + self.port, data)

        data = self.handleCleartextForwardPart(data, True)
        self.forwardData(data, True)

        self.stream = ''

        self.finish()

    def handleGzippedForwardPart(self, data, end=False):
        if not self.obj.client_supports_gzip:
            data = self.unzip(data, end)

        return data

    def handleCleartextForwardPart(self, data, end=False):
        if self.obj.client_supports_gzip:
            data = self.zip(data, end)

        return data

    def handleForwardPart(self, data):
        if self.obj.server_response_is_gzip:
            data = self.handleGzippedForwardPart(data)
        else:
            data = self.handleCleartextForwardPart(data)

        self.forwardData(data)

    def handleForwardEnd(self, data):
        if self.obj.server_response_is_gzip:
            data = self.handleGzippedForwardPart(data, True)
        else:
            data = self.handleCleartextForwardPart(data, True)

        self.forwardData(data, True)
        self.finish()

    def contentFinish(self, data):
        if self.obj.client_supports_gzip:
            self.setHeader(b'content-encoding', b'gzip')
            data = self.zip(data, True)

        self.setHeader(b'content-length', intToBytes(len(data)))

        if self.isSecure():
            self.setHeader(b'strict-transport-security', b'max-age=31536000')
            self.setHeader(b'Content-Security-Policy', b'upgrade-insecure-requests')

        if config.mode == 'TRANSLATION':
            # no additional headers are injected when in translation mode
            return

        if config.deny_caching:
            self.setHeader(b'cache-control', b'no-cache no-store, must-revalidate')
            self.setHeader(b'pragma', 'no-cache')
            self.setHeader(b'expires', b'-1')

{'fraction_non_alphanumeric': 0.0957938986872079, 'fraction_numerical': 0.005307531528146356, 'mean_word_length': 2.2567495219885276, 'pattern_counts': {'":': 4, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 56, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4176129', 'n_tokens_mistral': 12225, 'n_tokens_neox': 11795, 'n_words': 2696}