<commit_before>#!/usr/bin/env python

import getopt
import httplib
import os
import re
import select
import socket
import struct
import sys
import time
import traceback
import urllib
import xml.sax.saxutils

DEFAULT_REMOTE_ADDRESS = "0.0.0.0"
DEFAULT_REMOTE_PORT = 9000
DEFAULT_LOCAL_ADDRESS = "127.0.0.1"
DEFAULT_LOCAL_PORT = 9001
DEFAULT_FACILITATOR_PORT = 9002

LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"

class options(object):
    local_addr = None
    remote_addr = None
    facilitator_addr = None

    log_filename = None
    log_file = sys.stdout
    daemonize = False
    pid_filename = None

# We accept up to this many bytes from a socket not yet matched with a partner
# before disconnecting it.
UNCONNECTED_BUFFER_LIMIT = 10240

def usage(f = sys.stdout):
    print >> f, """\
Usage: %(progname)s -f FACILITATOR[:PORT] [LOCAL][:PORT] [REMOTE][:PORT]
Wait for connections on a local and a remote port. When any pair of connections
exists, data is ferried between them until one side is closed. By default
LOCAL is "%(local)s" and REMOTE is "%(remote)s".

The local connection acts as a SOCKS4a proxy, but the host and port in the SOCKS
request are ignored and the local connection is always joined to a remote
connection.

If the -f option is given, then the REMOTE address is advertised to the given
FACILITATOR.
  --daemon                       daemonize (Unix only).
  -f, --facilitator=HOST[:PORT]  advertise willingness to receive connections to
                                   HOST:PORT. By default PORT is %(fac_port)d.
  -h, --help                     show this help.
  -l, --log FILENAME             write log to FILENAME (default stdout).
      --pidfile FILENAME         write PID to FILENAME after daemonizing.\
""" % {
    "progname": sys.argv[0],
    "local": format_addr((DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT)),
    "remote": format_addr((DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT)),
    "fac_port": DEFAULT_FACILITATOR_PORT,
}

def log(msg):
    print >> options.log_file, (u"%s %s" % (time.strftime(LOG_DATE_FORMAT), msg)).encode("UTF-8")
    options.log_file.flush()

def parse_addr_spec(spec, defhost = None, defport = None):
    host = None
    port = None
    m = None
    # IPv6 syntax.
    if not m:
        m = re.match(ur'^\[(.+)\]:(\d+)$', spec)
        if m:
            host, port = m.groups()
            af = socket.AF_INET6
    if not m:
        m = re.match(ur'^\[(.+)\]:?$', spec)
        if m:
            host, = m.groups()
            af = socket.AF_INET6
    # IPv4 syntax.
    if not m:
        m = re.match(ur'^(.+):(\d+)$', spec)
        if m:
            host, port = m.groups()
            af = socket.AF_INET
    if not m:
        m = re.match(ur'^:?(\d+)$', spec)
        if m:
            port, = m.groups()
            af = 0
    if not m:
        host = spec
        af = 0
    host = host or defhost
    port = port or defport
    if not (host and port):
        raise ValueError("Bad address specification \"%s\"" % spec)
    return host, int(port)

def format_addr(addr):
    host, port = addr
    if not host:
        return u":%d" % port
    # Numeric IPv6 address?
    try:
        addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP, socket.AI_NUMERICHOST)
        af = addrs[0][0]
    except socket.gaierror, e:
        af = 0
    if af == socket.AF_INET6:
        return u"[%s]:%d" % (host, port)
    else:
        return u"%s:%d" % (host, port)

opts, args = getopt.gnu_getopt(sys.argv[1:], "f:hl:", ["daemon", "facilitator=", "help", "log=", "pidfile="])
for o, a in opts:
    if o == "--daemon":
        options.daemonize = True
    elif o == "-f" or o == "--facilitator":
        options.facilitator_addr = parse_addr_spec(a, None, DEFAULT_FACILITATOR_PORT)
    elif o == "-h" or o == "--help":
        usage()
        sys.exit()
    elif o == "-l" or o == "--log":
        options.log_filename = a
    elif o == "--pidfile":
        options.pid_filename = a

if len(args) == 0:
    options.local_addr = (DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT)
    options.remote_addr = (DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT)
elif len(args) == 1:
    options.local_addr = parse_addr_spec(args[0], DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT)
    options.remote_addr = (DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT)
elif len(args) == 2:
    options.local_addr = parse_addr_spec(args[0], DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT)
    options.remote_addr = parse_addr_spec(args[1], DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT)
else:
    usage(sys.stderr)
    sys.exit(1)

if options.log_filename:
    options.log_file = open(options.log_filename, "a")
    # Send error tracebacks to the log.
    sys.stderr = options.log_file
else:
    options.log_file = sys.stdout


class BufferSocket(object):
    """A socket containing a time of creation and a buffer of data received. The
    buffer stores data to make the socket selectable again."""
    def __init__(self, fd):
        self.fd = fd
        self.birthday = time.time()
        self.buf = ""

    def __getattr__(self, name):
        return getattr(self.fd, name)

    def is_expired(self, timeout):
        return time.time() - self.birthday > timeout

def listen_socket(addr):
    """Return a nonblocking socket listening on the given address."""
    addrinfo = socket.getaddrinfo(addr[0], addr[1], 0, socket.SOCK_STREAM, socket.IPPROTO_TCP)[0]
    s = socket.socket(addrinfo[0], addrinfo[1], addrinfo[2])
    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    s.bind(addr)
    s.listen(10)
    s.setblocking(0)
    return s

def format_peername(s):
    try:
        return format_addr(s.getpeername())
    except socket.error, e:
        return "<unconnected>"

# How long to wait for a crossdomain policy request before deciding that this is
# a normal socket.
CROSSDOMAIN_TIMEOUT = 2.0

# Local socket, accepting SOCKS requests from localhost
local_s = listen_socket(options.local_addr)
# Remote socket, accepting both crossdomain policy requests and remote proxy
# connections.
remote_s = listen_socket(options.remote_addr)

# Sockets that may be crossdomain policy requests or may be normal remote
# connections.
crossdomain_pending = []
# Remote connection sockets.
remotes = []
# New local sockets waiting to finish their SOCKS negotiation.
socks_pending = []
# Local Tor sockets, after SOCKS negotiation.
locals = []

# Bidirectional mapping between local sockets and remote sockets.
local_for = {}
remote_for = {}


def handle_policy_request(fd):
    """Returns True iff the socket is still open and usable (wasn't a
    crossdomain request and wasn't closed."""
    log(u"handle_policy_request")
    try:<commit_msg>Reduce the crossdomain timeout from 2.0 s to 0.5 s.<commit_after>#!/usr/bin/env python

import getopt
import httplib
import os
import re
import select
import socket
import struct
import sys
import time
import traceback
import urllib
import xml.sax.saxutils

DEFAULT_REMOTE_ADDRESS = "0.0.0.0"
DEFAULT_REMOTE_PORT = 9000
DEFAULT_LOCAL_ADDRESS = "127.0.0.1"
DEFAULT_LOCAL_PORT = 9001
DEFAULT_FACILITATOR_PORT = 9002

LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"

class options(object):
    local_addr = None
    remote_addr = None
    facilitator_addr = None

    log_filename = None
    log_file = sys.stdout
    daemonize = False
    pid_filename = None

# We accept up to this many bytes from a socket not yet matched with a partner
# before disconnecting it.
UNCONNECTED_BUFFER_LIMIT = 10240

def usage(f = sys.stdout):
    print >> f, """\
Usage: %(progname)s -f FACILITATOR[:PORT] [LOCAL][:PORT] [REMOTE][:PORT]
Wait for connections on a local and a remote port. When any pair of connections
exists, data is ferried between them until one side is closed. By default
LOCAL is "%(local)s" and REMOTE is "%(remote)s".

The local connection acts as a SOCKS4a proxy, but the host and port in the SOCKS
request are ignored and the local connection is always joined to a remote
connection.

If the -f option is given, then the REMOTE address is advertised to the given
FACILITATOR.
  --daemon                       daemonize (Unix only).
  -f, --facilitator=HOST[:PORT]  advertise willingness to receive connections to
                                   HOST:PORT. By default PORT is %(fac_port)d.
  -h, --help                     show this help.
  -l, --log FILENAME             write log to FILENAME (default stdout).
      --pidfile FILENAME         write PID to FILENAME after daemonizing.\
""" % {
    "progname": sys.argv[0],
    "local": format_addr((DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT)),
    "remote": format_addr((DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT)),
    "fac_port": DEFAULT_FACILITATOR_PORT,
}

def log(msg):
    print >> options.log_file, (u"%s %s" % (time.strftime(LOG_DATE_FORMAT), msg)).encode("UTF-8")
    options.log_file.flush()

def parse_addr_spec(spec, defhost = None, defport = None):
    host = None
    port = None
    m = None
    # IPv6 syntax.
    if not m:
        m = re.match(ur'^\[(.+)\]:(\d+)$', spec)
        if m:
            host, port = m.groups()
            af = socket.AF_INET6
    if not m:
        m = re.match(ur'^\[(.+)\]:?$', spec)
        if m:
            host, = m.groups()
            af = socket.AF_INET6
    # IPv4 syntax.
    if not m:
        m = re.match(ur'^(.+):(\d+)$', spec)
        if m:
            host, port = m.groups()
            af = socket.AF_INET
    if not m:
        m = re.match(ur'^:?(\d+)$', spec)
        if m:
            port, = m.groups()
            af = 0
    if not m:
        host = spec
        af = 0
    host = host or defhost
    port = port or defport
    if not (host and port):
        raise ValueError("Bad address specification \"%s\"" % spec)
    return host, int(port)

def format_addr(addr):
    host, port = addr
    if not host:
        return u":%d" % port
    # Numeric IPv6 address?
    try:
        addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP, socket.AI_NUMERICHOST)
        af = addrs[0][0]
    except socket.gaierror, e:
        af = 0
    if af == socket.AF_INET6:
        return u"[%s]:%d" % (host, port)
    else:
        return u"%s:%d" % (host, port)

opts, args = getopt.gnu_getopt(sys.argv[1:], "f:hl:", ["daemon", "facilitator=", "help", "log=", "pidfile="])
for o, a in opts:
    if o == "--daemon":
        options.daemonize = True
    elif o == "-f" or o == "--facilitator":
        options.facilitator_addr = parse_addr_spec(a, None, DEFAULT_FACILITATOR_PORT)
    elif o == "-h" or o == "--help":
        usage()
        sys.exit()
    elif o == "-l" or o == "--log":
        options.log_filename = a
    elif o == "--pidfile":
        options.pid_filename = a

if len(args) == 0:
    options.local_addr = (DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT)
    options.remote_addr = (DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT)
elif len(args) == 1:
    options.local_addr = parse_addr_spec(args[0], DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT)
    options.remote_addr = (DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT)
elif len(args) == 2:
    options.local_addr = parse_addr_spec(args[0], DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT)
    options.remote_addr = parse_addr_spec(args[1], DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT)
else:
    usage(sys.stderr)
    sys.exit(1)

if options.log_filename:
    options.log_file = open(options.log_filename, "a")
    # Send error tracebacks to the log.
    sys.stderr = options.log_file
else:
    options.log_file = sys.stdout


class BufferSocket(object):
    """A socket containing a time of creation and a buffer of data received. The
    buffer stores data to make the socket selectable again."""
    def __init__(self, fd):
        self.fd = fd
        self.birthday = time.time()
        self.buf = ""

    def __getattr__(self, name):
        return getattr(self.fd, name)

    def is_expired(self, timeout):
        return time.time() - self.birthday > timeout

def listen_socket(addr):
    """Return a nonblocking socket listening on the given address."""
    addrinfo = socket.getaddrinfo(addr[0], addr[1], 0, socket.SOCK_STREAM, socket.IPPROTO_TCP)[0]
    s = socket.socket(addrinfo[0], addrinfo[1], addrinfo[2])
    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    s.bind(addr)
    s.listen(10)
    s.setblocking(0)
    return s

def format_peername(s):
    try:
        return format_addr(s.getpeername())
    except socket.error, e:
        return "<unconnected>"

# How long to wait for a crossdomain policy request before deciding that this is
# a normal socket. In 160 crossdomain connections to the public facilitator, the
# mean time before receiving a crossdomain request is only 0.05625 s. But that
# was measured from log entries with a one-second resolution, so give a healthy
# margin above that.
CROSSDOMAIN_TIMEOUT = 0.5

# Local socket, accepting SOCKS requests from localhost
local_s = listen_socket(options.local_addr)
# Remote socket, accepting both crossdomain policy requests and remote proxy
# connections.
remote_s = listen_socket(options.remote_addr)

# Sockets that may be crossdomain policy requests or may be normal remote
# connections.
crossdomain_pending = []
# Remote connection sockets.
remotes = []
# New local sockets waiting to finish their SOCKS negotiation.
socks_pending = []
# Local Tor sockets, after SOCKS negotiation.
locals = []

# Bidirectional mapping between local sockets and remote sockets.
local_for = {}
remote_for = {}


def handle_policy_request(fd):
    """Returns True iff the socket is still open and usable (wasn't a
    crossdomain request and wasn't closed."""
    log(u"handle_policy_request")
    try:
{'fraction_non_alphanumeric': 0.11364967779730521, 'fraction_numerical': 0.010178676039835969, 'mean_word_length': 2.9909409701928698, 'pattern_counts': {'":': 20, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6044719', 'n_tokens_mistral': 4608, 'n_tokens_neox': 4383, 'n_words': 1354}