<commit_before>import sys
import os
import threading
import socket
import time
import subprocess
import re

CHILDTIMEOUT = 60

class EOFException(Exception): pass
class LineReader:
    buf = ''
    def __init__(self, socket, CANCELAT):
        self.s = socket
        self.CANCELAT = CANCELAT
    def readline(self):
        while True:
            pos = self.buf.find("\n")
            if pos != -1:
                line, self.buf = self.buf[:pos], self.buf[pos + 1:]
                return line
            if len(self.buf) > self.CANCELAT:
                # cause a panic
                try: self.s.close()
                except: pass
                return ''
            frag = self.s.recv(8192)
            if frag == None: raise EOFException()
            if len(frag) == 0: raise EOFException()
            frag = frag.replace("\r", "")
            if len(frag) == 0:
                buf, self.buf = self.buf, ''
                return buf # EOF
            self.buf = self.buf + frag

def die(reason):
    sys.stderr.write(reason)
    sys.stderr.write("\n")
    sys.exit(1)      

def usage():
    die("USAGE: %s <MAXPROCS> <ALLOWEDIP> [<ALLOWEDIP>] ..." % sys.argv[0])

def mksock():
    return socket.socket(socket.AF_INET, socket.SOCK_STREAM)

class CloseHandler(threading.Thread):
    def __init__(self, cs, addr):
        threading.Thread.__init__(self)
        self.cs = cs
        self.addr = addr
    def run(self):
        self.cs.sendall("400 You are not authorized to connect\n")
        self.cs.close()

class DieException(Exception): pass
class SocketHandler(threading.Thread):
    def __init__(self, cs, addr, allowedscripts, scriptpath, statuscallback):
        threading.Thread.__init__(self)
        self.cs = cs
        self.addr = addr
        self.allowedscripts = allowedscripts
        self.scriptpath = "scripts/"
        self.statuscallback = statuscallback
    def run(self):
        nr = self.statuscallback.incrunning()
        if nr == -1:
            self.cs.sendall("900 Too many children\n")
            self.cs.close()
            return
        try:
            self.handle(nr)
        except DieException, de:
            self.cs.sendall("%s\n" % de.message)
            self.cs.close()
        finally: self.statuscallback.decrunning()
    def die(self, reason):
        raise DieException(reason)
    def handle(self, nr):
        try: self.cs.settimeout(60)
        except: return    # If we can't set a timeout, return
        self.cs.sendall("200 %d Welcome\n" % nr)
        self.lr = LineReader(self.cs, 8192)
        env = os.environ
        validenvkey = re.compile("^[A-Za-z][A-Za-z0-9]*$")
        while True:
            cmdline = self.lr.readline().split(" ", 1)
            if len(cmdline) < 2: self.die("300 Invalid parameter count")
            if cmdline[0] == 'ENV':
                try: [key, value] = cmdline[1].split(" ", 1)
                except: self.die("300 Invalid parameter count")
                if validenvkey.match(key) == None: self.die("300 Invalid key")
                env['CTFGAME_' + key] = value
            else: break
        cmd = cmdline[0]
        if cmd not in self.allowedscripts: self.die("400 Invalid script specified")
        parms = cmdline[1].split(" ")
        self.runscript(cmd, parms, env)
    def runscript(self, cmd, parms, env):
        lendtime = time.time() + CHILDTIMEOUT
        cmdlist = ['./' + cmd] + parms
        print "[FORK] %s" % " ".join(cmdlist)
        try: process = subprocess.Popen(cmdlist, stdout=subprocess.PIPE, cwd=self.scriptpath)
        except Exception, e:
            print "[e] %s" % e
            self.die("500 %s" % e)
        print "[%d] launched" % process.pid
        while True:
            time.sleep(1)<commit_msg>important fixes<commit_after>import sys
import os
import threading
import socket
import time
import subprocess
import re

CHILDTIMEOUT = 60

class EOFException(Exception): pass
class LineReader:
    buf = ''
    def __init__(self, socket, CANCELAT):
        self.s = socket
        self.CANCELAT = CANCELAT
    def readline(self):
        while True:
            pos = self.buf.find("\n")
            if pos != -1:
                line, self.buf = self.buf[:pos], self.buf[pos + 1:]
                return line
            if len(self.buf) > self.CANCELAT:
                # cause a panic
                try: self.s.close()
                except: pass
                return ''
            frag = self.s.recv(8192)
            if frag == None: raise EOFException()
            if len(frag) == 0: raise EOFException()
            frag = frag.replace("\r", "")
            if len(frag) == 0:
                buf, self.buf = self.buf, ''
                return buf # EOF
            self.buf = self.buf + frag

def die(reason):
    sys.stderr.write(reason)
    sys.stderr.write("\n")
    sys.exit(1)      

def usage():
    die("USAGE: %s <MAXPROCS> <ALLOWEDIP> [<ALLOWEDIP>] ..." % sys.argv[0])

def mksock():
    return socket.socket(socket.AF_INET, socket.SOCK_STREAM)

class CloseHandler(threading.Thread):
    def __init__(self, cs, addr):
        threading.Thread.__init__(self)
        self.cs = cs
        self.addr = addr
    def run(self):
        self.cs.sendall("400 You are not authorized to connect\n")
        self.cs.close()

class DieException(Exception): pass
class SocketHandler(threading.Thread):
    def __init__(self, cs, addr, allowedscripts, scriptpath, statuscallback):
        threading.Thread.__init__(self)
        self.cs = cs
        self.addr = addr
        self.allowedscripts = allowedscripts
        self.scriptpath = "scripts/"
        self.statuscallback = statuscallback
    def run(self):
        nr = self.statuscallback.incrunning()
        if nr == -1:
            self.cs.sendall("900 Too many children\n")
            self.cs.close()
            return
        try:
            self.handle(nr)
        except DieException, de:
            self.cs.sendall("%s\n" % de.message)
            self.cs.close()
        finally: self.statuscallback.decrunning()
    def die(self, reason):
        raise DieException(reason)
    def handle(self, nr):
        try: self.cs.settimeout(60)
        except: return    # If we can't set a timeout, return
        self.cs.sendall("200 %d Welcome\n" % nr)
        self.lr = LineReader(self.cs, 8192)
        env = dict(os.environ)
        validenvkey = re.compile("^[A-Za-z][A-Za-z0-9]*$")
        while True:
            cmdline = self.lr.readline().split(" ", 1)
            if len(cmdline) < 2: self.die("300 Invalid parameter count")
            if cmdline[0] == 'ENV':
                try: [key, value] = cmdline[1].split(" ", 1)
                except: self.die("300 Invalid parameter count")
                if validenvkey.match(key) == None: self.die("300 Invalid key")
                env['CTFGAME_' + key] = value
            else: break
        cmd = cmdline[0]
        if cmd not in self.allowedscripts: self.die("400 Invalid script specified")
        parms = cmdline[1].split(" ")
        self.runscript(cmd, parms, env)
    def runscript(self, cmd, parms, env):
        lendtime = time.time() + CHILDTIMEOUT
        cmdlist = ['./' + cmd] + parms
        print "[FORK] %s" % " ".join(cmdlist)
        try: process = subprocess.Popen(cmdlist, stdout=subprocess.PIPE, cwd=self.scriptpath, env=env, close_fds=True)
        except Exception, e:
            print "[e] %s" % e
            self.die("500 %s" % e)
        print "[%d] launched" % process.pid
        while True:
            time.sleep(1)
{'fraction_non_alphanumeric': 0.12786841756201087, 'fraction_numerical': 0.014060220188353893, 'mean_word_length': 1.9134466769706338, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4271642', 'n_tokens_mistral': 2389, 'n_tokens_neox': 2294, 'n_words': 576}