<commit_before>import json
import socket
import threading
import struct
import traceback
import time
from sysinfo import load_fair
from executors import executors


class JudgeAuthenticationFailed(Exception):
    pass


class PacketManager(object):
    SIZE_PACK = struct.Struct('!I')

    def __init__(self, host, port, judge):
        self.host = host
        self.port = port
        self.judge = judge
        self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.conn.connect((host, port))
        self.input = self.conn.makefile('r')
        self.output = self.conn.makefile('w', 0)

    def __del__(self):
        self.conn.shutdown()

    def _read_async(self):
        try:
            while True:
                self._recieve_packet(self._read_single())
        except KeyboardInterrupt:
            pass
        except Exception:  # connection reset by peer
            traceback.print_exc()
            raise SystemExit(1)

    def _read_single(self):
        size = PacketManager.SIZE_PACK.unpack(self.input.read(PacketManager.SIZE_PACK.size))[0]
        packet = self.input.read(size).decode('zlib')
        return json.loads(packet)

    def run(self):
        self._read_async()

    def run_async(self):
        threading.Thread(target=self._read_async).start()

    def _send_packet(self, packet):
        if packet['name'] not in ['ping-response', 'handshake']:
            if 'key' in packet:
                temp = packet.copy()
                temp['key'] = '<secret>'
            else:
                temp = packet
            print '%s:%s => %s' % (self.host, self.port, json.dumps(temp, indent=4))

        raw = json.dumps(packet).encode('zlib')
        self.output.write(PacketManager.SIZE_PACK.pack(len(raw)))
        self.output.write(raw)

    def _recieve_packet(self, packet):
        if packet['name'] != 'ping':
            print '%s:%s <= %s' % (self.host, self.port, json.dumps(packet, indent=4))

        name = packet['name']
        if name == 'ping':
            self.ping_packet(time.time())
        elif name == 'get-current-submission':
            self.current_submission_packet()
        elif name == 'submission-request':
            self.judge.current_submission = packet['submission-id']
            self.judge.begin_grading(
                packet['problem-id'],
                packet['language'],
                packet['source'],
                int(packet['time-limit']),
                int(packet['memory-limit']),
                packet['short-circuit'],
                packet['grader-id'],
                packet['grader-args']
            )
        elif name == 'terminate-submission':
            self.judge.terminate_grading()
        else:
            print 'ERROR: unknown packet %s, payload %s' % (name, packet)
    
    def handshake(self, problems, id, key):
        self._send_packet({'name': 'handshake',
                           'problems': problems,
                           'executors': executors.keys(),
                           'id': id,
                           'key': key})
        try:
            resp = self._read_single()
        except Exception:
            traceback.print_exc()
            raise JudgeAuthenticationFailed()
        else:
            if resp['name'] != 'handshake-success':
                raise JudgeAuthenticationFailed()

    def test_case_status_packet(self, position, points, total_points, status, time, memory, output):
        self._send_packet({'name': 'test-case-status',
                           'submission-id': self.judge.current_submission,
                           'position': position,
                           'status': status,
                           'time': time,
                           'points': points,
                           'total-points': total_points,
                           'memory': memory,
                           'output': output})

    def compile_error_packet(self, log):
        self._send_packet({'name': 'compile-error',
                           'submission-id': self.judge.current_submission,
                           'log': log})

    def problem_not_exist_packet(self, problem):
        self._send_packet({'name': 'problem-not-exist',
                           'submission-id': self.judge.current_submission,
                           'problem': problem})

    def begin_grading_packet(self):
        self._send_packet({'name': 'grading-begin',
                           'submission-id': self.judge.current_submission})

    def grading_end_packet(self):
        self._send_packet({'name': 'grading-end',
                           'submission-id': self.judge.current_submission})
        self.judge.current_submission = None

    def begin_batch_packet(self):
        self._send_packet({'name': 'batch-begin',
                           'submission-id': self.judge.current_submission})

    def batch_end_packet(self):
        self._send_packet({'name': 'batch-end',
                           'submission-id': self.judge.current_submission})

    def current_submission_packet(self):
        self._send_packet({'name': 'current-submission-id',
                           'submission-id': self.judge.current_submission})

    def submission_terminated_packet(self):
        self._send_packet({'name': 'submission-terminated',
                           'submission-id': self.judge.current_submission})

    def ping_packet(self, when):
        self._send_packet({'name': 'ping-response',
                           'time': time.time() - when,
                           'load': load_fair()})<commit_msg>OK, let's traceback this and see where the abort came from<commit_after>import json
import socket
import threading
import struct
import traceback
import time
from sysinfo import load_fair
from executors import executors


class JudgeAuthenticationFailed(Exception):
    pass


class PacketManager(object):
    SIZE_PACK = struct.Struct('!I')

    def __init__(self, host, port, judge):
        self.host = host
        self.port = port
        self.judge = judge
        self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.conn.connect((host, port))
        self.input = self.conn.makefile('r')
        self.output = self.conn.makefile('w', 0)

    def __del__(self):
        self.conn.shutdown()

    def _read_async(self):
        try:
            while True:
                self._recieve_packet(self._read_single())
        except KeyboardInterrupt:
            pass
        except Exception:  # connection reset by peer
            traceback.print_exc()
            raise SystemExit(1)

    def _read_single(self):
        size = PacketManager.SIZE_PACK.unpack(self.input.read(PacketManager.SIZE_PACK.size))[0]
        packet = self.input.read(size).decode('zlib')
        return json.loads(packet)

    def run(self):
        self._read_async()

    def run_async(self):
        threading.Thread(target=self._read_async).start()

    def _send_packet(self, packet):
        if packet['name'] not in ['ping-response', 'handshake']:
            if 'key' in packet:
                temp = packet.copy()
                temp['key'] = '<secret>'
            else:
                temp = packet
            print '%s:%s => %s' % (self.host, self.port, json.dumps(temp, indent=4))

        raw = json.dumps(packet).encode('zlib')
        self.output.write(PacketManager.SIZE_PACK.pack(len(raw)))
        self.output.write(raw)

    def _recieve_packet(self, packet):
        if packet['name'] != 'ping':
            print '%s:%s <= %s' % (self.host, self.port, json.dumps(packet, indent=4))

        name = packet['name']
        if name == 'ping':
            self.ping_packet(time.time())
        elif name == 'get-current-submission':
            self.current_submission_packet()
        elif name == 'submission-request':
            self.judge.current_submission = packet['submission-id']
            self.judge.begin_grading(
                packet['problem-id'],
                packet['language'],
                packet['source'],
                int(packet['time-limit']),
                int(packet['memory-limit']),
                packet['short-circuit'],
                packet['grader-id'],
                packet['grader-args']
            )
        elif name == 'terminate-submission':
            self.judge.terminate_grading()
        else:
            print 'ERROR: unknown packet %s, payload %s' % (name, packet)
    
    def handshake(self, problems, id, key):
        self._send_packet({'name': 'handshake',
                           'problems': problems,
                           'executors': executors.keys(),
                           'id': id,
                           'key': key})
        try:
            resp = self._read_single()
        except Exception:
            traceback.print_exc()
            raise JudgeAuthenticationFailed()
        else:
            if resp['name'] != 'handshake-success':
                raise JudgeAuthenticationFailed()

    def test_case_status_packet(self, position, points, total_points, status, time, memory, output):
        self._send_packet({'name': 'test-case-status',
                           'submission-id': self.judge.current_submission,
                           'position': position,
                           'status': status,
                           'time': time,
                           'points': points,
                           'total-points': total_points,
                           'memory': memory,
                           'output': output})

    def compile_error_packet(self, log):
        self._send_packet({'name': 'compile-error',
                           'submission-id': self.judge.current_submission,
                           'log': log})

    def problem_not_exist_packet(self, problem):
        self._send_packet({'name': 'problem-not-exist',
                           'submission-id': self.judge.current_submission,
                           'problem': problem})

    def begin_grading_packet(self):
        self._send_packet({'name': 'grading-begin',
                           'submission-id': self.judge.current_submission})

    def grading_end_packet(self):
        self._send_packet({'name': 'grading-end',
                           'submission-id': self.judge.current_submission})
        self.judge.current_submission = None

    def begin_batch_packet(self):
        self._send_packet({'name': 'batch-begin',
                           'submission-id': self.judge.current_submission})

    def batch_end_packet(self):
        self._send_packet({'name': 'batch-end',
                           'submission-id': self.judge.current_submission})

    def current_submission_packet(self):
        self._send_packet({'name': 'current-submission-id',
                           'submission-id': self.judge.current_submission})

    def submission_terminated_packet(self):
        traceback.print_exc()
        try:
            raise ValueError()
        except ValueError:
            traceback.print_exc()
        self._send_packet({'name': 'submission-terminated',
                           'submission-id': self.judge.current_submission})

    def ping_packet(self, when):
        self._send_packet({'name': 'ping-response',
                           'time': time.time() - when,
                           'load': load_fair()})
{'fraction_non_alphanumeric': 0.12159550166930241, 'fraction_numerical': 0.0008785802143735723, 'mean_word_length': 1.8231646825396826, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2512983', 'n_tokens_mistral': 3173, 'n_tokens_neox': 3053, 'n_words': 590}