<commit_before>import datetime
import serial
import struct
import sys
import threading
import time
import traceback

class Commands:
    init, initPool, read, write, inputAvailable, inputRequest, inputPeek, ping = range(0, 8)

class State:
    initialized = False
    processState = 'idle'
    initValue, memoryPool = None, None
    inputData = bytearray()
    inputLock = threading.Lock()
    doQuit = False
    outdev = None

serInterface = serial.Serial()

def blockedRead(size):
    ret = bytearray()
    while size > 0:
        bytes = serInterface.read(size)
        if bytes:
            ret += bytes
            size -= len(bytes)
    return ret

def readInt():
    return struct.unpack('i', blockedRead(4))[0]

def writeInt(i):
    serInterface.write(struct.pack('i', i))

def sendCommand(cmd):
    serInterface.write(bytes([State.initValue]))
    serInterface.write(bytes([cmd]))
#    print("send: ", State.initValue, "/", bytes([cmd]))

def processByte(byte, printunknown=True):
    val = ord(byte)
    if State.processState == 'gotinit':
        handleCommand(val)
        State.processState = 'idle'
    elif val == State.initValue:
        assert(State.processState == 'idle')
#        print("Got init!")
        State.processState = 'gotinit'
    else:
        assert(State.processState == 'idle')
        if printunknown:
#            State.outdev.write(byte.decode('ascii', errors='ignore'))
            State.outdev.write(byte)

def handleCommand(command):
#    print("command: ", command)
    if command == Commands.ping:
        sendCommand(Commands.ping)
    elif command == Commands.init:
        State.initialized = True
        State.memoryPool = None # remove pool
        sendCommand(Commands.init) # reply
    elif not State.initialized:
        pass
    elif command == Commands.initPool:<commit_msg>flush output<commit_after>import datetime
import serial
import struct
import sys
import threading
import time
import traceback

class Commands:
    init, initPool, read, write, inputAvailable, inputRequest, inputPeek, ping = range(0, 8)

class State:
    initialized = False
    processState = 'idle'
    initValue, memoryPool = None, None
    inputData = bytearray()
    inputLock = threading.Lock()
    doQuit = False
    outdev = None

serInterface = serial.Serial()

def blockedRead(size):
    ret = bytearray()
    while size > 0:
        bytes = serInterface.read(size)
        if bytes:
            ret += bytes
            size -= len(bytes)
    return ret

def readInt():
    return struct.unpack('i', blockedRead(4))[0]

def writeInt(i):
    serInterface.write(struct.pack('i', i))

def sendCommand(cmd):
    serInterface.write(bytes([State.initValue]))
    serInterface.write(bytes([cmd]))
#    print("send: ", State.initValue, "/", bytes([cmd]))

def processByte(byte, printunknown=True):
    val = ord(byte)
    if State.processState == 'gotinit':
        handleCommand(val)
        State.processState = 'idle'
    elif val == State.initValue:
        assert(State.processState == 'idle')
#        print("Got init!")
        State.processState = 'gotinit'
    else:
        assert(State.processState == 'idle')
        if printunknown:
#            State.outdev.write(byte.decode('ascii', errors='ignore'))
            State.outdev.write(byte)
            State.outdev.flush()

def handleCommand(command):
#    print("command: ", command)
    if command == Commands.ping:
        sendCommand(Commands.ping)
    elif command == Commands.init:
        State.initialized = True
        State.memoryPool = None # remove pool
        sendCommand(Commands.init) # reply
    elif not State.initialized:
        pass
    elif command == Commands.initPool:
{'fraction_non_alphanumeric': 0.11984815618221258, 'fraction_numerical': 0.0027114967462039045, 'mean_word_length': 2.911983032873807, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1847247', 'n_tokens_mistral': 1083, 'n_tokens_neox': 1090, 'n_words': 267}