<commit_before>import json

def jsonencode(commands):
    out = []
    for item in commands:
        out.append([item.line,item.opcode,item.arguments])
    return(json.dumps(out))
        
def jsondecode(jsonstring):
    out = []
    for item in json.loads(jsonstring):
        out.append(Command(item[0], item[1], item[2])) 
    return out

class Command:
    def __init__(self, line, opcode, arguments):
        self.line = line
        self.opcode = opcode
        self.arguments = arguments
        self.size = -1
        
        if False:pass
        elif self.opcode == "SET": self.size = 4 #FORMAT: SET *x y
        elif self.opcode == "JUMP" or self.opcode == "JUMPL": self.size = 3 #FORMAT: JUMP *x      
        elif self.opcode == "IFE" or self.opcode == "IFEL": self.size = 7 #FORMAT: IFE *x *y *z  
        elif self.opcode == "ADD" or self.opcode == "SUB" : self.size = 5 #FORMAT: ADD/SUB *x *y
        else:
            print(self.line + ": Invalid opcode of " + self.opcode)<commit_msg>Modified cmdio to quit on invalid opcode<commit_after>import sys
import json

def jsonencode(commands):
    out = []
    for item in commands:
        out.append([item.line,item.opcode,item.arguments])
    return(json.dumps(out))
        
def jsondecode(jsonstring):
    out = []
    for item in json.loads(jsonstring):
        out.append(Command(item[0], item[1], item[2])) 
    return out

class Command:
    def __init__(self, line, opcode, arguments):
        self.line = line
        self.opcode = opcode
        self.arguments = arguments
        self.size = -1
        
        if False:pass
        elif self.opcode == "SET": self.size = 4 #FORMAT: SET *x y
        elif self.opcode == "JUMP" or self.opcode == "JUMPL": self.size = 3 #FORMAT: JUMP *x      
        elif self.opcode == "IFE" or self.opcode == "IFEL": self.size = 7 #FORMAT: IFE *x *y *z  
        elif self.opcode == "ADD" or self.opcode == "SUB" : self.size = 5 #FORMAT: ADD/SUB *x *y
        else:
            print(str(self.line) + ": Invalid opcode of " + str(self.opcode))
            sys.exit(1)
{'fraction_non_alphanumeric': 0.13618864292589028, 'fraction_numerical': 0.008180943214629451, 'mean_word_length': 2.2843601895734595, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3237032', 'n_tokens_mistral': 677, 'n_tokens_neox': 679, 'n_words': 190}