<commit_before>#!/usr/bin/env python

from nodes import Node
import json

class Dictionary(Node):
    char = ".d"
    args = 0
    results = 1
    
    def __init__(self, word_ids:Node.IntList):
        if not hasattr(Dictionary, "word_list"):
            Dictionary.word_list = init_words()
        self.words = " ".join(Dictionary.word_list[i] for i in word_ids)
        
    def func(self):
        return self.words
    
    def compress(inp):
        words = init_words()
        inp_words = [word.lower()for word in inp.split(" ")]
        rtn = chr(len(inp_words))
        for word in inp_words:
            assert(word in words)
            rtn += chr(words.index(word))
        return rtn
    <commit_msg>Add some exception handling for dict<commit_after>#!/usr/bin/env python

from nodes import Node
import json

class Dictionary(Node):
    char = ".d"
    args = 0
    results = 1
    
    def __init__(self, word_ids:Node.IntList):
        if not hasattr(Dictionary, "word_list"):
            Dictionary.word_list = init_words()
        self.words = " ".join(Dictionary.word_list[i] for i in word_ids)
        
    def func(self):
        return self.words
    
    def compress(inp):
        words = init_words()
        inp_words = [word.lower()for word in inp.split(" ")]
        rtn = chr(len(inp_words))
        for word in inp_words:
            if word not in words:
                rtn += "Word %s not in wordlist" % word
            else:
                rtn += chr(words.index(word))
        return rtn
    
{'fraction_non_alphanumeric': 0.10725964682799215, 'fraction_numerical': 0.002616088947024199, 'mean_word_length': 2.0, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2803389', 'n_tokens_mistral': 486, 'n_tokens_neox': 479, 'n_words': 129}