<commit_before>import Stemmer
import string

#Tokenise the documents                 
class PorterTokeniser(object):
    def __init__(self):
        self.stemmer = Stemmer.Stemmer('english')
        self.minWordLength = 2
     
    def __call__(self, doc):
        doc = str(doc.lower())
        doc = doc.translate(string.maketrans("",""), string.punctuation)
        tokens =  [self.stemmer.stemWord(t) for t in doc.split()]  
        return [token for token in tokens if len(token) >= self.minWordLength]<commit_msg>Fix for unicode error <commit_after>import Stemmer
import string

#Tokenise the documents                 
class PorterTokeniser(object):
    def __init__(self):
        self.stemmer = Stemmer.Stemmer('english')
        self.minWordLength = 2
     
    def __call__(self, doc):
        doc = doc.lower().encode('utf-8').strip()
        doc = doc.translate(string.maketrans("",""), string.punctuation).decode("utf-8")
        tokens =  [self.stemmer.stemWord(t) for t in doc.split()]  
        return [token for token in tokens if len(token) >= self.minWordLength]
{'fraction_non_alphanumeric': 0.12292051756007394, 'fraction_numerical': 0.0036968576709796672, 'mean_word_length': 2.813380281690141, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7219384', 'n_tokens_mistral': 342, 'n_tokens_neox': 326, 'n_words': 79}