<commit_before># WordNet handles most lemmatisation, but it struggles with a few things.

# During lemmatisation, words will be corrected via this list.

# insert the word and what you want it to be turned into.

wordlist = {u"felt": u"feel",
            u"'s": u"is",
            u"'re": u"are",
            u"'m": u"am",
            u"'d": u"had",
            u"'ve": u"have",
            u"n't": u"not",
            u"smelt": u"smell",
            u"saw": u"see",
            u"hated": u"hate",
            u"people": u"person",
            u"women": u"woman",
            u"men": u"man",
            u"will": u"would",
            u"wo": u"would",
            u"'ll": u"would",
            u"ca": u"could",<commit_msg>teen-ager added<commit_after># WordNet handles most lemmatisation, but it struggles with a few things.

# During lemmatisation, words will be corrected via this list.

# insert the word and what you want it to be turned into.

wordlist = {u"felt": u"feel",
            u"'s": u"is",
            u"'re": u"are",
            u"'m": u"am",
            u"'d": u"had",
            u"'ve": u"have",
            u"n't": u"not",
            u"smelt": u"smell",
            u"saw": u"see",
            u"hated": u"hate",
            u"people": u"person",
            u"women": u"woman",
            u"men": u"man",
            u"teen-ager": u"teenager",
            u"teen-agers": u"teenager",
            u"will": u"would",
            u"wo": u"would",
            u"'ll": u"would",
            u"ca": u"could",
{'fraction_non_alphanumeric': 0.16983606557377048, 'fraction_numerical': 0.0, 'mean_word_length': 1.6818980667838312, 'pattern_counts': {'":': 36, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6936214', 'n_tokens_mistral': 534, 'n_tokens_neox': 505, 'n_words': 140}