<commit_before>import codecs
from encodings import normalize_encoding

_cache = {}

# Define some aliases for 'utf-8-variants'
UTF8_VAR_NAMES = (
    'utf_8_variants', 'utf8_variants',
    'utf_8_variant', 'utf8_variant',
    'utf_8_var', 'utf8_var',
    'cesu_8', 'cesu8',
    'java_utf_8', 'java_utf8'
)


def search_function(encoding):
    if encoding in _cache:
        return _cache[encoding]

    norm_encoding = normalize_encoding(encoding)
    codec = None
    if norm_encoding in UTF8_VAR_NAMES:
        from ftfy.bad_codecs.utf8_variants import CODEC_INFO
        codec = CODEC_INFO
    elif norm_encoding.startswith('sloppy_'):
        from ftfy.bad_codecs.sloppy import CODECS
        return CODECS.get(norm_encoding)

    if codec is not None:
        _cache[encoding] = codec

    return codec


codecs.register(search_function)<commit_msg>allow sloppy codecs to be cached<commit_after>import codecs
from encodings import normalize_encoding

_cache = {}

# Define some aliases for 'utf-8-variants'
UTF8_VAR_NAMES = (
    'utf_8_variants', 'utf8_variants',
    'utf_8_variant', 'utf8_variant',
    'utf_8_var', 'utf8_var',
    'cesu_8', 'cesu8',
    'java_utf_8', 'java_utf8'
)


def search_function(encoding):
    if encoding in _cache:
        return _cache[encoding]

    norm_encoding = normalize_encoding(encoding)
    codec = None
    if norm_encoding in UTF8_VAR_NAMES:
        from ftfy.bad_codecs.utf8_variants import CODEC_INFO
        codec = CODEC_INFO
    elif norm_encoding.startswith('sloppy_'):
        from ftfy.bad_codecs.sloppy import CODECS
        codec = CODECS.get(norm_encoding)

    if codec is not None:
        _cache[encoding] = codec

    return codec


codecs.register(search_function)
{'fraction_non_alphanumeric': 0.08738425925925926, 'fraction_numerical': 0.016203703703703703, 'mean_word_length': 3.5620052770448547, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4010016', 'n_tokens_mistral': 629, 'n_tokens_neox': 591, 'n_words': 130}