<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import

import os
from cryptography.fernet import Fernet, InvalidToken

fernet = Fernet(os.environ['KOMBU_FERNET_KEY'])
fallback_fernet = None
try:
    fallback_fernet = Fernet(os.environ['OLD_KOMBU_FERNET_KEY'])
except KeyError:
    pass


def fernet_encode(func):
    def inner(message):
        return fernet.encrypt(func(message))
    return inner


def fernet_decode(func):
    def inner(encoded_message):
        if isinstance(encoded_message, unicode):
            encoded_message = encoded_message.encode('utf-8')
        try:
            message = fernet.decrypt(encoded_message)
        except InvalidToken:
            message = fallback_fernet.decrypt(encoded_message)
        return func(message)
    return inner<commit_msg>Use MultiFernet provided by cryptography lib<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import

import os
from cryptography.fernet import Fernet, MultiFernet

fernet = Fernet(os.environ['KOMBU_FERNET_KEY'])
fallback_fernet = None
try:
    fallback_fernet = Fernet(os.environ['OLD_KOMBU_FERNET_KEY'])
except KeyError:
    pass
else:
    fernet = MultiFernet([fernet, fallback_fernet])


def fernet_encode(func):
    def inner(message):
        return fernet.encrypt(func(message))
    return inner


def fernet_decode(func):
    def inner(encoded_message):
        if isinstance(encoded_message, unicode):
            encoded_message = encoded_message.encode('utf-8')
        message = fernet.decrypt(encoded_message)
        return func(message)
    return inner
{'fraction_non_alphanumeric': 0.09124311083894672, 'fraction_numerical': 0.002449479485609308, 'mean_word_length': 3.8922155688622753, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5033990', 'n_tokens_mistral': 518, 'n_tokens_neox': 508, 'n_words': 109}