<commit_before>import datetime
import uuid

from datetime import timedelta
from functools import wraps, update_wrapper

from flask import current_app, jsonify, make_response, request


class Auth(object):
    """Simple handler for auth tokens"""
    auth_tokens = {}

    def generate_token(self):
        return str(uuid.uuid4())

    def generate_expiration(self):
        return datetime.datetime.now() + datetime.timedelta(minutes=30)

    def set_token(self, token=None):
        if token is None:
            token = self.generate_token()
        self.auth_tokens[token] = self.generate_expiration()
        return token

    def valid_token(self, token):
        """Check is token is still valid

        If token is still live, update expiration
        otherwise remove it
        """
        if token in self.auth_tokens:
            if datetime.datetime.now() < self.auth_tokens[token]:
                self.auth_tokens[token] = self.generate_expiration()
                return True
            self.remove_token(token)
        return False

    def remove_token(self, token):
        self.auth_tokens.pop(token)
        # need to periodically clean out auth tokens that
        # have expired
        for token, expiration in self.auth_tokens.iteritems():<commit_msg>handle empty/none auth token when attempting to remove it<commit_after>import datetime
import uuid

from datetime import timedelta
from functools import wraps, update_wrapper

from flask import current_app, jsonify, make_response, request


class Auth(object):
    """Simple handler for auth tokens"""
    auth_tokens = {}

    def generate_token(self):
        return str(uuid.uuid4())

    def generate_expiration(self):
        return datetime.datetime.now() + datetime.timedelta(minutes=30)

    def set_token(self, token=None):
        if token is None:
            token = self.generate_token()
        self.auth_tokens[token] = self.generate_expiration()
        return token

    def valid_token(self, token):
        """Check is token is still valid

        If token is still live, update expiration
        otherwise remove it
        """
        if token in self.auth_tokens:
            if datetime.datetime.now() < self.auth_tokens[token]:
                self.auth_tokens[token] = self.generate_expiration()
                return True
            self.remove_token(token)
        return False

    def remove_token(self, token):
        try:
            self.auth_tokens.pop(token)
        except KeyError:
            # appears we don't actually have an auth token
            pass
        # need to periodically clean out auth tokens that
        # have expired
        for token, expiration in self.auth_tokens.iteritems():
{'fraction_non_alphanumeric': 0.07825128581925055, 'fraction_numerical': 0.002204261572373255, 'mean_word_length': 2.5045045045045047, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7179442', 'n_tokens_mistral': 733, 'n_tokens_neox': 697, 'n_words': 223}