<commit_before>import time
import fnmatch
from datetime import datetime, timedelta

from twisted.trial import unittest

from vumi.tests.utils import get_stubbed_worker
from vumi.workers.failures.workers import FailureWorker


RETRY_TIMESTAMPS_KEY = "failures:vas2nets#retry_timestamps"


class FakeRedis(object):
    def __init__(self):
        self.data = {}

    def get(self, key):
        return self.data.get(key)

    def set(self, key, value):
        self.data[key] = value

    def exists(self, key):
        return key in self.data

    def keys(self, pattern='*'):
        return fnmatch.filter(self.data.keys(), pattern)

    def flushdb(self):
        self.data = {}


def mktimestamp(delta=0):
    timestamp = datetime.utcnow() + timedelta(seconds=delta)
    return timestamp.isoformat().split('.')[0]


class FailureWorkerTestCase(unittest.TestCase):

    def setUp(self):
        self.config = {
            'transport_name': 'vas2nets'
        }
        self.worker = get_stubbed_worker(FailureWorker, self.config)
        self.worker.startWorker()
        # self.worker.r_server = FakeRedis()
        self.worker.r_server.flushdb()
        self.redis = self.worker.r_server

    def assert_write_timestamp(self, expected, delta, now):
        self.assertEqual(expected,
                         self.worker.get_next_write_timestamp(delta, now=now))

    def assert_zcard(self, expected, key):
        self.assertEqual(expected, self.redis.zcard(key))

    def assert_stored_timestamps(self, *expected):
        self.assertEqual(list(expected),
                         self.redis.zrange(RETRY_TIMESTAMPS_KEY, 0, -1))

    def store_failure(self, reason=None, message=None):
        if not reason:
            reason = "bad stuff happened"
        if not message:
            message = {'message': 'foo', 'reason': reason}
        return self.worker.store_failure(message, reason)

    def test_redis_access(self):
        """
        Sanity check that we can put stuff in redis (or our fake) and
        get it out again.
        """
        self.assertEqual(None, self.worker.r_get("foo"))
        self.assertEqual([], self.redis.keys())
        self.worker.r_set("foo", "bar")
        self.assertEqual("bar", self.worker.r_get("foo"))
        self.assertEqual(['failures:vas2nets#foo'], self.redis.keys())

    def test_store_failure(self):
        """
        Store a failure in redis and make sure we can get at it again.
        """
        key = self.store_failure()
        self.assertEqual(set([key]), self.worker.get_failure_keys())
        self.assertEqual({
                "message": str({"message": "foo"}),
                "retry_delay": "None",
                "reason": "bad stuff happened",
                }, self.redis.hgetall(key))

    def test_write_timestamp(self):
        """
        We need granular timestamps.
        """
        start = datetime.utcnow().isoformat()
        timestamp = self.worker.get_next_write_timestamp(0)
        end = (datetime.utcnow() + timedelta(seconds=6)).isoformat()
        self.assertTrue(start < timestamp < end)

        self.assert_write_timestamp("1970-01-01T00:00:05", 0, 0)
        self.assert_write_timestamp("1970-01-01T00:00:05", 0, 4)
        self.assert_write_timestamp("1970-01-01T00:00:10", 0, 5)
        self.assert_write_timestamp("1970-01-01T00:00:10", 0, 9)

        self.assert_write_timestamp("1970-01-01T00:00:05", 2, 0)
        self.assert_write_timestamp("1970-01-01T00:00:10", 2, 4)
        self.assert_write_timestamp("1970-01-01T00:00:10", 2, 5)<commit_msg>Fix the test I broke in the last commit.<commit_after>import time
import fnmatch
from datetime import datetime, timedelta

from twisted.trial import unittest

from vumi.tests.utils import get_stubbed_worker
from vumi.workers.failures.workers import FailureWorker


RETRY_TIMESTAMPS_KEY = "failures:vas2nets#retry_timestamps"


class FakeRedis(object):
    def __init__(self):
        self.data = {}

    def get(self, key):
        return self.data.get(key)

    def set(self, key, value):
        self.data[key] = value

    def exists(self, key):
        return key in self.data

    def keys(self, pattern='*'):
        return fnmatch.filter(self.data.keys(), pattern)

    def flushdb(self):
        self.data = {}


def mktimestamp(delta=0):
    timestamp = datetime.utcnow() + timedelta(seconds=delta)
    return timestamp.isoformat().split('.')[0]


class FailureWorkerTestCase(unittest.TestCase):

    def setUp(self):
        self.config = {
            'transport_name': 'vas2nets'
        }
        self.worker = get_stubbed_worker(FailureWorker, self.config)
        self.worker.startWorker()
        # self.worker.r_server = FakeRedis()
        self.worker.r_server.flushdb()
        self.redis = self.worker.r_server

    def assert_write_timestamp(self, expected, delta, now):
        self.assertEqual(expected,
                         self.worker.get_next_write_timestamp(delta, now=now))

    def assert_zcard(self, expected, key):
        self.assertEqual(expected, self.redis.zcard(key))

    def assert_stored_timestamps(self, *expected):
        self.assertEqual(list(expected),
                         self.redis.zrange(RETRY_TIMESTAMPS_KEY, 0, -1))

    def store_failure(self, reason=None, message=None):
        if not reason:
            reason = "bad stuff happened"
        if not message:
            message = {'message': 'foo', 'reason': reason}
        return self.worker.store_failure(message, reason)

    def test_redis_access(self):
        """
        Sanity check that we can put stuff in redis (or our fake) and
        get it out again.
        """
        self.assertEqual(None, self.worker.r_get("foo"))
        self.assertEqual([], self.redis.keys())
        self.worker.r_set("foo", "bar")
        self.assertEqual("bar", self.worker.r_get("foo"))
        self.assertEqual(['failures:vas2nets#foo'], self.redis.keys())

    def test_store_failure(self):
        """
        Store a failure in redis and make sure we can get at it again.
        """
        key = self.store_failure(reason="reason")
        self.assertEqual(set([key]), self.worker.get_failure_keys())
        self.assertEqual({
                "message": str({"message": "foo", "reason": "reason"}),
                "retry_delay": "None",
                "reason": "reason",
                }, self.redis.hgetall(key))

    def test_write_timestamp(self):
        """
        We need granular timestamps.
        """
        start = datetime.utcnow().isoformat()
        timestamp = self.worker.get_next_write_timestamp(0)
        end = (datetime.utcnow() + timedelta(seconds=6)).isoformat()
        self.assertTrue(start < timestamp < end)

        self.assert_write_timestamp("1970-01-01T00:00:05", 0, 0)
        self.assert_write_timestamp("1970-01-01T00:00:05", 0, 4)
        self.assert_write_timestamp("1970-01-01T00:00:10", 0, 5)
        self.assert_write_timestamp("1970-01-01T00:00:10", 0, 9)

        self.assert_write_timestamp("1970-01-01T00:00:05", 2, 0)
        self.assert_write_timestamp("1970-01-01T00:00:10", 2, 4)
        self.assert_write_timestamp("1970-01-01T00:00:10", 2, 5)
{'fraction_non_alphanumeric': 0.12831241283124128, 'fraction_numerical': 0.03375174337517434, 'mean_word_length': 2.684994861253854, 'pattern_counts': {'":': 9, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '403241', 'n_tokens_mistral': 2409, 'n_tokens_neox': 2205, 'n_words': 443}