<commit_before>'''Test throttle-centric operations'''

import redis
import code
from common import TestQless

class TestThrottle(TestQless):
  '''Test setting throttle data'''
  def test_set(self):
    self.lua('throttle.set', 0, 'tid', 5, 0)
    self.assertEqual(self.redis.hmget('ql:th:tid', 'id')[0], 'tid')
    self.assertEqual(self.redis.hmget('ql:th:tid', 'maximum')[0], '5')
    self.assertEqual(self.redis.ttl('ql:th:tid'), None)

  '''Test setting a expiring throttle'''
  def test_set_with_expiration(self):
    self.lua('throttle.set', 0, 'tid', 5, 1000)
    self.assertNotEqual(self.redis.ttl('ql:th:tid'), None)

  '''Test retrieving throttle ttl'''
  def test_retrieve_ttl(self):
    self.lua('throttle.set', 0, 'tid', 5, 1000)
    self.assertEqual(self.lua('throttle.ttl', 1, 'tid'), self.redis.ttl('ql:th:tid'))

  '''Test retrieving throttle data'''
  def test_get(self):
    self.redis.hmset('ql:th:tid', {'id': 'tid', 'maximum' : 5})
    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 5})

  '''Test retrieving uninitiailized throttle data'''
  def test_get(self):
    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 0})

  '''Test deleting the throttle data'''
  def test_delete(self):
    self.lua('throttle.set', 0, 'tid', 5)
    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 5})
    self.lua('throttle.delete', 0, 'tid')
    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 0})

class TestAcquire(TestQless):
  '''Test that a job has a default queue throttle'''
  def test_default_queue_throttle(self):
    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0)
    self.assertEqual(self.lua('get', 0, 'jid')['throttles'], ['ql:q:queue'])

  '''Test that job can specify a throttle'''
  def test_specify_throttle(self):
    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
    self.assertEqual(self.lua('get', 0, 'jid')['throttles'], ['tid', 'ql:q:queue'])

  '''Test that a job can acquire a throttle'''
  def test_acquire_throttle(self):
    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('pop', 0, 'queue', 'worker', 1)
    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])

  '''Test that acquiring of a throttle lock properly limits the number of jobs'''
  def test_limit_number_of_locks(self):
    self.lua('throttle.set', 0, 'tid', 1)
    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('put', 2, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('put', 3, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('pop', 0, 'queue', 'worker', 4)
    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), ['jid2', 'jid3', 'jid4'])

class TestRelease(TestQless):
  '''Test that when there are no pending jobs lock is properly released'''
  def test_no_pending_jobs(self):
    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('pop', 0, 'queue', 'worker', 1)
    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
    self.lua('complete', 0, 'jid', 'worker', 'queue', {})
    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])

  '''Test that releasing a lock properly another job in the work queue'''
  def test_next_job_is_moved_into_work_qeueue(self):
    self.lua('throttle.set', 0, 'tid', 1)
    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('pop', 2, 'queue', 'worker', 2)
    self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
    self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
    self.lua('complete', 5, 'jid1', 'worker', 'queue', {})
    # Lock should be empty until another job is popped
    self.assertEqual(self.lua('throttle.locks', 6, 'tid'), [])
    self.assertEqual(self.lua('jobs', 7, 'throttled', 'queue'), ['jid2'])
    self.lua('pop', 8, 'queue', 'worker', 1)
    self.assertEqual(self.lua('throttle.locks', 9, 'tid'), ['jid2'])<commit_msg>test exposing cancel bug<commit_after>'''Test throttle-centric operations'''

import redis
import code
from common import TestQless

class TestThrottle(TestQless):
  '''Test setting throttle data'''
  def test_set(self):
    self.lua('throttle.set', 0, 'tid', 5, 0)
    self.assertEqual(self.redis.hmget('ql:th:tid', 'id')[0], 'tid')
    self.assertEqual(self.redis.hmget('ql:th:tid', 'maximum')[0], '5')
    self.assertEqual(self.redis.ttl('ql:th:tid'), None)

  '''Test setting a expiring throttle'''
  def test_set_with_expiration(self):
    self.lua('throttle.set', 0, 'tid', 5, 1000)
    self.assertNotEqual(self.redis.ttl('ql:th:tid'), None)

  '''Test retrieving throttle ttl'''
  def test_retrieve_ttl(self):
    self.lua('throttle.set', 0, 'tid', 5, 1000)
    self.assertEqual(self.lua('throttle.ttl', 1, 'tid'), self.redis.ttl('ql:th:tid'))

  '''Test retrieving throttle data'''
  def test_get(self):
    self.redis.hmset('ql:th:tid', {'id': 'tid', 'maximum' : 5})
    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 5})

  '''Test retrieving uninitiailized throttle data'''
  def test_get(self):
    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 0})

  '''Test deleting the throttle data'''
  def test_delete(self):
    self.lua('throttle.set', 0, 'tid', 5)
    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 5})
    self.lua('throttle.delete', 0, 'tid')
    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 0})

class TestAcquire(TestQless):
  '''Test that a job has a default queue throttle'''
  def test_default_queue_throttle(self):
    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0)
    self.assertEqual(self.lua('get', 0, 'jid')['throttles'], ['ql:q:queue'])

  '''Test that job can specify a throttle'''
  def test_specify_throttle(self):
    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
    self.assertEqual(self.lua('get', 0, 'jid')['throttles'], ['tid', 'ql:q:queue'])

  '''Test that a job can acquire a throttle'''
  def test_acquire_throttle(self):
    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('pop', 0, 'queue', 'worker', 1)
    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])

  '''Test that acquiring of a throttle lock properly limits the number of jobs'''
  def test_limit_number_of_locks(self):
    self.lua('throttle.set', 0, 'tid', 1)
    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('put', 2, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('put', 3, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('pop', 0, 'queue', 'worker', 4)
    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), ['jid2', 'jid3', 'jid4'])

class TestRelease(TestQless):
  '''Test that when there are no pending jobs lock is properly released'''
  def test_no_pending_jobs(self):
    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('pop', 0, 'queue', 'worker', 1)
    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
    self.lua('complete', 0, 'jid', 'worker', 'queue', {})
    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])

  '''Test that releasing a lock properly another job in the work queue'''
  def test_next_job_is_moved_into_work_qeueue(self):
    self.lua('throttle.set', 0, 'tid', 1)
    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('pop', 2, 'queue', 'worker', 2)
    self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
    self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
    self.lua('complete', 5, 'jid1', 'worker', 'queue', {})
    # Lock should be empty until another job is popped
    self.assertEqual(self.lua('throttle.locks', 6, 'tid'), [])
    self.assertEqual(self.lua('jobs', 7, 'throttled', 'queue'), ['jid2'])
    self.lua('pop', 8, 'queue', 'worker', 1)
    self.assertEqual(self.lua('throttle.locks', 9, 'tid'), ['jid2'])
    self.assertEqual(self.lua('jobs', 10, 'throttled', 'queue'), [])


  '''Test that cancelling a job properly adds another job in the work queue'''
  def test_on_cancel_next_job_is_moved_into_work_queue(self):
    self.lua('throttle.set', 0, 'tid', 1)
    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
    self.lua('pop', 2, 'queue', 'worker', 2)
    self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
    self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
    self.lua('cancel', 5, 'jid1', 'worker', 'queue', {})
    # Lock should be empty until another job is popped
    self.assertEqual(self.lua('throttle.locks', 6, 'tid'), [])
    self.assertEqual(self.lua('jobs', 7, 'throttled', 'queue'), ['jid2'])
    self.lua('pop', 8, 'queue', 'worker', 1)
    self.assertEqual(self.lua('throttle.locks', 9, 'tid'), ['jid2'])
{'fraction_non_alphanumeric': 0.23620723749258454, 'fraction_numerical': 0.0214554083448685, 'mean_word_length': 4.840069284064665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '772736', 'n_tokens_mistral': 3986, 'n_tokens_neox': 3717, 'n_words': 884}