<commit_before>

template <class T>
class Queue
{
	public:
		Queue()
		{
			m_maxsize = 0;
		}

		~Queue()
		{
			Flush();
		}

		bool Add(T item)
		{
			ScopedLock lock(&m_mutex);
			if (m_maxsize != 0 && m_queue.size() >= m_maxsize)
				abort();
			m_queue.push_back(item);
			m_mutex.WakeUp();
			return true;
		}

		T GetNext()
		{
			void *tmp = NULL;
			ScopedLock lock(&m_mutex);
			while(tmp == NULL)
			{
				if (m_queue.empty() == false)
				{
					tmp = m_queue.front();
					m_queue.pop_front();
					return tmp;
				}
				else
				{
					m_mutex.Wait();
				}
			}
			return NULL;
		}

		T GetNext(struct timespec *ts)
		{
			T tmp = NULL;
			ScopedLock lock(&m_mutex);
			while(tmp == NULL)
			{
				if (m_queue.empty() == false)
				{
					tmp = m_queue.front();
					m_queue.pop_front();
					return tmp;
				}
				else
				{
					if (m_mutex.Wait(ts) == -ETIMEDOUT)
						return NULL;
				}
			}
			return NULL;
		}

		void Flush()
		{
			m_mutex.Lock();
			while(m_queue.empty() == false)
			{
				m_mutex.Unlock();
				sleep(1);	//FIXME: Remove sleep
				m_mutex.Lock();
			}
			m_mutex.Unlock();
		}

		size_t GetCount()
		{
			ScopedLock lock(&m_mutex);
			return m_queue.size();
		}

		void SetMaxItems(size_t size)
		{
			ScopedLock lock(&m_mutex);
			m_maxsize = size;
		}

		size_t GetMaxItems(size_t size)
		{
			ScopedLock lock(&m_mutex);
			return m_maxsize;
		}

	private:
		size_t m_maxsize;
		std::list<T> m_queue;
		Mutex m_mutex;
};
<commit_msg>Fixed Queue flush sleep by removing the sleep and using a timeout instead, Added a high water size count to queue, Fixed Add error handling<commit_after>

template <class T>
class Queue
{
	public:
		Queue()
		{
			m_maxsize = 0;
			m_hwsize = 0;
			m_flushing = false;
		}

		~Queue()
		{
			Flush(); //This prevents the queue from being destryoed when it contains items
		}

		bool Add(T item)
		{
			ScopedLock lock(&m_mutex);
			if (m_maxsize != 0 && m_queue.size() >= m_maxsize)
				return false;

			if (m_queue.size() > m_hwsize)
				m_hwsize = m_queue.size();

			m_count++;
			m_queue.push_back(item);
			m_mutex.WakeUp();
			return true;
		}

		T GetNext()
		{
			void *tmp = NULL;
			ScopedLock lock(&m_mutex);
			while(tmp == NULL)
			{
				if (m_queue.empty() == false)
				{
					tmp = m_queue.front();
					m_queue.pop_front();
					return tmp;
				}
				else
				{
					m_mutex.Wait();
				}
			}
			return NULL;
		}

		T GetNext(struct timespec *ts)
		{
			T tmp = NULL;
			ScopedLock lock(&m_mutex);
			while(tmp == NULL)
			{
				if (m_queue.empty() == false)
				{
					tmp = m_queue.front();
					m_queue.pop_front();
					//Since we are removing items. If the queue is being flushed also do a wakeup
					if (m_flushing && m_queue.empty() == true)
						m_mutex.WakeUp();
					return tmp;
				}
				else
				{
					if (m_mutex.Wait(ts) == -ETIMEDOUT)
						return NULL;
				}
			}
			return NULL;
		}

		void Flush()
		{
			ScopedLock lock(&m_mutex);
			m_flushing = true;
			while(m_queue.empty() == false)
			{
				struct timespec ts = {1, 0};
				m_mutex.Wait(&ts);
			}
			m_flushing = false;
		}

		size_t GetCount()
		{
			ScopedLock lock(&m_mutex);
			return m_queue.size();
		}

		size_t GetHWCount()
		{
			ScopedLock lock(&m_mutex);
			return m_hwsize;
		}

		uint64_t GetProcCount()
		{
			ScopedLock lock(&m_mutex);
			return m_count;
		}

		void SetMaxItems(size_t size)
		{
			ScopedLock lock(&m_mutex);
			m_maxsize = size;
		}

		size_t GetMaxItems(size_t size)
		{
			ScopedLock lock(&m_mutex);
			return m_maxsize;
		}

	private:
		size_t m_maxsize;
		size_t m_hwsize; //High water mark for size
		uint64_t m_count; //Number of items processed
		bool m_flushing;
		std::list<T> m_queue;
		Mutex m_mutex;
};

{'fraction_non_alphanumeric': 0.13292158968850698, 'fraction_numerical': 0.00322234156820623, 'mean_word_length': 2.4049360146252283, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2245728', 'n_tokens_mistral': 1891, 'n_tokens_neox': 1470, 'n_words': 325}