<commit_before>#include <deque>
#include <vector>
#include <mutex>
#include <functional>
#include <future>
#include <memory>

namespace snf {

template<typename T>
class sync_queue
{
private:
	std::deque<T>           m_queue;
	bool                    m_valid = true;
	size_t                  m_waiting = 0;
	std::mutex              m_lock;
	std::condition_variable m_cv;

public:
	sync_queue() {}
	virtual ~sync_queue() { invalidate(); }

	sync_queue(const sync_queue<T> &) = delete;
	sync_queue(sync_queue<T> &&) = delete;

	const sync_queue<T> & operator= (const sync_queue<T> &) = delete;
	sync_queue<T> & operator= (sync_queue<T> &&) = delete;

	bool empty()
	{
		std::lock_guard<std::mutex> guard(m_lock);
		return m_queue.empty();
	}

	size_t size()
	{
		std::lock_guard<std::mutex> guard(m_lock);
		return m_queue.size();
	}

	size_t waiting()
	{
		std::lock_guard<std::mutex> guard(m_lock);
		return m_waiting;
	}

	void put(const T &elem)
	{
		std::lock_guard<std::mutex> guard(m_lock);
		if (m_valid) {
			m_queue.push_back(elem);
			m_cv.notify_one();
		}
	}

	void put(T &&elem)
	{
		std::lock_guard<std::mutex> guard(m_lock);
		if (m_valid) {
			m_queue.emplace_back(std::move(elem));
			m_cv.notify_one();
		}
	}

	void get(T &elem)
	{
		std::unique_lock<std::mutex> lock(m_lock);
		m_waiting++;
		m_cv.wait(lock, [this] { return !(m_valid && m_queue.empty()); });
		m_waiting--;
		if (m_valid && !m_queue.empty()) {
			elem = std::move(m_queue.front());
			m_queue.pop_front();
		}
	}

	void invalidate()
	{
		std::lock_guard<std::mutex> guard(m_lock);
		m_valid = false;
		m_cv.notify_all();
	}

	void clear()
	{
		std::lock_guard<std::mutex> guard(m_lock);
		while (!m_queue.empty())
			m_queue.pop_back();
	}

	void notify_all()
	{
		std::lock_guard<std::mutex> guard(m_lock);
		m_cv.notify_all();
	}
};

class thread_pool
{
private:
	using fcn_type = std::function<void()>;

	std::atomic<bool>                           m_done { false };
	std::vector<std::unique_ptr<std::thread>>   m_threads;
	sync_queue<fcn_type>                        m_queue;

	void start(size_t n)
	{
		auto worker = [this] {
			while (!m_done) {
				fcn_type fcn;
				m_queue.get(fcn);
				(fcn)();
			}
		};

		for (size_t i = 0; i < n; ++i) {
			std::unique_ptr<std::thread> uniq_thrd{new std::thread{worker}};
			m_threads.emplace_back(std::move(uniq_thrd));
		}
	}

public:
	thread_pool()
	{
		size_t hc = std::thread::hardware_concurrency();<commit_msg>Removed unused functions.<commit_after>#include <deque>
#include <vector>
#include <mutex>
#include <functional>
#include <future>
#include <memory>

namespace snf {

template<typename T>
class sync_queue
{
private:
	std::deque<T>           m_queue;
	bool                    m_valid = true;
	size_t                  m_waiting = 0;
	std::mutex              m_lock;
	std::condition_variable m_cv;

public:
	sync_queue() {}
	virtual ~sync_queue() { invalidate(); }

	sync_queue(const sync_queue<T> &) = delete;
	sync_queue(sync_queue<T> &&) = delete;

	const sync_queue<T> & operator= (const sync_queue<T> &) = delete;
	sync_queue<T> & operator= (sync_queue<T> &&) = delete;

	bool empty()
	{
		std::lock_guard<std::mutex> guard(m_lock);
		return m_queue.empty();
	}

	size_t size()
	{
		std::lock_guard<std::mutex> guard(m_lock);
		return m_queue.size();
	}

	size_t waiting()
	{
		std::lock_guard<std::mutex> guard(m_lock);
		return m_waiting;
	}

	void put(const T &elem)
	{
		std::lock_guard<std::mutex> guard(m_lock);
		if (m_valid) {
			m_queue.push_back(elem);
			m_cv.notify_one();
		}
	}

	void put(T &&elem)
	{
		std::lock_guard<std::mutex> guard(m_lock);
		if (m_valid) {
			m_queue.emplace_back(std::move(elem));
			m_cv.notify_one();
		}
	}

	void get(T &elem)
	{
		std::unique_lock<std::mutex> lock(m_lock);
		m_waiting++;
		m_cv.wait(lock, [this] { return !(m_valid && m_queue.empty()); });
		m_waiting--;
		if (m_valid && !m_queue.empty()) {
			elem = std::move(m_queue.front());
			m_queue.pop_front();
		}
	}

	void invalidate()
	{
		std::lock_guard<std::mutex> guard(m_lock);
		m_valid = false;
		while (!m_queue.empty())
			m_queue.pop_back();
		m_cv.notify_all();
	}
};

class thread_pool
{
private:
	using fcn_type = std::function<void()>;

	std::atomic<bool>                           m_done { false };
	std::vector<std::unique_ptr<std::thread>>   m_threads;
	sync_queue<fcn_type>                        m_queue;

	void start(size_t n)
	{
		auto worker = [this] {
			while (!m_done) {
				fcn_type fcn;
				this->m_queue.get(fcn);
				(fcn)();
			}
		};

		for (size_t i = 0; i < n; ++i) {
			std::unique_ptr<std::thread> uniq_thrd{new std::thread{worker}};
			m_threads.emplace_back(std::move(uniq_thrd));
		}
	}

public:
	thread_pool()
	{
		size_t hc = std::thread::hardware_concurrency();
{'fraction_non_alphanumeric': 0.16988660226795463, 'fraction_numerical': 0.0008399832003359933, 'mean_word_length': 3.593056894889103, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2100768', 'n_tokens_mistral': 2077, 'n_tokens_neox': 1921, 'n_words': 324}