<commit_before>#ifndef STINGRAY_TOOLKIT_DATABUFFER_H
#define STINGRAY_TOOLKIT_DATABUFFER_H


#include <stingray/log/Logger.h>
#include <stingray/toolkit/BithreadCircularBuffer.h>
#include <stingray/toolkit/IDataSource.h>

namespace stingray
{

	class DataBufferBase : public virtual IDataConsumer
	{
	protected:
		static NamedLogger		s_logger;
		bool					_discardOnOverflow;
		BithreadCircularBuffer	_buffer;
		const size_t			_inputPacketSize;

		Mutex					_bufferMutex;
		Mutex					_writeMutex;

		WaitToken				_bufferEmpty;
		WaitToken				_bufferFull;
		bool					_eod;

	protected:
		DataBufferBase(bool discardOnOverflow, size_t size, size_t inputPacketSize) :
			_discardOnOverflow(discardOnOverflow), _buffer(size),
			_inputPacketSize(inputPacketSize), _eod(false)
		{
			TOOLKIT_CHECK(inputPacketSize != 0, ArgumentException("inputPacketSize", inputPacketSize));
			TOOLKIT_CHECK(size % inputPacketSize == 0, "Buffer size is not a multiple of input packet size!");
		}

	public:
		void Clear()
		{
			MutexLock l(_bufferMutex);
			while (true)
			{
				BithreadCircularBuffer::Reader r = _buffer.Read();
				if (r.size() == 0)
					return;<commit_msg>Added status functions to DataBuffer<commit_after>#ifndef STINGRAY_TOOLKIT_DATABUFFER_H
#define STINGRAY_TOOLKIT_DATABUFFER_H


#include <stingray/log/Logger.h>
#include <stingray/toolkit/BithreadCircularBuffer.h>
#include <stingray/toolkit/IDataSource.h>

namespace stingray
{

	class DataBufferBase : public virtual IDataConsumer
	{
	protected:
		static NamedLogger		s_logger;
		bool					_discardOnOverflow;
		BithreadCircularBuffer	_buffer;
		const size_t			_inputPacketSize;

		Mutex					_bufferMutex;
		Mutex					_writeMutex;

		WaitToken				_bufferEmpty;
		WaitToken				_bufferFull;
		bool					_eod;

	protected:
		DataBufferBase(bool discardOnOverflow, size_t size, size_t inputPacketSize) :
			_discardOnOverflow(discardOnOverflow), _buffer(size),
			_inputPacketSize(inputPacketSize), _eod(false)
		{
			TOOLKIT_CHECK(inputPacketSize != 0, ArgumentException("inputPacketSize", inputPacketSize));
			TOOLKIT_CHECK(size % inputPacketSize == 0, "Buffer size is not a multiple of input packet size!");
		}

	public:
		size_t GetDataSize()			{ return _buffer.GetDataSize(); }
		size_t GetFreeSize()			{ return _buffer.GetFreeSize(); }
		size_t GetStorageSize() const	{ return _buffer.GetTotalSize(); }

		void Clear()
		{
			MutexLock l(_bufferMutex);
			while (true)
			{
				BithreadCircularBuffer::Reader r = _buffer.Read();
				if (r.size() == 0)
					return;
{'fraction_non_alphanumeric': 0.0895049504950495, 'fraction_numerical': 0.002376237623762376, 'mean_word_length': 4.6891891891891895, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3455639', 'n_tokens_mistral': 1011, 'n_tokens_neox': 875, 'n_words': 176}