<commit_before>#ifndef SILICIUM_SINK_HPP
#define SILICIUM_SINK_HPP

#include <silicium/override.hpp>
#include <boost/range/iterator_range.hpp>
#include <boost/range/algorithm/copy.hpp>
#include <boost/filesystem/path.hpp>
#include <ostream>
#include <array>
#include <memory>

namespace Si
{
	template <class Element>
	struct sink
	{
		virtual ~sink()
		{
		}

		virtual boost::iterator_range<Element *> make_append_space(std::size_t size) = 0;
		virtual void flush_append_space() = 0;
		virtual void append(boost::iterator_range<Element const *> data) = 0;
	};

	template <class Element>
	struct flushable_sink : sink<Element>
	{
		virtual void flush() = 0;
	};

	template <class Element>
	void commit(sink<Element> &destination, std::size_t count)
	{
		destination.make_append_space(count);
		destination.flush_append_space();
	}

	template <class Element, class Buffer = std::array<Element, ((1U << 13U) / sizeof(Element))>>
	struct buffering_sink : flushable_sink<Element>
	{
		explicit buffering_sink(sink<Element> &destination, Buffer buffer = Buffer())
			: m_destination(destination)
			, m_fallback_buffer(std::move(buffer))
			, m_buffer_used(0)
		{
		}

		boost::iterator_range<Element *> make_append_space(std::size_t size) SILICIUM_OVERRIDE
		{
			auto first_try = m_destination.make_append_space(size);
			if (!first_try.empty())
			{
				auto const copied = (std::min)(static_cast<std::ptrdiff_t>(m_buffer_used), first_try.size());
				std::copy(m_fallback_buffer.begin(), m_fallback_buffer.begin() + copied, first_try.begin());
				m_buffer_used = 0;
				return first_try;
			}
			m_buffer_used = (std::min)(size, m_fallback_buffer.size());
			return boost::make_iterator_range(m_fallback_buffer.data(), m_fallback_buffer.data() + m_buffer_used);
		}

		void flush_append_space() SILICIUM_OVERRIDE
		{
			if (m_buffer_used)
			{
				flush();
			}
			else
			{
				m_destination.flush_append_space();
			}
		}

		void append(boost::iterator_range<Element const *> data) SILICIUM_OVERRIDE
		{
			if (data.size() <= (m_fallback_buffer.size() - m_buffer_used))
			{
				boost::range::copy(data, m_fallback_buffer.begin() + m_buffer_used);
				m_buffer_used += data.size();
				return;
			}

			flush();
			m_destination.append(data);
		}

		void flush() SILICIUM_OVERRIDE
		{
			m_destination.append(boost::make_iterator_range(m_fallback_buffer.data(), m_fallback_buffer.data() + m_buffer_used));
			m_buffer_used = 0;
		}

	private:

		sink<Element> &m_destination;
		Buffer m_fallback_buffer;
		std::size_t m_buffer_used;
	};

	template <class Element, class OutputIterator>
	struct iterator_sink : sink<Element>
	{
		explicit iterator_sink(OutputIterator out)
			: m_out(std::move(out))
		{
		}

		virtual boost::iterator_range<Element *> make_append_space(std::size_t) SILICIUM_OVERRIDE
		{
			return {};
		}

		virtual void flush_append_space() SILICIUM_OVERRIDE
		{
		}

		virtual void append(boost::iterator_range<Element const *> data) SILICIUM_OVERRIDE
		{
			boost::range::copy(data, m_out);
		}

	private:

		OutputIterator m_out;
	};

	template <class Element, class OutputIterator>
	auto make_iterator_sink(OutputIterator out)
		-> iterator_sink<Element, typename std::decay<OutputIterator>::type>
	{
		return iterator_sink<Element, typename std::decay<OutputIterator>::type>(std::move(out));
	}

	template <class Container>
	auto make_container_sink(Container &destination)
		-> iterator_sink<typename Container::value_type, std::back_insert_iterator<Container>>
	{
		return make_iterator_sink<typename Container::value_type>(std::back_inserter(destination));
	}
<commit_msg>add sink that append to an std::ostream &<commit_after>#ifndef SILICIUM_SINK_HPP
#define SILICIUM_SINK_HPP

#include <silicium/override.hpp>
#include <boost/range/iterator_range.hpp>
#include <boost/range/algorithm/copy.hpp>
#include <boost/filesystem/path.hpp>
#include <ostream>
#include <array>
#include <memory>

namespace Si
{
	template <class Element>
	struct sink
	{
		virtual ~sink()
		{
		}

		virtual boost::iterator_range<Element *> make_append_space(std::size_t size) = 0;
		virtual void flush_append_space() = 0;
		virtual void append(boost::iterator_range<Element const *> data) = 0;
	};

	template <class Element>
	struct flushable_sink : sink<Element>
	{
		virtual void flush() = 0;
	};

	template <class Element>
	void commit(sink<Element> &destination, std::size_t count)
	{
		destination.make_append_space(count);
		destination.flush_append_space();
	}

	template <class Element, class Buffer = std::array<Element, ((1U << 13U) / sizeof(Element))>>
	struct buffering_sink : flushable_sink<Element>
	{
		explicit buffering_sink(sink<Element> &destination, Buffer buffer = Buffer())
			: m_destination(destination)
			, m_fallback_buffer(std::move(buffer))
			, m_buffer_used(0)
		{
		}

		boost::iterator_range<Element *> make_append_space(std::size_t size) SILICIUM_OVERRIDE
		{
			auto first_try = m_destination.make_append_space(size);
			if (!first_try.empty())
			{
				auto const copied = (std::min)(static_cast<std::ptrdiff_t>(m_buffer_used), first_try.size());
				std::copy(m_fallback_buffer.begin(), m_fallback_buffer.begin() + copied, first_try.begin());
				m_buffer_used = 0;
				return first_try;
			}
			m_buffer_used = (std::min)(size, m_fallback_buffer.size());
			return boost::make_iterator_range(m_fallback_buffer.data(), m_fallback_buffer.data() + m_buffer_used);
		}

		void flush_append_space() SILICIUM_OVERRIDE
		{
			if (m_buffer_used)
			{
				flush();
			}
			else
			{
				m_destination.flush_append_space();
			}
		}

		void append(boost::iterator_range<Element const *> data) SILICIUM_OVERRIDE
		{
			if (data.size() <= (m_fallback_buffer.size() - m_buffer_used))
			{
				boost::range::copy(data, m_fallback_buffer.begin() + m_buffer_used);
				m_buffer_used += data.size();
				return;
			}

			flush();
			m_destination.append(data);
		}

		void flush() SILICIUM_OVERRIDE
		{
			m_destination.append(boost::make_iterator_range(m_fallback_buffer.data(), m_fallback_buffer.data() + m_buffer_used));
			m_buffer_used = 0;
		}

	private:

		sink<Element> &m_destination;
		Buffer m_fallback_buffer;
		std::size_t m_buffer_used;
	};

	template <class Element, class OutputIterator>
	struct iterator_sink : sink<Element>
	{
		explicit iterator_sink(OutputIterator out)
			: m_out(std::move(out))
		{
		}

		virtual boost::iterator_range<Element *> make_append_space(std::size_t) SILICIUM_OVERRIDE
		{
			return {};
		}

		virtual void flush_append_space() SILICIUM_OVERRIDE
		{
		}

		virtual void append(boost::iterator_range<Element const *> data) SILICIUM_OVERRIDE
		{
			boost::range::copy(data, m_out);
		}

	private:

		OutputIterator m_out;
	};

	template <class Element, class OutputIterator>
	auto make_iterator_sink(OutputIterator out)
		-> iterator_sink<Element, typename std::decay<OutputIterator>::type>
	{
		return iterator_sink<Element, typename std::decay<OutputIterator>::type>(std::move(out));
	}

	template <class Container>
	auto make_container_sink(Container &destination)
		-> iterator_sink<typename Container::value_type, std::back_insert_iterator<Container>>
	{
		return make_iterator_sink<typename Container::value_type>(std::back_inserter(destination));
	}

	struct ostream_ref_sink : flushable_sink<char>
	{
		explicit ostream_ref_sink(std::ostream &file)
		   : m_file(file)
		{
		}

		virtual boost::iterator_range<char *> make_append_space(std::size_t) SILICIUM_OVERRIDE
		{
		   return {};
		}

		virtual void flush_append_space() SILICIUM_OVERRIDE
		{
		}

		virtual void append(boost::iterator_range<char const *> data) SILICIUM_OVERRIDE
		{
		   m_file.write(data.begin(), data.size());
		}

		virtual void flush() SILICIUM_OVERRIDE
		{
		   m_file.flush();
		}

	private:

		std::ostream &m_file;
	};

{'fraction_non_alphanumeric': 0.12668636772452782, 'fraction_numerical': 0.002569703199280483, 'mean_word_length': 5.100313479623824, 'pattern_counts': {'":': 0, '<': 82, '<?xml version=': 0, '>': 80, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1308781', 'n_tokens_mistral': 3121, 'n_tokens_neox': 2767, 'n_words': 461}