<commit_before>#ifndef SILICIUM_WIN32_PROCESS_HPP
#define SILICIUM_WIN32_PROCESS_HPP

#include <silicium/process_parameters.hpp>
#include <silicium/win32/win32.hpp>
#include <silicium/error_code.hpp>
#include <silicium/sink/ptr_sink.hpp>
#include <silicium/sink/buffering_sink.hpp>
#include <cassert>
#include <stdexcept>

namespace Si
{
	namespace win32
	{
		typedef std::basic_string<WCHAR> winapi_string;

		inline winapi_string utf8_to_winapi_string(char const *original, size_t length)
		{
			if (length > static_cast<size_t>((std::numeric_limits<int>::max)()))
			{
				throw std::invalid_argument("Input string is too long for WinAPI");
			}
			int const output_size = MultiByteToWideChar(CP_UTF8, 0, original, static_cast<int>(length), nullptr, 0);
			assert(output_size >= 0);
			if (output_size == 0)
			{
				assert(GetLastError() == ERROR_NO_UNICODE_TRANSLATION);
				throw std::invalid_argument("Input string is not UTF-8");
			}
			winapi_string result;
			result.resize(static_cast<size_t>(output_size));
			if (!result.empty())
			{
				MultiByteToWideChar(CP_UTF8, 0, original, static_cast<int>(length), &result[0], output_size);
			}
			return result;
		}

		inline winapi_string utf8_to_winapi_string(std::string const &str)
		{
			return utf8_to_winapi_string(str.data(), str.size());
		}

		inline winapi_string build_command_line(std::vector<std::string> const &arguments)
		{
			winapi_string command_line;
			for (auto a = begin(arguments); a != end(arguments); ++a)
			{
				if ((a + 1) != end(arguments))
				{
					command_line += L" ";
				}
				command_line += utf8_to_winapi_string(a->data(), a->size());
			}
			return command_line;
		}

		struct pipe
		{
			unique_handle read, write;

			pipe() BOOST_NOEXCEPT
			{
			}

			pipe(pipe &&other) BOOST_NOEXCEPT
			{
				swap(other);
			}

			pipe &operator = (pipe &&other) BOOST_NOEXCEPT
			{
				swap(other);
				return *this;
			}<commit_msg>fix a bug in the Windows run_process<commit_after>#ifndef SILICIUM_WIN32_PROCESS_HPP
#define SILICIUM_WIN32_PROCESS_HPP

#include <silicium/process_parameters.hpp>
#include <silicium/win32/win32.hpp>
#include <silicium/error_code.hpp>
#include <silicium/sink/ptr_sink.hpp>
#include <silicium/sink/buffering_sink.hpp>
#include <cassert>
#include <stdexcept>

namespace Si
{
	namespace win32
	{
		typedef std::basic_string<WCHAR> winapi_string;

		inline winapi_string utf8_to_winapi_string(char const *original, size_t length)
		{
			if (length > static_cast<size_t>((std::numeric_limits<int>::max)()))
			{
				throw std::invalid_argument("Input string is too long for WinAPI");
			}
			int const output_size = MultiByteToWideChar(CP_UTF8, 0, original, static_cast<int>(length), nullptr, 0);
			assert(output_size >= 0);
			if (output_size == 0)
			{
				assert(GetLastError() == ERROR_NO_UNICODE_TRANSLATION);
				throw std::invalid_argument("Input string is not UTF-8");
			}
			winapi_string result;
			result.resize(static_cast<size_t>(output_size));
			if (!result.empty())
			{
				MultiByteToWideChar(CP_UTF8, 0, original, static_cast<int>(length), &result[0], output_size);
			}
			return result;
		}

		inline winapi_string utf8_to_winapi_string(std::string const &str)
		{
			return utf8_to_winapi_string(str.data(), str.size());
		}

		inline winapi_string build_command_line(std::vector<std::string> const &arguments)
		{
			winapi_string command_line;
			for (auto a = begin(arguments); a != end(arguments); ++a)
			{
				if (a != begin(arguments))
				{
					command_line += L" ";
				}
				command_line += utf8_to_winapi_string(a->data(), a->size());
			}
			return command_line;
		}

		struct pipe
		{
			unique_handle read, write;

			pipe() BOOST_NOEXCEPT
			{
			}

			pipe(pipe &&other) BOOST_NOEXCEPT
			{
				swap(other);
			}

			pipe &operator = (pipe &&other) BOOST_NOEXCEPT
			{
				swap(other);
				return *this;
			}
{'fraction_non_alphanumeric': 0.13052468338071854, 'fraction_numerical': 0.012147841819591625, 'mean_word_length': 4.3232462173315, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4908214', 'n_tokens_mistral': 1671, 'n_tokens_neox': 1392, 'n_words': 265}