<commit_before>/**
@file IO/unicode.hpp
@brief Unicode IO operations.

@author <NAME>
@copyright 2010-2013 <NAME> under the MIT license;
see @ref index or the accompanying LICENSE file for full text.
*/

#ifndef DUCT_IO_UNICODE_HPP_
#define DUCT_IO_UNICODE_HPP_

// TODO: Unicode read/write variants for generic iterators
// (to supply support for narrow and wide string literals)

#include "../config.hpp"
#include "../debug.hpp"
#include "../string.hpp"
#include "../detail/string_traits.hpp"
#include "../EndianUtils.hpp"

#include <cstring>
#include <limits>
#include <iostream>

namespace duct {
namespace IO {

/**
	@addtogroup io
	@{
*/
/**<commit_msg>IO/unicode: added missing #include "./arithmetic.hpp".<commit_after>/**
@file IO/unicode.hpp
@brief Unicode IO operations.

@author <NAME>
@copyright 2010-2013 <NAME> under the MIT license;
see @ref index or the accompanying LICENSE file for full text.
*/

#ifndef DUCT_IO_UNICODE_HPP_
#define DUCT_IO_UNICODE_HPP_

// TODO: Unicode read/write variants for generic iterators
// (to supply support for narrow and wide string literals)

#include "../config.hpp"
#include "../debug.hpp"
#include "../string.hpp"
#include "../detail/string_traits.hpp"
#include "../EndianUtils.hpp"
#include "./arithmetic.hpp"

#include <cstring>
#include <limits>
#include <iostream>

namespace duct {
namespace IO {

/**
	@addtogroup io
	@{
*/
/**
{'fraction_non_alphanumeric': 0.14203316510454217, 'fraction_numerical': 0.011535688536409516, 'mean_word_length': 4.359073359073359, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1313114', 'n_tokens_mistral': 488, 'n_tokens_neox': 448, 'n_words': 144}