<commit_before>
#include <SAX/ArabicaConfig.h>
#ifndef ARABICA_NO_CODECVT_SPECIALISATIONS
#include <Utils/impl/codecvt_specialisations.h>

namespace std
{

codecvt<char, wchar_t, mbstate_t>::
~codecvt() 
{
} // ~codecvt

////////////////////////////////////////////////////////////////////////
codecvt_base::result 
codecvt<char, wchar_t, mbstate_t>:: 
do_out(mbstate_t&, const char* from, const char* from_end, const char*& from_next,
       wchar_t* to, wchar_t* to_limit, wchar_t*& to_next) const
{
  int limit = std::max<int>(from_end - from, to_limit - to);
  from_next = from;
  to_next = to;

  while(limit--)
    *to_next++ = static_cast<wchar_t>(*from_next++);

  return codecvt_base::ok;
} // do_out

codecvt_base::result 
codecvt<char, wchar_t, mbstate_t>:: 
do_in(mbstate_t&, const wchar_t* from, const wchar_t* from_end, const wchar_t*& from_next,
                  char* to, char* to_limit, char*& to_next)  const
{
  int limit = std::max<int>(from_end - from, to_limit - to);
  from_next = from;
  to_next = to;

  while(limit--)
    *to_next++ = static_cast<char>(*from_next++);

  return codecvt_base::ok;
} // do_in

codecvt_base::result 
codecvt<char, wchar_t, mbstate_t>:: 
do_unshift(mbstate_t&, wchar_t* to, wchar_t* to_limit, wchar_t*& to_next) const
{
  to_next = to;
  return codecvt_base::noconv;
} // do_unshift

int 
codecvt<char, wchar_t, mbstate_t>:: 
do_encoding() const throw()
{
  return 1;
} // do_encoding

bool 
codecvt<char, wchar_t, mbstate_t>:: 
do_always_noconv() const throw()
{
  return false;
} // do_always_noconv

int 
codecvt<char, wchar_t, mbstate_t>:: 
do_length(const mbstate_t&, const wchar_t* from, const wchar_t* end, size_t max) const
{
  return std::min<int>(max, (end - from));
} // do_length

int 
codecvt<char, wchar_t, mbstate_t>:: 
do_max_length() const throw()
{
  return 1;
} // do_max_length

} // namespace std

#endif<commit_msg>exclude from build if now wchar_t<commit_after>
#include <SAX/ArabicaConfig.h>
#ifndef ARABICA_NO_CODECVT_SPECIALISATIONS 
#ifndef ARABICA_NO_WCHAR_T
#include <Utils/impl/codecvt_specialisations.h>

namespace std
{

codecvt<char, wchar_t, mbstate_t>::
~codecvt() 
{
} // ~codecvt

////////////////////////////////////////////////////////////////////////
codecvt_base::result 
codecvt<char, wchar_t, mbstate_t>:: 
do_out(mbstate_t&, const char* from, const char* from_end, const char*& from_next,
       wchar_t* to, wchar_t* to_limit, wchar_t*& to_next) const
{
  int limit = std::max<int>(from_end - from, to_limit - to);
  from_next = from;
  to_next = to;

  while(limit--)
    *to_next++ = static_cast<wchar_t>(*from_next++);

  return codecvt_base::ok;
} // do_out

codecvt_base::result 
codecvt<char, wchar_t, mbstate_t>:: 
do_in(mbstate_t&, const wchar_t* from, const wchar_t* from_end, const wchar_t*& from_next,
                  char* to, char* to_limit, char*& to_next)  const
{
  int limit = std::max<int>(from_end - from, to_limit - to);
  from_next = from;
  to_next = to;

  while(limit--)
    *to_next++ = static_cast<char>(*from_next++);

  return codecvt_base::ok;
} // do_in

codecvt_base::result 
codecvt<char, wchar_t, mbstate_t>:: 
do_unshift(mbstate_t&, wchar_t* to, wchar_t* to_limit, wchar_t*& to_next) const
{
  to_next = to;
  return codecvt_base::noconv;
} // do_unshift

int 
codecvt<char, wchar_t, mbstate_t>:: 
do_encoding() const throw()
{
  return 1;
} // do_encoding

bool 
codecvt<char, wchar_t, mbstate_t>:: 
do_always_noconv() const throw()
{
  return false;
} // do_always_noconv

int 
codecvt<char, wchar_t, mbstate_t>:: 
do_length(const mbstate_t&, const wchar_t* from, const wchar_t* end, size_t max) const
{
  return std::min<int>(max, (end - from));
} // do_length

int 
codecvt<char, wchar_t, mbstate_t>:: 
do_max_length() const throw()
{
  return 1;
} // do_max_length

} // namespace std

#endif
#endif
{'fraction_non_alphanumeric': 0.17391304347826086, 'fraction_numerical': 0.001041395469929706, 'mean_word_length': 3.754950495049505, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3000697', 'n_tokens_mistral': 1568, 'n_tokens_neox': 1575, 'n_words': 323}