<commit_before>////////////////////////////////////////////////////////////////////////
// iconvstreamcpp
//

#include "iconvstream.h"
#include <string.h>

iconvstreambuf* iconvstreambuf::open(std::ostream& sink_,
  const char* tocode, const char* fromcode)
{
  sink = &sink_;
  cd = iconv_open(tocode, fromcode);

  if (cd == (iconv_t)-1)
    return 0;

  setp(buffer, buffer + (sizeof(buffer) - 1));
  return this;
}

iconvstreambuf* iconvstreambuf::close() throw()
{
  if (cd != (iconv_t)-1)
  {
    int r = iconv_close(cd);
    if (r == 0)
    {
      cd = 0;
      return this;
    }
    else
      return 0;
  }
  else
    return this;
}

iconvstreambuf::int_type iconvstreambuf::overflow(int_type c)
{
  if (pptr() == pbase())
  {
    // 1. Aufruf: initialisiere Puffer
    setp(buffer, buffer + (sizeof(buffer) - 1));
    if (c != traits_type::eof())
    {
      *pptr() = (char_type)c;
      pbump(1);
    }
    return 0;
  }
  else
  {
    // Ausgabepuffer voll - konvertiere Zeichen
    char outbuf[32];

    size_t inbytesleft = pptr() - buffer;
    if (c != traits_type::eof())
    {
      *pptr() = (char_type)c;
      ++inbytesleft;
    }
    size_t outbytesleft = sizeof(outbuf);

    char* inbufptr = buffer;
    char* outbufptr = outbuf;

    // Konvertiere so viele Zeichen wie mglich
    iconv(cd,
          &inbufptr, &inbytesleft,
          &outbufptr, &outbytesleft);

    if (errno != 0 && errno != EINVAL && errno != E2BIG)
    {
      // Konvertierung fehlgeschlagen
      return traits_type::eof();
    }
    errno = 0;

    // Konvertierung erfolgreich

    // Schreibe outbuf in sink
    sink->write(outbuf, outbufptr - outbuf);

    if (sink->fail())
      return traits_type::eof();

    // verschiebe brig gebliebene Zeichen an den Anfang des Puffers
    // und setze Puffer neu.
    if (inbytesleft > 0)
      memmove(buffer, inbufptr, inbytesleft);

    setp(buffer + inbytesleft, buffer + sizeof(buffer) - 1);
    return 0;
  }
}

iconvstream::int_type iconvstreambuf::underflow()
{
  return traits_type::eof();
}

int iconvstreambuf::sync()
{
  return overflow(traits_type::eof());
}

void iconvstream::open(std::ostream& sink_,
  const char* tocode, const char* fromcode)
{
  if (!fail() && streambuf.open(sink_, tocode, fromcode) == 0)
    setstate(std::ios::failbit);
}<commit_msg>iconvstreambuf::sync ruft jetzt flush auf das Ziel auf<commit_after>////////////////////////////////////////////////////////////////////////
// iconvstreamcpp
//

#include "iconvstream.h"
#include <string.h>

iconvstreambuf* iconvstreambuf::open(std::ostream& sink_,
  const char* tocode, const char* fromcode)
{
  sink = &sink_;
  cd = iconv_open(tocode, fromcode);

  if (cd == (iconv_t)-1)
    return 0;

  setp(buffer, buffer + (sizeof(buffer) - 1));
  return this;
}

iconvstreambuf* iconvstreambuf::close() throw()
{
  if (cd != (iconv_t)-1)
  {
    int r = iconv_close(cd);
    if (r == 0)
    {
      cd = 0;
      return this;
    }
    else
      return 0;
  }
  else
    return this;
}

iconvstreambuf::int_type iconvstreambuf::overflow(int_type c)
{
  if (sink == 0)
    return traits_type::eof();
  else if (pptr() == pbase())
  {
    // 1. Aufruf: initialisiere Puffer
    setp(buffer, buffer + (sizeof(buffer) - 1));
    if (c != traits_type::eof())
    {
      *pptr() = (char_type)c;
      pbump(1);
    }
    return 0;
  }
  else
  {
    // Ausgabepuffer voll - konvertiere Zeichen
    char outbuf[32];

    size_t inbytesleft = pptr() - buffer;
    if (c != traits_type::eof())
    {
      *pptr() = (char_type)c;
      ++inbytesleft;
    }
    size_t outbytesleft = sizeof(outbuf);

    char* inbufptr = buffer;
    char* outbufptr = outbuf;

    // Konvertiere so viele Zeichen wie mglich
    iconv(cd,
          &inbufptr, &inbytesleft,
          &outbufptr, &outbytesleft);

    if (errno != 0 && errno != EINVAL && errno != E2BIG)
    {
      // Konvertierung fehlgeschlagen
      return traits_type::eof();
    }
    errno = 0;

    // Konvertierung erfolgreich

    // Schreibe outbuf in sink
    sink->write(outbuf, outbufptr - outbuf);

    if (sink->fail())
      return traits_type::eof();

    // verschiebe brig gebliebene Zeichen an den Anfang des Puffers
    // und setze Puffer neu.
    if (inbytesleft > 0)
      memmove(buffer, inbufptr, inbytesleft);

    setp(buffer + inbytesleft, buffer + sizeof(buffer) - 1);
    return 0;
  }
}

iconvstream::int_type iconvstreambuf::underflow()
{
  return traits_type::eof();
}

int iconvstreambuf::sync()
{
  if (sink)
  {
    int_type ret = overflow(traits_type::eof());
    sink->flush();
    return ret == traits_type::eof() || sink->fail() ? -1 : 0;
  }
  else
    return -1;
}

void iconvstream::open(std::ostream& sink_,
  const char* tocode, const char* fromcode)
{
  if (!fail() && streambuf.open(sink_, tocode, fromcode) == 0)
    setstate(std::ios::failbit);
}
{'fraction_non_alphanumeric': 0.17152834839769926, 'fraction_numerical': 0.009038619556285949, 'mean_word_length': 2.5053995680345573, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7560970', 'n_tokens_mistral': 1811, 'n_tokens_neox': 1783, 'n_words': 402}