<commit_before>#include <cstdlib>

#include "buffer.hh"

namespace mimosa
{
  namespace stream
  {
    Buffer::Buffer(uint64_t size)
      : size_(size),
        data_(reinterpret_cast<char *>(::malloc(size + 4)))
    {
      data_[size_ - 3] = 0;
      data_[size_ - 2] = 0;
      data_[size_ - 1] = 0;
      data_[size_]     = 0;
    }

    Buffer::~Buffer()
    {
      free(data_);
      size_ = 0;
      data_ = nullptr;
    }

    void
    Buffer::resize(uint64_t size)
    {
      data_            = reinterpret_cast<char *>(::realloc(data_, size + 4));
      size_            = size;
      data_[size_ - 3] = 0;
      data_[size_ - 2] = 0;
      data_[size_ - 1] = 0;
      data_[size_]     = 0;
    }
  }
}
<commit_msg>Buffer has now 4 trailling null bytes which make more sense in case of utf-16 or utf-32 buffer.<commit_after>#include <cstdlib>

#include "buffer.hh"

namespace mimosa
{
  namespace stream
  {
    Buffer::Buffer(uint64_t size)
      : size_(size),
        data_(reinterpret_cast<char *>(::malloc(size + 4)))
    {
      data_[size_ + 3] = 0;
      data_[size_ + 2] = 0;
      data_[size_ + 1] = 0;
      data_[size_]     = 0;
    }

    Buffer::~Buffer()
    {
      free(data_);
      size_ = 0;
      data_ = nullptr;
    }

    void
    Buffer::resize(uint64_t size)
    {
      data_            = reinterpret_cast<char *>(::realloc(data_, size + 4));
      size_            = size;
      data_[size_ + 3] = 0;
      data_[size_ + 2] = 0;
      data_[size_ + 1] = 0;
      data_[size_]     = 0;
    }
  }
}

{'fraction_non_alphanumeric': 0.14369310793237972, 'fraction_numerical': 0.0305591677503251, 'mean_word_length': 1.8552875695732838, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '626061', 'n_tokens_mistral': 602, 'n_tokens_neox': 556, 'n_words': 121}