<commit_before>#include "unpack_mwm.hpp"
#include "../coding/file_container.hpp"
#include "../coding/file_writer.hpp"
#include "../base/logging.hpp"
#include "../base/stl_add.hpp"
#include "../std/algorithm.hpp"
#include "../std/vector.hpp"

void UnpackMwm(string const & filePath)
{
  LOG(LINFO, ("Unpacking mwm sections..."));
  FilesContainerR container(filePath);
  vector<string> tags;
  container.ForEachTag(MakeBackInsertFunctor<vector<string> >(tags));
  for (size_t i = 0; i < tags.size(); ++i)
  {
    LOG(LINFO, ("Unpacking", tags[i]));
    FilesContainerR::ReaderT reader = container.GetReader(tags[i]);
    FileWriter writer(filePath + "." + tags[i]);

    uint64_t const size = reader.Size();
    uint64_t pos = 0;
    while (pos < size)
    {
      vector<char> buffer(static_cast<size_t>(min(size - pos, 1024 * 1024 * 1ULL)));<commit_msg>[linux] Fixed build error<commit_after>#include "unpack_mwm.hpp"
#include "../coding/file_container.hpp"
#include "../coding/file_writer.hpp"
#include "../base/logging.hpp"
#include "../base/stl_add.hpp"
#include "../std/algorithm.hpp"
#include "../std/vector.hpp"

void UnpackMwm(string const & filePath)
{
  LOG(LINFO, ("Unpacking mwm sections..."));
  FilesContainerR container(filePath);
  vector<string> tags;
  container.ForEachTag(MakeBackInsertFunctor<vector<string> >(tags));
  for (size_t i = 0; i < tags.size(); ++i)
  {
    LOG(LINFO, ("Unpacking", tags[i]));
    FilesContainerR::ReaderT reader = container.GetReader(tags[i]);
    FileWriter writer(filePath + "." + tags[i]);

    uint64_t const size = reader.Size();
    uint64_t pos = 0;
    while (pos < size)
    {
      vector<char> buffer(min(static_cast<size_t>(size - pos), static_cast<size_t>(1024 * 1024)));
{'fraction_non_alphanumeric': 0.18050749711649366, 'fraction_numerical': 0.016724336793540944, 'mean_word_length': 4.289634146341464, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6403775', 'n_tokens_mistral': 632, 'n_tokens_neox': 597, 'n_words': 121}