<commit_before>#pragma once

#include <boost/thread/mutex.hpp>
#include <boost/unordered_map.hpp>
#include <iostream>
#include <map>

#include "common/definitions.h"
#include "common/file_stream.h"
#include "common/logging.h"
#include "data/alignment.h"

namespace marian {

class ScoreCollector {
public:
  ScoreCollector() : nextId_(0), outStrm_(new OutputFileStream(std::cout)){};

  virtual void Write(long id, const std::string& message) {
    boost::mutex::scoped_lock lock(mutex_);
    if(id == nextId_) {
      ((std::ostream&)*outStrm_) << message << std::endl;

      ++nextId_;

      typename Outputs::const_iterator iter, iterNext;
      iter = outputs_.begin();
      while(iter != outputs_.end()) {
        long currId = iter->first;

        if(currId == nextId_) {
          // 1st element in the map is the next
          ((std::ostream&)*outStrm_) << iter->second << std::endl;

          ++nextId_;

          // delete current record, move iter on 1
          iterNext = iter;
          ++iterNext;
          outputs_.erase(iter);
          iter = iterNext;
        } else {
          // not the next. stop iterating
          assert(nextId_ < currId);
          break;
        }
      }

    } else {
      // save for later
      outputs_[id] = message;
    }
  }

  virtual void Write(long id, float value) { Write(id, std::to_string(value)); }

  virtual void Write(long id,
                     float value,
                     const std::vector<data::SoftAlignment>& softAlign) {
    auto align = data::ConvertSoftAlignToHardAlign(softAlign, 1.f, false);

    std::stringstream str;
    str << value << " |||";
    for(auto p = align.begin(); p != align.end() - 1; ++p) {
      str << " " << p->first << "-" << p->second;
    }
    Write(id, str.str());
  }

protected:
  long nextId_{0};
  UPtr<OutputFileStream> outStrm_;
  boost::mutex mutex_;<commit_msg>Fix scorer output without alignment<commit_after>#pragma once

#include <boost/thread/mutex.hpp>
#include <boost/unordered_map.hpp>
#include <iostream>
#include <map>

#include "common/definitions.h"
#include "common/file_stream.h"
#include "common/logging.h"
#include "data/alignment.h"

namespace marian {

class ScoreCollector {
public:
  ScoreCollector() : nextId_(0), outStrm_(new OutputFileStream(std::cout)){};

  virtual void Write(long id, const std::string& message) {
    boost::mutex::scoped_lock lock(mutex_);
    if(id == nextId_) {
      ((std::ostream&)*outStrm_) << message << std::endl;

      ++nextId_;

      typename Outputs::const_iterator iter, iterNext;
      iter = outputs_.begin();
      while(iter != outputs_.end()) {
        long currId = iter->first;

        if(currId == nextId_) {
          // 1st element in the map is the next
          ((std::ostream&)*outStrm_) << iter->second << std::endl;

          ++nextId_;

          // delete current record, move iter on 1
          iterNext = iter;
          ++iterNext;
          outputs_.erase(iter);
          iter = iterNext;
        } else {
          // not the next. stop iterating
          assert(nextId_ < currId);
          break;
        }
      }

    } else {
      // save for later
      outputs_[id] = message;
    }
  }

  virtual void Write(long id, float value) { Write(id, std::to_string(value)); }

  virtual void Write(long id,
                     float value,
                     const std::vector<data::SoftAlignment>& softAlign) {
    if(softAlign.empty()) {
      Write(id, value);
      return;
    }

    auto align = data::ConvertSoftAlignToHardAlign(softAlign, 1.f, false);

    std::stringstream str;
    str << value << " |||";
    for(auto p = align.begin(); p != align.end() - 1; ++p) {
      str << " " << p->first << "-" << p->second;
    }
    Write(id, str.str());
  }

protected:
  long nextId_{0};
  UPtr<OutputFileStream> outStrm_;
  boost::mutex mutex_;
{'fraction_non_alphanumeric': 0.14843547970002585, 'fraction_numerical': 0.0031031807602792862, 'mean_word_length': 2.4878268710550047, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5875929', 'n_tokens_mistral': 1236, 'n_tokens_neox': 1204, 'n_words': 287}