<commit_before>#pragma once

#include "3rd_party/threadpool.h"
#include "common/config.h"
#include "common/timer.h"
#include "common/utils.h"
#include "common/regex.h"
#include "common/utils.h"
#include "data/batch_generator.h"
#include "data/corpus.h"
#include "graph/expression_graph.h"
#include "training/training_state.h"
#include "translator/beam_search.h"
#include "translator/history.h"
#include "translator/output_collector.h"
#include "translator/output_printer.h"
#include "translator/scorers.h"

#include <cstdio>
#include <cstdlib>
#include <limits>

namespace marian {

/**
 * @brief Base class for validators
 */
class ValidatorBase : public TrainingObserver {
protected:
  bool lowerIsBetter_{true};
  float lastBest_;
  size_t stalled_{0};
  std::mutex mutex_;
  ThreadPool threadPool_;

public:
  ValidatorBase(bool lowerIsBetter) : lowerIsBetter_(lowerIsBetter), lastBest_{initScore()} {}

  virtual float validate(const std::vector<Ptr<ExpressionGraph>>& graphs) = 0;
  virtual std::string type() = 0;

  float lastBest() { return lastBest_; }
  size_t stalled() { return stalled_; }

  virtual float initScore() {
    return lowerIsBetter_ ? std::numeric_limits<float>::max()
                          : std::numeric_limits<float>::lowest();
  }

  virtual void actAfterLoaded(TrainingState& state) override {
    if(state.validators[type()]) {
      lastBest_ = state.validators[type()]["last-best"].as<float>();
      stalled_ = state.validators[type()]["stalled"].as<size_t>();
    }
  }
};

template <class DataSet>
class Validator : public ValidatorBase {
public:
  Validator(std::vector<Ptr<Vocab>> vocabs, Ptr<Options> options, bool lowerIsBetter = true)
      : ValidatorBase(lowerIsBetter),
        vocabs_(vocabs),
        // options_ is a clone of global options, so it can be safely modified within the class
        options_(New<Options>(options->clone())) {
    // set options common for all validators
    options_->set("inference", true);
    if(options_->has("valid-max-length"))
      options_->set("max-length", options_->get<size_t>("valid-max-length"));
    if(options_->has("valid-mini-batch"))
      options_->set("mini-batch", options_->get<size_t>("valid-mini-batch"));
    options_->set("mini-batch-sort", "src");
    options_->set("maxi-batch", 10);
  }

protected:
  void createBatchGenerator(bool isTranslating) {
    // Create the BatchGenerator. Note that ScriptValidator does not use batchGenerator_.

    // Update validation options
    auto opts = New<Options>();
    opts->merge(options_);
    opts->set("inference", true);

    if (isTranslating) { // TranslationValidator and BleuValidator
      opts->set("max-length", 1000);
      opts->set("mini-batch", options_->get<int>("valid-mini-batch"));
      opts->set("maxi-batch", 10);
    }
    else { // CrossEntropyValidator
      opts->set("max-length", options_->get<size_t>("valid-max-length"));
      if(options_->has("valid-mini-batch"))
        opts->set("mini-batch", options_->get<size_t>("valid-mini-batch"));
      opts->set("mini-batch-sort", "src");
    }

    // Create corpus
    auto validPaths = options_->get<std::vector<std::string>>("valid-sets");
    auto corpus = New<DataSet>(validPaths, vocabs_, options_);

    // Create batch generator
    batchGenerator_ = New<data::BatchGenerator<DataSet>>(corpus, opts);
  }
public:

  virtual float validate(const std::vector<Ptr<ExpressionGraph>>& graphs) override {

    for(auto graph : graphs)
      graph->setInference(true);

    batchGenerator_->prepare(false);

    // Validate on batches
    float val = validateBG(graphs);
    updateStalled(graphs, val);

    for(auto graph : graphs)
      graph->setInference(false);

    return val;
  };

protected:
  std::vector<Ptr<Vocab>> vocabs_;
  Ptr<Options> options_;
  Ptr<models::ModelBase> builder_;
  Ptr<data::BatchGenerator<DataSet>> batchGenerator_;

  virtual float validateBG(const std::vector<Ptr<ExpressionGraph>>&)
      = 0;

  void updateStalled(const std::vector<Ptr<ExpressionGraph>>& graphs,
                     float val) {
    if((lowerIsBetter_ && lastBest_ > val)
       || (!lowerIsBetter_ && lastBest_ < val)) {
      stalled_ = 0;
      lastBest_ = val;
      if(options_->get<bool>("keep-best"))
        keepBest(graphs);
    } else if (lastBest_ != val) { // (special case 0 at start)  @TODO: needed? Seems stall count gets reset each time it does improve. If not needed, remove "if(...)" again.
      stalled_++;
    }
  }
<commit_msg>restore old validation behaviour<commit_after>#pragma once

#include "3rd_party/threadpool.h"
#include "common/config.h"
#include "common/timer.h"
#include "common/utils.h"
#include "common/regex.h"
#include "common/utils.h"
#include "data/batch_generator.h"
#include "data/corpus.h"
#include "graph/expression_graph.h"
#include "training/training_state.h"
#include "translator/beam_search.h"
#include "translator/history.h"
#include "translator/output_collector.h"
#include "translator/output_printer.h"
#include "translator/scorers.h"

#include <cstdio>
#include <cstdlib>
#include <limits>

namespace marian {

/**
 * @brief Base class for validators
 */
class ValidatorBase : public TrainingObserver {
protected:
  bool lowerIsBetter_{true};
  float lastBest_;
  size_t stalled_{0};
  std::mutex mutex_;
  ThreadPool threadPool_;

public:
  ValidatorBase(bool lowerIsBetter) : lowerIsBetter_(lowerIsBetter), lastBest_{initScore()} {}

  virtual float validate(const std::vector<Ptr<ExpressionGraph>>& graphs) = 0;
  virtual std::string type() = 0;

  float lastBest() { return lastBest_; }
  size_t stalled() { return stalled_; }

  virtual float initScore() {
    return lowerIsBetter_ ? std::numeric_limits<float>::max()
                          : std::numeric_limits<float>::lowest();
  }

  virtual void actAfterLoaded(TrainingState& state) override {
    if(state.validators[type()]) {
      lastBest_ = state.validators[type()]["last-best"].as<float>();
      stalled_ = state.validators[type()]["stalled"].as<size_t>();
    }
  }
};

template <class DataSet>
class Validator : public ValidatorBase {
public:
  Validator(std::vector<Ptr<Vocab>> vocabs, Ptr<Options> options, bool lowerIsBetter = true)
      : ValidatorBase(lowerIsBetter),
        vocabs_(vocabs),
        // options_ is a clone of global options, so it can be safely modified within the class
        options_(New<Options>(options->clone())) {
    // set options common for all validators
    options_->set("inference", true);
    if(options_->has("valid-max-length"))
      options_->set("max-length", options_->get<size_t>("valid-max-length"));
    if(options_->has("valid-mini-batch"))
      options_->set("mini-batch", options_->get<size_t>("valid-mini-batch"));
    options_->set("mini-batch-sort", "src");
    options_->set("maxi-batch", 10);
  }

protected:
  void createBatchGenerator(bool isTranslating) {
    // Create the BatchGenerator. Note that ScriptValidator does not use batchGenerator_.

    // Update validation options
    auto opts = New<Options>();
    opts->merge(options_);
    opts->set("inference", true);

    if (isTranslating) { // TranslationValidator and BleuValidator
      opts->set("max-length", 1000);
      opts->set("mini-batch", options_->get<int>("valid-mini-batch"));
      opts->set("maxi-batch", 10);
    }
    else { // CrossEntropyValidator
      opts->set("max-length", options_->get<size_t>("valid-max-length"));
      if(options_->has("valid-mini-batch"))
        opts->set("mini-batch", options_->get<size_t>("valid-mini-batch"));
      opts->set("mini-batch-sort", "src");
    }

    // Create corpus
    auto validPaths = options_->get<std::vector<std::string>>("valid-sets");
    auto corpus = New<DataSet>(validPaths, vocabs_, options_);

    // Create batch generator
    batchGenerator_ = New<data::BatchGenerator<DataSet>>(corpus, opts);
  }
public:

  virtual float validate(const std::vector<Ptr<ExpressionGraph>>& graphs) override {

    for(auto graph : graphs)
      graph->setInference(true);

    batchGenerator_->prepare(false);

    // Validate on batches
    float val = validateBG(graphs);
    updateStalled(graphs, val);

    for(auto graph : graphs)
      graph->setInference(false);

    return val;
  };

protected:
  std::vector<Ptr<Vocab>> vocabs_;
  Ptr<Options> options_;
  Ptr<models::ModelBase> builder_;
  Ptr<data::BatchGenerator<DataSet>> batchGenerator_;

  virtual float validateBG(const std::vector<Ptr<ExpressionGraph>>&)
      = 0;

  void updateStalled(const std::vector<Ptr<ExpressionGraph>>& graphs,
                     float val) {
    if((lowerIsBetter_ && lastBest_ > val)
       || (!lowerIsBetter_ && lastBest_ < val)) {
      stalled_ = 0;
      lastBest_ = val;
      if(options_->get<bool>("keep-best"))
        keepBest(graphs);
    } else /* if (lastBest_ != val) */ { // (special case 0 at start)  @TODO: needed? Seems stall count gets reset each time it does improve. If not needed, remove "if(...)" again.
      stalled_++;
    }
  }

{'fraction_non_alphanumeric': 0.1515050538709319, 'fraction_numerical': 0.003332222592469177, 'mean_word_length': 3.9526952695269526, 'pattern_counts': {'":': 0, '<': 81, '<?xml version=': 0, '>': 135, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5596984', 'n_tokens_mistral': 2884, 'n_tokens_neox': 2767, 'n_words': 597}