<commit_before>#include "common/config_parser.h"

#include "common/definitions.h"
#include "common/cli_helper.h"
#include "common/config_validator.h"
#include "common/file_stream.h"
#include "common/logging.h"
#include "common/options.h"
#include "common/config.h"
#include "common/utils.h"
#include "common/regex.h"
#include <algorithm>
#include <set>
#include <stdexcept>
#include <string>

#if MKL_FOUND
#include <mkl.h>
#else
#if BLAS_FOUND
#include <cblas.h>
#endif
#endif

namespace marian {

// TODO: Move this to CLIWrapper and allow to mark options as paths in the same place they are
// defined
// clang-format off
const std::set<std::string> PATHS = {
  "model",
  "models",
  "train-sets",
  "vocabs",
  "embedding-vectors",
  "valid-sets",
  "valid-script-path",
  "valid-log",
  "valid-translation-output",
  "input",            // except: stdin
  "output",           // except: stdout
  "pretrained-model",
  "data-weighting",
  "log"
  // TODO: Handle the special value in helper functions
  //"sqlite",         // except: temporary
  // TODO: This is a vector with a path and some numbers, handle this in helper
  // functions or separate shortlist path to a separate command-line option
  //"shortlist",
};
// clang-format on

std::string escapeCmdLine(int argc, char** argv){
  std::string cmdLine;
  for(int i = 0; i < argc; i++) {
    std::string arg = argv[i];
    std::string quote; // attempt to quote special chars
    if(arg.empty() || arg.find_first_of(" #`\"'\\${}|&^?*!()%><") != std::string::npos)
      quote = "'";
    arg = regex::regex_replace(arg, regex::regex("'"), "'\\''");
    if(!cmdLine.empty())
      cmdLine.push_back(' ');
    cmdLine += quote + arg + quote;
  }
  return cmdLine;
}

std::string const& ConfigParser::cmdLine() const {
  return cmdLine_;
}

ConfigParser::ConfigParser(cli::mode mode)
  : cli_(config_,"Marian: Fast Neural Machine Translation in C++",
         "General options", "", 40),
    // modeServer_(mode == cli::mode::server),
    mode_(mode == cli::mode::server ? cli::mode::translation : mode) {

  addOptionsGeneral(cli_);
  if (mode == cli::mode::server)
    addOptionsServer(cli_);
  addOptionsModel(cli_);

  // clang-format off
  switch(mode_) {
    case cli::mode::training:
      addOptionsTraining(cli_);
      addOptionsValidation(cli_);
      break;<commit_msg>Call addAliases() when initializing ConfigParser.<commit_after>#include "common/config_parser.h"

#include "common/definitions.h"
#include "common/cli_helper.h"
#include "common/config_validator.h"
#include "common/file_stream.h"
#include "common/logging.h"
#include "common/options.h"
#include "common/config.h"
#include "common/utils.h"
#include "common/regex.h"
#include <algorithm>
#include <set>
#include <stdexcept>
#include <string>

#if MKL_FOUND
#include <mkl.h>
#else
#if BLAS_FOUND
#include <cblas.h>
#endif
#endif

namespace marian {

// TODO: Move this to CLIWrapper and allow to mark options as paths in the same place they are
// defined
// clang-format off
const std::set<std::string> PATHS = {
  "model",
  "models",
  "train-sets",
  "vocabs",
  "embedding-vectors",
  "valid-sets",
  "valid-script-path",
  "valid-log",
  "valid-translation-output",
  "input",            // except: stdin
  "output",           // except: stdout
  "pretrained-model",
  "data-weighting",
  "log"
  // TODO: Handle the special value in helper functions
  //"sqlite",         // except: temporary
  // TODO: This is a vector with a path and some numbers, handle this in helper
  // functions or separate shortlist path to a separate command-line option
  //"shortlist",
};
// clang-format on

std::string escapeCmdLine(int argc, char** argv){
  std::string cmdLine;
  for(int i = 0; i < argc; i++) {
    std::string arg = argv[i];
    std::string quote; // attempt to quote special chars
    if(arg.empty() || arg.find_first_of(" #`\"'\\${}|&^?*!()%><") != std::string::npos)
      quote = "'";
    arg = regex::regex_replace(arg, regex::regex("'"), "'\\''");
    if(!cmdLine.empty())
      cmdLine.push_back(' ');
    cmdLine += quote + arg + quote;
  }
  return cmdLine;
}

std::string const& ConfigParser::cmdLine() const {
  return cmdLine_;
}

ConfigParser::ConfigParser(cli::mode mode)
  : cli_(config_,"Marian: Fast Neural Machine Translation in C++",
         "General options", "", 40),
    // modeServer_(mode == cli::mode::server),
    mode_(mode == cli::mode::server ? cli::mode::translation : mode) {

  addOptionsGeneral(cli_);
  if (mode == cli::mode::server)
    addOptionsServer(cli_);
  addOptionsModel(cli_);

  // clang-format off
  switch(mode_) {
    case cli::mode::training:
      addOptionsTraining(cli_);
      addOptionsValidation(cli_);
      addAliases(cli_);
      break;
{'fraction_non_alphanumeric': 0.16617460986925348, 'fraction_numerical': 0.0012652889076339097, 'mean_word_length': 3.7382617382617385, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3839590', 'n_tokens_mistral': 1612, 'n_tokens_neox': 1586, 'n_words': 400}