<commit_before>#ifndef __TEST__MODELS__UTILITY_HPP__
#define __TEST__MODELS__UTILITY_HPP__

#include <stdexcept>
#include <boost/algorithm/string.hpp>

/** 
 * Gets the path separator for the OS.
 * 
 * @return '\' for Windows, '/' otherwise.
 */
char get_path_separator() {
  static char path_separator = 0;
  if (path_separator == 0) {
    FILE *in;
    if(!(in = popen("make path_separator --no-print-directory", "r")))
      throw std::runtime_error("\"make path_separator\" has failed.");
    path_separator = fgetc(in);
    pclose(in);
  }
  return path_separator;
}

/** 
 * Returns the path as a string with the appropriate
 * path separator.
 * 
 * @param model_path vector of strings representing path to the model
 * 
 * @return the string representation of the path with the appropriate
 *    path separator.
 */
std::string convert_model_path(const std::vector<std::string>& model_path) {
  std::string path;
  if (model_path.size() > 0) {
    path.append(model_path[0]);
    for (size_t i = 1; i < model_path.size(); i++) {
      path.append(1, get_path_separator());
      path.append(model_path[i]);
    }
  }
  return path;
}


/** 
 * Runs the command provided and returns the system output
 * as a string.
 * 
 * @param command A command that can be run from the shell
 * @return the system output of the command
 */  
std::string run_command(const std::string& command) {
  FILE *in;
  if(!(in = popen(command.c_str(), "r"))) {
    std::string err_msg;
    err_msg = "Could not run: \"";
    err_msg+= command;
    err_msg+= "\"";
    throw std::runtime_error(err_msg.c_str());
  }
  
  std::string output;
  char buf[1024];
  size_t count = fread(&buf, 1, 1024, in);
  while (count > 0) {
    output += std::string(&buf[0], &buf[count]);
    count = fread(&buf, 1, 1024, in);
  }
  pclose(in);

  return output;
}

/** 
 * Returns the help options from the string provided.
 * Help options start with "--".
 * 
 * @param help_output output from "model/command --help"
 * @return a vector of strings of the help options
 */
std::vector<std::string> parse_help_options(const std::string& help_output) {
  std::vector<std::string> help_options;
  
  size_t option_start = help_output.find("--");
  while (option_start != std::string::npos) {
    // find the option name (skip two characters for "--")
    option_start += 2;
    size_t option_end = help_output.find_first_of("= ", option_start);
    help_options.push_back(help_output.substr(option_start, option_end-option_start));
    option_start = help_output.find("--", option_start+1);
  }<commit_msg>test-models: updated run command<commit_after>#ifndef __TEST__MODELS__UTILITY_HPP__
#define __TEST__MODELS__UTILITY_HPP__

#include <stdexcept>
#include <boost/algorithm/string.hpp>

/** 
 * Gets the path separator for the OS.
 * 
 * @return '\' for Windows, '/' otherwise.
 */
char get_path_separator() {
  static char path_separator = 0;
  if (path_separator == 0) {
    FILE *in;
    if(!(in = popen("make path_separator --no-print-directory", "r")))
      throw std::runtime_error("\"make path_separator\" has failed.");
    path_separator = fgetc(in);
    pclose(in);
  }
  return path_separator;
}

/** 
 * Returns the path as a string with the appropriate
 * path separator.
 * 
 * @param model_path vector of strings representing path to the model
 * 
 * @return the string representation of the path with the appropriate
 *    path separator.
 */
std::string convert_model_path(const std::vector<std::string>& model_path) {
  std::string path;
  if (model_path.size() > 0) {
    path.append(model_path[0]);
    for (size_t i = 1; i < model_path.size(); i++) {
      path.append(1, get_path_separator());
      path.append(model_path[i]);
    }
  }
  return path;
}


/** 
 * Runs the command provided and returns the system output
 * as a string.
 * 
 * @param command A command that can be run from the shell
 * @return the system output of the command
 */  
std::string run_command(std::string command) {
  FILE *in;
  command += " 2>&1"; // capture stderr
  if(!(in = popen(command.c_str(), "r"))) {
    std::string err_msg;
    err_msg = "Could not run: \"";
    err_msg+= command;
    err_msg+= "\"";
    throw std::runtime_error(err_msg.c_str());
  }
  
  std::string output;
  char buf[1024];
  size_t count = fread(&buf, 1, 1024, in);
  while (count > 0) {
    output += std::string(&buf[0], &buf[count]);
    count = fread(&buf, 1, 1024, in);
  }
  
  if (pclose(in) != 0) {
    std::string err_msg;
    err_msg = "Could not run: \"";
    err_msg+= command;
    err_msg+= "\"";
    throw std::runtime_error(err_msg.c_str());
  }

  return output;
}

/** 
 * Returns the help options from the string provided.
 * Help options start with "--".
 * 
 * @param help_output output from "model/command --help"
 * @return a vector of strings of the help options
 */
std::vector<std::string> parse_help_options(const std::string& help_output) {
  std::vector<std::string> help_options;
  
  size_t option_start = help_output.find("--");
  while (option_start != std::string::npos) {
    // find the option name (skip two characters for "--")
    option_start += 2;
    size_t option_end = help_output.find_first_of("= ", option_start);
    help_options.push_back(help_output.substr(option_start, option_end-option_start));
    option_start = help_output.find("--", option_start+1);
  }
{'fraction_non_alphanumeric': 0.1458643815201192, 'fraction_numerical': 0.009500745156482861, 'mean_word_length': 3.6364421416234887, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5367171', 'n_tokens_mistral': 1859, 'n_tokens_neox': 1769, 'n_words': 483}