<commit_before>
#include <string>
#include <vector>
#include <algorithm>
#include <sstream>
#include <set>
#include <iterator>
#include <memory>

#include "parse.h"

// the EOF macro wreaks havok on our TokType enum which has an EOF member. So undefine it and the
// redefine it at the end of this file.
#define TMPEOF EOF
#undef EOF

namespace hit
{

// split breaks input into a vector treating whitespace as a delimiter.  Consecutive whitespace
// characters are treated as a single delimiter.
std::vector<std::string>
split(const std::string & input)
{
  std::istringstream buffer(input);
  std::vector<std::string> ret((std::istream_iterator<std::string>(buffer)),
                               std::istream_iterator<std::string>());
  return ret;
}

// lower converts all characters in str to their lower-case versions.
std::string
lower(const std::string & str)
{
  std::string l = str;
  std::transform(l.begin(), l.end(), l.begin(), ::tolower);
  return l;
}

// trim removes consecutive whitespace characters from the beginning and end of str.
std::string
trim(const std::string & str)
{
  size_t first = str.find_first_not_of(" \r\n\t");
  if (std::string::npos == first)
    return str;
  size_t last = str.find_last_not_of(" \r\n\t");
  return str.substr(first, (last - first + 1));
}

// toBool converts the given val to a boolean value which is stored in dst.  It returns true if
// val was successfully converted to a boolean and returns false otherwise.
bool
toBool(const std::string & val, bool * dst)
{
  std::vector<std::string> trues = {"true", "on", "yes"};
  std::vector<std::string> falses = {"false", "off", "no"};
  auto s = lower(trim(val));
  for (auto & v : trues)
  {
    if (s == v)
    {
      *dst = true;
      return true;
    }
  }
  for (auto & v : falses)
  {
    if (s == v)
    {
      *dst = false;
      return true;
    }
  }
  return false;
}

std::string
nodeTypeName(NodeType t)
{
// clang-format off
  #define nodecase(type) case NodeType::type: return #type;
  switch (t)
    {
      nodecase(Root)
      nodecase(Section)
      nodecase(Comment)
      nodecase(Field)
      default : return std::to_string((int)t);
    }
  #undef nodecase
  // clang-format on
}

Error::Error(const std::string & msg) : msg(msg) {}

const char *
Error::what() const throw()
{
  return msg.c_str();
};

ParseError::ParseError(const std::string & msg) : Error(msg) {}

std::string
strRepeat(const std::string & s, int n)
{
  std::string rs;
  for (int i = 0; i < n; i++)
    rs += s;
  return rs;
}

std::string
pathNorm(const std::string & path)
{
  std::string norm;
  size_t pos = 0;
  while (pos < path.size())
  {
    if (path[pos] == '/')
    {
      while (true)
      {
        if (path[pos] == '/')
          pos++;
        else if (path.find("./", pos) == pos)
          pos += 2;
        else
          break;
      }
      norm += "/";
    }
    else
      norm += path[pos++];
  }
  if (norm.find("./", 0) == 0)
    return norm.substr(2, norm.size() - 2);
  return norm;
}

std::string
pathJoin(const std::vector<std::string> & paths)
{
  static std::string fullpath;
  fullpath.clear();
  for (auto & p : paths)
  {
    if (p == "")
      continue;
    fullpath += "/" + p;
  }<commit_msg>remove static<commit_after>
#include <string>
#include <vector>
#include <algorithm>
#include <sstream>
#include <set>
#include <iterator>
#include <memory>

#include "parse.h"

// the EOF macro wreaks havok on our TokType enum which has an EOF member. So undefine it and the
// redefine it at the end of this file.
#define TMPEOF EOF
#undef EOF

namespace hit
{

// split breaks input into a vector treating whitespace as a delimiter.  Consecutive whitespace
// characters are treated as a single delimiter.
std::vector<std::string>
split(const std::string & input)
{
  std::istringstream buffer(input);
  std::vector<std::string> ret((std::istream_iterator<std::string>(buffer)),
                               std::istream_iterator<std::string>());
  return ret;
}

// lower converts all characters in str to their lower-case versions.
std::string
lower(const std::string & str)
{
  std::string l = str;
  std::transform(l.begin(), l.end(), l.begin(), ::tolower);
  return l;
}

// trim removes consecutive whitespace characters from the beginning and end of str.
std::string
trim(const std::string & str)
{
  size_t first = str.find_first_not_of(" \r\n\t");
  if (std::string::npos == first)
    return str;
  size_t last = str.find_last_not_of(" \r\n\t");
  return str.substr(first, (last - first + 1));
}

// toBool converts the given val to a boolean value which is stored in dst.  It returns true if
// val was successfully converted to a boolean and returns false otherwise.
bool
toBool(const std::string & val, bool * dst)
{
  std::vector<std::string> trues = {"true", "on", "yes"};
  std::vector<std::string> falses = {"false", "off", "no"};
  auto s = lower(trim(val));
  for (auto & v : trues)
  {
    if (s == v)
    {
      *dst = true;
      return true;
    }
  }
  for (auto & v : falses)
  {
    if (s == v)
    {
      *dst = false;
      return true;
    }
  }
  return false;
}

std::string
nodeTypeName(NodeType t)
{
// clang-format off
  #define nodecase(type) case NodeType::type: return #type;
  switch (t)
    {
      nodecase(Root)
      nodecase(Section)
      nodecase(Comment)
      nodecase(Field)
      default : return std::to_string((int)t);
    }
  #undef nodecase
  // clang-format on
}

Error::Error(const std::string & msg) : msg(msg) {}

const char *
Error::what() const throw()
{
  return msg.c_str();
};

ParseError::ParseError(const std::string & msg) : Error(msg) {}

std::string
strRepeat(const std::string & s, int n)
{
  std::string rs;
  for (int i = 0; i < n; i++)
    rs += s;
  return rs;
}

std::string
pathNorm(const std::string & path)
{
  std::string norm;
  size_t pos = 0;
  while (pos < path.size())
  {
    if (path[pos] == '/')
    {
      while (true)
      {
        if (path[pos] == '/')
          pos++;
        else if (path.find("./", pos) == pos)
          pos += 2;
        else
          break;
      }
      norm += "/";
    }
    else
      norm += path[pos++];
  }
  if (norm.find("./", 0) == 0)
    return norm.substr(2, norm.size() - 2);
  return norm;
}

std::string
pathJoin(const std::vector<std::string> & paths)
{
  std::string fullpath;
  for (auto & p : paths)
  {
    if (p == "")
      continue;
    fullpath += "/" + p;
  }
{'fraction_non_alphanumeric': 0.15773948144698027, 'fraction_numerical': 0.0024840863219996894, 'mean_word_length': 2.8924471299093657, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4015135', 'n_tokens_mistral': 2256, 'n_tokens_neox': 2224, 'n_words': 655}