<commit_before>#ifndef __STOUT_STRINGS_HPP__
#define __STOUT_STRINGS_HPP__

#include <algorithm>
#include <string>
#include <map>
#include <vector>

#include "foreach.hpp"
#include "format.hpp"
#include "stringify.hpp"

namespace strings {

// Flags indicating how remove should operate.
enum Mode {
  PREFIX,
  SUFFIX,
  ANY
};


inline std::string remove(
    const std::string& from,
    const std::string& substring,
    Mode mode = ANY)
{
  std::string result = from;

  if (mode == PREFIX) {
    if (from.find(substring) == 0) {
      result = from.substr(substring.size());
    }
  } else if (mode == SUFFIX) {
    if (from.rfind(substring) == from.size() - substring.size()) {
      result = from.substr(0, from.size() - substring.size());
    }
  } else {
    size_t index;
    while ((index = result.find(substring)) != std::string::npos) {
      result = result.erase(index, substring.size());
    }
  }

  return result;
}


inline std::string trim(
    const std::string& from,
    const std::string& chars = " \t\n\r")
{
  size_t start = from.find_first_not_of(chars);
  size_t end = from.find_last_not_of(chars);
  if (start == std::string::npos) { // Contains only characters in chars.
    return "";
  }

  return from.substr(start, end + 1 - start);
}


// Replaces all the occurrences of the 'from' string with the 'to' string.
inline std::string replace(
    const std::string& s,
    const std::string& from,
    const std::string& to)
{
  std::string result = s;
  size_t index = 0;

  if (from.empty()) {
    return result;
  }

  while ((index = result.find(from, index)) != std::string::npos) {
    result.replace(index, from.length(), to);
    index += to.length();
  }
  return result;
}


// Tokenizes the string using the delimiters.
// Empty tokens will not be included in the result.
inline std::vector<std::string> tokenize(
    const std::string& s,
    const std::string& delims)
{
  size_t offset = 0;
  std::vector<std::string> tokens;

  while (true) {
    size_t i = s.find_first_not_of(delims, offset);
    if (std::string::npos == i) {
      offset = s.length();
      return tokens;
    }

    size_t j = s.find_first_of(delims, i);
    if (std::string::npos == j) {
      tokens.push_back(s.substr(i));
      offset = s.length();
      continue;
    }

    tokens.push_back(s.substr(i, j - i));
    offset = j;
  }
  return tokens;
}


// Splits the string using the provided delimiters.
// Empty tokens are allowed in the result.
inline std::vector<std::string> split(
    const std::string& s,
    const std::string& delims)
{
  std::vector<std::string> tokens;
  size_t offset = 0;
  size_t next = 0;

  while (true) {
    next = s.find_first_of(delims, offset);
    if (next == std::string::npos) {
      tokens.push_back(s.substr(offset));<commit_msg>Cleaned up strings::tokenize.<commit_after>#ifndef __STOUT_STRINGS_HPP__
#define __STOUT_STRINGS_HPP__

#include <algorithm>
#include <string>
#include <map>
#include <vector>

#include "foreach.hpp"
#include "format.hpp"
#include "stringify.hpp"

namespace strings {

// Flags indicating how remove should operate.
enum Mode {
  PREFIX,
  SUFFIX,
  ANY
};


inline std::string remove(
    const std::string& from,
    const std::string& substring,
    Mode mode = ANY)
{
  std::string result = from;

  if (mode == PREFIX) {
    if (from.find(substring) == 0) {
      result = from.substr(substring.size());
    }
  } else if (mode == SUFFIX) {
    if (from.rfind(substring) == from.size() - substring.size()) {
      result = from.substr(0, from.size() - substring.size());
    }
  } else {
    size_t index;
    while ((index = result.find(substring)) != std::string::npos) {
      result = result.erase(index, substring.size());
    }
  }

  return result;
}


inline std::string trim(
    const std::string& from,
    const std::string& chars = " \t\n\r")
{
  size_t start = from.find_first_not_of(chars);
  size_t end = from.find_last_not_of(chars);
  if (start == std::string::npos) { // Contains only characters in chars.
    return "";
  }

  return from.substr(start, end + 1 - start);
}


// Replaces all the occurrences of the 'from' string with the 'to' string.
inline std::string replace(
    const std::string& s,
    const std::string& from,
    const std::string& to)
{
  std::string result = s;
  size_t index = 0;

  if (from.empty()) {
    return result;
  }

  while ((index = result.find(from, index)) != std::string::npos) {
    result.replace(index, from.length(), to);
    index += to.length();
  }
  return result;
}


// Tokenizes the string using the delimiters.
// Empty tokens will not be included in the result.
inline std::vector<std::string> tokenize(
    const std::string& s,
    const std::string& delims)
{
  size_t offset = 0;
  std::vector<std::string> tokens;

  while (true) {
    size_t i = s.find_first_not_of(delims, offset);
    if (std::string::npos == i) {
      break;
    }

    size_t j = s.find_first_of(delims, i);
    if (std::string::npos == j) {
      tokens.push_back(s.substr(i));
      offset = s.length();
      continue;
    }

    tokens.push_back(s.substr(i, j - i));
    offset = j;
  }
  return tokens;
}


// Splits the string using the provided delimiters.
// Empty tokens are allowed in the result.
inline std::vector<std::string> split(
    const std::string& s,
    const std::string& delims)
{
  std::vector<std::string> tokens;
  size_t offset = 0;
  size_t next = 0;

  while (true) {
    next = s.find_first_of(delims, offset);
    if (next == std::string::npos) {
      tokens.push_back(s.substr(offset));
{'fraction_non_alphanumeric': 0.15169445938676707, 'fraction_numerical': 0.002510310202617895, 'mean_word_length': 3.197140707298721, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1691347', 'n_tokens_mistral': 1988, 'n_tokens_neox': 1923, 'n_words': 506}