<commit_before>//
// Copyleft RIME Developers
// License: GPLv3
//
// 2013-01-30 <NAME> <<EMAIL>>
//
#include <fstream>
#include <boost/algorithm/string.hpp>
#include <rime/algo/utilities.h>

namespace rime {

int CompareVersionString(const std::string& x, const std::string& y) {
  if (x.empty() && y.empty()) return 0;
  if (x.empty()) return -1;
  if (y.empty()) return 1;
  std::vector<std::string> xx, yy;
  boost::split(xx, x, boost::is_any_of("."));
  boost::split(yy, y, boost::is_any_of("."));
  size_t i = 0;
  for (; i < xx.size() && i < yy.size(); ++i) {
    int dx = atoi(xx[i].c_str());
    int dy = atoi(yy[i].c_str());
    if (dx != dy) return dx - dy;
    int c = xx[i].compare(yy[i]);
    if (c != 0) return c;
  }
  if (i < xx.size()) return 1;
  if (i < yy.size()) return -1;
  return 0;
}

void ChecksumComputer::ProcessFile(const std::string& file_name) {
  std::ifstream fin(file_name.c_str());
  std::string file_content((std::istreambuf_iterator<char>(fin)),
                           std::istreambuf_iterator<char>());
  crc_.process_bytes(file_content.data(), file_content.length());
}

uint32_t ChecksumComputer::Checksum() {
  return crc_.checksum();
}

}  // namespace rime
<commit_msg>fix compile errors with clang.<commit_after>//
// Copyleft RIME Developers
// License: GPLv3
//
// 2013-01-30 <NAME> <<EMAIL>>
//
#include <fstream>
#include <vector>
#include <boost/algorithm/string.hpp>
#include <rime/algo/utilities.h>

namespace rime {

int CompareVersionString(const std::string& x, const std::string& y) {
  if (x.empty() && y.empty()) return 0;
  if (x.empty()) return -1;
  if (y.empty()) return 1;
  std::vector<std::string> xx, yy;
  boost::split(xx, x, boost::is_any_of("."));
  boost::split(yy, y, boost::is_any_of("."));
  size_t i = 0;
  for (; i < xx.size() && i < yy.size(); ++i) {
    int dx = atoi(xx[i].c_str());
    int dy = atoi(yy[i].c_str());
    if (dx != dy) return dx - dy;
    int c = xx[i].compare(yy[i]);
    if (c != 0) return c;
  }
  if (i < xx.size()) return 1;
  if (i < yy.size()) return -1;
  return 0;
}

void ChecksumComputer::ProcessFile(const std::string& file_name) {
  std::ifstream fin(file_name.c_str());
  std::string file_content((std::istreambuf_iterator<char>(fin)),
                           std::istreambuf_iterator<char>());
  crc_.process_bytes(file_content.data(), file_content.length());
}

uint32_t ChecksumComputer::Checksum() {
  return crc_.checksum();
}

}  // namespace rime

{'fraction_non_alphanumeric': 0.20089104900769542, 'fraction_numerical': 0.01539084649655731, 'mean_word_length': 3.642857142857143, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6472922', 'n_tokens_mistral': 984, 'n_tokens_neox': 953, 'n_words': 214}