<commit_before>#ifndef IV_CONVERSIONS_H_
#define IV_CONVERSIONS_H_
#include <cstdio>
#include <cmath>
#include <string>
#include <limits>
#include <tr1/array>
#include <tr1/cstdint>
#include "chars.h"
#include "dtoa.h"
#include "ustringpiece.h"
#include "none.h"
namespace iv {
namespace core {
namespace detail {
template<typename T>
class Conversions {
 public:
  static const double kNaN;
  static const int kMaxSignificantDigits = 772;
  static const std::string kInfinity;
  static const double DoubleToInt32_Two32;
  static const double DoubleToInt32_Two31;
  static const char* kHex;
  static const int kMaxDoubleToStringWithRadixBufferSize = 2200;
};
template<typename T>
const double Conversions<T>::kNaN = std::numeric_limits<double>::quiet_NaN();

template<typename T>
const std::string Conversions<T>::kInfinity = "Infinity";

template<typename T>
const double Conversions<T>::DoubleToInt32_Two32 = 4294967296.0;

template<typename T>
const double Conversions<T>::DoubleToInt32_Two31 = 2147483648.0;

template<typename T>
const char* Conversions<T>::kHex = "0123456789abcdefghijklmnopqrstuvwxyz";

}  // namespace iv::core::detail

typedef detail::Conversions<None> Conversions;

template<typename Iter>
inline double StringToDouble(Iter it, Iter last) {
  bool is_decimal = true;
  bool is_signed = false;
  std::size_t pos = 0;
  int significant_digits = 0;
  int insignificant_digits = 0;
  std::tr1::array<char, Conversions::kMaxSignificantDigits+10> buffer;

  // empty string ""
  if (it == last) {
    return 0;
  }

  while (it != last &&
         (Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
    ++it;
  }

  // white space only "  "
  if (it == last) {
    return 0;
  }

  if (*it == '-') {
    buffer[pos++] = '-';
    ++it;
    is_signed = true;
  } else if (*it == '+') {
    ++it;
  }

  if (it == last) {
    return Conversions::kNaN;
  }

  if (Chars::IsDecimalDigit(*it)) {
    if (*it == '0') {
      ++it;
      if (it == last) {
        return 0;
      }
      if (*it == 'x' || *it == 'X') {
        is_decimal = false;
        buffer[pos++] = '0';
        buffer[pos++] = *it;
        ++it;
        ++significant_digits;
        if (it == last || !Chars::IsHexDigit(*it)) {
          return Conversions::kNaN;
        }
        // waste leading zero<commit_msg>fix signed hex number in ToNumber<commit_after>#ifndef IV_CONVERSIONS_H_
#define IV_CONVERSIONS_H_
#include <cstdio>
#include <cmath>
#include <string>
#include <limits>
#include <tr1/array>
#include <tr1/cstdint>
#include "chars.h"
#include "dtoa.h"
#include "ustringpiece.h"
#include "none.h"
namespace iv {
namespace core {
namespace detail {
template<typename T>
class Conversions {
 public:
  static const double kNaN;
  static const int kMaxSignificantDigits = 772;
  static const std::string kInfinity;
  static const double DoubleToInt32_Two32;
  static const double DoubleToInt32_Two31;
  static const char* kHex;
  static const int kMaxDoubleToStringWithRadixBufferSize = 2200;
};
template<typename T>
const double Conversions<T>::kNaN = std::numeric_limits<double>::quiet_NaN();

template<typename T>
const std::string Conversions<T>::kInfinity = "Infinity";

template<typename T>
const double Conversions<T>::DoubleToInt32_Two32 = 4294967296.0;

template<typename T>
const double Conversions<T>::DoubleToInt32_Two31 = 2147483648.0;

template<typename T>
const char* Conversions<T>::kHex = "0123456789abcdefghijklmnopqrstuvwxyz";

}  // namespace iv::core::detail

typedef detail::Conversions<None> Conversions;

template<typename Iter>
inline double StringToDouble(Iter it, Iter last) {
  bool is_decimal = true;
  bool is_signed = false;
  bool is_sign_found = false;
  std::size_t pos = 0;
  int significant_digits = 0;
  int insignificant_digits = 0;
  std::tr1::array<char, Conversions::kMaxSignificantDigits+10> buffer;

  // empty string ""
  if (it == last) {
    return 0;
  }

  while (it != last &&
         (Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
    ++it;
  }

  // white space only "  "
  if (it == last) {
    return 0;
  }

  if (*it == '-') {
    buffer[pos++] = '-';
    ++it;
    is_signed = true;
    is_sign_found = true;
  } else if (*it == '+') {
    ++it;
    is_sign_found = true;
  }

  if (it == last) {
    return Conversions::kNaN;
  }

  if (Chars::IsDecimalDigit(*it)) {
    if (*it == '0') {
      ++it;
      if (it == last) {
        return 0;
      }
      if (*it == 'x' || *it == 'X') {
        if (is_sign_found) {
          return Conversions::kNaN;
        }
        is_decimal = false;
        buffer[pos++] = '0';
        buffer[pos++] = *it;
        ++it;
        ++significant_digits;
        if (it == last || !Chars::IsHexDigit(*it)) {
          return Conversions::kNaN;
        }
        // waste leading zero
{'fraction_non_alphanumeric': 0.14261919633562356, 'fraction_numerical': 0.028315636060795336, 'mean_word_length': 3.5320754716981133, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3577783', 'n_tokens_mistral': 1783, 'n_tokens_neox': 1656, 'n_words': 405}