<commit_before>#include "serializer.h"
#include <cmath>
#include <cstdint>
#include <cstdlib>
#include <limits>

namespace rapscallion {

void serializer<std::uint_least64_t>::write(Serializer& s, std::uint_least64_t value) {
  while (value >= 0x80) {
    s.addByte((uint8_t)(value & 0x7F) | 0x80);
    value >>= 7;
  }
  s.addByte((uint8_t)value);
}
void serializer<long>::write(Serializer& s, const long v) {
  std::uint_least64_t val = (std::abs(v) << 1) | (v < 0 ? 1 : 0);
  serializer<std::uint_least64_t>::write(s, val);
}
void serializer<int>::write(Serializer& s, const int v) {
  serializer<long>::write(s, v);
}
void serializer<std::string>::write(Serializer& s, const std::string& value) {
  serializer<std::uint_least64_t>::write(s, value.size());
  for (auto& c : value) s.addByte(c);
}
void serializer<bool>::write(Serializer& s, const bool b) {
  serializer<std::uint_least64_t>::write(s, b ? 1 : 0);
}

namespace {
  enum Type {
    // First position of enum, because it is zero, must be covered with a value for which the sign doesn't matter
    NaN,
    Infinity,
    Zero,
    Normal
  };
  struct float_repr {
    // Can fit an exponent of 32-2=20 bits, can support octuple-precision IEEE754 floats
    std::int_least32_t exponent;
    // Can support double precision IEEE754 floats (quadruple requires 112 bits, octuple 236 bits)
    std::uint_least64_t fraction;
    // One bit reserved for sign bit
    static constexpr unsigned fraction_bits = 63 - 1;
    bool is_negative;
    Type type;
  };
  auto bitreverse(std::uint_least64_t b) -> decltype(b)
  {
    b = ((b & 0xFFFFFFFF00000000ULL) >> 32) | ((b & 0x00000000FFFFFFFFULL) << 32);
    b = ((b & 0xFFFF0000FFFF0000ULL) >> 16) | ((b & 0x0000FFFF0000FFFFULL) << 16);<commit_msg>Don't leave space for sign bit in fraction<commit_after>#include "serializer.h"
#include <cmath>
#include <cstdint>
#include <cstdlib>
#include <limits>

namespace rapscallion {

void serializer<std::uint_least64_t>::write(Serializer& s, std::uint_least64_t value) {
  while (value >= 0x80) {
    s.addByte((uint8_t)(value & 0x7F) | 0x80);
    value >>= 7;
  }
  s.addByte((uint8_t)value);
}
void serializer<long>::write(Serializer& s, const long v) {
  std::uint_least64_t val = (std::abs(v) << 1) | (v < 0 ? 1 : 0);
  serializer<std::uint_least64_t>::write(s, val);
}
void serializer<int>::write(Serializer& s, const int v) {
  serializer<long>::write(s, v);
}
void serializer<std::string>::write(Serializer& s, const std::string& value) {
  serializer<std::uint_least64_t>::write(s, value.size());
  for (auto& c : value) s.addByte(c);
}
void serializer<bool>::write(Serializer& s, const bool b) {
  serializer<std::uint_least64_t>::write(s, b ? 1 : 0);
}

namespace {
  enum Type {
    // First position of enum, because it is zero, must be covered with a value for which the sign doesn't matter
    NaN,
    Infinity,
    Zero,
    Normal
  };
  struct float_repr {
    // Can fit an exponent of 32-2=20 bits, can support octuple-precision IEEE754 floats
    std::int_least32_t exponent;
    // Can support double precision IEEE754 floats (quadruple requires 112 bits, octuple 236 bits)
    std::uint_least64_t fraction;
    static constexpr unsigned fraction_bits = 64;
    bool is_negative;
    Type type;
  };
  auto bitreverse(std::uint_least64_t b) -> decltype(b)
  {
    b = ((b & 0xFFFFFFFF00000000ULL) >> 32) | ((b & 0x00000000FFFFFFFFULL) << 32);
    b = ((b & 0xFFFF0000FFFF0000ULL) >> 16) | ((b & 0x0000FFFF0000FFFFULL) << 16);
{'fraction_non_alphanumeric': 0.1457975986277873, 'fraction_numerical': 0.05631789594053745, 'mean_word_length': 4.317629179331307, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2052400', 'n_tokens_mistral': 1418, 'n_tokens_neox': 1227, 'n_words': 329}