<commit_before>// Licensed under the Apache License 2.0 (see LICENSE file).

// Constants and functions.

#pragma once

#include "types.h"

namespace cheesebase {

constexpr uint16_t kVersion{ 0x0001 };
constexpr uint64_t k_magic{ 0x0000455342534843 + // CHSBSExx
                            (static_cast<uint64_t>(kVersion) << 48) };

// size of a memory page
const size_t k_page_size_power{ 12 };
const size_t k_page_size{ 1u << k_page_size_power };

const size_t k_default_cache_size{ k_page_size * 1024 * 10 }; // 40 MB - test

using Byte = gsl::byte;

template <class T>
using Span = gsl::span<T, gsl::dynamic_range>;

struct PageNr {
  explicit PageNr(uint64_t nr) : value{ nr } {}
  uint64_t addr() const noexcept { return value * k_page_size; }

  //! Provide hash callable for use in unordered_map.
  struct Hash {
    size_t operator()(PageNr n) const noexcept {
      return std::hash<uint64_t>{}(n.value);
    }
  };

  bool operator==(PageNr o) const noexcept { return value == o.value; }
  bool operator!=(PageNr o) const noexcept { return value != o.value; }
  bool operator<(PageNr o) const noexcept { return value < o.value; }
  bool operator>(PageNr o) const noexcept { return value > o.value; }

  uint64_t value;
};

struct Addr {
  explicit Addr() = default;
  explicit Addr(uint64_t addr) : value{ addr } {}

  //! Get \c PageNr
  PageNr pageNr() const noexcept { return PageNr(value >> k_page_size_power); }

  //! Get offset in page: address % page_size.
  auto pageOffset() const noexcept {
    return gsl::narrow_cast<std::ptrdiff_t>(value & (k_page_size - 1));
  }

  //! Provide hash callable for use in unordered_map.
  struct Hash {
    size_t operator()(Addr a) const noexcept {
      return std::hash<uint64_t>{}(a.value);
    }
  };

  //! True if address is 0.
  bool isNull() const noexcept { return value == 0; }

  bool operator==(Addr o) const noexcept { return value == o.value; }
  bool operator!=(Addr o) const noexcept { return value != o.value; }
  bool operator<(Addr o) const noexcept { return value < o.value; }
  bool operator>(Addr o) const noexcept { return value > o.value; }

  uint64_t value;
};

//! Signed wrapper for sizeof().
template <typename T>
constexpr auto ssizeof() {
  static_assert(sizeof(T) == gsl::narrow_cast<std::ptrdiff_t>(sizeof(T)),
                "Signed cast changes value!");
  return gsl::narrow_cast<std::ptrdiff_t>(sizeof(T));
}

//! Signed wrapper for sizeof().
template <typename T>
constexpr auto ssizeof(const T&) {
  return ssizeof<T>();
}

//! Represents a write to disk.
struct Write {
  Addr addr;
  Span<const Byte> data;
};

using Writes = std::vector<Write>;<commit_msg>Maybe fix VC compilation error<commit_after>// Licensed under the Apache License 2.0 (see LICENSE file).

// Constants and functions.

#pragma once

#include "types.h"

namespace cheesebase {

constexpr uint16_t kVersion{ 0x0001 };
constexpr uint64_t k_magic{ 0x0000455342534843 + // CHSBSExx
                            (static_cast<uint64_t>(kVersion) << 48) };

// size of a memory page
const size_t k_page_size_power{ 12 };
const size_t k_page_size{ 1u << k_page_size_power };

const size_t k_default_cache_size{ k_page_size * 1024 * 10 }; // 40 MB - test

using Byte = gsl::byte;

template <class T>
using Span = gsl::span<T, gsl::dynamic_range>;

struct PageNr {
  explicit PageNr(uint64_t nr) : value{ nr } {}
  uint64_t addr() const noexcept { return value * k_page_size; }

  //! Provide hash callable for use in unordered_map.
  struct Hash {
    size_t operator()(PageNr n) const noexcept {
      return std::hash<uint64_t>{}(n.value);
    }
  };

  bool operator==(PageNr o) const noexcept { return value == o.value; }
  bool operator!=(PageNr o) const noexcept { return value != o.value; }
  bool operator<(PageNr o) const noexcept { return value < o.value; }
  bool operator>(PageNr o) const noexcept { return value > o.value; }

  uint64_t value;
};

struct Addr {
  explicit Addr() = default;
  explicit Addr(uint64_t addr) : value{ addr } {}

  //! Get \c PageNr
  PageNr pageNr() const noexcept { return PageNr(value >> k_page_size_power); }

  //! Get offset in page: address % page_size.
  auto pageOffset() const noexcept {
    return gsl::narrow_cast<std::ptrdiff_t>(value & (k_page_size - 1));
  }

  //! Provide hash callable for use in unordered_map.
  struct Hash {
    size_t operator()(Addr a) const noexcept {
      return std::hash<uint64_t>{}(a.value);
    }
  };

  //! True if address is 0.
  bool isNull() const noexcept { return value == 0; }

  bool operator==(Addr o) const noexcept { return value == o.value; }
  bool operator!=(Addr o) const noexcept { return value != o.value; }
  bool operator<(Addr o) const noexcept { return value < o.value; }
  bool operator>(Addr o) const noexcept { return value > o.value; }

  uint64_t value;
};

//! Signed wrapper for sizeof().
template <typename T>
constexpr auto ssizeof() {
  static_assert(sizeof(T) == static_cast<std::ptrdiff_t>(sizeof(T)),
                "Signed sizeof changes value!");
  return static_cast<std::ptrdiff_t>(sizeof(T));
}

//! Signed wrapper for sizeof().
template <typename T>
constexpr auto ssizeof(const T&) {
  return ssizeof<T>();
}

//! Represents a write to disk.
struct Write {
  Addr addr;
  Span<const Byte> data;
};

using Writes = std::vector<Write>;
{'fraction_non_alphanumeric': 0.12835428785888534, 'fraction_numerical': 0.02251829611559392, 'mean_word_length': 3.630755864465682, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2039481', 'n_tokens_mistral': 1914, 'n_tokens_neox': 1804, 'n_words': 541}