<commit_before>#include <bf/bloom_filter/basic.hpp>

#include <cassert>
#include <cmath>

namespace bf {

size_t basic_bloom_filter::m(double fp, size_t capacity) {
  auto ln2 = std::log(2);
  return std::ceil(-(capacity * std::log(fp) / ln2 / ln2));
}

size_t basic_bloom_filter::k(size_t cells, size_t capacity) {
  auto frac = static_cast<double>(cells) / static_cast<double>(capacity);
  return std::ceil(frac * std::log(2));
}

basic_bloom_filter::basic_bloom_filter(hasher h, size_t cells, bool partition)
    : hasher_(std::move(h)), bits_(cells), partition_(partition) {
}

basic_bloom_filter::basic_bloom_filter(double fp, size_t capacity, size_t seed,
                                       bool double_hashing, bool partition)
    : partition_(partition) {
  auto required_cells = m(fp, capacity);
  auto optimal_k = k(required_cells, capacity);
  if (partition_)
    required_cells += optimal_k - required_cells % optimal_k;
  bits_.resize(required_cells);
  hasher_ = make_hasher(optimal_k, seed, double_hashing);
}

basic_bloom_filter::basic_bloom_filter(hasher h, bitvector b)
    : hasher_(std::move(h)), bits_(std::move(b)) {
}

basic_bloom_filter::basic_bloom_filter(basic_bloom_filter&& other)
    : hasher_(std::move(other.hasher_)), bits_(std::move(other.bits_)) {
}

void basic_bloom_filter::add(object const& o) {
  auto digests = hasher_(o);
  assert(bits_.size() % digests.size() == 0);
  if (partition_) {
    auto parts = bits_.size() / digests.size();
    for (size_t i = 0; i < digests.size(); ++i)
      bits_.set(i * parts + (digests[i] % parts));
  } else {
    for (auto d : digests)
      bits_.set(d % bits_.size());
  }
}

size_t basic_bloom_filter::lookup(object const& o) const {
  auto digests = hasher_(o);
  assert(bits_.size() % digests.size() == 0);
  if (partition_) {
    auto parts = bits_.size() / digests.size();
    for (size_t i = 0; i < digests.size(); ++i)
      if (!bits_[i * parts + (digests[i] % parts)])
        return 0;
  } else {
    for (auto d : digests)<commit_msg>assert alignment only if partition is true<commit_after>#include <bf/bloom_filter/basic.hpp>

#include <cassert>
#include <cmath>

namespace bf {

size_t basic_bloom_filter::m(double fp, size_t capacity) {
  auto ln2 = std::log(2);
  return std::ceil(-(capacity * std::log(fp) / ln2 / ln2));
}

size_t basic_bloom_filter::k(size_t cells, size_t capacity) {
  auto frac = static_cast<double>(cells) / static_cast<double>(capacity);
  return std::ceil(frac * std::log(2));
}

basic_bloom_filter::basic_bloom_filter(hasher h, size_t cells, bool partition)
    : hasher_(std::move(h)), bits_(cells), partition_(partition) {
}

basic_bloom_filter::basic_bloom_filter(double fp, size_t capacity, size_t seed,
                                       bool double_hashing, bool partition)
    : partition_(partition) {
  auto required_cells = m(fp, capacity);
  auto optimal_k = k(required_cells, capacity);
  if (partition_)
    required_cells += optimal_k - required_cells % optimal_k;
  bits_.resize(required_cells);
  hasher_ = make_hasher(optimal_k, seed, double_hashing);
}

basic_bloom_filter::basic_bloom_filter(hasher h, bitvector b)
    : hasher_(std::move(h)), bits_(std::move(b)) {
}

basic_bloom_filter::basic_bloom_filter(basic_bloom_filter&& other)
    : hasher_(std::move(other.hasher_)), bits_(std::move(other.bits_)) {
}

void basic_bloom_filter::add(object const& o) {
  auto digests = hasher_(o);
  if (partition_) {
    assert(bits_.size() % digests.size() == 0);
    auto parts = bits_.size() / digests.size();
    for (size_t i = 0; i < digests.size(); ++i)
      bits_.set(i * parts + (digests[i] % parts));
  } else {
    for (auto d : digests)
      bits_.set(d % bits_.size());
  }
}

size_t basic_bloom_filter::lookup(object const& o) const {
  auto digests = hasher_(o);
  if (partition_) {
    assert(bits_.size() % digests.size() == 0);
    auto parts = bits_.size() / digests.size();
    for (size_t i = 0; i < digests.size(); ++i)
      if (!bits_[i * parts + (digests[i] % parts)])
        return 0;
  } else {
    for (auto d : digests)
{'fraction_non_alphanumeric': 0.15260454878943508, 'fraction_numerical': 0.004891171435558816, 'mean_word_length': 4.106117353308364, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5294406', 'n_tokens_mistral': 1497, 'n_tokens_neox': 1461, 'n_words': 309}