<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Copyright (C) 2015 ScyllaDB
 *
 * Modified by ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include "exceptions/exceptions.hh"

namespace utils {

/**
 * The following calculations are taken from:
 * http://www.cs.wisc.edu/~cao/papers/summary-cache/node8.html
 * "Bloom Filters - the math"
 *
 * This class's static methods are meant to facilitate the use of the Bloom
 * Filter class by helping to choose correct values of 'bits per element' and
 * 'number of hash functions, k'.
 */
namespace bloom_calculations {

    /**
     * A wrapper class that holds two key parameters for a Bloom Filter: the
     * number of hash functions used, and the number of buckets per element used.
     */
    struct bloom_specification final {
        int K; // number of hash functions.
        int buckets_per_element;

        bloom_specification(int k, int buckets_per_element) : K(k), buckets_per_element(buckets_per_element) { }

        operator sstring() {
            return format("bloom_specification(K={:d}, buckets_per_element={:d})", K, buckets_per_element);
        }
    };

    int constexpr min_buckets = 2;
    int constexpr min_k = 1;
    int constexpr EXCESS = 20;

    extern std::vector<std::vector<double>> probs;
    extern std::vector<int> opt_k_per_buckets;

    /**
     * Given the number of buckets that can be used per element, return a
     * specification that minimizes the false positive rate.
     *
     * @param buckets_per_element The number of buckets per element for the filter.
     * @return A spec that minimizes the false positive rate.
     */
    inline bloom_specification compute_bloom_spec(int buckets_per_element) {
        assert(buckets_per_element >= 1);
        assert(buckets_per_element <= int(probs.size()) - 1);
        return bloom_specification(opt_k_per_buckets[buckets_per_element], buckets_per_element);
    }

    /**
     * Given a maximum tolerable false positive probability, compute a Bloom
     * specification which will give less than the specified false positive rate,
     * but minimize the number of buckets per element and the number of hash
     * functions used.  Because bandwidth (and therefore total bitvector size)
     * is considered more expensive than computing power, preference is given
     * to minimizing buckets per element rather than number of hash functions.
     *
     * @param max_buckets_per_element The maximum number of buckets available for the filter.
     * @param max_false_pos_prob The maximum tolerable false positive rate.
     * @return A Bloom Specification which would result in a false positive rate
     * less than specified by the function call
     * @throws unsupported_operation_exception if a filter satisfying the parameters cannot be met
     */
    inline bloom_specification compute_bloom_spec(int max_buckets_per_element, double max_false_pos_prob) {
        assert(max_buckets_per_element >= 1);
        assert(max_buckets_per_element <= int(probs.size()) - 1);

        auto max_k = int(probs[max_buckets_per_element].size()) - 1;

        // Handle the trivial cases
        if(max_false_pos_prob >= probs[min_buckets][min_k]) {
            return bloom_specification(2, opt_k_per_buckets[2]);
        }

        if (max_false_pos_prob < probs[max_buckets_per_element][max_k]) {
            throw exceptions::unsupported_operation_exception(format("Unable to satisfy {:f} with {:d} buckets per element", max_false_pos_prob, max_buckets_per_element));
        }

        // First find the minimal required number of buckets:
        int buckets_per_element = 2;
        int K = opt_k_per_buckets[2];

        while(probs[buckets_per_element][K] > max_false_pos_prob){
            buckets_per_element++;
            K = opt_k_per_buckets[buckets_per_element];
        }
        // Now that the number of buckets is sufficient, see if we can relax K
        // without losing too much precision.
        while(probs[buckets_per_element][K - 1] <= max_false_pos_prob){
            K--;
        }

        return bloom_specification(K, buckets_per_element);
    }

    /**
     * Calculates the maximum number of buckets per element that this implementation
     * can support.  Crucially, it will lower the bucket count if necessary to meet
     * BitSet's size restrictions.
     */
    inline int max_buckets_per_element(long num_elements) {
        num_elements = std::max(1l, num_elements);

        double v = std::numeric_limits<long>::max() - EXCESS;
        v = v / num_elements;

        if (v < 1.0) {
            throw exceptions::unsupported_operation_exception(format("Cannot compute probabilities for {:d} elements.", num_elements));
        }
        return std::min(probs.size() - 1, size_t(v));
    }
}
<commit_msg>utils: bloom_calculations: avoid gratuitous conversion to double<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * Copyright (C) 2015 ScyllaDB
 *
 * Modified by ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include "exceptions/exceptions.hh"

namespace utils {

/**
 * The following calculations are taken from:
 * http://www.cs.wisc.edu/~cao/papers/summary-cache/node8.html
 * "Bloom Filters - the math"
 *
 * This class's static methods are meant to facilitate the use of the Bloom
 * Filter class by helping to choose correct values of 'bits per element' and
 * 'number of hash functions, k'.
 */
namespace bloom_calculations {

    /**
     * A wrapper class that holds two key parameters for a Bloom Filter: the
     * number of hash functions used, and the number of buckets per element used.
     */
    struct bloom_specification final {
        int K; // number of hash functions.
        int buckets_per_element;

        bloom_specification(int k, int buckets_per_element) : K(k), buckets_per_element(buckets_per_element) { }

        operator sstring() {
            return format("bloom_specification(K={:d}, buckets_per_element={:d})", K, buckets_per_element);
        }
    };

    int constexpr min_buckets = 2;
    int constexpr min_k = 1;
    int constexpr EXCESS = 20;

    extern std::vector<std::vector<double>> probs;
    extern std::vector<int> opt_k_per_buckets;

    /**
     * Given the number of buckets that can be used per element, return a
     * specification that minimizes the false positive rate.
     *
     * @param buckets_per_element The number of buckets per element for the filter.
     * @return A spec that minimizes the false positive rate.
     */
    inline bloom_specification compute_bloom_spec(int buckets_per_element) {
        assert(buckets_per_element >= 1);
        assert(buckets_per_element <= int(probs.size()) - 1);
        return bloom_specification(opt_k_per_buckets[buckets_per_element], buckets_per_element);
    }

    /**
     * Given a maximum tolerable false positive probability, compute a Bloom
     * specification which will give less than the specified false positive rate,
     * but minimize the number of buckets per element and the number of hash
     * functions used.  Because bandwidth (and therefore total bitvector size)
     * is considered more expensive than computing power, preference is given
     * to minimizing buckets per element rather than number of hash functions.
     *
     * @param max_buckets_per_element The maximum number of buckets available for the filter.
     * @param max_false_pos_prob The maximum tolerable false positive rate.
     * @return A Bloom Specification which would result in a false positive rate
     * less than specified by the function call
     * @throws unsupported_operation_exception if a filter satisfying the parameters cannot be met
     */
    inline bloom_specification compute_bloom_spec(int max_buckets_per_element, double max_false_pos_prob) {
        assert(max_buckets_per_element >= 1);
        assert(max_buckets_per_element <= int(probs.size()) - 1);

        auto max_k = int(probs[max_buckets_per_element].size()) - 1;

        // Handle the trivial cases
        if(max_false_pos_prob >= probs[min_buckets][min_k]) {
            return bloom_specification(2, opt_k_per_buckets[2]);
        }

        if (max_false_pos_prob < probs[max_buckets_per_element][max_k]) {
            throw exceptions::unsupported_operation_exception(format("Unable to satisfy {:f} with {:d} buckets per element", max_false_pos_prob, max_buckets_per_element));
        }

        // First find the minimal required number of buckets:
        int buckets_per_element = 2;
        int K = opt_k_per_buckets[2];

        while(probs[buckets_per_element][K] > max_false_pos_prob){
            buckets_per_element++;
            K = opt_k_per_buckets[buckets_per_element];
        }
        // Now that the number of buckets is sufficient, see if we can relax K
        // without losing too much precision.
        while(probs[buckets_per_element][K - 1] <= max_false_pos_prob){
            K--;
        }

        return bloom_specification(K, buckets_per_element);
    }

    /**
     * Calculates the maximum number of buckets per element that this implementation
     * can support.  Crucially, it will lower the bucket count if necessary to meet
     * BitSet's size restrictions.
     */
    inline int max_buckets_per_element(long num_elements) {
        num_elements = std::max(1l, num_elements);

        auto v = std::numeric_limits<long>::max() - EXCESS;
        v = v / num_elements;

        if (v < 1) {
            throw exceptions::unsupported_operation_exception(format("Cannot compute probabilities for {:d} elements.", num_elements));
        }
        return std::min(probs.size() - 1, size_t(v));
    }
}

{'fraction_non_alphanumeric': 0.07713212373111662, 'fraction_numerical': 0.0043961314043641595, 'mean_word_length': 3.1986577181208053, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2616866', 'n_tokens_mistral': 3576, 'n_tokens_neox': 3374, 'n_words': 1300}