<commit_before>#include "ms/isocalc.hpp"

#include <array>
#include <cassert>
#include <cmath>
#include <cstdint>
#include <cstring>
#include <functional>
#include <limits>
#include <map>
#include <memory>
#include <mutex>
#include <new>
#include <queue>
#include <sstream>
#include <stdexcept>
#include <unordered_set>
#include <vector>

namespace ms {

struct LogFacTable {
  double table[1024];

  LogFacTable() {
    for (int i = 0; i < 1024; i++) table[i] = std::lgamma(i + 1);
  }

  double operator[](size_t i) const { return table[i]; }
};

static LogFacTable log_fac_table;

double logFactorial(int n) {
  if (n >= 1024) {
    const double PI = 3.141592653589793;
    double x = n + 1;
    return (x - 0.5) * log(x) - x + 0.5 * log(2 * PI) + 1.0 / (12.0 * x);
  } else {
    return log_fac_table[n];
  }
}

class SingleElementConf {
public:
  // 10 is the maximum number of isotopes known (Sn element)
  using CountVec = std::array<uint16_t, 10>;

private:
  const ms::Element* element_;
  CountVec counts_;
  uint8_t n_isotopes_;

public:
  SingleElementConf(const ms::Element* element, const CountVec& counts={}) :
    element_{element},
    counts_(counts),
    n_isotopes_{static_cast<uint8_t>(element->isotope_pattern.size())}
  {
  }

  SingleElementConf(const SingleElementConf& other) :
    element_(other.element_),
    counts_(other.counts_),
    n_isotopes_(other.n_isotopes_)
  {
  }

  bool operator==(const SingleElementConf& other) const {
    assert(element_->abbr == other.element_->abbr);
    return counts_ == other.counts_;
  }

  bool operator<(const SingleElementConf& other) const {
    assert(element_->abbr == other.element_->abbr);
    return counts_ < other.counts_;
  }

  // multinomial distribution
  //
  // according to the paper, we don't try to perform log prob updates on neighboring moves
  // because that would lead to accumulating errors
  double getLogProbability() const {
    double result = 0.0;
    int total_count = 0;

    for (int i = 0; i < size(); i++) {
      int count = counts_[i];
      double log_prob = element_->logProbability(i);
      result += count * log_prob - logFactorial(count);
      total_count += count;
    }

    return result + logFactorial(total_count);
  }

  double getMass() const {<commit_msg>removed useless code<commit_after>#include "ms/isocalc.hpp"

#include <array>
#include <cassert>
#include <cmath>
#include <cstdint>
#include <cstring>
#include <functional>
#include <limits>
#include <map>
#include <memory>
#include <mutex>
#include <new>
#include <queue>
#include <sstream>
#include <stdexcept>
#include <unordered_set>
#include <vector>

namespace ms {

struct LogFacTable {
  double table[1024];

  LogFacTable() {
    for (int i = 0; i < 1024; i++) table[i] = std::lgamma(i + 1);
  }

  double operator[](size_t i) const { return table[i]; }
};

static LogFacTable log_fac_table;

double logFactorial(int n) {
  if (n >= 1024) {
    const double PI = 3.141592653589793;
    double x = n + 1;
    return (x - 0.5) * log(x) - x + 0.5 * log(2 * PI) + 1.0 / (12.0 * x);
  } else {
    return log_fac_table[n];
  }
}

class SingleElementConf {
public:
  // 10 is the maximum number of isotopes known (Sn element)
  using CountVec = std::array<uint16_t, 10>;

private:
  const ms::Element* element_;
  CountVec counts_;
  uint8_t n_isotopes_;

public:
  SingleElementConf(const ms::Element* element, const CountVec& counts={}) :
    element_{element},
    counts_(counts),
    n_isotopes_{static_cast<uint8_t>(element->isotope_pattern.size())}
  {
  }

  bool operator==(const SingleElementConf& other) const {
    assert(element_->abbr == other.element_->abbr);
    return counts_ == other.counts_;
  }

  bool operator<(const SingleElementConf& other) const {
    assert(element_->abbr == other.element_->abbr);
    return counts_ < other.counts_;
  }

  // multinomial distribution
  //
  // according to the paper, we don't try to perform log prob updates on neighboring moves
  // because that would lead to accumulating errors
  double getLogProbability() const {
    double result = 0.0;
    int total_count = 0;

    for (int i = 0; i < size(); i++) {
      int count = counts_[i];
      double log_prob = element_->logProbability(i);
      result += count * log_prob - logFactorial(count);
      total_count += count;
    }

    return result + logFactorial(total_count);
  }

  double getMass() const {
{'fraction_non_alphanumeric': 0.1337116912599319, 'fraction_numerical': 0.024063564131668557, 'mean_word_length': 3.647679324894515, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3165367', 'n_tokens_mistral': 1599, 'n_tokens_neox': 1487, 'n_words': 410}