<commit_before>#include <microscopes/models/bbnc.hpp>
#include <microscopes/io/schema.pb.h>
#include <distributions/random.hpp>
#include <distributions/special.hpp>

#include <stdexcept>
#include <sstream>
#include <cmath>

using namespace std;
using namespace distributions;
using namespace microscopes::io;
using namespace microscopes::common;
using namespace microscopes::models;

typedef BetaBernoulliNonConj::Group group_message_type;
typedef BetaBernoulliNonConj::Shared shared_message_type;

void
bbnc_feature_group::add_value(const model &m, const row_accessor &value, rng_t &rng)
{
  if (value.get<bool>())
    heads_++;
  else
    tails_++;
}

void
bbnc_feature_group::remove_value(const model &m, const row_accessor &value, rng_t &rng)
{
  if (value.get<bool>()) {
    MICROSCOPES_ASSERT(heads_ > 0);
    heads_--;
  } else {
    MICROSCOPES_ASSERT(tails_ > 0);
    tails_--;
  }
}

float
bbnc_feature_group::score_value(const model &m, const row_accessor &value, rng_t &rng) const
{
  return value.get<bool>() ? fast_log(p_) : fast_log(1.-p_);
}

static inline float
fast_lbeta(float a, float b)
{
  return fast_lgamma(a) + fast_lgamma(b) - fast_lgamma(a + b);
}

float
bbnc_feature_group::score_data(const model &m, rng_t &rng) const
{
  const float alpha = static_cast<const bbnc_model &>(m).alpha_;
  const float beta = static_cast<const bbnc_model &>(m).beta_;
  const float score_prior =
    (alpha-1.)*fast_log(p_) + (beta-1.)*fast_log(1.-p_) - fast_lbeta(alpha, beta);
  const float score_likelihood =
    float(heads_)*fast_log(p_) + float(tails_)*fast_log(1.-p_);
  return score_prior + score_likelihood;
}

void
bbnc_feature_group::sample_value(const model &m, row_mutator &value, rng_t &rng) const
{
  value.set<bool>(sample_bernoulli(rng, p_));
}

suffstats_bag_t
bbnc_feature_group::get_ss() const
{
  group_message_type m;
  m.set_p(p_);
  ostringstream out;
  m.SerializeToOstream(&out);
  return out.str();
}

void
bbnc_feature_group::set_ss(const suffstats_bag_t &ss)
{
  istringstream inp(ss);
  group_message_type m;
  m.ParseFromIstream(&inp);
  p_ = m.p();
}

void *
bbnc_feature_group::get_ss_raw_ptr(const string &key)
{
  if (key == "p")
    return &p_;
  throw runtime_error("unknown key: " + key);<commit_msg>handle negative p values for bbnc<commit_after>#include <microscopes/models/bbnc.hpp>
#include <microscopes/io/schema.pb.h>
#include <distributions/random.hpp>
#include <distributions/special.hpp>

#include <stdexcept>
#include <sstream>
#include <cmath>
#include <limits>

using namespace std;
using namespace distributions;
using namespace microscopes::io;
using namespace microscopes::common;
using namespace microscopes::models;

typedef BetaBernoulliNonConj::Group group_message_type;
typedef BetaBernoulliNonConj::Shared shared_message_type;

void
bbnc_feature_group::add_value(const model &m, const row_accessor &value, rng_t &rng)
{
  if (value.get<bool>())
    heads_++;
  else
    tails_++;
}

void
bbnc_feature_group::remove_value(const model &m, const row_accessor &value, rng_t &rng)
{
  if (value.get<bool>()) {
    MICROSCOPES_ASSERT(heads_ > 0);
    heads_--;
  } else {
    MICROSCOPES_ASSERT(tails_ > 0);
    tails_--;
  }
}

float
bbnc_feature_group::score_value(const model &m, const row_accessor &value, rng_t &rng) const
{
  MICROSCOPES_ASSERT(p_ >= 0.0 && p_ <= 1.0);
  return value.get<bool>() ? fast_log(p_) : fast_log(1.-p_);
}

static inline float
fast_lbeta(float a, float b)
{
  return fast_lgamma(a) + fast_lgamma(b) - fast_lgamma(a + b);
}

float
bbnc_feature_group::score_data(const model &m, rng_t &rng) const
{
  if (p_ < 0.0 || p_ > 1.0)
    return -numeric_limits<float>::infinity();
  const float alpha = static_cast<const bbnc_model &>(m).alpha_;
  const float beta = static_cast<const bbnc_model &>(m).beta_;
  const float score_prior =
    (alpha-1.)*fast_log(p_) + (beta-1.)*fast_log(1.-p_) - fast_lbeta(alpha, beta);
  const float score_likelihood =
    float(heads_)*fast_log(p_) + float(tails_)*fast_log(1.-p_);
  return score_prior + score_likelihood;
}

void
bbnc_feature_group::sample_value(const model &m, row_mutator &value, rng_t &rng) const
{
  MICROSCOPES_ASSERT(p_ >= 0.0 && p_ <= 1.0);
  value.set<bool>(sample_bernoulli(rng, p_));
}

suffstats_bag_t
bbnc_feature_group::get_ss() const
{
  group_message_type m;
  m.set_p(p_);
  ostringstream out;
  m.SerializeToOstream(&out);
  return out.str();
}

void
bbnc_feature_group::set_ss(const suffstats_bag_t &ss)
{
  istringstream inp(ss);
  group_message_type m;
  m.ParseFromIstream(&inp);
  p_ = m.p();
}

void *
bbnc_feature_group::get_ss_raw_ptr(const string &key)
{
  if (key == "p")
    return &p_;
  throw runtime_error("unknown key: " + key);
{'fraction_non_alphanumeric': 0.1368376250266014, 'fraction_numerical': 0.005533092147265376, 'mean_word_length': 5.325706594885599, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5653163', 'n_tokens_mistral': 1873, 'n_tokens_neox': 1808, 'n_words': 367}