<commit_before>#include <LightGBM/utils/common.h>
#include <LightGBM/bin.h>

#include "dense_bin.hpp"
#include "sparse_bin.hpp"

#include <cmath>
#include <cstring>
#include <cstdint>

#include <limits>
#include <vector>
#include <algorithm>

namespace LightGBM {

BinMapper::BinMapper()
  :bin_upper_bound_(nullptr) {
}

// deep copy function for BinMapper
BinMapper::BinMapper(const BinMapper& other)
  : bin_upper_bound_(nullptr) {
  num_bin_ = other.num_bin_;
  is_trival_ = other.is_trival_;
  sparse_rate_ = other.sparse_rate_;
  bin_upper_bound_ = new float[num_bin_];
  for (int i = 0; i < num_bin_; ++i) {
    bin_upper_bound_[i] = other.bin_upper_bound_[i];
  }
}

BinMapper::BinMapper(const void* memory)
  :bin_upper_bound_(nullptr) {
  CopyFrom(reinterpret_cast<const char*>(memory));
}

BinMapper::~BinMapper() {
  delete[] bin_upper_bound_;
}

void BinMapper::FindBin(std::vector<float>* values, int max_bin) {
  std::vector<float>& ref_values = (*values);
  size_t sample_size = values->size();
  // find distinct_values first
  std::vector<float> distinct_values;
  std::vector<int> counts;
  
  std::sort(ref_values.begin(), ref_values.end());
  distinct_values.push_back(ref_values[0]);
  counts.push_back(1);
  for (size_t i = 1; i < ref_values.size(); ++i) {
    if (ref_values[i] != ref_values[i - 1]) {
      distinct_values.push_back(ref_values[i]);
      counts.push_back(1);
    } else {
      ++counts.back();
    }
  }
  int num_values = static_cast<int>(distinct_values.size());
  int cnt_in_bin0 = 0;

  if (num_values <= max_bin) {
    // use distinct value is enough
    num_bin_ = num_values;
    bin_upper_bound_ = new float[num_values];
    for (int i = 0; i < num_values - 1; ++i) {
      bin_upper_bound_[i] = (distinct_values[i] + distinct_values[i + 1]) / 2;
    }
    cnt_in_bin0 = counts[0];
    bin_upper_bound_[num_values - 1] = std::numeric_limits<float>::infinity();
  } else {
    // mean size for one bin
    float mean_bin_size = sample_size / static_cast<float>(max_bin);
    int rest_sample_cnt = static_cast<int>(sample_size);
    int bin_cnt = 0;

    num_bin_ = max_bin;
    std::vector<float> upper_bounds(max_bin, std::numeric_limits<float>::infinity());
    std::vector<float> lower_bounds(max_bin, std::numeric_limits<float>::infinity());
    // sort by count, descent
    Common::SortForPair(counts, distinct_values, 0, true);
    // fetch big slot as unique bin 
    while (counts[bin_cnt] > mean_bin_size) {
      upper_bounds[bin_cnt] = distinct_values[bin_cnt];
      lower_bounds[bin_cnt] = distinct_values[bin_cnt];
      rest_sample_cnt -= counts[bin_cnt];
      ++bin_cnt;
    }
    // process reminder bins
    if (bin_cnt < max_bin) {
      // sort rest by values
      Common::SortForPair<float, int>(distinct_values, counts, bin_cnt, false);
      mean_bin_size = rest_sample_cnt / static_cast<float>(max_bin - bin_cnt);
      lower_bounds[bin_cnt] = distinct_values[bin_cnt];
      int cur_cnt_inbin = 0;
      for (int i = bin_cnt; i < num_values - 1; ++i) {
        rest_sample_cnt -= counts[i];
        cur_cnt_inbin += counts[i];
        // need a new bin
        if (cur_cnt_inbin >= mean_bin_size) {
          upper_bounds[bin_cnt] = distinct_values[i];
          if (bin_cnt == 0) { cnt_in_bin0 = cur_cnt_inbin; }
          ++bin_cnt;
          lower_bounds[bin_cnt] = distinct_values[i + 1];
          if (bin_cnt >= max_bin - 1) break;
          cur_cnt_inbin = 0;
          mean_bin_size = rest_sample_cnt / static_cast<float>(max_bin - bin_cnt);
        }
      }
      cur_cnt_inbin += counts[num_values - 1];
      
    }
    Common::SortForPair<float, float>(lower_bounds, upper_bounds, 0, false);
    // update bin upper bound
    bin_upper_bound_ = new float[bin_cnt];
    for (int i = 0; i < bin_cnt - 1; ++i) {
      bin_upper_bound_[i] = (upper_bounds[i] + lower_bounds[i + 1]) / 2.0f;
    }

    // last bin upper bound
    bin_upper_bound_[bin_cnt - 1] = std::numeric_limits<float>::infinity();
    
    CHECK(bin_cnt <= max_bin);

  }
  // check trival(num_bin_ == 1) feature
  if (num_bin_ <= 1) {
    is_trival_ = true;
  } else {
    is_trival_ = false;
  }
  // calculate sparse rate
  sparse_rate_ = static_cast<float>(cnt_in_bin0) / static_cast<float>(sample_size);
}


int BinMapper::SizeForSpecificBin(int bin) {
  int size = 0;
  size += sizeof(int);
  size += sizeof(bool);
  size += sizeof(float);
  size += bin * sizeof(float);
  return size;
}
<commit_msg>fixed a bug in bin.cpp<commit_after>#include <LightGBM/utils/common.h>
#include <LightGBM/bin.h>

#include "dense_bin.hpp"
#include "sparse_bin.hpp"

#include <cmath>
#include <cstring>
#include <cstdint>

#include <limits>
#include <vector>
#include <algorithm>

namespace LightGBM {

BinMapper::BinMapper()
  :bin_upper_bound_(nullptr) {
}

// deep copy function for BinMapper
BinMapper::BinMapper(const BinMapper& other)
  : bin_upper_bound_(nullptr) {
  num_bin_ = other.num_bin_;
  is_trival_ = other.is_trival_;
  sparse_rate_ = other.sparse_rate_;
  bin_upper_bound_ = new float[num_bin_];
  for (int i = 0; i < num_bin_; ++i) {
    bin_upper_bound_[i] = other.bin_upper_bound_[i];
  }
}

BinMapper::BinMapper(const void* memory)
  :bin_upper_bound_(nullptr) {
  CopyFrom(reinterpret_cast<const char*>(memory));
}

BinMapper::~BinMapper() {
  delete[] bin_upper_bound_;
}

void BinMapper::FindBin(std::vector<float>* values, int max_bin) {
  std::vector<float>& ref_values = (*values);
  size_t sample_size = values->size();
  // find distinct_values first
  std::vector<float> distinct_values;
  std::vector<int> counts;
  
  std::sort(ref_values.begin(), ref_values.end());
  distinct_values.push_back(ref_values[0]);
  counts.push_back(1);
  for (size_t i = 1; i < ref_values.size(); ++i) {
    if (ref_values[i] != ref_values[i - 1]) {
      distinct_values.push_back(ref_values[i]);
      counts.push_back(1);
    } else {
      ++counts.back();
    }
  }
  int num_values = static_cast<int>(distinct_values.size());
  int cnt_in_bin0 = 0;

  if (num_values <= max_bin) {
    // use distinct value is enough
    num_bin_ = num_values;
    bin_upper_bound_ = new float[num_values];
    for (int i = 0; i < num_values - 1; ++i) {
      bin_upper_bound_[i] = (distinct_values[i] + distinct_values[i + 1]) / 2;
    }
    cnt_in_bin0 = counts[0];
    bin_upper_bound_[num_values - 1] = std::numeric_limits<float>::infinity();
  } else {
    // mean size for one bin
    float mean_bin_size = sample_size / static_cast<float>(max_bin);
    int rest_sample_cnt = static_cast<int>(sample_size);
    int bin_cnt = 0;

    num_bin_ = max_bin;
    std::vector<float> upper_bounds(max_bin, std::numeric_limits<float>::infinity());
    std::vector<float> lower_bounds(max_bin, std::numeric_limits<float>::infinity());
    // sort by count, descent
    Common::SortForPair(counts, distinct_values, 0, true);
    // fetch big slot as unique bin 
    while (counts[bin_cnt] > mean_bin_size) {
      upper_bounds[bin_cnt] = distinct_values[bin_cnt];
      lower_bounds[bin_cnt] = distinct_values[bin_cnt];
      rest_sample_cnt -= counts[bin_cnt];
      ++bin_cnt;
    }
    // process reminder bins
    if (bin_cnt < max_bin) {
      // sort rest by values
      Common::SortForPair<float, int>(distinct_values, counts, bin_cnt, false);
      mean_bin_size = rest_sample_cnt / static_cast<float>(max_bin - bin_cnt);
      lower_bounds[bin_cnt] = distinct_values[bin_cnt];
      int cur_cnt_inbin = 0;
      for (int i = bin_cnt; i < num_values - 1; ++i) {
        rest_sample_cnt -= counts[i];
        cur_cnt_inbin += counts[i];
        // need a new bin
        if (cur_cnt_inbin >= mean_bin_size) {
          upper_bounds[bin_cnt] = distinct_values[i];
          if (bin_cnt == 0) { cnt_in_bin0 = cur_cnt_inbin; }
          ++bin_cnt;
          lower_bounds[bin_cnt] = distinct_values[i + 1];
          if (bin_cnt >= max_bin - 1) break;
          cur_cnt_inbin = 0;
          mean_bin_size = rest_sample_cnt / static_cast<float>(max_bin - bin_cnt);
        }
      }
      cur_cnt_inbin += counts[num_values - 1];
      
    }
    Common::SortForPair<float, float>(lower_bounds, upper_bounds, 0, false);
    // update bin upper bound
    bin_upper_bound_ = new float[bin_cnt];
    num_bin_ = bin_cnt;
    for (int i = 0; i < bin_cnt - 1; ++i) {
      bin_upper_bound_[i] = (upper_bounds[i] + lower_bounds[i + 1]) / 2.0f;
    }
    // last bin upper bound
    bin_upper_bound_[bin_cnt - 1] = std::numeric_limits<float>::infinity();
  }
  // check trival(num_bin_ == 1) feature
  if (num_bin_ <= 1) {
    is_trival_ = true;
  } else {
    is_trival_ = false;
  }
  // calculate sparse rate
  sparse_rate_ = static_cast<float>(cnt_in_bin0) / static_cast<float>(sample_size);
}


int BinMapper::SizeForSpecificBin(int bin) {
  int size = 0;
  size += sizeof(int);
  size += sizeof(bool);
  size += sizeof(float);
  size += bin * sizeof(float);
  return size;
}

{'fraction_non_alphanumeric': 0.13813207758717344, 'fraction_numerical': 0.008072653884964682, 'mean_word_length': 3.5073269327943404, 'pattern_counts': {'":': 0, '<': 76, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5725293', 'n_tokens_mistral': 3310, 'n_tokens_neox': 3239, 'n_words': 642}