<commit_before>#include <LightGBM/dataset.h>

#include <LightGBM/utils/common.h>

#include <vector>
#include <string>

namespace LightGBM {

Metadata::Metadata()
  :label_(nullptr), label_int_(nullptr), weights_(nullptr), 
  query_boundaries_(nullptr),
  query_weights_(nullptr), init_score_(nullptr), queries_(nullptr){

}

void Metadata::Init(const char * data_filename, const char* init_score_filename) {
  data_filename_ = data_filename;
  init_score_filename_ = init_score_filename;
  // for lambdarank, it needs query data for partition data in parallel learning
  LoadQueryBoundaries();
  LoadWeights();
  LoadQueryWeights();
  LoadInitialScore();
}

void Metadata::Init(const char* init_score_filename) {
  init_score_filename_ = init_score_filename;
  LoadInitialScore();
}


Metadata::~Metadata() {
  if (label_ != nullptr) { delete[] label_; }
  if (weights_ != nullptr) { delete[] weights_; }
  if (query_boundaries_ != nullptr) { delete[] query_boundaries_; }
  if (query_weights_ != nullptr) { delete[] query_weights_; }
  if (init_score_ != nullptr) { delete[] init_score_; }
  if (queries_ != nullptr) { delete[] queries_; }
}


void Metadata::Init(data_size_t num_data, int weight_idx, int query_idx) {
  num_data_ = num_data;
  label_ = new float[num_data_];
  if (weight_idx >= 0) {
    if (weights_ != nullptr) {
      Log::Info("using weight in data file, and ignore additional weight file");
      delete[] weights_; 
    }
    weights_ = new float[num_data_];
    num_weights_ = num_data_;
    memset(weights_, 0, sizeof(float) * num_data_);
  }
  if (query_idx >= 0) {
    if (query_boundaries_ != nullptr) {
      Log::Info("using query id in data file, and ignore additional query file");
      delete[] query_boundaries_;
    }
    if (query_weights_ != nullptr) { delete[] query_weights_; }
    queries_ = new data_size_t[num_data_];
    memset(queries_, 0, sizeof(data_size_t) * num_data_);
  }
}

void Metadata::PartitionLabel(const std::vector<data_size_t>& used_indices) {
  if (used_indices.size() <= 0) {
    return;
  }
  float* old_label = label_;
  num_data_ = static_cast<data_size_t>(used_indices.size());
  label_ = new float[num_data_];
  for (data_size_t i = 0; i < num_data_; ++i) {
    label_[i] = old_label[used_indices[i]];
  }
  delete[] old_label;
}

void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector<data_size_t>& used_data_indices) {
  if (used_data_indices.size() == 0) {
    if (queries_ != nullptr) {
      // need convert query_id to boundaries
      std::vector<data_size_t> tmp_buffer;
      data_size_t last_qid = -1;
      data_size_t cur_cnt = 0;
      for (data_size_t i = 0; i < num_data_; ++i) {
        if (last_qid != queries_[i]) {
          if (cur_cnt > 0) {
            tmp_buffer.push_back(cur_cnt);
          }
          cur_cnt = 0;
          last_qid = queries_[i];
        }
        ++cur_cnt;
      }
      tmp_buffer.push_back(cur_cnt);
      query_boundaries_ = new data_size_t[tmp_buffer.size() + 1];
      num_queries_ = static_cast<data_size_t>(tmp_buffer.size());
      query_boundaries_[0] = 0;
      for (size_t i = 0; i < tmp_buffer.size(); ++i) {
        query_boundaries_[i + 1] = query_boundaries_[i] + tmp_buffer[i];
      }
      LoadQueryWeights();
      delete[] queries_;
      queries_ = nullptr;
    }
    // check weights
    if (weights_ != nullptr && num_weights_ != num_data_) {
      Log::Error("Initial weight size doesn't equal to data, weights will be ignored");
      delete[] weights_;
      num_weights_ = 0;
      weights_ = nullptr;
    }

    // check query boundries
    if (query_boundaries_ != nullptr && query_boundaries_[num_queries_] != num_data_) {
      Log::Error("Initial query size doesn't equal to data, queies will be ignored");
      delete[] query_boundaries_;
      num_queries_ = 0;
      query_boundaries_ = nullptr;
    }

    // contain initial score file
    if (init_score_ != nullptr && num_init_score_ != num_data_) {
      delete[] init_score_;
      Log::Error("Initial score size doesn't equal to data, score file will be ignored");
      num_init_score_ = 0;
    }
  } else {
    data_size_t num_used_data = static_cast<data_size_t>(used_data_indices.size());
    // check weights
    if (weights_ != nullptr && num_weights_ != num_all_data) {
      Log::Error("Initial weights size doesn't equal to data, weights will be ignored");
      delete[] weights_;
      num_weights_ = 0;
      weights_ = nullptr;
    }
    // check query boundries
    if (query_boundaries_ != nullptr && query_boundaries_[num_queries_] != num_all_data) {
      Log::Error("Initial query size doesn't equal to data , queries will be ignored");
      delete[] query_boundaries_;
      num_queries_ = 0;
      query_boundaries_ = nullptr;
    }

    // contain initial score file
    if (init_score_ != nullptr && num_init_score_ != num_all_data) {
      Log::Error("Initial score size doesn't equal to data , initial scores will be ignored");
      delete[] init_score_;
      num_init_score_ = 0;
    }

    // get local weights
    if (weights_ != nullptr) {
      float* old_weights = weights_;
      num_weights_ = num_data_;
      weights_ = new float[num_data_];
      for (size_t i = 0; i < used_data_indices.size(); ++i) {
        weights_[i] = old_weights[used_data_indices[i]];
      }
      delete[] old_weights;
    }

    // get local query boundaries
    if (query_boundaries_ != nullptr) {
      std::vector<data_size_t> used_query;
      data_size_t data_idx = 0;
      for (data_size_t qid = 0; qid < num_queries_ && data_idx < num_used_data; ++qid) {
        data_size_t start = query_boundaries_[qid];
        data_size_t end = query_boundaries_[qid + 1];
        data_size_t len = end - start;
        if (used_data_indices[data_idx] > start) {
          continue;
        } else if (used_data_indices[data_idx] == start) {
          if (num_used_data >= data_idx + len && used_data_indices[data_idx + len - 1] == end - 1) {
            used_query.push_back(qid);
            data_idx += len;
          } else {<commit_msg>set init_score_ = nullptr after delete<commit_after>#include <LightGBM/dataset.h>

#include <LightGBM/utils/common.h>

#include <vector>
#include <string>

namespace LightGBM {

Metadata::Metadata()
  :label_(nullptr), label_int_(nullptr), weights_(nullptr), 
  query_boundaries_(nullptr),
  query_weights_(nullptr), init_score_(nullptr), queries_(nullptr){

}

void Metadata::Init(const char * data_filename, const char* init_score_filename) {
  data_filename_ = data_filename;
  init_score_filename_ = init_score_filename;
  // for lambdarank, it needs query data for partition data in parallel learning
  LoadQueryBoundaries();
  LoadWeights();
  LoadQueryWeights();
  LoadInitialScore();
}

void Metadata::Init(const char* init_score_filename) {
  init_score_filename_ = init_score_filename;
  LoadInitialScore();
}


Metadata::~Metadata() {
  if (label_ != nullptr) { delete[] label_; }
  if (weights_ != nullptr) { delete[] weights_; }
  if (query_boundaries_ != nullptr) { delete[] query_boundaries_; }
  if (query_weights_ != nullptr) { delete[] query_weights_; }
  if (init_score_ != nullptr) { delete[] init_score_; }
  if (queries_ != nullptr) { delete[] queries_; }
}


void Metadata::Init(data_size_t num_data, int weight_idx, int query_idx) {
  num_data_ = num_data;
  label_ = new float[num_data_];
  if (weight_idx >= 0) {
    if (weights_ != nullptr) {
      Log::Info("using weight in data file, and ignore additional weight file");
      delete[] weights_; 
    }
    weights_ = new float[num_data_];
    num_weights_ = num_data_;
    memset(weights_, 0, sizeof(float) * num_data_);
  }
  if (query_idx >= 0) {
    if (query_boundaries_ != nullptr) {
      Log::Info("using query id in data file, and ignore additional query file");
      delete[] query_boundaries_;
    }
    if (query_weights_ != nullptr) { delete[] query_weights_; }
    queries_ = new data_size_t[num_data_];
    memset(queries_, 0, sizeof(data_size_t) * num_data_);
  }
}

void Metadata::PartitionLabel(const std::vector<data_size_t>& used_indices) {
  if (used_indices.size() <= 0) {
    return;
  }
  float* old_label = label_;
  num_data_ = static_cast<data_size_t>(used_indices.size());
  label_ = new float[num_data_];
  for (data_size_t i = 0; i < num_data_; ++i) {
    label_[i] = old_label[used_indices[i]];
  }
  delete[] old_label;
}

void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector<data_size_t>& used_data_indices) {
  if (used_data_indices.size() == 0) {
    if (queries_ != nullptr) {
      // need convert query_id to boundaries
      std::vector<data_size_t> tmp_buffer;
      data_size_t last_qid = -1;
      data_size_t cur_cnt = 0;
      for (data_size_t i = 0; i < num_data_; ++i) {
        if (last_qid != queries_[i]) {
          if (cur_cnt > 0) {
            tmp_buffer.push_back(cur_cnt);
          }
          cur_cnt = 0;
          last_qid = queries_[i];
        }
        ++cur_cnt;
      }
      tmp_buffer.push_back(cur_cnt);
      query_boundaries_ = new data_size_t[tmp_buffer.size() + 1];
      num_queries_ = static_cast<data_size_t>(tmp_buffer.size());
      query_boundaries_[0] = 0;
      for (size_t i = 0; i < tmp_buffer.size(); ++i) {
        query_boundaries_[i + 1] = query_boundaries_[i] + tmp_buffer[i];
      }
      LoadQueryWeights();
      delete[] queries_;
      queries_ = nullptr;
    }
    // check weights
    if (weights_ != nullptr && num_weights_ != num_data_) {
      Log::Error("Initial weight size doesn't equal to data, weights will be ignored");
      delete[] weights_;
      num_weights_ = 0;
      weights_ = nullptr;
    }

    // check query boundries
    if (query_boundaries_ != nullptr && query_boundaries_[num_queries_] != num_data_) {
      Log::Error("Initial query size doesn't equal to data, queies will be ignored");
      delete[] query_boundaries_;
      num_queries_ = 0;
      query_boundaries_ = nullptr;
    }

    // contain initial score file
    if (init_score_ != nullptr && num_init_score_ != num_data_) {
      delete[] init_score_;
      Log::Error("Initial score size doesn't equal to data, score file will be ignored");
      init_score_ = nullptr;
      num_init_score_ = 0;
    }
  } else {
    data_size_t num_used_data = static_cast<data_size_t>(used_data_indices.size());
    // check weights
    if (weights_ != nullptr && num_weights_ != num_all_data) {
      Log::Error("Initial weights size doesn't equal to data, weights will be ignored");
      delete[] weights_;
      num_weights_ = 0;
      weights_ = nullptr;
    }
    // check query boundries
    if (query_boundaries_ != nullptr && query_boundaries_[num_queries_] != num_all_data) {
      Log::Error("Initial query size doesn't equal to data , queries will be ignored");
      delete[] query_boundaries_;
      num_queries_ = 0;
      query_boundaries_ = nullptr;
    }

    // contain initial score file
    if (init_score_ != nullptr && num_init_score_ != num_all_data) {
      Log::Error("Initial score size doesn't equal to data , initial scores will be ignored");
      delete[] init_score_;
      num_init_score_ = 0;
      init_score_ = nullptr;
    }

    // get local weights
    if (weights_ != nullptr) {
      float* old_weights = weights_;
      num_weights_ = num_data_;
      weights_ = new float[num_data_];
      for (size_t i = 0; i < used_data_indices.size(); ++i) {
        weights_[i] = old_weights[used_data_indices[i]];
      }
      delete[] old_weights;
    }

    // get local query boundaries
    if (query_boundaries_ != nullptr) {
      std::vector<data_size_t> used_query;
      data_size_t data_idx = 0;
      for (data_size_t qid = 0; qid < num_queries_ && data_idx < num_used_data; ++qid) {
        data_size_t start = query_boundaries_[qid];
        data_size_t end = query_boundaries_[qid + 1];
        data_size_t len = end - start;
        if (used_data_indices[data_idx] > start) {
          continue;
        } else if (used_data_indices[data_idx] == start) {
          if (num_used_data >= data_idx + len && used_data_indices[data_idx + len - 1] == end - 1) {
            used_query.push_back(qid);
            data_idx += len;
          } else {
{'fraction_non_alphanumeric': 0.11931400571661903, 'fraction_numerical': 0.004736627194773377, 'mean_word_length': 3.222758620689655, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7349325', 'n_tokens_mistral': 4087, 'n_tokens_neox': 4016, 'n_words': 966}