<commit_before>#include "copy_root_tree.hh"
#include "HdfTuple.hh"

#include "TFile.h"
#include "TTree.h"
#include "TLeaf.h"

#include <iostream>

class DataBuffers: public std::vector<data_buffer_t*> {
public:
  ~DataBuffers() {
    for (auto buf : *this) {
      delete buf;
    }
  }
};

// functions to convert union types back to whatever they are
template <typename T>
T& get_ref(data_buffer_t* buf);
template<> int& get_ref<int>(data_buffer_t* buf) {
  return buf->_int;
}
template<> float& get_ref<float>(data_buffer_t* buf) {
  return buf->_float;
}
template<> double& get_ref<double>(data_buffer_t* buf) {
  return buf->_double;
}
template<> bool& get_ref<bool>(data_buffer_t* buf) {
  return buf->_bool;
}

// copy function
template <typename T>
void set_branch(VariableFillers& vars, TTree& tt, DataBuffers& buffer,
                const std::string& name) {
  buffer.push_back(new data_buffer_t);
  T& buf = get_ref<T>(buffer.back());
  tt.SetBranchAddress(name.c_str(), &buf);
  vars.add<T>(name, [&buf](){return buf;});
}

// main function to do all the things
void copy_root_tree(TTree& tt, H5::CommonFG& fg) {
  const std::string tree_name = tt.GetName();

  DataBuffers buffer;
  std::set<std::string> skipped;

  TIter next(tt.GetListOfLeaves());
  VariableFillers vars;
  TLeaf* leaf;
  while ((leaf = dynamic_cast<TLeaf*>(next()))) {
    std::string leaf_name = leaf->GetName();
    std::string leaf_type = leaf->GetTypeName();
    if (leaf_type == "Int_t") {
      set_branch<int>(vars, tt, buffer, leaf_name);
    } else if (leaf_type == "Float_t") {
      set_branch<float>(vars, tt, buffer, leaf_name);
    } else if (leaf_type == "Double_t") {
      set_branch<double>(vars, tt, buffer, leaf_name);
    } else if (leaf_type == "Bool_t") {
      set_branch<bool>(vars, tt, buffer, leaf_name);
    } else {
      skipped.insert(leaf_type);
    }
  }

  Writer writer(fg, tree_name, vars);
  size_t n_entries = tt.GetEntries();
  for (size_t iii = 0; iii < n_entries; iii++) {
    tt.GetEntry(iii);
    writer.fill();
  }
  writer.flush();
  writer.close();

  for (const auto& name: skipped) {
    std::cerr << "skipped " << name << std::endl;
  }
}<commit_msg>Prevent adding identical leaf names<commit_after>#include "copy_root_tree.hh"
#include "HdfTuple.hh"

#include "TFile.h"
#include "TTree.h"
#include "TLeaf.h"

#include <iostream>

class DataBuffers: public std::vector<data_buffer_t*> {
public:
  ~DataBuffers() {
    for (auto buf : *this) {
      delete buf;
    }
  }
};

// functions to convert union types back to whatever they are
template <typename T>
T& get_ref(data_buffer_t* buf);
template<> int& get_ref<int>(data_buffer_t* buf) {
  return buf->_int;
}
template<> float& get_ref<float>(data_buffer_t* buf) {
  return buf->_float;
}
template<> double& get_ref<double>(data_buffer_t* buf) {
  return buf->_double;
}
template<> bool& get_ref<bool>(data_buffer_t* buf) {
  return buf->_bool;
}

// copy function
template <typename T>
void set_branch(VariableFillers& vars, TTree& tt, DataBuffers& buffer,
                const std::string& name) {
  buffer.push_back(new data_buffer_t);
  T& buf = get_ref<T>(buffer.back());
  tt.SetBranchAddress(name.c_str(), &buf);
  vars.add<T>(name, [&buf](){return buf;});
}

// main function to do all the things
void copy_root_tree(TTree& tt, H5::CommonFG& fg) {
  const std::string tree_name = tt.GetName();

  DataBuffers buffer;
  std::set<std::string> skipped;

  TIter next(tt.GetListOfLeaves());
  TLeaf* leaf;
  std::set<std::string> leaf_names;
  while ((leaf = dynamic_cast<TLeaf*>(next()))) {
    leaf_names.insert(leaf->GetName());
  }
  VariableFillers vars;
  for (const auto& leaf_name: leaf_names) {
    leaf = tt.GetLeaf(leaf_name.c_str());
    std::string leaf_type = leaf->GetTypeName();
    if (leaf_type == "Int_t") {
      set_branch<int>(vars, tt, buffer, leaf_name);
    } else if (leaf_type == "Float_t") {
      set_branch<float>(vars, tt, buffer, leaf_name);
    } else if (leaf_type == "Double_t") {
      set_branch<double>(vars, tt, buffer, leaf_name);
    } else if (leaf_type == "Bool_t") {
      set_branch<bool>(vars, tt, buffer, leaf_name);
    } else {
      skipped.insert(leaf_type);
    }
  }

  Writer writer(fg, tree_name, vars);
  size_t n_entries = tt.GetEntries();
  for (size_t iii = 0; iii < n_entries; iii++) {
    tt.GetEntry(iii);
    writer.fill();
  }
  writer.flush();
  writer.close();

  for (const auto& name: skipped) {
    std::cerr << "skipped " << name << std::endl;
  }
}
{'fraction_non_alphanumeric': 0.16153334810547307, 'fraction_numerical': 0.0008863283846665189, 'mean_word_length': 4.054871220604703, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '200509', 'n_tokens_mistral': 1670, 'n_tokens_neox': 1644, 'n_words': 379}