<commit_before>#include <iostream>

#include <SmurffCpp/Utils/HDF5.h>

#include <Utils/Error.h>
#include <Utils/StringUtils.h>


namespace smurff {

bool HDF5::hasDataSet(const std::string& section, const std::string& tag) const
{
   if (!m_group.exist(section)) return false;
   auto section_group = m_group.getGroup(section);
   return (section_group.exist(tag));
}

bool HDF5::hasSection(const std::string& section) const
{
   return m_group.exist(section);
}

h5::Group HDF5::addGroup(const std::string& name)
{
   if (!m_group.exist(name)) m_group.createGroup(name);
   return getGroup(name);
}

h5::Group HDF5::getGroup(const std::string& name) const
{
   return m_group.getGroup(name);
}

void HDF5::read(const std::string& section, const std::string& tag, Matrix &X) const
{
   auto dataset = m_group.getGroup(section).getDataSet(tag);
   std::vector<size_t> dims = dataset.getDimensions();
   X.resize(dims[0], dims[1]);
   dataset.read(X.data());

   // convert to ColMajor if needed (HDF5 always stores row-major)
   if (!X.IsRowMajor)
   {
      X = Eigen::Map<Eigen::Matrix<
            typename Matrix::Scalar,
            Matrix::RowsAtCompileTime,
            Matrix::ColsAtCompileTime,
               Eigen::RowMajor,
            Matrix::MaxRowsAtCompileTime,
               Matrix::MaxColsAtCompileTime>>(X.data(), dims[0], dims[1]);
   }
}

void HDF5::read(const std::string& section, const std::string& tag, Vector &X) const
{
   auto dataset = m_group.getGroup(section).getDataSet(tag);
   std::vector<size_t> dims = dataset.getDimensions();
   THROWERROR_ASSERT(dims[1] == 1);
   X.resize(dims[0]);
   dataset.read(X.data());
}

void HDF5::read(const std::string& section, const std::string& tag, SparseMatrix &X) const
{
   auto sparse_group = m_group.getGroup(section).getGroup(tag); 

   std::string format;
   sparse_group.getAttribute("h5sparse_format").read(format);
   THROWERROR_ASSERT(( SparseMatrix::IsRowMajor && format == "csr") || \
                     (!SparseMatrix::IsRowMajor && format == "csc"));
   
   std::vector<Eigen::Index> shape(2);
   sparse_group.getAttribute("h5sparse_shape").read(shape);<commit_msg>FIX: fix conversion col-major <-> row-major<commit_after>#include <iostream>

#include <SmurffCpp/Utils/HDF5.h>

#include <Utils/Error.h>
#include <Utils/StringUtils.h>


namespace smurff {

bool HDF5::hasDataSet(const std::string& section, const std::string& tag) const
{
   if (!m_group.exist(section)) return false;
   auto section_group = m_group.getGroup(section);
   return (section_group.exist(tag));
}

bool HDF5::hasSection(const std::string& section) const
{
   return m_group.exist(section);
}

h5::Group HDF5::addGroup(const std::string& name)
{
   if (!m_group.exist(name)) m_group.createGroup(name);
   return getGroup(name);
}

h5::Group HDF5::getGroup(const std::string& name) const
{
   return m_group.getGroup(name);
}

void HDF5::read(const std::string& section, const std::string& tag, Matrix &X) const
{
   auto dataset = m_group.getGroup(section).getDataSet(tag);
   std::vector<size_t> dims = dataset.getDimensions();
   X.resize(dims[0], dims[1]);
   dataset.read(X.data());

   if (!X.IsRowMajor)
   {
      Matrix colMajor = Eigen::Map<Eigen::Matrix<
            typename Matrix::Scalar,
            Matrix::RowsAtCompileTime,
            Matrix::ColsAtCompileTime,
            Matrix::ColsAtCompileTime==1?Eigen::ColMajor:Eigen::RowMajor,
            Matrix::MaxRowsAtCompileTime,
            Matrix::MaxColsAtCompileTime>>(X.data(), dims[0], dims[1]);

      std::swap(colMajor, X);
   }
}

void HDF5::read(const std::string& section, const std::string& tag, Vector &X) const
{
   auto dataset = m_group.getGroup(section).getDataSet(tag);
   std::vector<size_t> dims = dataset.getDimensions();
   THROWERROR_ASSERT(dims[1] == 1);
   X.resize(dims[0]);
   dataset.read(X.data());
}

void HDF5::read(const std::string& section, const std::string& tag, SparseMatrix &X) const
{
   auto sparse_group = m_group.getGroup(section).getGroup(tag); 

   std::string format;
   sparse_group.getAttribute("h5sparse_format").read(format);
   THROWERROR_ASSERT(( SparseMatrix::IsRowMajor && format == "csr") || \
                     (!SparseMatrix::IsRowMajor && format == "csc"));
   
   std::vector<Eigen::Index> shape(2);
   sparse_group.getAttribute("h5sparse_shape").read(shape);
{'fraction_non_alphanumeric': 0.1569754079521949, 'fraction_numerical': 0.009652953344058838, 'mean_word_length': 4.379480840543882, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3922917', 'n_tokens_mistral': 1526, 'n_tokens_neox': 1503, 'n_words': 271}