<commit_before>/**
 * @file h5_pool.cpp
 * @brief implementation of #h5_pool_iface
 * @author <NAME>
 * @version 1.0
 * @date 2011-03-05
*/
#ifdef BSPY_EXPORTING_PLUGIN
#include <boost/python.hpp>
#endif

#include <hdf5.h>
#include <stdio.h>
#include <iomanip>

#include "h5_pool.hpp"
#include "h5_helper.h"

#include "hdf5_type.h"
#include "hdf5_type_to_hid.hpp"
#include "hdf5_hid_holder.hpp"
#include "hdf5_functions.h"
#include "date_helper.h"
#include "bos_report.h"

using namespace std;
using namespace boost::python;


namespace blue_sky
{
  error_h5_no_array::error_h5_no_array (std::string const &array_name)
  : bs_exception ("error_h5_no_array", array_name)
  {
  }

  h5_pool::h5_pool (bs_type_ctor_param)
        : h5_pool_iface ()
    {
      init_all ();
      mute_flag = 0;

    }
  h5_pool::h5_pool (const h5_pool & /*src*/) : bs_refcounter ()
    {
      init_all ();
      mute_flag = 0;
    }
  h5_pool::~h5_pool ()
    {
      close_file ();
    }

  herr_t
  h5_pool::it_group (hid_t g_id, const char *name, const H5L_info_t * /*info*/, void * m)
    {
      hid_t dset = detail::open_dataset (g_id, name);
      hid_t dspace = H5Dget_space (dset);
      if (dspace < 0)
        {
          bs_throw_exception (boost::format ("Can't get dataspace for dataset %s in group %d") % name % g_id);
        }

      hid_t dtype = H5Dget_type (dset);
      if (dtype < 0)
        {
          bs_throw_exception (boost::format ("Can't get datatype for dataset %s in group %d") % name % g_id);
        }

      hid_t plist = H5Dget_create_plist (dset);
      if (plist < 0)
        {
          bs_throw_exception (boost::format ("Can't get property list for dataset %s in group %d") % name % g_id);
        }

      hsize_t dims_[10] = {0};
      // FIXME: check
      int n_dims = H5Sget_simple_extent_dims (dspace, dims_, NULL);
      map_t::iterator it = ((h5_pool *)m)->add_node (name, g_id, dset, dspace, dtype, plist, n_dims, dims_, 0);

      BOSOUT (section::h5, level::low)
        << "Node from file: " << name << bs_end;
      return 0;
    }

  template <class T> h5_pool::map_t::iterator
  h5_pool::add_node (const std::string &name, const hid_t group_id,
                     const hid_t dset, const hid_t dspace,
                     const hid_t dtype, const hid_t plist,
                     const int n_dims, const T *dims, const bool var_dims)
    {
      BS_ASSERT (dtype >= 0) (name);
      BS_ASSERT (plist >= 0) (name);
      pair_t p;

      p.first = name;
      p.second.dset = dset;
      p.second.dspace = dspace;
      p.second.dtype = dtype;
      p.second.plist = plist;
      p.second.n_dims = n_dims;
      p.second.var_dims = var_dims;
      p.second.size = 0;
      p.second.group_id = group_id;
      if (var_dims)
        {
          for (int i = 0; i < 6; ++i)
            {
              p.second.src_dims[i] = (hsize_t)dims[i];
            }
        }
      else
        {
          p.second.size = 1;
          // FIXME: check n_dims
          for (int i = 0; i < n_dims; ++i)
            {
              p.second.py_dims[i] = (npy_intp)dims[i];
              p.second.h5_dims[i] = (hsize_t)dims[i];

              p.second.size *= dims[i];
            }
        }
      return h5_map.insert (p).first;
    }

  void
  h5_pool::fill_map (hid_t group)
  {
    BS_ASSERT (group);
    // dont walk inside group whose name begins with '_'
    char gname[3] = {0, 0, 0};
    H5Iget_name(group, &gname[0], 3);
    if(gname[1] == '_') return;

    clear_map ();
    herr_t r = H5Literate (group, H5_INDEX_NAME, H5_ITER_NATIVE, NULL, &(h5_pool::it_group), this);
    if (r < 0)
      {
        bs_throw_exception (boost::format ("iterate fails"));
      }
  }

  void
  h5_pool::close_node (h5_pair &p)
    {
      if (p.dset >= 0)
        {
          H5Dclose (p.dset);
          p.dset = -1;
        }
      if (p.dspace >= 0)
        {
          H5Sclose (p.dspace);
          p.dspace = -1;
        }
      if (p.dtype >= 0)
        {
          H5Tclose (p.dtype);
          p.dtype = -1;
        }
      if (p.plist >= 0)
        {
          H5Pclose (p.plist);
          p.plist = -1;
        }
    }

  void
  h5_pool::clear_map ()
    {
      map_t::iterator i, e;
      e = h5_map.end ();
      for (i = h5_map.begin (); i != e; ++i)
        {
          close_node (i->second);
        }
      h5_map.clear ();
    }

  void
  h5_pool::open_file (const std::string& fname_)
    {
	  //this function is not really open, it creates file
      edit_base = true;

      // FIXME:
      if (file_id >= 0)
        {
          close_file ();
        }

      file_id = H5Fcreate (fname_.c_str (), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
      if (file_id < 0)
        {
          bs_throw_exception (boost::format ("Can't create h5 file: %s !") % fname_.c_str ());
        }

      fname = fname_;

      //TODO: Open existing file: remove H5F_ACC_TRUNC, parse and add existing groups
      group_id.clear ();
      //std::string base_name = "base_" + get_date_time_str();
      //group_id.insert (pair<std::string, hid_t>(base_name.c_str(), -1));
      group_id.insert (pair<std::string, hid_t>("actual", -1));

      map_hid_t::iterator i;
      for (i = group_id.begin(); i != group_id.end(); ++i)
        {
          std::string path_ = i->first;<commit_msg>h5_pool: possibly FIX "Can't create H5 file" bug!<commit_after>/**
 * @file h5_pool.cpp
 * @brief implementation of #h5_pool_iface
 * @author <NAME>
 * @version 1.0
 * @date 2011-03-05
*/
#ifdef BSPY_EXPORTING_PLUGIN
#include <boost/python.hpp>
#endif

#include <hdf5.h>
#include <stdio.h>
#include <iomanip>
#include <cstdio>

#include "h5_pool.hpp"
#include "h5_helper.h"

#include "hdf5_type.h"
#include "hdf5_type_to_hid.hpp"
#include "hdf5_hid_holder.hpp"
#include "hdf5_functions.h"
#include "date_helper.h"
#include "bos_report.h"

using namespace std;
using namespace boost::python;


namespace blue_sky
{
  error_h5_no_array::error_h5_no_array (std::string const &array_name)
  : bs_exception ("error_h5_no_array", array_name)
  {
  }

  h5_pool::h5_pool (bs_type_ctor_param)
        : h5_pool_iface ()
    {
      init_all ();
      mute_flag = 0;

    }
  h5_pool::h5_pool (const h5_pool & /*src*/) : bs_refcounter ()
    {
      init_all ();
      mute_flag = 0;
    }
  h5_pool::~h5_pool ()
    {
      close_file ();
    }

  herr_t
  h5_pool::it_group (hid_t g_id, const char *name, const H5L_info_t * /*info*/, void * m)
    {
      hid_t dset = detail::open_dataset (g_id, name);
      hid_t dspace = H5Dget_space (dset);
      if (dspace < 0)
        {
          bs_throw_exception (boost::format ("Can't get dataspace for dataset %s in group %d") % name % g_id);
        }

      hid_t dtype = H5Dget_type (dset);
      if (dtype < 0)
        {
          bs_throw_exception (boost::format ("Can't get datatype for dataset %s in group %d") % name % g_id);
        }

      hid_t plist = H5Dget_create_plist (dset);
      if (plist < 0)
        {
          bs_throw_exception (boost::format ("Can't get property list for dataset %s in group %d") % name % g_id);
        }

      hsize_t dims_[10] = {0};
      // FIXME: check
      int n_dims = H5Sget_simple_extent_dims (dspace, dims_, NULL);
      map_t::iterator it = ((h5_pool *)m)->add_node (name, g_id, dset, dspace, dtype, plist, n_dims, dims_, 0);

      BOSOUT (section::h5, level::low)
        << "Node from file: " << name << bs_end;
      return 0;
    }

  template <class T> h5_pool::map_t::iterator
  h5_pool::add_node (const std::string &name, const hid_t group_id,
                     const hid_t dset, const hid_t dspace,
                     const hid_t dtype, const hid_t plist,
                     const int n_dims, const T *dims, const bool var_dims)
    {
      BS_ASSERT (dtype >= 0) (name);
      BS_ASSERT (plist >= 0) (name);
      pair_t p;

      p.first = name;
      p.second.dset = dset;
      p.second.dspace = dspace;
      p.second.dtype = dtype;
      p.second.plist = plist;
      p.second.n_dims = n_dims;
      p.second.var_dims = var_dims;
      p.second.size = 0;
      p.second.group_id = group_id;
      if (var_dims)
        {
          for (int i = 0; i < 6; ++i)
            {
              p.second.src_dims[i] = (hsize_t)dims[i];
            }
        }
      else
        {
          p.second.size = 1;
          // FIXME: check n_dims
          for (int i = 0; i < n_dims; ++i)
            {
              p.second.py_dims[i] = (npy_intp)dims[i];
              p.second.h5_dims[i] = (hsize_t)dims[i];

              p.second.size *= dims[i];
            }
        }
      return h5_map.insert (p).first;
    }

  void
  h5_pool::fill_map (hid_t group)
  {
    BS_ASSERT (group);
    // dont walk inside group whose name begins with '_'
    char gname[3] = {0, 0, 0};
    H5Iget_name(group, &gname[0], 3);
    if(gname[1] == '_') return;

    clear_map ();
    herr_t r = H5Literate (group, H5_INDEX_NAME, H5_ITER_NATIVE, NULL, &(h5_pool::it_group), this);
    if (r < 0)
      {
        bs_throw_exception (boost::format ("iterate fails"));
      }
  }

  void
  h5_pool::close_node (h5_pair &p)
    {
      if (p.dset >= 0)
        {
          H5Dclose (p.dset);
          p.dset = -1;
        }
      if (p.dspace >= 0)
        {
          H5Sclose (p.dspace);
          p.dspace = -1;
        }
      if (p.dtype >= 0)
        {
          H5Tclose (p.dtype);
          p.dtype = -1;
        }
      if (p.plist >= 0)
        {
          H5Pclose (p.plist);
          p.plist = -1;
        }
    }

  void
  h5_pool::clear_map ()
    {
      map_t::iterator i, e;
      e = h5_map.end ();
      for (i = h5_map.begin (); i != e; ++i)
        {
          close_node (i->second);
        }
      h5_map.clear ();
    }

  void
  h5_pool::open_file (const std::string& fname_)
    {
	  //this function is not really open, it creates file
      edit_base = true;

      // FIXME:
      if (file_id >= 0)
        {
          close_file ();
        }

      // check if file already exists, if so -- delete it
      FILE* f;
      if((f = fopen(fname_.c_str(), "r")) != 0) {
        fclose(f);
        remove(fname_.c_str());
      }

      file_id = H5Fcreate (fname_.c_str (), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
      if (file_id < 0)
        {
          bs_throw_exception (boost::format ("Can't create h5 file: %s !") % fname_.c_str ());
        }

      fname = fname_;

      //TODO: Open existing file: remove H5F_ACC_TRUNC, parse and add existing groups
      group_id.clear ();
      //std::string base_name = "base_" + get_date_time_str();
      //group_id.insert (pair<std::string, hid_t>(base_name.c_str(), -1));
      group_id.insert (pair<std::string, hid_t>("actual", -1));

      map_hid_t::iterator i;
      for (i = group_id.begin(); i != group_id.end(); ++i)
        {
          std::string path_ = i->first;
{'fraction_non_alphanumeric': 0.14077893379983428, 'fraction_numerical': 0.01905901850658319, 'mean_word_length': 1.9906387665198237, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3368780', 'n_tokens_mistral': 4254, 'n_tokens_neox': 4158, 'n_words': 905}