<commit_before>

#include "HDF5/H5Utilities.h"
#include "HDF5/H5Lite.h"
#include "Utilities/StringUtils.h"
#include "Common/LogTime.h"

// C++ Includes
#include <iostream>


#define CheckValidLocId(locId)\
  if (locId < 0 ) {std::cout << "Invalid HDF Location ID: " << locId << std::endl;return -1;}

#define CloseH5T(tid, err, retError)\
  err = H5Tclose(tid);\
  if (err < 0 ) {std::cout << "Error closing DataType" << std::endl; retErr = err;}


// *****************************************************************************************
// * H5Utilities Implementation Methods
// *****************************************************************************************

// -----------------------------------------------------------------------------
//  Returns the full path to the object referred to by the
// -----------------------------------------------------------------------------
std::string H5Utilities::getObjectPath(hid_t loc_id, bool trim)
{
  //char *obj_name;
  size_t name_size;
  name_size = 1 + H5Iget_name(loc_id, NULL, 0);
  std::vector<char> obj_name(name_size, 0);
  H5Iget_name(loc_id, &(obj_name.front()), name_size);
  std::string objPath(&(obj_name.front()));

  if ((objPath != "/") &&
      (objPath.at(0) == '/')) {
    objPath.erase(0, 1); 
  }
  
  return objPath;
}


// -----------------------------------------------------------------------------
// @brief Retrieves the HDF object type for obj_name at loc_id and stores
//    it in the parameter obj_type passed in.
// -----------------------------------------------------------------------------
herr_t H5Utilities::getObjectType(hid_t loc_id, std::string objName, 
				    int32 *obj_type)
{
  herr_t err=0;
  H5G_stat_t obj_info;
  
  err = H5Gget_objinfo(loc_id, objName.c_str(), 1, &obj_info);
  if (err < 0) {
    return err;
  }

  (* obj_type) = obj_info.type;

  return err;
}



// Opens and returns the HDF object (since the HDF api requires
//  different open and close methods for different types of objects
hid_t H5Utilities::openHDF5Object(hid_t loc_id, std::string objName)
{
  int32 obj_type;
  hid_t obj_id;
  herr_t err=0;
  HDF_ERROR_HANDLER_OFF;
  err = getObjectType(loc_id, objName, &obj_type);
  if (err < 0) {
    std::cout << "Error: Unable to get object type for object: " << objName << std::endl;
    HDF_ERROR_HANDLER_ON;
    return -1;
  }

  switch (obj_type) {
    case H5G_GROUP:
      obj_id = H5Gopen(loc_id, objName.c_str());
      break;
    case H5G_DATASET:
      obj_id = H5Dopen(loc_id, objName.c_str());
      break;
    default:
      std::cout << "Unknonwn HDF Type: " << obj_type << std::endl;
      obj_id = -1;
  }
  HDF_ERROR_HANDLER_ON;
  return obj_id;
}

// -----------------------------------------------------------------------------
//  
// -----------------------------------------------------------------------------
herr_t H5Utilities::closeHDF5Object(hid_t obj_id)
{
  H5I_type_t obj_type;
  herr_t err=-1;  // default to an error 

  obj_type = H5Iget_type(obj_id);
  
  switch(obj_type) {
  case H5I_FILE:
    err = H5Fclose(obj_id);
    break;
  case H5I_GROUP:
    err = H5Gclose(obj_id);
    break;
  case H5I_DATASET:
    err = H5Dclose(obj_id);
    break;
  case H5I_ATTR:
    err = H5Aclose(obj_id);
    break;
  case H5I_DATATYPE:
    err = H5Tclose(obj_id);
    break;
  case H5I_DATASPACE:
    err = H5Sclose(obj_id);
    break;
  default:
    std::cout << "Error unknown HDF object for closing: " 
	      << obj_type << std::endl;
    err = -1;
  }

  return err;
}
  
  


//--------------------------------------------------------------------//
// HDF Group Methods
//--------------------------------------------------------------------//

/*! @brief Returns a std::list of std::strings containing the names
 *   of all objects attached to the group referred to by loc_id
 * 
 * @parameter typeFilter is one of HDF5_GROUP, HDF5_DATASET, HDF5_TYPE,
 *  or HDF5_LINK or any combination of these using the bitwise or | 
 *  command.  Or you can pass in HDF5_ANY (equivalent to: HDF5_GROUP |
 *  HDF5_DATASET | HDF5_TYPE | HDF5_LINK) to not filter at all
 */
herr_t H5Utilities::getGroupObjects(hid_t loc_id, int32 typeFilter, std::list<std::string>& names)
{
  herr_t err=0;
  hsize_t numObjs = 0;
  
  err = H5Gget_num_objs(loc_id, &numObjs);
  if (err < 0) {
    std::cout << "Error getting number of objects for group: " 
	      << loc_id << std::endl;
    return err;
  }
  
  if (numObjs <= 0) {
    return 0; // no objects in group
  }
   
  size_t size=0;
  int32 type=-1;
  
  for (hsize_t i=0; i<numObjs; i++) {
    size = 1 + H5Gget_objname_by_idx(loc_id, i, NULL, 0);
    std::vector<char> name(size * sizeof(char), 0);
      
    H5Gget_objname_by_idx(loc_id, i, &(name.front()), size);
    if (typeFilter == MXA_ANY) {
      std::string objName( &(name.front()) );
      names.push_back(objName);
    } else {
      type = H5Gget_objtype_by_idx(loc_id, i);
      if ((type == H5G_GROUP) && (MXA_GROUP & typeFilter) ||
          (type == H5G_DATASET) && (MXA_DATASET & typeFilter) ||
          (type == H5G_TYPE) && (MXA_TYPE & typeFilter) ||
          (type == H5G_LINK) && (MXA_LINK & typeFilter)) 
      {
        std::string objName( &(name.front()) );
        names.push_back(objName);
      }
    }
  }
  
  return err;
}


// HDF Creation/Modification Methods
hid_t H5Utilities::createGroup(hid_t loc_id, std::string group) 
{
  hid_t grp_id;
  herr_t err = -1;

  // Suspend the HDF Error Handlers
  herr_t (*_oldHDF_error_func)(void *);
  void *_oldHDF_error_client_data;

  // Store the current HDF error handlers
  H5Eget_auto(&_oldHDF_error_func, &_oldHDF_error_client_data);

  // Turn off error handling
  H5Eset_auto(NULL, NULL);

  err = H5Gget_objinfo(loc_id, group.c_str(), 0, NULL);
//  std::cout << "H5Gget_objinfo = " << err << " for " << group << std::endl;  
  if (err == 0)
  {
    grp_id = H5Gopen(loc_id, group.c_str());
  } else 
  {
    grp_id = H5Gcreate(loc_id, group.c_str(), 0);
  }
  // Turn the HDF Error handlers back on
  H5Eset_auto(_oldHDF_error_func, _oldHDF_error_client_data);
  return grp_id;
}

// -----------------------------------------------------------------------------
//  
// -----------------------------------------------------------------------------
int32 H5Utilities::createGroupsFromPath(std::string path, hid_t parent)
{
  
  hid_t gid = 1;
  herr_t err = -1;
  std::string first;
  std::string second;
  
  if (parent <= 0) {
    std::cout << "Bad parent Id. Returning from createGroupsFromPath" << std::endl;
    return -1;
  }
  // remove any front slash
  std::string::size_type pos = path.find_first_of("/", 0);
  if ( 0 == pos ) 
  {
    path = path.substr(1, path.size());
  } 
  else if (pos == std::string::npos) // Path contains only one element
  {
    gid = H5Utilities::createGroup(parent, path);
    if (gid < 0)
    {
      std::cout << "Error creating group: " << path << " err:" << gid << std::endl;
      return gid;
    }
    err = H5Gclose(gid);
    if (err < 0) { std::cout << logTime() << "Error closing group during group creation." << std::endl; return err; }
    return err; //Now return here as this was a special case.
  }
  
  //Remove any trailing slash
  pos = path.find_last_of("/");
  if ( pos == (path.size() - 1) ) // slash was in the last position
  {
    path = path.substr(0, pos);
  }
  
  if (path.size() == 0) 
  {
    return -1; // The path that was passed in was only a slash.. 
  }

  
  pos = path.find_first_of("/", 0);
  while (pos != std::string::npos )
  {
    first = path.substr(0, pos);
    second = path.substr(pos+1, path.length() );
    gid = H5Utilities::createGroup(parent,first);
    if (gid < 0)
    {
      std::cout << "Error creating group:" << gid << std::endl;
      return gid;
    }
    err = H5Gclose(gid);
    pos = path.find_first_of("/", pos+1);
    if (pos == std::string::npos) 
    {
      first += "/" + second;
      gid = createGroup(parent,first);
      if (gid < 0)
      {
        std::cout << "Error creating group:" << gid << std::endl;<commit_msg>Fixed bug in createGroupsFromPath(...) method where a path with a single element  would return an error.<commit_after>

#include "HDF5/H5Utilities.h"
#include "HDF5/H5Lite.h"
#include "Utilities/StringUtils.h"
#include "Common/LogTime.h"

// C++ Includes
#include <iostream>


#define CheckValidLocId(locId)\
  if (locId < 0 ) {std::cout << "Invalid HDF Location ID: " << locId << std::endl;return -1;}

#define CloseH5T(tid, err, retError)\
  err = H5Tclose(tid);\
  if (err < 0 ) {std::cout << "Error closing DataType" << std::endl; retErr = err;}


// *****************************************************************************************
// * H5Utilities Implementation Methods
// *****************************************************************************************

// -----------------------------------------------------------------------------
//  Returns the full path to the object referred to by the
// -----------------------------------------------------------------------------
std::string H5Utilities::getObjectPath(hid_t loc_id, bool trim)
{
  //char *obj_name;
  size_t name_size;
  name_size = 1 + H5Iget_name(loc_id, NULL, 0);
  std::vector<char> obj_name(name_size, 0);
  H5Iget_name(loc_id, &(obj_name.front()), name_size);
  std::string objPath(&(obj_name.front()));

  if ((objPath != "/") &&
      (objPath.at(0) == '/')) {
    objPath.erase(0, 1); 
  }
  
  return objPath;
}


// -----------------------------------------------------------------------------
// @brief Retrieves the HDF object type for obj_name at loc_id and stores
//    it in the parameter obj_type passed in.
// -----------------------------------------------------------------------------
herr_t H5Utilities::getObjectType(hid_t loc_id, std::string objName, 
				    int32 *obj_type)
{
  herr_t err=0;
  H5G_stat_t obj_info;
  
  err = H5Gget_objinfo(loc_id, objName.c_str(), 1, &obj_info);
  if (err < 0) {
    return err;
  }

  (* obj_type) = obj_info.type;

  return err;
}



// Opens and returns the HDF object (since the HDF api requires
//  different open and close methods for different types of objects
hid_t H5Utilities::openHDF5Object(hid_t loc_id, std::string objName)
{
  int32 obj_type;
  hid_t obj_id;
  herr_t err=0;
  HDF_ERROR_HANDLER_OFF;
  err = getObjectType(loc_id, objName, &obj_type);
  if (err < 0) {
    std::cout << "Error: Unable to get object type for object: " << objName << std::endl;
    HDF_ERROR_HANDLER_ON;
    return -1;
  }

  switch (obj_type) {
    case H5G_GROUP:
      obj_id = H5Gopen(loc_id, objName.c_str());
      break;
    case H5G_DATASET:
      obj_id = H5Dopen(loc_id, objName.c_str());
      break;
    default:
      std::cout << "Unknonwn HDF Type: " << obj_type << std::endl;
      obj_id = -1;
  }
  HDF_ERROR_HANDLER_ON;
  return obj_id;
}

// -----------------------------------------------------------------------------
//  
// -----------------------------------------------------------------------------
herr_t H5Utilities::closeHDF5Object(hid_t obj_id)
{
  H5I_type_t obj_type;
  herr_t err=-1;  // default to an error 

  obj_type = H5Iget_type(obj_id);
  
  switch(obj_type) {
  case H5I_FILE:
    err = H5Fclose(obj_id);
    break;
  case H5I_GROUP:
    err = H5Gclose(obj_id);
    break;
  case H5I_DATASET:
    err = H5Dclose(obj_id);
    break;
  case H5I_ATTR:
    err = H5Aclose(obj_id);
    break;
  case H5I_DATATYPE:
    err = H5Tclose(obj_id);
    break;
  case H5I_DATASPACE:
    err = H5Sclose(obj_id);
    break;
  default:
    std::cout << "Error unknown HDF object for closing: " 
	      << obj_type << std::endl;
    err = -1;
  }

  return err;
}
  
  


//--------------------------------------------------------------------//
// HDF Group Methods
//--------------------------------------------------------------------//

/*! @brief Returns a std::list of std::strings containing the names
 *   of all objects attached to the group referred to by loc_id
 * 
 * @parameter typeFilter is one of HDF5_GROUP, HDF5_DATASET, HDF5_TYPE,
 *  or HDF5_LINK or any combination of these using the bitwise or | 
 *  command.  Or you can pass in HDF5_ANY (equivalent to: HDF5_GROUP |
 *  HDF5_DATASET | HDF5_TYPE | HDF5_LINK) to not filter at all
 */
herr_t H5Utilities::getGroupObjects(hid_t loc_id, int32 typeFilter, std::list<std::string>& names)
{
  herr_t err=0;
  hsize_t numObjs = 0;
  
  err = H5Gget_num_objs(loc_id, &numObjs);
  if (err < 0) {
    std::cout << "Error getting number of objects for group: " 
	      << loc_id << std::endl;
    return err;
  }
  
  if (numObjs <= 0) {
    return 0; // no objects in group
  }
   
  size_t size=0;
  int32 type=-1;
  
  for (hsize_t i=0; i<numObjs; i++) {
    size = 1 + H5Gget_objname_by_idx(loc_id, i, NULL, 0);
    std::vector<char> name(size * sizeof(char), 0);
      
    H5Gget_objname_by_idx(loc_id, i, &(name.front()), size);
    if (typeFilter == MXA_ANY) {
      std::string objName( &(name.front()) );
      names.push_back(objName);
    } else {
      type = H5Gget_objtype_by_idx(loc_id, i);
      if ((type == H5G_GROUP) && (MXA_GROUP & typeFilter) ||
          (type == H5G_DATASET) && (MXA_DATASET & typeFilter) ||
          (type == H5G_TYPE) && (MXA_TYPE & typeFilter) ||
          (type == H5G_LINK) && (MXA_LINK & typeFilter)) 
      {
        std::string objName( &(name.front()) );
        names.push_back(objName);
      }
    }
  }
  
  return err;
}


// HDF Creation/Modification Methods
hid_t H5Utilities::createGroup(hid_t loc_id, std::string group) 
{
  hid_t grp_id;
  herr_t err = -1;

  // Suspend the HDF Error Handlers
  herr_t (*_oldHDF_error_func)(void *);
  void *_oldHDF_error_client_data;

  // Store the current HDF error handlers
  H5Eget_auto(&_oldHDF_error_func, &_oldHDF_error_client_data);

  // Turn off error handling
  H5Eset_auto(NULL, NULL);

  err = H5Gget_objinfo(loc_id, group.c_str(), 0, NULL);
//  std::cout << "H5Gget_objinfo = " << err << " for " << group << std::endl;  
  if (err == 0)
  {
    grp_id = H5Gopen(loc_id, group.c_str());
  } else 
  {
    grp_id = H5Gcreate(loc_id, group.c_str(), 0);
  }
  // Turn the HDF Error handlers back on
  H5Eset_auto(_oldHDF_error_func, _oldHDF_error_client_data);
  return grp_id;
}

// -----------------------------------------------------------------------------
//  
// -----------------------------------------------------------------------------
int32 H5Utilities::createGroupsFromPath(std::string path, hid_t parent)
{
  
  hid_t gid = 1;
  herr_t err = -1;
  std::string first;
  std::string second;
  
  if (parent <= 0) {
    std::cout << "Bad parent Id. Returning from createGroupsFromPath" << std::endl;
    return -1;
  }
  // remove any front slash
  std::string::size_type pos = path.find_first_of("/", 0);
  if ( 0 == pos ) 
  {
    path = path.substr(1, path.size());
  } 
  else if (pos == std::string::npos) // Path contains only one element
  {
    gid = H5Utilities::createGroup(parent, path);
    if (gid < 0)
    {
      std::cout << "Error creating group: " << path << " err:" << gid << std::endl;
      return gid;
    }
    err = H5Gclose(gid);
    if (err < 0) { std::cout << logTime() << "Error closing group during group creation." << std::endl; return err; }
    return err; //Now return here as this was a special case.
  }
  
  //Remove any trailing slash
  pos = path.find_last_of("/");
  if ( pos == (path.size() - 1) ) // slash was in the last position
  {
    path = path.substr(0, pos);
  }
  
  if (path.size() == 0) 
  {
    return -1; // The path that was passed in was only a slash.. 
  }

  
  pos = path.find_first_of("/", 0);
  if (pos == std::string::npos) // Only one element in the path
  {
    gid = H5Utilities::createGroup(parent, path);
    if (gid < 0)
    {
      std::cout << "Error creating group '" << path << "' for group id " << gid << std::endl;
      return gid;
    }
    err = H5Gclose(gid);
    return err;
  }
  
  while (pos != std::string::npos )
  {
    first = path.substr(0, pos);
    second = path.substr(pos+1, path.length() );
    gid = H5Utilities::createGroup(parent,first);
    if (gid < 0)
    {
      std::cout << "Error creating group:" << gid << std::endl;
      return gid;
    }
    err = H5Gclose(gid);
    pos = path.find_first_of("/", pos+1);
    if (pos == std::string::npos) 
    {
      first += "/" + second;
      gid = createGroup(parent,first);
      if (gid < 0)
      {
        std::cout << "Error creating group:" << gid << std::endl;
{'fraction_non_alphanumeric': 0.24222060782177018, 'fraction_numerical': 0.01567986439036203, 'mean_word_length': 2.1464761904761906, 'pattern_counts': {'":': 0, '<': 198, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5076324', 'n_tokens_mistral': 5886, 'n_tokens_neox': 5739, 'n_words': 1377}