<commit_before>/**
 * @file poh5.c
 * @brief Source file for poh5 API.
 * @author <NAME> @ RIST
 * @date 2016/08/12
 */


#include <stdlib.h>

#include "poh5.h"
#include "hdf5.h"

typedef struct{
  int64_t s;
  int64_t e;
} tp_t;

#ifndef DBGOUT
/* #define DBGOUT stderr */
#define DBGOUT stdout
#endif


static hid_t hlng_tid; /** datatype for long strings */
static hid_t hmid_tid; /** datatype for middle strings */
static hid_t hsht_tid; /** datatype for short strings */


/*========================================================================*/
/*
 * Declaration of internal routines
 *
 */
int write_gattr(const hid_t file_id, const char *name, const hid_t f_tid, const hid_t sid,
                const hid_t m_tid, const void *vals);
int update_num_of_var( hid_t file_id );
int extend_dataset( hid_t v_gid, char *dname);
int write_dataset( const hid_t   v_gid, const int32_t step,  const char   *dname, 
                   const hid_t   type,  const void   *data);
int write_dataset_1rgn( const hid_t   v_gid, const int32_t step,  const int32_t ll,   
                        const hid_t   type, const void   *data);
int read_dataset( const hid_t   v_gid, const int32_t step,  const char   *dname,
                  const hid_t   type,  void         *data);

void check_h5( int );

/*========================================================================*/
/*
 * Implementation of public routines
 *
 */

/*========================================================================*/
/** @brief Setup poh5 library.
 *
 * Call this before everything.
 *
 * @return 0 if success.
 */
int poh5_setup(){

#ifdef DEBUG
  fprintf(DBGOUT,"dbg:poh5_setup:\n");
#endif

  /* strings type */
  hlng_tid = H5Tcopy(H5T_C_S1);  H5Tset_size(hlng_tid, POH5_HLNG);
  hmid_tid = H5Tcopy(H5T_C_S1);  H5Tset_size(hmid_tid, POH5_HMID);
  hsht_tid = H5Tcopy(H5T_C_S1);  H5Tset_size(hsht_tid, POH5_HSHT);

  /* Turn off error handling permanently */
  H5Eset_auto (H5E_DEFAULT, NULL, NULL);

  return 0;
}


/*========================================================================*/
/** \brief Open poh5 file.
 *
 * Open poh5 file with given `fname`.
 *
 * `mode` must be one of below:
 * - `POH5_FREAD`: read only (no modification allowed)
 * - `POH5_FWRITE`: newly created (existing file will be discarded)
 * - `POH5_FAPPEND`: open to append/modify
 * .
 *
 * \return hdf5 file id.
 *
 */
hid_t poh5_open_file(
                     const char* fname, /**<[in] file name */
                     int mode )         /**<[in] file open mode */
{
  /* mode is one of [POH5_FREAD, POH5_FWRITE, POH5_FAPPEND] */

  hid_t fid;

#ifdef DEBUG
  fprintf(DBGOUT,"dbg:poh5_open_file:mode=%d,fname=%s\n",mode,fname);
#endif

  switch (mode){
  case POH5_FWRITE:   /* existing content will be vanished on open.*/
    fid = H5Fcreate(fname,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT);    break;
  case POH5_FREAD:    /* avoid overwrite action */
    fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);    break;
  case POH5_FAPPEND:  /* overwrite mode */
    fid = H5Fopen(fname, H5F_ACC_RDWR, H5P_DEFAULT);    break;
  default:
    fprintf(stderr,"Err:poh5_open_file:invalid mode:%d\n",mode);
    exit (1);
  }

  check_h5( fid > 0 );
#ifdef DEBUG
  fprintf(DBGOUT,"dbg:poh5_open_file:fid=%d\n",fid);
#endif
 
  return fid;
}


/*========================================================================*/
/** \brief Open poh5 file.
 *
 * Close poh5 file.
 *
 */
int poh5_close_file(
                    const hid_t fid ) /**<[in] file_id to close */
{

#ifdef DEBUG
  fprintf(DBGOUT,"dbg:poh5_close_file:fid=%d\n",fid);
#endif
     
  int res = H5Fclose(fid);
<commit_msg>Check routine for the unclosed objects<commit_after>/**
 * @file poh5.c
 * @brief Source file for poh5 API.
 * @author <NAME> @ RIST
 * @date 2016/08/12
 */


#include <stdlib.h>

#include "poh5.h"
#include "hdf5.h"

typedef struct{
  int64_t s;
  int64_t e;
} tp_t;

#ifndef DBGOUT
/* #define DBGOUT stderr */
#define DBGOUT stdout
#endif


static hid_t hlng_tid; /** datatype for long strings */
static hid_t hmid_tid; /** datatype for middle strings */
static hid_t hsht_tid; /** datatype for short strings */


/*========================================================================*/
/*
 * Declaration of internal routines
 *
 */
int write_gattr(const hid_t file_id, const char *name, const hid_t f_tid, const hid_t sid,
                const hid_t m_tid, const void *vals);
int update_num_of_var( hid_t file_id );
int extend_dataset( hid_t v_gid, char *dname);
int write_dataset( const hid_t   v_gid, const int32_t step,  const char   *dname, 
                   const hid_t   type,  const void   *data);
int write_dataset_1rgn( const hid_t   v_gid, const int32_t step,  const int32_t ll,   
                        const hid_t   type, const void   *data);
int read_dataset( const hid_t   v_gid, const int32_t step,  const char   *dname,
                  const hid_t   type,  void         *data);

void check_h5( int );

/*========================================================================*/
/*
 * Implementation of public routines
 *
 */

/*========================================================================*/
/** @brief Setup poh5 library.
 *
 * Call this before everything.
 *
 * @return 0 if success.
 */
int poh5_setup(){

#ifdef DEBUG
  fprintf(DBGOUT,"dbg:poh5_setup:\n");
#endif

  /* strings type */
  hlng_tid = H5Tcopy(H5T_C_S1);  H5Tset_size(hlng_tid, POH5_HLNG);
  hmid_tid = H5Tcopy(H5T_C_S1);  H5Tset_size(hmid_tid, POH5_HMID);
  hsht_tid = H5Tcopy(H5T_C_S1);  H5Tset_size(hsht_tid, POH5_HSHT);

  /* Turn off error handling permanently */
  H5Eset_auto (H5E_DEFAULT, NULL, NULL);

  return 0;
}


/*========================================================================*/
/** \brief Open poh5 file.
 *
 * Open poh5 file with given `fname`.
 *
 * `mode` must be one of below:
 * - `POH5_FREAD`: read only (no modification allowed)
 * - `POH5_FWRITE`: newly created (existing file will be discarded)
 * - `POH5_FAPPEND`: open to append/modify
 * .
 *
 * \return hdf5 file id.
 *
 */
hid_t poh5_open_file(
                     const char* fname, /**<[in] file name */
                     int mode )         /**<[in] file open mode */
{
  /* mode is one of [POH5_FREAD, POH5_FWRITE, POH5_FAPPEND] */

  hid_t fid;

#ifdef DEBUG
  fprintf(DBGOUT,"dbg:poh5_open_file:mode=%d,fname=%s\n",mode,fname);
#endif

  switch (mode){
  case POH5_FWRITE:   /* existing content will be vanished on open.*/
    fid = H5Fcreate(fname,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT);    break;
  case POH5_FREAD:    /* avoid overwrite action */
    fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);    break;
  case POH5_FAPPEND:  /* overwrite mode */
    fid = H5Fopen(fname, H5F_ACC_RDWR, H5P_DEFAULT);    break;
  default:
    fprintf(stderr,"Err:poh5_open_file:invalid mode:%d\n",mode);
    exit (1);
  }

  check_h5( fid > 0 );
#ifdef DEBUG
  fprintf(DBGOUT,"dbg:poh5_open_file:fid=%d\n",fid);
#endif
 
  return fid;
}


/*========================================================================*/
/** \brief Open poh5 file.
 *
 * Close poh5 file.
 *
 */
int poh5_close_file(
                    const hid_t fid ) /**<[in] file_id to close */
{
  ssize_t    cnt;
  hid_t     *objs;
  hid_t      anobj;
  H5I_type_t ot;
  herr_t     res;

  int        objids;
  char      *objtype;
  char       objname[1024];

#ifdef DEBUG
  fprintf(DBGOUT,"dbg:poh5_close_file:fid=%d\n",fid);
#endif

  cnt = H5Fget_obj_count(fid,H5F_OBJ_ALL);

  if ( cnt > 1 ){
    fprintf(stderr,"%d objects are not closed before closing file.\n", cnt);

    objs   = malloc(cnt * sizeof(hid_t));
    objids = H5Fget_obj_ids(fid,H5F_OBJ_ALL,cnt,objs);

    for (int i = 0; i < objids; i++ ) {
      anobj = *objs++;
      ot    = H5Iget_type(anobj);
      res   = H5Iget_name(anobj,objname,1024);
      check_h5( res > 0 );

      switch (ot){
      case H5I_FILE:
        objtype = "FILE "; break;
      case H5I_GROUP:
        objtype = "GROUP"; break;
      case H5I_DATATYPE:
        objtype = "DTTYP"; break;
      case H5I_DATASPACE:
        objtype = "DTSPC"; break;
      case H5I_DATASET:
        objtype = "DTSET"; break;
      case H5I_ATTR:
        objtype = "ATTR "; break;
      default:
        objtype = "UNDEF";
      }

      fprintf(stderr," %d: type %s, name %s\n",i,objtype,objname);
    }
    exit (1);
  }

  res = H5Fclose(fid);

#ifdef DEBUG
  fprintf(DBGOUT,"dbg:poh5_close_file:res=%d\n",res);
#endif

{'fraction_non_alphanumeric': 0.20278893878515716, 'fraction_numerical': 0.022571496100212717, 'mean_word_length': 3.1958353991075854, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2023005', 'n_tokens_mistral': 3220, 'n_tokens_neox': 3088, 'n_words': 752}