<commit_before>#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <cassert>
#include <cstring>
#include <string>
#include <sstream>
#include "phIO.h"
#include "phComm.h"
#include "phio_base.h"
#include "ph_mpi_help.h"

#ifndef PHASTAIO_TIMERS_ON
#define PHASTAIO_TIMERS_ON 0
#endif
struct phastaio_stats {
  double readTime;
  double writeTime;
  double openTime;
  double closeTime;
  size_t readBytes;
  size_t writeBytes;
};
phastaio_stats phio_global_stats;

namespace {
  inline double getTime() {
    return MPI_Wtime();
  }
  inline size_t getSize(const char* t) {
    std::string type(t);
    if(type == "integer")
      return sizeof(int);
    else if(type == "double")
      return sizeof(double);
    else {
      assert(0);
      exit(EXIT_FAILURE);
    }
  }
}

#define PHIO_TRACING 0
namespace {
  void trace(const char* key, const char* aux="", void* obj=NULL) {
    if(PHIO_TRACING)
      fprintf(stderr, "PHIO_TRACE entering %s %s %p\n", key, aux, obj);
  }
  void printMinMaxAvg(const char* key, size_t v) {
    int val = static_cast<int>(v);
    int min = ph_min_int(val);
    int max = ph_max_int(val);
    long tot = ph_add_long(static_cast<long>(val));
    double avg = tot/static_cast<double>(ph_peers());
    if(!ph_self())
      fprintf(stderr, "phio_%s min max avg %d %d %f\n",
          key, min, max, avg);
  }

  void printMinMaxAvg(const char* key, double v) {
    double min = ph_min_double(v);
    double max = ph_max_double(v);
    double tot = ph_add_double(v);
    double avg = tot/ph_peers();
    if(!ph_self())
      fprintf(stderr, "phio_%s min max avg %f %f %f\n",
          key, min, max, avg);
  }
}

#ifdef __cplusplus
extern "C" {
#endif

void phio_printStats() {
  printMinMaxAvg("readTime",phio_getReadTime());
  printMinMaxAvg("writeTime", phio_getWriteTime());
  printMinMaxAvg("openTime", phio_getOpenTime());
  printMinMaxAvg("closeTime", phio_getCloseTime());
  printMinMaxAvg("readBytes", phio_getReadBytes());
  printMinMaxAvg("writeBytes", phio_getWriteBytes());<commit_msg>common: write io stat units and r/w bandwidth<commit_after>#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <cassert>
#include <cstring>
#include <string>
#include <sstream>
#include "phIO.h"
#include "phComm.h"
#include "phio_base.h"
#include "ph_mpi_help.h"


#ifndef PHASTAIO_TIMERS_ON
#define PHASTAIO_TIMERS_ON 0
#endif
struct phastaio_stats {
  double readTime;
  double writeTime;
  double openTime;
  double closeTime;
  size_t readBytes;
  size_t writeBytes;
};
phastaio_stats phio_global_stats;

namespace {
  inline double getTime() {
    return MPI_Wtime();
  }
  inline size_t getSize(const char* t) {
    std::string type(t);
    if(type == "integer")
      return sizeof(int);
    else if(type == "double")
      return sizeof(double);
    else {
      assert(0);
      exit(EXIT_FAILURE);
    }
  }
}

#define PHIO_TRACING 0
namespace {
  void trace(const char* key, const char* aux="", void* obj=NULL) {
    if(PHIO_TRACING)
      fprintf(stderr, "PHIO_TRACE entering %s %s %p\n", key, aux, obj);
  }
  void printMinMaxAvg(const char* key, size_t v) {
    int val = static_cast<int>(v);
    int min = ph_min_int(val);
    int max = ph_max_int(val);
    long tot = ph_add_long(static_cast<long>(val));
    double avg = tot/static_cast<double>(ph_peers());
    if(!ph_self())
      fprintf(stderr, "phio_%s min max avg %d %d %f\n",
          key, min, max, avg);
  }

  void printMinMaxAvg(const char* key, double v) {
    double min = ph_min_double(v);
    double max = ph_max_double(v);
    double tot = ph_add_double(v);
    double avg = tot/ph_peers();
    if(!ph_self())
      fprintf(stderr, "phio_%s min max avg %f %f %f\n",
          key, min, max, avg);
  }
}

#ifdef __cplusplus
extern "C" {
#endif

void phio_printStats() {
  const int mebi=1024*1024;
  printMinMaxAvg("readTime (s)",phio_getReadTime());
  printMinMaxAvg("writeTime (s)", phio_getWriteTime());
  printMinMaxAvg("openTime (s)", phio_getOpenTime());
  printMinMaxAvg("closeTime (s)", phio_getCloseTime());
  printMinMaxAvg("readBytes (B)", phio_getReadBytes());
  printMinMaxAvg("writeBytes (B)", phio_getWriteBytes());
  printMinMaxAvg("readBandwidth (MiB/s)",
      (phio_getReadBytes()/phio_getReadTime())/mebi);
  printMinMaxAvg("writeBandwidth (MiB/s)",
      (phio_getWriteBytes()/phio_getWriteTime())/mebi);
{'fraction_non_alphanumeric': 0.1437414030261348, 'fraction_numerical': 0.0032095369096744614, 'mean_word_length': 4.126909518213866, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '626176', 'n_tokens_mistral': 1686, 'n_tokens_neox': 1667, 'n_words': 371}