<commit_before>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <errno.h>
#include <sys/uio.h>

#include "common/Clock.h"

#include <iostream>
using namespace std;

int main(int argc, char **argv)
{
  void   *buf;
  int     dev_id, count, loop = 0, ret;
  
  if (argc != 3) {
    fprintf(stderr, "Usage: %s device mb\n", argv[0]);
    exit (0);
  }
  
  count = atoi(argv[2]);
  int bsize = 1048576;
  
  posix_memalign(&buf, sysconf(_SC_PAGESIZE), bsize);
  
  if ((dev_id = open(argv[1], O_DIRECT|O_RDWR)) < 0) {
    fprintf(stderr, "Can't open device %s\n", argv[1]);
    exit (4);
  }
  
  fprintf(stderr, "device is %s, dev_id is %d\n", argv[1], dev_id);
  
  utime_t start = g_clock.now();
  while (loop++ < count) {
    ret = ::write(dev_id, buf, bsize);
    if ((loop % 100) == 0) 
      fprintf(stderr, ".");
  }
  utime_t end = g_clock.now();
  end -= start;

  int mb = count;

  cout << mb << " MB, " << end << " seconds, " << ((double)mb / (double)end) << " MB/sec" << std::endl;
}<commit_msg>improved test_disk_bw<commit_after>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <errno.h>
#include <sys/uio.h>

#include "common/Clock.h"

#include <iostream>
using namespace std;

int main(int argc, char **argv)
{
  void   *buf;
  int     fd, count, loop = 0, ret;
  
  if (argc != 4) {
    fprintf(stderr, "Usage: %s device bsize count\n", argv[0]);
    exit (0);
  }
  
  int bsize = atoi(argv[2]);
  count = atoi(argv[3]);
  
  posix_memalign(&buf, sysconf(_SC_PAGESIZE), bsize);
  
  //if ((fd = open(argv[1], O_SYNC|O_RDWR)) < 0) {  
  if ((fd = open(argv[1], O_DIRECT|O_RDWR)) < 0) {

    fprintf(stderr, "Can't open device %s\n", argv[1]);
    exit (4);
  }
  
 
  utime_t start = g_clock.now();
  while (loop++ < count) {
    ret = ::write(fd, buf, bsize);
    //if ((loop % 100) == 0) 
    //fprintf(stderr, ".");
  }
  ::fsync(fd);
  ::close(fd);
  utime_t end = g_clock.now();
  end -= start;


  char hostname[80];
  gethostname(hostname, 80);
  
  double mb = bsize*count/1024/1024;

  cout << hostname << "\t" << mb << " MB\t" << end << " seconds\t" << (mb / (double)end) << " MB/sec" << std::endl;
}
{'fraction_non_alphanumeric': 0.20530079932688264, 'fraction_numerical': 0.020614219604543543, 'mean_word_length': 3.186619718309859, 'pattern_counts': {'":': 0, '<': 62, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '223123', 'n_tokens_mistral': 1042, 'n_tokens_neox': 1003, 'n_words': 234}