<commit_before>#define _GNU_SOURCE
#include <stdlib.h>
#include <fcntl.h>
#include <sys/time.h>
#include <linux/fs.h>
#include <sys/ioctl.h>
#include <malloc.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <syslog.h>

#include "utils.h"

extern int keepRunning;


struct timeval gettm() {
  struct timeval now;
  gettimeofday(&now, NULL);
  return now;
}

double timedouble() {
  struct timeval now = gettm();
  double tm = (now.tv_sec * 1000000 + now.tv_usec);
  return tm/1000000.0;
}


size_t blockDeviceSize(char *path) {

  int fd = open(path, O_RDONLY );
  if (fd < 0) {
    perror(path);
    return 0;
  }
  size_t file_size_in_bytes = 0;
  ioctl(fd, BLKGETSIZE64, &file_size_in_bytes);
  close(fd);
  return file_size_in_bytes;
}


void doChunks(int fd, char *label, int *chunkSizes, int numChunks, size_t maxTime, logSpeedType *l, size_t maxBufSize, size_t outputEvery, int writeAction, int sequential, int direct, int verifyWrites, float flushEverySecs) {
  void *buf = aligned_alloc(65536, maxBufSize);
  if (!buf) { // O_DIRECT requires aligned memory
	fprintf(stderr,"memory allocation failed\n");exit(1);
  }
  srand((int)timedouble());

  char *charbuf = (char*) buf;
  size_t checksum = 0;
  const size_t startChar = rand() % 255;
  const size_t startMod = 20 + rand() % 40;
  for (size_t i = 0; i < maxBufSize; i++ ) {
    charbuf[i] = 'A' + (char) ((startChar + i ) % startMod);
    checksum += ('A' + (startChar + i) % startMod);
  }

  size_t maxblocks = 0;
  if (!sequential) {
    if (isBlockDevice(label) && ((maxblocks = blockDeviceSize(label)/4096) > 0)) {
       fprintf(stderr,"max blocks on %s is %zd\n", label, maxblocks);
    } else {
      fprintf(stderr,"error: need to be a block device with the -r option\n");
      exit(1);
    }
  } 
  int wbytes = 0;
  double lastg = timedouble();
  int chunkIndex = 0;
  double startTime = timedouble();
  double lastFdatasync = startTime;
  int resetCount = 5;
  logSpeedType previousSpeeds;

  size_t countValues = 0, allocValues = 20000, sumBytes = 0;
  double *allValues, *allTimes, *allTotal;

  allValues = malloc(allocValues * sizeof(double));
  allTimes = malloc(allocValues * sizeof(double));
  allTotal = malloc(allocValues * sizeof(double));

  logSpeedInit(&previousSpeeds);
  
  while (keepRunning) {
    if (!sequential) {
	size_t pos = (rand() % maxblocks) * 4096;
 	//fprintf(stderr,"%zd\n", pos);	
 	off_t ret = lseek(fd, pos, SEEK_SET);
	if (ret < 0) {
	  perror("seek error");
	}
    }

    if (writeAction) {
      wbytes = write(fd, charbuf, chunkSizes[0]);
    } else {
      wbytes = read(fd, charbuf, chunkSizes[0]);
    }
    if (wbytes == 0) {
      break;
    }
    if (wbytes < 0) {
      perror("problem");
      break;
    }

    const double tt = timedouble();
    
    logSpeedAdd(l, wbytes);

    sumBytes += wbytes;
    allValues[countValues] = wbytes;
    allTimes[countValues] = tt;
    allTotal[countValues] = sumBytes;
    countValues++;
    
    if (countValues >= allocValues) {
      allocValues = allocValues * 2 + 2;
      allValues = realloc(allValues, allocValues * sizeof(double));
      allTimes = realloc(allTimes, allocValues * sizeof(double));
      allTotal = realloc(allTotal, allocValues * sizeof(double));
    }

    if ((flushEverySecs > 0) && (tt - lastFdatasync > flushEverySecs)) {
      fprintf(stderr,"fdatasync() at %.1lf seconds\n", tt - startTime);
      fdatasync(fd);
      lastFdatasync = tt;
    }
    
    if ((tt - lastg) >= outputEvery) {
      lastg = tt;
      fprintf(stderr,"%s '%s': %.1lf GiB, mean %.1f MiB/s, median %.1f MiB/s, 1%% %.1f MiB/s, 99%% %.1f MiB/s, n=%zd, %.1fs\n", writeAction ? "write" : "read", label, l->total / 1024.0 / 1024 / 1024, logSpeedMean(l) / 1024.0 / 1024, logSpeedMedian(l) / 1024.0 / 1024, logSpeedRank(l, 0.01) / 1024.0 / 1024, logSpeedRank(l, 0.99) /1024.0/1024, l->num, tt - l->starttime);
      logSpeedAdd(&previousSpeeds, logSpeedMean(l));
      if (logSpeedN(&previousSpeeds) >= 10) { // at least 10 data points before a reset
	if (keepRunning) {
	  double low = logSpeedRank(&previousSpeeds, .1);
	  double high = logSpeedRank(&previousSpeeds, .9);
	  double mean = logSpeedMean(&previousSpeeds);
	  if ((high / low > 1.05) && (mean < low || mean > high) && (resetCount > 0)) { // must be over 5% difference
	    fprintf(stderr,"  [ %.1lf < %.1lf MiB/s < %.1lf ]\n", low / 1024.0 / 1024, mean / 1024.0 / 1024, high / 1024.0 /1024);
	    fprintf(stderr,"resetting due to volatile timings (%d resets remain)\n", resetCount);
	    resetCount--;
	    logSpeedReset(l);
	    logSpeedReset(&previousSpeeds);
	    startTime = tt;
	  }
	}
      }
    }
    if (chunkIndex >= numChunks) {
      chunkIndex = 0;
    }
    if (tt - startTime > maxTime) {
      //fprintf(stderr,"timer triggered after %zd seconds\n", maxTime);
      break;
    }
  } // while loop
  if (writeAction) {
    fprintf(stderr,"flushing and closing...\n");
  }
  fdatasync(fd);
  fsync(fd);
  close(fd);

  // add the very last value
  allValues[countValues] = wbytes;
  allTimes[countValues] = timedouble();
  allTotal[countValues] = sumBytes;
  countValues++;

  l->lasttime = timedouble();
  if (resetCount > 0) {
    char s[1000];
    sprintf(s, "Total %s speed '%s': %.1lf GiB in %.1f s, mean %.2f MiB/s, %d bytes, %s, %s, n=%zd (stutools %s)%s\n", writeAction ? "write" : "read", label, l->total / 1024.0 / 1024 / 1024, logSpeedTime(l), logSpeedMean(l) / 1024.0 / 1024, chunkSizes[0], sequential ? "sequential" : "random", direct ? "O_DIRECT" : "pagecache", logSpeedN(l), VERSION, keepRunning ? "" : " - interrupted");
    fprintf(stderr, "%s", s);
    char *user = username();
    syslog(LOG_INFO, "%s - %s", user, s);
    free(user);
  } else {
    fprintf(stderr,"error: results too volatile. Perhaps the machine is busy?\n");
  }

  // dump all values to a log file
  char s[1000];
  sprintf(s, "log-%s", label);
  for (size_t i = 0; i < strlen(s); i++) {
    if (s[i] == '/') {
      s[i] = '_';
    }
  }
  double firsttime = 0;
  if (countValues > 0) {<commit_msg>Reset if not in [25,75%] range<commit_after>#define _GNU_SOURCE
#include <stdlib.h>
#include <fcntl.h>
#include <sys/time.h>
#include <linux/fs.h>
#include <sys/ioctl.h>
#include <malloc.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <syslog.h>

#include "utils.h"

extern int keepRunning;


struct timeval gettm() {
  struct timeval now;
  gettimeofday(&now, NULL);
  return now;
}

double timedouble() {
  struct timeval now = gettm();
  double tm = (now.tv_sec * 1000000 + now.tv_usec);
  return tm/1000000.0;
}


size_t blockDeviceSize(char *path) {

  int fd = open(path, O_RDONLY );
  if (fd < 0) {
    perror(path);
    return 0;
  }
  size_t file_size_in_bytes = 0;
  ioctl(fd, BLKGETSIZE64, &file_size_in_bytes);
  close(fd);
  return file_size_in_bytes;
}


void doChunks(int fd, char *label, int *chunkSizes, int numChunks, size_t maxTime, logSpeedType *l, size_t maxBufSize, size_t outputEvery, int writeAction, int sequential, int direct, int verifyWrites, float flushEverySecs) {
  void *buf = aligned_alloc(65536, maxBufSize);
  if (!buf) { // O_DIRECT requires aligned memory
	fprintf(stderr,"memory allocation failed\n");exit(1);
  }
  srand((int)timedouble());

  char *charbuf = (char*) buf;
  size_t checksum = 0;
  const size_t startChar = rand() % 255;
  const size_t startMod = 20 + rand() % 40;
  for (size_t i = 0; i < maxBufSize; i++ ) {
    charbuf[i] = 'A' + (char) ((startChar + i ) % startMod);
    checksum += ('A' + (startChar + i) % startMod);
  }

  size_t maxblocks = 0;
  if (!sequential) {
    if (isBlockDevice(label) && ((maxblocks = blockDeviceSize(label)/4096) > 0)) {
       fprintf(stderr,"max blocks on %s is %zd\n", label, maxblocks);
    } else {
      fprintf(stderr,"error: need to be a block device with the -r option\n");
      exit(1);
    }
  } 
  int wbytes = 0;
  double lastg = timedouble();
  int chunkIndex = 0;
  double startTime = timedouble();
  double lastFdatasync = startTime;
  int resetCount = 5;
  logSpeedType previousSpeeds;

  size_t countValues = 0, allocValues = 20000, sumBytes = 0;
  double *allValues, *allTimes, *allTotal;

  allValues = malloc(allocValues * sizeof(double));
  allTimes = malloc(allocValues * sizeof(double));
  allTotal = malloc(allocValues * sizeof(double));

  logSpeedInit(&previousSpeeds);
  
  while (keepRunning) {
    if (!sequential) {
	size_t pos = (rand() % maxblocks) * 4096;
 	//fprintf(stderr,"%zd\n", pos);	
 	off_t ret = lseek(fd, pos, SEEK_SET);
	if (ret < 0) {
	  perror("seek error");
	}
    }

    if (writeAction) {
      wbytes = write(fd, charbuf, chunkSizes[0]);
    } else {
      wbytes = read(fd, charbuf, chunkSizes[0]);
    }
    if (wbytes == 0) {
      break;
    }
    if (wbytes < 0) {
      perror("problem");
      break;
    }

    const double tt = timedouble();
    
    logSpeedAdd(l, wbytes);

    sumBytes += wbytes;
    allValues[countValues] = wbytes;
    allTimes[countValues] = tt;
    allTotal[countValues] = sumBytes;
    countValues++;
    
    if (countValues >= allocValues) {
      allocValues = allocValues * 2 + 2;
      allValues = realloc(allValues, allocValues * sizeof(double));
      allTimes = realloc(allTimes, allocValues * sizeof(double));
      allTotal = realloc(allTotal, allocValues * sizeof(double));
    }

    if ((flushEverySecs > 0) && (tt - lastFdatasync > flushEverySecs)) {
      fprintf(stderr,"fdatasync() at %.1lf seconds\n", tt - startTime);
      fdatasync(fd);
      lastFdatasync = tt;
    }
    
    if ((tt - lastg) >= outputEvery) {
      lastg = tt;
      fprintf(stderr,"%s '%s': %.1lf GiB, mean %.1f MiB/s, median %.1f MiB/s, 1%% %.1f MiB/s, 99%% %.1f MiB/s, n=%zd, %.1fs\n", writeAction ? "write" : "read", label, l->total / 1024.0 / 1024 / 1024, logSpeedMean(l) / 1024.0 / 1024, logSpeedMedian(l) / 1024.0 / 1024, logSpeedRank(l, 0.01) / 1024.0 / 1024, logSpeedRank(l, 0.99) /1024.0/1024, l->num, tt - l->starttime);
      logSpeedAdd(&previousSpeeds, logSpeedMean(l));
      if (logSpeedN(&previousSpeeds) >= 10) { // at least 10 data points before a reset
	if (keepRunning) {
	  double low = logSpeedRank(&previousSpeeds, .25);
	  double high = logSpeedRank(&previousSpeeds, .75);
	  double mean = logSpeedMean(&previousSpeeds);
	  if ((high / low > 1.05) && (mean < low || mean > high) && (resetCount > 0)) { // must be over 5% difference
	    fprintf(stderr,"  [ %.1lf < %.1lf MiB/s < %.1lf ]\n", low / 1024.0 / 1024, mean / 1024.0 / 1024, high / 1024.0 /1024);
	    fprintf(stderr,"resetting due to volatile timings (%d resets remain)\n", resetCount);
	    resetCount--;
	    logSpeedReset(l);
	    logSpeedReset(&previousSpeeds);
	    startTime = tt;
	  }
	}
      }
    }
    if (chunkIndex >= numChunks) {
      chunkIndex = 0;
    }
    if (tt - startTime > maxTime) {
      //fprintf(stderr,"timer triggered after %zd seconds\n", maxTime);
      break;
    }
  } // while loop
  if (writeAction) {
    fprintf(stderr,"flushing and closing...\n");
  }
  fdatasync(fd);
  fsync(fd);
  close(fd);

  // add the very last value
  allValues[countValues] = wbytes;
  allTimes[countValues] = timedouble();
  allTotal[countValues] = sumBytes;
  countValues++;

  l->lasttime = timedouble(); // change time after closing
  if (resetCount > 0) {
    char s[1000];
    sprintf(s, "Total %s speed '%s': %.1lf GiB in %.1f s, mean %.2f MiB/s, %d bytes, %s, %s, n=%zd (stutools %s)%s\n", writeAction ? "write" : "read", label, l->total / 1024.0 / 1024 / 1024, logSpeedTime(l), logSpeedMean(l) / 1024.0 / 1024, chunkSizes[0], sequential ? "sequential" : "random", direct ? "O_DIRECT" : "pagecache", logSpeedN(l), VERSION, keepRunning ? "" : " - interrupted");
    fprintf(stderr, "%s", s);
    char *user = username();
    syslog(LOG_INFO, "%s - %s", user, s);
    free(user);
  } else {
    fprintf(stderr,"error: results too volatile. Perhaps the machine is busy?\n");
  }

  // dump all values to a log file
  char s[1000];
  sprintf(s, "log-%s", label);
  for (size_t i = 0; i < strlen(s); i++) {
    if (s[i] == '/') {
      s[i] = '_';
    }
  }
  double firsttime = 0;
  if (countValues > 0) {
{'fraction_non_alphanumeric': 0.160828811050814, 'fraction_numerical': 0.034862687058049666, 'mean_word_length': 3.1782892476812092, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1625683', 'n_tokens_mistral': 4786, 'n_tokens_neox': 4387, 'n_words': 1143}