<commit_before>#define _GNU_SOURCE
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <math.h>

#include "utils.h"
#include "positions.h"

extern int verbose;

// sorting function, used by qsort
static int poscompare(const void *p1, const void *p2)
{
  const positionType *pos1 = (positionType*)p1;
  const positionType *pos2 = (positionType*)p2;
  if (pos1->pos < pos2->pos) return -1;
  else if (pos1->pos > pos2->pos) return 1;
  else return 0;
}



positionType *createPositions(size_t num) {
  positionType *p;
  CALLOC(p, num, sizeof(positionType));
  return p;
}


void checkPositionArray(const positionType *positions, size_t num, size_t bdSizeBytes) {
  //  fprintf(stderr,"... checking\n");
  size_t rcount = 0, wcount = 0;
  size_t sizelow = -1, sizehigh = 0;
  positionType *p = (positionType *)positions;
  for (size_t j = 0; j < num; j++) {
    if (p->len < sizelow) {
      sizelow = p->len;
    }
    if (p->len > sizehigh) {
      sizehigh = p->len;
    }
    if (p->action == 'R') {
      rcount++;
    } else {
      wcount++;
    }
    p++;
  }

  // check all positions are aligned to low and high lengths
  p = (positionType *) positions;
  if (sizelow > 0) {
    for (size_t j = 0; j < num; j++) {
      if (p->pos + sizehigh > bdSizeBytes) {
	fprintf(stderr,"eek off the end of the array %zd!\n", p->pos);
      }
      if ((p->pos % sizelow) != 0) {
	fprintf(stderr,"eek no %zd aligned position at %zd!\n", sizelow, p->pos);
      }
      if ((p->pos % sizehigh) != 0) {
	fprintf(stderr,"eek no %zd aligned position at %zd!\n", sizehigh, p->pos);
      }
      p++;
    }
  }

  // duplicate, sort the array. Count unique positions
  positionType *copy;
  CALLOC(copy, num, sizeof(positionType));
  memcpy(copy, positions, num * sizeof(positionType));
  qsort(copy, num, sizeof(positionType), poscompare);
  // check order
  size_t unique = 0;
  for (size_t i = 1; i <num; i++) {
    if (copy[i].pos != copy[i-1].pos) {
      unique++;
      if (i==1) { // if the first number checked is different then really it's 2 unique values.
	unique++;
      }
    }
    if (copy[i].pos < copy[i-1].pos) {
      fprintf(stderr,"not sorted %zd %zd, unique %zd\n",i, copy[i].pos, unique);
    }
  }
  free(copy);

  fprintf(stderr,"action summary: reads %zd, writes %zd, checked ratio %.1lf, len [%zd, %zd], unique positions %zd\n", rcount, wcount, rcount*1.0/(rcount+wcount), sizelow, sizehigh, unique);
}


// lots of checks
void dumpPositions(const char *name, positionType *positions, size_t num, size_t bdSizeBytes) {
  if (name) {
    FILE *fp = fopen(name, "wt");
    if (!fp) {
      perror(name); return;
    }
    for (size_t i = 0; i <num; i++) {
      fprintf(fp, "%8d\t%10zd\t%.2lf GiB\t%c\t%zd\t%zd\t%.2lf GiB\t(%.1lf%%)\n", positions[i].fd, positions[i].pos, TOGiB(positions[i].pos), positions[i].action, positions[i].len, bdSizeBytes, TOGiB(bdSizeBytes), positions[i].pos * 100.0 / bdSizeBytes);
    }
    fclose(fp);
  }
}


// create the position array
void setupPositions(positionType *positions,
		    size_t *num,
		          const int *fdArray,
		          const size_t fdSize,
			  const size_t bdSizeBytes,
			  const int sf,
			  const double readorwrite,
			  const size_t lowbs,
			  const size_t bs,
		          const size_t alignment,
			  const size_t singlePosition,
		          const int    jumpStep,
		          const size_t startAtZero
		    ) {
  assert(lowbs <= bs);
  assert(positions);
  assert(fdArray);
  if (bdSizeBytes < bs) {
    fprintf(stderr, "*warning* size of device is less than block size!\n");
    return;
  }

  // list of possibles positions
  positionType *poss;
  size_t possAlloc = 100, startPos = 0, count = 0;
  CALLOC(poss, possAlloc, sizeof(positionType));
  while ((startPos + bs < bdSizeBytes) && (count < *num)) {
    int alignbits = (int)(log(alignment)/log(2) + 0.01);
  //  fprintf(stderr,"%zd %d\n", alignment, alignbits);
    assert((1<<alignbits) == alignment);
    
    poss[count].pos = startPos;
    poss[count].len = randomBlockSize(lowbs, bs, alignbits);
    
    startPos += poss[count].len;
    count++;
    if (count >= possAlloc) {
      possAlloc = possAlloc * 4 / 3 + 1;
      poss = realloc(poss, possAlloc * sizeof(positionType));
      if (!poss) {
	fprintf(stderr,"OOM: breaking from setup array\n");
	break;
      }
      //      fprintf(stderr,"realloc\n");
    }
  } // find a position across the disks
  if (verbose > 1) {
    fprintf(stderr,"*info* %zd unique positions, allocated %zd positions\n", count, *num);
  }
  if (*num > count) {
    if (verbose > 1) {
      fprintf(stderr,"*warning* there are %zd unique positions on the device\n", count);
    }
    *num = count;
  }

  int fdPos = 0;
  if (singlePosition) {
    // set a few fixed positions
    fprintf(stderr, "Using a single block position: singlePosition value of %zd\n", singlePosition);
    
    size_t sinPos = lrand48() % count;
    if (startAtZero) sinPos = 0;<commit_msg>Verbose MB usage<commit_after>#define _GNU_SOURCE
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <math.h>

#include "utils.h"
#include "positions.h"

extern int verbose;

// sorting function, used by qsort
static int poscompare(const void *p1, const void *p2)
{
  const positionType *pos1 = (positionType*)p1;
  const positionType *pos2 = (positionType*)p2;
  if (pos1->pos < pos2->pos) return -1;
  else if (pos1->pos > pos2->pos) return 1;
  else return 0;
}



positionType *createPositions(size_t num) {
  positionType *p;
  CALLOC(p, num, sizeof(positionType));
  return p;
}


void checkPositionArray(const positionType *positions, size_t num, size_t bdSizeBytes) {
  //  fprintf(stderr,"... checking\n");
  size_t rcount = 0, wcount = 0;
  size_t sizelow = -1, sizehigh = 0;
  positionType *p = (positionType *)positions;
  for (size_t j = 0; j < num; j++) {
    if (p->len < sizelow) {
      sizelow = p->len;
    }
    if (p->len > sizehigh) {
      sizehigh = p->len;
    }
    if (p->action == 'R') {
      rcount++;
    } else {
      wcount++;
    }
    p++;
  }

  // check all positions are aligned to low and high lengths
  p = (positionType *) positions;
  if (sizelow > 0) {
    for (size_t j = 0; j < num; j++) {
      if (p->pos + sizehigh > bdSizeBytes) {
	fprintf(stderr,"eek off the end of the array %zd!\n", p->pos);
      }
      if ((p->pos % sizelow) != 0) {
	fprintf(stderr,"eek no %zd aligned position at %zd!\n", sizelow, p->pos);
      }
      if ((p->pos % sizehigh) != 0) {
	fprintf(stderr,"eek no %zd aligned position at %zd!\n", sizehigh, p->pos);
      }
      p++;
    }
  }

  // duplicate, sort the array. Count unique positions
  positionType *copy;
  CALLOC(copy, num, sizeof(positionType));
  memcpy(copy, positions, num * sizeof(positionType));
  qsort(copy, num, sizeof(positionType), poscompare);
  // check order
  size_t unique = 0;
  for (size_t i = 1; i <num; i++) {
    if (copy[i].pos != copy[i-1].pos) {
      unique++;
      if (i==1) { // if the first number checked is different then really it's 2 unique values.
	unique++;
      }
    }
    if (copy[i].pos < copy[i-1].pos) {
      fprintf(stderr,"not sorted %zd %zd, unique %zd\n",i, copy[i].pos, unique);
    }
  }
  free(copy);

  fprintf(stderr,"action summary: reads %zd, writes %zd, checked ratio %.1lf, len [%zd, %zd], unique positions %zd\n", rcount, wcount, rcount*1.0/(rcount+wcount), sizelow, sizehigh, unique);
}


// lots of checks
void dumpPositions(const char *name, positionType *positions, size_t num, size_t bdSizeBytes) {
  if (name) {
    FILE *fp = fopen(name, "wt");
    if (!fp) {
      perror(name); return;
    }
    for (size_t i = 0; i <num; i++) {
      fprintf(fp, "%8d\t%10zd\t%.2lf GiB\t%c\t%zd\t%zd\t%.2lf GiB\t(%.1lf%%)\n", positions[i].fd, positions[i].pos, TOGiB(positions[i].pos), positions[i].action, positions[i].len, bdSizeBytes, TOGiB(bdSizeBytes), positions[i].pos * 100.0 / bdSizeBytes);
    }
    fclose(fp);
  }
}


// create the position array
void setupPositions(positionType *positions,
		    size_t *num,
		          const int *fdArray,
		          const size_t fdSize,
			  const size_t bdSizeBytes,
			  const int sf,
			  const double readorwrite,
			  const size_t lowbs,
			  const size_t bs,
		          const size_t alignment,
			  const size_t singlePosition,
		          const int    jumpStep,
		          const size_t startAtZero
		    ) {
  assert(lowbs <= bs);
  assert(positions);
  assert(fdArray);
  if (bdSizeBytes < bs) {
    fprintf(stderr, "*warning* size of device is less than block size!\n");
    return;
  }

  // list of possibles positions
  positionType *poss, *poss2;
  size_t possAlloc = 100, startPos = 0, count = 0;
  CALLOC(poss, possAlloc, sizeof(positionType));
  while ((startPos + bs < bdSizeBytes) && (count < *num)) {
    int alignbits = (int)(log(alignment)/log(2) + 0.01);
  //  fprintf(stderr,"%zd %d\n", alignment, alignbits);
    assert((1<<alignbits) == alignment);
    
    poss[count].pos = startPos;
    poss[count].len = randomBlockSize(lowbs, bs, alignbits);
    
    startPos += poss[count].len;
    count++;
    if (count >= possAlloc) {
      possAlloc = possAlloc * 4 / 3 + 1; // grow by a 1/3 each time
      poss2 = realloc(poss, possAlloc * sizeof(positionType));
      if (!poss) {
	fprintf(stderr,"OOM: breaking from setup array\n");
	break;
      } else {
	if (verbose >= 1) {
	  fprintf(stderr,"*info*: new position size %.1lf MB array\n", TOMiB(possAlloc * sizeof(positionType)));
	}
	poss = poss2; // point to the new array
      }
      //      fprintf(stderr,"realloc\n");
    }
  } // find a position across the disks
  if (verbose > 1) {
    fprintf(stderr,"*info* %zd unique positions, allocated %zd positions\n", count, *num);
  }
  if (*num > count) {
    if (verbose > 1) {
      fprintf(stderr,"*warning* there are %zd unique positions on the device\n", count);
    }
    *num = count;
  }

  int fdPos = 0;
  if (singlePosition) {
    // set a few fixed positions
    fprintf(stderr, "Using a single block position: singlePosition value of %zd\n", singlePosition);
    
    size_t sinPos = lrand48() % count;
    if (startAtZero) sinPos = 0;
{'fraction_non_alphanumeric': 0.15379330650701736, 'fraction_numerical': 0.012660712533123957, 'mean_word_length': 2.922247882986913, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '467447', 'n_tokens_mistral': 3786, 'n_tokens_neox': 3605, 'n_words': 1012}