<commit_before>#define _POSIX_C_SOURCE 200809L
#define _GNU_SOURCE

#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <signal.h>

/**
 * verify.c
 *
 * the main() for ./verify, a standalone position verifier
 *
 */
#include <stdlib.h>
#include <string.h>

#include "positions.h"
#include "utils.h"
#include "blockVerify.h"

int verbose = 1;
int keepRunning = 1;
size_t waitEvery = 0;


void intHandler(int d)
{
  if (d) {}
  fprintf(stderr,"got signal\n");
  keepRunning = 0;
}

/**
 * main
 *
 */
int main(int argc, char *argv[])
{
  signal(SIGTERM, intHandler);
  signal(SIGINT, intHandler);

  if (argc <= 1) {
    fprintf(stdout,"*usage* ./spitchecker [ -D ] [ -t 256 ] [ -n (don't sort)] positions*\n");
    exit(1);
  }

  size_t threads = 256;

  int opt = 0;
  size_t o_direct = O_DIRECT;
  size_t sort = 1;
  size_t displayJSON = 0;

  while ((opt = getopt(argc, argv, "Dt:nj")) != -1) {
    switch (opt) {
    case 'D':
      o_direct = 0;
      break;
    case 'j':
      displayJSON = 1;
      break;
    case 't':
      threads = atoi(optarg);
      break;
    case 'n':
      sort = 0;
      break;
    default:
      break;
    }
  }

  size_t numFiles = argc -optind;
  fprintf(stderr,"*info* number of files %zd, threads set to %zd, sort %zd\n", numFiles, threads, sort);

  positionContainer *origpc;
  CALLOC(origpc, numFiles, sizeof(positionContainer));

  jobType job;

  for (int i= optind; i < argc; i++) {
    fprintf(stderr,"*info* position file: %s\n", argv[i]);
    FILE *fp = NULL;
    if (strcmp(argv[i], "-") == 0) { 
      fp = stdin;
    } else {
      fp = fopen(argv[i], "rt");
    }
    if (fp) {
      if (fp == stdin) {
	size_t correct = 0, incorrect = 0, ioerrors = 0, errors = 0, jc = 0;
	size_t tot_cor = 0;
	do {
	  job = positionContainerLoadLines(&origpc[i - optind], fp, 5000);
	  if (job.count) {
	    positionContainer pc = positionContainerMerge(origpc, numFiles);
	    //positionContainerCheckOverlap(&pc);
	    errors += verifyPositions(&pc, threads, &job, o_direct, 0, 0 /*runtime*/, &correct, &incorrect, &ioerrors);
	    tot_cor += correct;
	    positionContainerFree(&pc);
	  }
	  jc = job.count;
	  jobFree(&job);
	  fprintf(stderr,"Correct %zd, errors %zd\n", tot_cor, errors);
	} while (jc != 0);
	if (errors) {
	  exit(1);
	}
	//	exit(0);
      } else {
	job = positionContainerLoad(&origpc[i - optind], fp);
      }
      //      positionContainerInfo(&origpc[i]);
      fclose(fp);
    }
  }

  if (origpc->sz == 0) {
    fprintf(stderr,"*warning* no positions to verify\n");
    exit(-1);
  }<commit_msg>tidier output<commit_after>#define _POSIX_C_SOURCE 200809L
#define _GNU_SOURCE

#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <signal.h>

/**
 * verify.c
 *
 * the main() for ./verify, a standalone position verifier
 *
 */
#include <stdlib.h>
#include <string.h>

#include "positions.h"
#include "utils.h"
#include "blockVerify.h"

int verbose = 1;
int keepRunning = 1;
size_t waitEvery = 0;


void intHandler(int d)
{
  if (d) {}
  fprintf(stderr,"got signal\n");
  keepRunning = 0;
}

/**
 * main
 *
 */
int main(int argc, char *argv[])
{
  signal(SIGTERM, intHandler);
  signal(SIGINT, intHandler);

  if (argc <= 1) {
    fprintf(stdout,"*usage* ./spitchecker [ -D ] [ -t 256 ] [ -n (don't sort)] positions*\n");
    exit(1);
  }

  size_t threads = 256;

  int opt = 0;
  size_t o_direct = O_DIRECT;
  size_t sort = 1;
  size_t displayJSON = 0;

  while ((opt = getopt(argc, argv, "Dt:nj")) != -1) {
    switch (opt) {
    case 'D':
      o_direct = 0;
      break;
    case 'j':
      displayJSON = 1;
      break;
    case 't':
      threads = atoi(optarg);
      break;
    case 'n':
      sort = 0;
      break;
    default:
      break;
    }
  }

  size_t numFiles = argc -optind;
  fprintf(stderr,"*info* number of files %zd, threads set to %zd, sort %zd\n", numFiles, threads, sort);

  positionContainer *origpc;
  CALLOC(origpc, numFiles, sizeof(positionContainer));

  jobType job;

  for (int i= optind; i < argc; i++) {
    fprintf(stderr,"*info* position file: %s\n", argv[i]);
    FILE *fp = NULL;
    if (strcmp(argv[i], "-") == 0) { 
      fp = stdin;
    } else {
      fp = fopen(argv[i], "rt");
    }
    if (fp) {
      if (fp == stdin) {
	size_t correct = 0, incorrect = 0, ioerrors = 0, errors = 0, jc = 0;
	size_t tot_cor = 0;
	do {
	  job = positionContainerLoadLines(&origpc[i - optind], fp, 5000);
	  if (job.count) {
	    positionContainer pc = positionContainerMerge(origpc, numFiles);
	    //positionContainerCheckOverlap(&pc);
	    errors += verifyPositions(&pc, threads, &job, o_direct, 0, 0 /*runtime*/, &correct, &incorrect, &ioerrors);
	    tot_cor += correct;
	    positionContainerFree(&pc);
	    fprintf(stderr,"Correct %zd, errors %zd\n", tot_cor, errors);
	  }
	  jc = job.count;
	  jobFree(&job);
	} while (jc != 0);
	if (errors) {
	  exit(1);
	}
	exit(0);
      } else {
	job = positionContainerLoad(&origpc[i - optind], fp);
      }
      //      positionContainerInfo(&origpc[i]);
      fclose(fp);
    }
  }

  if (origpc->sz == 0) {
    fprintf(stderr,"*warning* no positions to verify\n");
    exit(-1);
  }
{'fraction_non_alphanumeric': 0.16856756234293446, 'fraction_numerical': 0.016624782524647205, 'mean_word_length': 2.829755736491488, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1944861', 'n_tokens_mistral': 2059, 'n_tokens_neox': 1935, 'n_words': 491}