<commit_before>



#include "NGR.h";
#include <stdio.h>;
#include <fcntl.h>;
#include <string.h>;
#include <stdlib.h>;
#include <sys/types.h>
#include <sys/uio.h>
#include <unistd.h>
#include <assert.h>
#include <sys/mman.h>
#include <errno.h>
#include <sys/stat.h>
#include <time.h>






/* file format is pretty simple
   4 bytes for the width of the fields (32/64)
   4 bytes for the version number of format
   4 bytes for the resolution in seconds (resolution is not changeable once a file is created
   4 bytes for the number of columns
   $width create_timestamp unix_time when the series start
   list of items, indexed of the time-create_time/interval
*/



struct NGR_metric_t * NGR_create(char *filename, time_t created_time, int resolution, int columns) {
  int   fd, write_len;
  int   size = sizeof(int);
  int   version = 1;
  columns = 1; // only support one for now

  if(!created_time)
    created_time = time(NULL);

  if(!resolution)
    resolution = 60;
  
  fd = open(filename, O_CREAT | O_RDWR | O_EXCL, 0755);

  if(fd == -1)
    printf("Cannot open %s (%s)\n", filename, strerror(errno));
  assert(fd != -1);

  
  write_len = write(fd, &size, 4);
  assert(write_len == 4);
  write_len = write(fd, &version, 4);
  assert(write_len == 4);
  write_len = write(fd, &resolution, 4);
  assert(write_len == 4);
  write_len = write(fd, &columns, 4);
  assert(write_len == 4);
  write_len = write(fd, &created_time, size);
  assert(write_len == size);
  close(fd);

  return NGR_open(filename);
}

struct NGR_metric_t * NGR_open(char *filename) {
  size_t read_len;
  char width_buf[4];
  struct NGR_metric_t *obj;

  obj = malloc(sizeof(struct NGR_metric_t));


  obj->fd = open(filename, O_RDWR);

  if(obj->fd == -1)
    printf("Cannot open %s (%s)\n", filename, strerror(errno));
  assert(obj->fd != -1);

  obj->base = 4;
  read_len = read(obj->fd, width_buf, 4);
  assert(read_len == 4);
  obj->width = *width_buf;

  obj->base += 4;
  read_len = read(obj->fd, &obj->version, 4);
  assert(read_len == 4);

  obj->base += 4;
  read_len = read(obj->fd, &obj->resolution, 4);
  assert(read_len == 4);

  obj->base += 4;
  read_len = read(obj->fd, &obj->columns, 4);
  assert(read_len == 4);
  

  obj->base += obj->width;
  read_len = read(obj->fd, &obj->created, obj->width);
  assert(read_len == obj->width);

  return obj;
}


/* 
   column     which column to write in
   timestamp  which timestamp this is for, defaults to now if not set
   value      value to store
*/<commit_msg>be more paranoid<commit_after>



#include "NGR.h";
#include <stdio.h>;
#include <fcntl.h>;
#include <string.h>;
#include <stdlib.h>;
#include <sys/types.h>
#include <sys/uio.h>
#include <unistd.h>
#include <assert.h>
#include <sys/mman.h>
#include <errno.h>
#include <sys/stat.h>
#include <time.h>






/* file format is pretty simple
   4 bytes for the width of the fields (32/64)
   4 bytes for the version number of format
   4 bytes for the resolution in seconds (resolution is not changeable once a file is created
   4 bytes for the number of columns
   $width create_timestamp unix_time when the series start
   list of items, indexed of the time-create_time/interval
*/



struct NGR_metric_t * NGR_create(char *filename, time_t created_time, int resolution, int columns) {
  int   fd, write_len;
  int   size = sizeof(int);
  int   version = 1;
  columns = 1; // only support one for now

  if(!created_time)
    created_time = time(NULL);

  if(!resolution)
    resolution = 60;
  
  fd = open(filename, O_CREAT | O_RDWR | O_EXCL, 0755);

  if(fd == -1)
    printf("Cannot open %s (%s)\n", filename, strerror(errno));
  assert(fd != -1);

  
  write_len = write(fd, &size, 4);
  assert(write_len == 4);
  write_len = write(fd, &version, 4);
  assert(write_len == 4);
  write_len = write(fd, &resolution, 4);
  assert(write_len == 4);
  write_len = write(fd, &columns, 4);
  assert(write_len == 4);
  write_len = write(fd, &created_time, size);
  assert(write_len == size);
  close(fd);

  return NGR_open(filename);
}

struct NGR_metric_t * NGR_open(char *filename) {
  size_t read_len;
  char width_buf[4];
  struct NGR_metric_t *obj;

  obj = malloc(sizeof(struct NGR_metric_t));


  obj->fd = open(filename, O_RDWR);

  if(obj->fd == -1)
    printf("Cannot open %s (%s)\n", filename, strerror(errno));
  assert(obj->fd != -1);

  obj->base = 4;
  read_len = read(obj->fd, width_buf, 4);
  assert(read_len == 4);
  obj->width = *width_buf;

  obj->base += 4;
  read_len = read(obj->fd, &obj->version, 4);
  assert(read_len == 4);

  obj->base += 4;
  read_len = read(obj->fd, &obj->resolution, 4);
  assert(read_len == 4);

  obj->base += 4;
  read_len = read(obj->fd, &obj->columns, 4);
  assert(read_len == 4);
  

  obj->base += obj->width;
  read_len = read(obj->fd, &obj->created, obj->width);
  assert(read_len == obj->width);

  assert(obj->width == 4 || obj->width == 8);

  return obj;
}


/* 
   column     which column to write in
   timestamp  which timestamp this is for, defaults to now if not set
   value      value to store
*/
{'fraction_non_alphanumeric': 0.15264612954186413, 'fraction_numerical': 0.016587677725118485, 'mean_word_length': 3.526362823949955, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5238100', 'n_tokens_mistral': 1968, 'n_tokens_neox': 1838, 'n_words': 526}