<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <arpa/inet.h>
#include "idx.h"

int main(int argc, char **argv){
   if(argc<2){
      printf("Usage: \n");
      printf("readlabels t10k-labels-idx1-ubyte \n");
      exit(1);
   }
   
   char *slabelfname = argv[1];
   
   IDX1_DATA idxdata;
   if(!fread_idx1_file( slabelfname, &idxdata)){
      printf("The datafile '%s' is not a valid IDX_1 file.\n", slabelfname);
      exit(2);
   }
   //printf("Length: %d\n", idxdata.length);
   int i;
   for(i=0;i<idxdata.length;i++){
//      printf("DATA[%05d] = %02d\n", i, idxdata.data[i]);
      printf("%d\n", idxdata.data[i]);
   }
   
   return 0;
}
<commit_msg>Implement a simple ASCII text image dumper.<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <arpa/inet.h>
#include "idx.h"

int main(int argc, char **argv){
   if(argc<2){
      printf("Usage: \n");
      printf("readimages t10k-images-idx3-ubyte \n");
      exit(1);
   }
   
   char *slabelfname = argv[1];
   
   IDX3_DATA idxdata;
   if(!fread_idx3_file( slabelfname, &idxdata)){
      printf("The datafile '%s' is not a valid IDX3 file.\n", slabelfname);
      exit(2);
   }
   //printf("Length: %d\n", idxdata.length);
   
   int i;
   int x;
   int y;
   int ibase;
   int rbase;
   unsigned char pxl;
   for(i=0;i<idxdata.nimages;i++){
      ibase = i * idxdata.nrows * idxdata.ncols;
      printf("Image %05d ( %010d ):\n", i + 1, ibase);
      for(y=0;y<idxdata.nrows;y++){
         rbase = ibase + y*idxdata.ncols;
         for(x=0;x<idxdata.ncols;x++){
            //      printf("DATA[%05d] = %02d\n", i, idxdata.data[i]);
            pxl = idxdata.data[ rbase + x ];
            if(pxl){
               printf("%02X", pxl);
            }else{
               printf("  ");
            }
         }
         printf("\n");
      }
      printf("========================================================\n");
   }
   
   return 0;
}

{'fraction_non_alphanumeric': 0.20812182741116753, 'fraction_numerical': 0.02131979695431472, 'mean_word_length': 2.3924268502581754, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4286396', 'n_tokens_mistral': 781, 'n_tokens_neox': 757, 'n_words': 152}