<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <libtorque/arch.h>
#include <libtorque/memory.h>
#include <libtorque/libtorque.h>

static const char *
memory_type(int mtype){
	switch(mtype){
		case MEMTYPE_DATA: return "data";
		case MEMTYPE_CODE: return "code";
		case MEMTYPE_UNIFIED: return "unified";
		default: return NULL;
	}
}

static const char *
tlb_type(int ttype){
	switch(ttype){
		case MEMTYPE_DATA: return "data";
		case MEMTYPE_CODE: return "code";
		case MEMTYPE_UNIFIED: return "mixed";
		default: return NULL;
	}
}

static const char *
x86_type(int x86type){
	switch(x86type){
		case PROCESSOR_X86_OEM: return "OEM";
		case PROCESSOR_X86_OVERDRIVE: return "OverDrive"; // FIXME: TM!
		case PROCESSOR_X86_DUAL: return "MP";
		default: return NULL;
	}
}

static int
fprintf_bunit(FILE *fp,const char *suffix,uintmax_t val){
	const char units[] = "KMGTPEZY",*unit = units;
	const uintmax_t SCALE = 1024;
	int r;

	if(val < SCALE || (val % SCALE)){
		return fprintf(fp,"%ju%s",val,suffix);
	}
	val /= SCALE;
	while(val >= SCALE && (val % SCALE == 0)){
		val /= SCALE;
		if(!*++unit){ // damn, chief!
			return -1;
		}
	}
	if((r = fprintf(fp,"%ju%c%s",val,*unit,suffix)) < 0){
		return r;
	}
	if((val >= SCALE) && (val % SCALE) && *++unit){
		return fprintf(fp," (%.3f%c%s)",(float)val / SCALE,*unit,suffix);
	}
	return 0;
}
<commit_msg>even better output for unclean values (3980532KB -> 3.796GB rather than 3887.238MB; 12292928KB -> 11.723GB rather than 0.011GB, etc)<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <libtorque/arch.h>
#include <libtorque/memory.h>
#include <libtorque/libtorque.h>

static const char *
memory_type(int mtype){
	switch(mtype){
		case MEMTYPE_DATA: return "data";
		case MEMTYPE_CODE: return "code";
		case MEMTYPE_UNIFIED: return "unified";
		default: return NULL;
	}
}

static const char *
tlb_type(int ttype){
	switch(ttype){
		case MEMTYPE_DATA: return "data";
		case MEMTYPE_CODE: return "code";
		case MEMTYPE_UNIFIED: return "mixed";
		default: return NULL;
	}
}

static const char *
x86_type(int x86type){
	switch(x86type){
		case PROCESSOR_X86_OEM: return "OEM";
		case PROCESSOR_X86_OVERDRIVE: return "OverDrive"; // FIXME: TM!
		case PROCESSOR_X86_DUAL: return "MP";
		default: return NULL;
	}
}

static int
fprintf_bunit(FILE *fp,const char *suffix,uintmax_t val){
	const char units[] = "KMGTPEZY",*unit = units;
	const uintmax_t SCALE = 1024;
	int r;

	if(val < SCALE || (val % SCALE)){
		return fprintf(fp,"%ju%s",val,suffix);
	}
	val /= SCALE;
	while(val >= SCALE && (val % SCALE == 0)){
		val /= SCALE;
		if(!*++unit){ // damn, chief!
			return -1;
		}
	}
	if((r = fprintf(fp,"%ju%c%s",val,*unit,suffix)) < 0){
		return r;
	}
	if(val >= SCALE && (val % SCALE)){
		while(*++unit && (val >= SCALE * SCALE) && (val % SCALE)){
			val /= SCALE;
		}
		if(*unit){
			return fprintf(fp," (%.3f%c%s)",(float)val / SCALE,*unit,suffix);
		}
	}
	return 0;
}

{'fraction_non_alphanumeric': 0.16909814323607428, 'fraction_numerical': 0.02553050397877984, 'mean_word_length': 4.436036036036036, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2732623', 'n_tokens_mistral': 1350, 'n_tokens_neox': 1218, 'n_words': 290}