<commit_before>#include <iostream>
#include <vector>
#include <climits>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <getopt.h>
#include <string>
#include <algorithm> 
#include <map>

using namespace std;

/* FLAGS */
static int brief_flag;
static int delimiter_flag;
static int transpose_flag;
static int q1_flag;
static int q3_flag;
static int n_flag;
static int min_flag;
static int max_flag;
static int mean_flag;
static int median_flag;
static int sum_flag;
static int sd_flag;
static int sderr_flag;
static int population_flag;
static int sample_flag;
static int comp_flag;
static int var_flag;
static int help_flag;
static int sort_flag; //redundent?
static int quartiles_flag;
static int all_flag;

/* VARIABLES */
string delimiter = "\t";

/* DATA STRUCTURES */
vector<long double> stats;
vector<long double> points;
map<string, long double> global_stats;
map<string, int> opts;

/* STATS COMPUTED AT THE LINE */
void compute_line_stats(){
	long double stat = stats.at(0);

	//sum
	stats.at(1) += stat;

	//N
	stats.at(4)+=1;

	//min
	if (stats.at(2) > stat){
		stats.at(2) = stat;
	}

	//max
	if (stats.at(3) < stat){
		stats.at(3) = stat; 
	}
}

void print_help(){
	cerr << " " << endl;	
	cerr << "Usage: sta [options] < file.txt" << endl;	
	cerr << " " << endl;	
	cerr << "--help		prints this help " << endl;	
	cerr << " " << endl;	
	cerr << "Desriptive Stastistics: " << endl;	
	cerr << "--mean		average" << endl;	
	cerr << "--median	median" << endl;	
	cerr << "--min		min point" << endl;	
	cerr << "--max		max point" << endl;	
	cerr << "--n		sample size" << endl;	
	cerr << "--sd		standard deviation" << endl;	
	cerr << "--sderr	standard error of the mean" << endl;	
	cerr << "--sum		sum" << endl;	
	cerr << "--q		quartiles" << endl;	
	cerr << "--q1		first quartile" << endl;	
	cerr << "--q3		third quartile" << endl;	
	cerr << "--var		variance" << endl;	
	cerr << " " << endl;	
	cerr << "Options: " << endl;	
	cerr << "--brief		brief mode; only values are output" << endl;	
	cerr << "--compensated		compenssated variant" << endl;	
	cerr << "--delimiter		specifiy string to use as delimiter" << endl;	
	cerr << "--population		unbiased estimator (n-1)" << endl;	
	cerr << "--transpose		transpose output" << endl;	
}

void compute_quartiles(long double &n){
	size_t const size = points.size();
	int const Q1 = size / 4;
	int const Q2 = size / 2;
	int const Q3 = Q1 + Q2;

	/** PARTIAL SORT **/
	nth_element(points.begin(),          points.begin() + Q1, points.end());
	nth_element(points.begin() + Q1 + 1, points.begin() + Q2, points.end());
	nth_element(points.begin() + Q2 + 1, points.begin() + Q3, points.end());

	long double firq = points[Q1];
	long double lasq = points[Q3];
	long double median = size%2!=0 ? points[Q2] : (points[Q2]+points[Q2-1])/2;
	
	global_stats["Q1"] = firq;
	global_stats["median"] = median; 
	global_stats["Q3"] = lasq;<commit_msg>typos<commit_after>#include <iostream>
#include <vector>
#include <climits>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <getopt.h>
#include <string>
#include <algorithm> 
#include <map>

using namespace std;

/* FLAGS */
static int brief_flag;
static int delimiter_flag;
static int transpose_flag;
static int q1_flag;
static int q3_flag;
static int n_flag;
static int min_flag;
static int max_flag;
static int mean_flag;
static int median_flag;
static int sum_flag;
static int sd_flag;
static int sderr_flag;
static int population_flag;
static int sample_flag;
static int comp_flag;
static int var_flag;
static int help_flag;
static int sort_flag; //redundent?
static int quartiles_flag;
static int all_flag;

/* VARIABLES */
string delimiter = "\t";

/* DATA STRUCTURES */
vector<long double> stats;
vector<long double> points;
map<string, long double> global_stats;
map<string, int> opts;

/* STATS COMPUTED AT THE LINE */
void compute_line_stats(){
	long double stat = stats.at(0);

	//sum
	stats.at(1) += stat;

	//N
	stats.at(4)+=1;

	//min
	if (stats.at(2) > stat){
		stats.at(2) = stat;
	}

	//max
	if (stats.at(3) < stat){
		stats.at(3) = stat; 
	}
}

void print_help(){
	cerr << " " << endl;	
	cerr << "Usage: sta [options] < file.txt" << endl;	
	cerr << " " << endl;	
	cerr << "--help		prints this help " << endl;	
	cerr << " " << endl;	
	cerr << "Descriptive Stastistics: " << endl;	
	cerr << "--mean		average" << endl;	
	cerr << "--median	median" << endl;	
	cerr << "--min		min point" << endl;	
	cerr << "--max		max point" << endl;	
	cerr << "--n		sample size" << endl;	
	cerr << "--sd		standard deviation" << endl;	
	cerr << "--sderr	standard error of the mean" << endl;	
	cerr << "--sum		sum" << endl;	
	cerr << "--q		quartiles" << endl;	
	cerr << "--q1		first quartile" << endl;	
	cerr << "--q3		third quartile" << endl;	
	cerr << "--var		variance" << endl;	
	cerr << " " << endl;	
	cerr << "Options: " << endl;	
	cerr << "--brief		brief mode; only values are output" << endl;	
	cerr << "--compensated		compensated variant" << endl;	
	cerr << "--delimiter		specify string to use as delimiter" << endl;	
	cerr << "--population		unbiased estimator (n-1)" << endl;	
	cerr << "--transpose		transpose output" << endl;	
}

void compute_quartiles(long double &n){
	size_t const size = points.size();
	int const Q1 = size / 4;
	int const Q2 = size / 2;
	int const Q3 = Q1 + Q2;

	/** PARTIAL SORT **/
	nth_element(points.begin(),          points.begin() + Q1, points.end());
	nth_element(points.begin() + Q1 + 1, points.begin() + Q2, points.end());
	nth_element(points.begin() + Q2 + 1, points.begin() + Q3, points.end());

	long double firq = points[Q1];
	long double lasq = points[Q3];
	long double median = size%2!=0 ? points[Q2] : (points[Q2]+points[Q2-1])/2;
	
	global_stats["Q1"] = firq;
	global_stats["median"] = median; 
	global_stats["Q3"] = lasq;
{'fraction_non_alphanumeric': 0.17150713180958926, 'fraction_numerical': 0.013060663344217219, 'mean_word_length': 3.6153846153846154, 'pattern_counts': {'":': 0, '<': 237, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2581943', 'n_tokens_mistral': 2290, 'n_tokens_neox': 2223, 'n_words': 637}