<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdbool.h>
#include <assert.h>
#include "debug.h"


#define NULL_ENCODED '\x1a'

#define BUFFER_SIZE 1024*1024
//#define BUFFER_SIZE 3


static char _buffer[BUFFER_SIZE];

struct {
	FILE* source;
	char separator;
	bool drop_header;
} config;

static void parse_config(int argc, char** argv);
static void do_pipe(size_t chars_read);

int main(int argc, char** argv) {
	parse_config(argc, argv);

	size_t chars_read;
	while ((chars_read = fread(_buffer, sizeof(char), BUFFER_SIZE, config.source)) > 0) {
		do_pipe(chars_read);
	}
	return 0;
}

static void print_help() {
	fprintf(stderr, "usage: csvpipe [OPTIONS] [FILE]");
	fprintf(stderr, "options:");
	fprintf(stderr, "-s character\n");
	fprintf(stderr, "  Which character is used as separator (default is ,)\n");
	fprintf(stderr, "-d\n");
	fprintf(stderr, "  drop header row\n");
}




static void parse_config(int argc, char** argv) {
	config.source = stdin;
	config.separator = ',';
	config.drop_header = false;
	char c;
	while ((c = getopt (argc, argv, "s:d")) != -1) {
		switch (c) {
			case 's': 
				config.separator = optarg[0];
				break;
			case 'd':
				config.drop_header = true;
				break;
			case '?':
			case 'h':
			default:
				print_help();
				exit(1);
				break;
		}
	}
	if (optind < argc) {
		config.source = fopen(argv[optind], "r");
		if (!config.source) {
			fprintf(stderr, "Could not open file %s for reading\n", argv[optind]);
			exit(1);<commit_msg>Removed need for specifying separator<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdbool.h>
#include <assert.h>
#include "debug.h"


#define NULL_ENCODED '\x1a'

#define BUFFER_SIZE 1024*1024
//#define BUFFER_SIZE 3


static char _buffer[BUFFER_SIZE];

struct {
	FILE* source;
	bool drop_header;
} config;

static void parse_config(int argc, char** argv);
static void do_pipe(size_t chars_read);

int main(int argc, char** argv) {
	parse_config(argc, argv);

	size_t chars_read;
	while ((chars_read = fread(_buffer, sizeof(char), BUFFER_SIZE, config.source)) > 0) {
		do_pipe(chars_read);
	}
	return 0;
}

static void print_help() {
	fprintf(stderr, "usage: csvpipe [OPTIONS] [FILE]");
	fprintf(stderr, "options:");
	fprintf(stderr, "-d\n");
	fprintf(stderr, "  drop header row\n");
}




static void parse_config(int argc, char** argv) {
	config.source = stdin;
	config.drop_header = false;
	char c;
	while ((c = getopt (argc, argv, "d")) != -1) {
		switch (c) {
			case 'd':
				config.drop_header = true;
				break;
			case '?':
			case 'h':
			default:
				print_help();
				exit(1);
				break;
		}
	}
	if (optind < argc) {
		config.source = fopen(argv[optind], "r");
		if (!config.source) {
			fprintf(stderr, "Could not open file %s for reading\n", argv[optind]);
			exit(1);
{'fraction_non_alphanumeric': 0.16137005649717515, 'fraction_numerical': 0.010946327683615819, 'mean_word_length': 4.113718411552346, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2702581', 'n_tokens_mistral': 1180, 'n_tokens_neox': 1076, 'n_words': 276}