<commit_before>/*
 ============================================================================
 Name        : parser.c
 Author      : <NAME>
 Version     : 0.1
 Copyright   : (c) 2015 <NAME>
 Description : Parser for the Claw language
 ============================================================================
 */

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>

#include "parser.h"
#include "lex/lex.h"
#include "rpn.h"
#include "cpl/cpl.h"
#include "error/error.h"

// test the lexer
int do_parse(char* file)
{
	FILE* fp = fopen(file, "r");

	if(!fp)
	    return ERR_INVALID_FILE;

	LexState ls;
	lex_init(&ls, fp);

	while (!feof(fp))
	{
		int tok = lex_next(&ls);
		printf("%i", tok);
		if (tok == TK_NAME)
			printf(" %s", ls.kf.name);
		if (tok == TK_NUMBER)
					printf(" %d", ls.kf.number);
		printf("\n");
	}

	fclose(fp);
	return ERR_NO_ERROR;
}<commit_msg>More descriptive output<commit_after>/*
 ============================================================================
 Name        : parser.c
 Author      : <NAME>
 Version     : 0.1
 Copyright   : (c) 2015 <NAME>
 Description : Parser for the Claw language
 ============================================================================
 */

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>

#include "parser.h"
#include "lex/lex.h"
#include "rpn.h"
#include "cpl/cpl.h"
#include "error/error.h"

// test the lexer
int do_parse(char* file)
{
	FILE* fp = fopen(file, "r");

	if(!fp)
	    return ERR_INVALID_FILE;

	LexState ls;
	lex_init(&ls, fp);

	while (!feof(fp))
	{
		int tok = lex_next(&ls);
		printf("%d", tok);
		if (tok == TK_NAME)
			printf(" name: %s", ls.kf.name);
		if (tok == TK_NUMBER)
			printf(" number: %d", ls.kf.number);
		printf("\n");
	}

	fclose(fp);
	return ERR_NO_ERROR;
}
{'fraction_non_alphanumeric': 0.3221476510067114, 'fraction_numerical': 0.006711409395973154, 'mean_word_length': 4.03943661971831, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4572843', 'n_tokens_mistral': 699, 'n_tokens_neox': 615, 'n_words': 150}