<commit_before>#include "lexer.h"

// this is just for debugging
static const char* TOKEN_NAMES[] = {
	"END_OF_FILE", "IDENTIFIER", "NUMBER",
	"OPERATOR", "SEPARATOR", "ERRORNEOUS",
	"STRING", "CHARACTER", "UNKNOWN"
};

token *create_token(lexer *lexer) {
	token *tok = safe_malloc(sizeof(*tok));
	tok->type = UNKNOWN;
	tok->content = NULL;
	tok->line_number = lexer->line_number;
	tok->char_number = lexer->char_number;
	return tok;
}

const char* get_token_name(token *tok) {
	return TOKEN_NAMES[tok->type];
}

void destroy_token(token *token) {
	if (token) {
		free(token);
	}
}

lexer *create_lexer(char* input) {
	lexer *lexer = safe_malloc(sizeof(*lexer));
	lexer->input = input;
	lexer->input_size = strlen(lexer->input);
	lexer->pos = 0;
	lexer->current_char = input[lexer->pos];
	lexer->token_stream = create_vector();
	lexer->running = true;
	lexer->line_number = 1;
	lexer->char_number = 1;
	return lexer;
}

void consume_character(lexer *lexer) {
	if (lexer->pos > lexer->input_size) {
		error_message("reached end of input");
		destroy_lexer(lexer);
		return;
	}
	if (lexer->current_char == '\n' || is_end_of_input(lexer->current_char)) {
		lexer->char_number = 0;	// reset the char number back to zero
		lexer->line_number++;
	}
	lexer->current_char = lexer->input[++lexer->pos];
	lexer->char_number++;
}

char* extract_token(lexer *lexer, int start, int length) {
	char* result = safe_malloc(sizeof(char) * (length + 1));
	strncpy(result, &lexer->input[start], length);<commit_msg>separate is_end_of_input check and handle accordingly<commit_after>#include "lexer.h"

// this is just for debugging
static const char* TOKEN_NAMES[] = {
	"END_OF_FILE", "IDENTIFIER", "NUMBER",
	"OPERATOR", "SEPARATOR", "ERRORNEOUS",
	"STRING", "CHARACTER", "UNKNOWN"
};

token *create_token(lexer *lexer) {
	token *tok = safe_malloc(sizeof(*tok));
	tok->type = UNKNOWN;
	tok->content = NULL;
	tok->line_number = lexer->line_number;
	tok->char_number = lexer->char_number;
	return tok;
}

const char* get_token_name(token *tok) {
	return TOKEN_NAMES[tok->type];
}

void destroy_token(token *token) {
	if (token) {
		free(token);
	}
}

lexer *create_lexer(char* input) {
	lexer *lexer = safe_malloc(sizeof(*lexer));
	lexer->input = input;
	lexer->input_size = strlen(lexer->input);
	lexer->pos = 0;
	lexer->current_char = input[lexer->pos];
	lexer->token_stream = create_vector();
	lexer->running = true;
	lexer->line_number = 1;
	lexer->char_number = 1;
	return lexer;
}

void consume_character(lexer *lexer) {
	if (lexer->pos > lexer->input_size) {
		error_message("reached end of input");
		destroy_lexer(lexer);
		return;
	}
	if (lexer->current_char == '\n') {
		lexer->char_number = 0;	// reset the char number back to zero
		lexer->line_number++;
	}
    else if(is_end_of_input(lexer->current_char)) {
        return;
    }
    
	lexer->current_char = lexer->input[++lexer->pos];
	lexer->char_number++;
}

char* extract_token(lexer *lexer, int start, int length) {
	char* result = safe_malloc(sizeof(char) * (length + 1));
	strncpy(result, &lexer->input[start], length);
{'fraction_non_alphanumeric': 0.16574405731032238, 'fraction_numerical': 0.0032562683165092803, 'mean_word_length': 4.976653696498055, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4278910', 'n_tokens_mistral': 1221, 'n_tokens_neox': 1180, 'n_words': 238}