<commit_before>#include "ex/lexer.h"
#include <ctype.h>
#include "array.h"
#include "io.h"
#include "utils.h"

LIP_IMPLEMENT_CONSTRUCTOR_AND_DESTRUCTOR(lip_lexer)

void
lip_lexer_init(lip_lexer_t* lexer, lip_allocator_t* allocator)
{
	lexer->allocator = allocator;
	lexer->capture_buffs = lip_array_create(allocator, lip_array(char), 0);
	lexer->capture_buff = NULL;
	lip_lexer_reset(lexer, NULL);
}

void
lip_lexer_cleanup(lip_lexer_t* lexer)
{
	lip_lexer_reset(lexer, NULL);
	lip_array_destroy(lexer->capture_buffs);
	if(lexer->capture_buff)
	{
		lip_array_destroy(lexer->capture_buff);
	}
}

void
lip_lexer_reset(lip_lexer_t* lexer, lip_in_t* input)
{
	lexer->location.line = 1;
	lexer->location.column = 1;
	memset(&lexer->error, 0, sizeof(lexer->error));
	lexer->read_buff = 0;
	lexer->capturing = false;
	lexer->buffered = false;
	lexer->eos = false;
	lexer->input = input;
	lip_array_foreach(lip_array(char), buff, lexer->capture_buffs)
	{
		lip_array_destroy(*buff);
	}
	lip_array_clear(lexer->capture_buffs);
	if(lexer->capture_buff)
	{
		lip_array_clear(lexer->capture_buff);
	}
}

static void
lip_lexer_begin_capture(lip_lexer_t* lexer)
{
	if(!lexer->capture_buff)
	{
		lexer->capture_buff = lip_array_create(lexer->allocator, char, 0);
	}
	lexer->capturing = true;
}

static void
lip_lexer_reset_capture(lip_lexer_t* lexer)
{
	if(lexer->capture_buff)
	{
		lip_array_clear(lexer->capture_buff);
	}
	lexer->capturing = false;
}

static lip_string_ref_t
lip_lexer_end_capture(lip_lexer_t* lexer)
{

	unsigned int len = lip_array_len(lexer->capture_buff);
	lip_array_push(lexer->capture_buff, 0); // null-terminate
	lip_string_ref_t ref = { len, lexer->capture_buff };
	lip_array_push(lexer->capture_buffs, lexer->capture_buff);
	lexer->capture_buff = NULL;
	lexer->capturing = false;

	return ref;
}

static lip_stream_status_t
lip_lexer_make_token(
	lip_lexer_t* lexer,
	lip_token_t* token,
	lip_token_type_t type
)
{
	token->type = type;
	token->lexeme = lip_lexer_end_capture(lexer);
	token->location.end = lexer->location;
	--token->location.end.column;

	return LIP_STREAM_OK;
}

static bool
lip_lexer_peek_char(lip_lexer_t* lexer, char* ch)
{
	if(lexer->buffered) { *ch = lexer->read_buff; return true; }

	if(lip_read(&lexer->read_buff, 1, lexer->input))
	{
		lexer->buffered = true;
		*ch = lexer->read_buff;
		return true;
	}
	else
	{
		lexer->eos = true;
		return false;
	}
}

static void
lip_lexer_consume_char(lip_lexer_t* lexer)
{
	lexer->buffered = false;
	if(lexer->capturing)
	{
		lip_array_push(lexer->capture_buff, lexer->read_buff);
	}
	++lexer->location.column;
}

static inline bool lip_lexer_is_separator(char ch)
{
	return isspace(ch) || ch == ')' || ch == '(' || ch == ';' || ch == '"';
}

static lip_stream_status_t
lip_lexer_error(lip_lexer_t* lexer, lip_lex_error_t error)
{
	lexer->error.code = error;
	lexer->error.location.end = lexer->location;
	--lexer->error.location.end.column;
	lip_lexer_reset_capture(lexer);

	return LIP_STREAM_ERROR;
}
<commit_msg>Clean up<commit_after>#include "ex/lexer.h"
#include <ctype.h>
#include "array.h"
#include "io.h"
#include "utils.h"

LIP_IMPLEMENT_CONSTRUCTOR_AND_DESTRUCTOR(lip_lexer)

void
lip_lexer_init(lip_lexer_t* lexer, lip_allocator_t* allocator)
{
	lexer->allocator = allocator;
	lexer->capture_buffs = lip_array_create(allocator, lip_array(char), 0);
	lexer->capture_buff = NULL;
	lip_lexer_reset(lexer, NULL);
}

void
lip_lexer_cleanup(lip_lexer_t* lexer)
{
	lip_lexer_reset(lexer, NULL);
	lip_array_destroy(lexer->capture_buffs);
	if(lexer->capture_buff)
	{
		lip_array_destroy(lexer->capture_buff);
	}
}

void
lip_lexer_reset(lip_lexer_t* lexer, lip_in_t* input)
{
	lexer->location.line = 1;
	lexer->location.column = 1;
	memset(&lexer->error, 0, sizeof(lexer->error));
	lexer->read_buff = 0;
	lexer->capturing = false;
	lexer->buffered = false;
	lexer->eos = false;
	lexer->input = input;
	lip_array_foreach(lip_array(char), buff, lexer->capture_buffs)
	{
		lip_array_destroy(*buff);
	}
	lip_array_clear(lexer->capture_buffs);
	if(lexer->capture_buff)
	{
		lip_array_clear(lexer->capture_buff);
	}
}

static void
lip_lexer_begin_capture(lip_lexer_t* lexer)
{
	if(!lexer->capture_buff)
	{
		lexer->capture_buff = lip_array_create(lexer->allocator, char, 0);
	}
	lexer->capturing = true;
}

static void
lip_lexer_reset_capture(lip_lexer_t* lexer)
{
	if(lexer->capture_buff)
	{
		lip_array_clear(lexer->capture_buff);
	}
	lexer->capturing = false;
}

static lip_string_ref_t
lip_lexer_end_capture(lip_lexer_t* lexer)
{

	unsigned int len = lip_array_len(lexer->capture_buff);
	lip_array_push(lexer->capture_buff, 0); // null-terminate
	lip_string_ref_t ref = { len, lexer->capture_buff };
	lip_array_push(lexer->capture_buffs, lexer->capture_buff);
	lexer->capture_buff = NULL;
	lexer->capturing = false;

	return ref;
}

static lip_stream_status_t
lip_lexer_make_token(
	lip_lexer_t* lexer,
	lip_token_t* token,
	lip_token_type_t type
)
{
	token->type = type;
	token->lexeme = lip_lexer_end_capture(lexer);
	token->location.end = lexer->location;
	--token->location.end.column;

	return LIP_STREAM_OK;
}

static bool
lip_lexer_peek_char(lip_lexer_t* lexer, char* ch)
{
	if(lexer->buffered) { *ch = lexer->read_buff; return true; }

	if(lip_read(&lexer->read_buff, 1, lexer->input))
	{
		lexer->buffered = true;
		*ch = lexer->read_buff;
		return true;
	}
	else
	{
		lexer->eos = true;
		return false;
	}
}

static void
lip_lexer_consume_char(lip_lexer_t* lexer)
{
	lexer->buffered = false;
	if(lexer->capturing)
	{
		lip_array_push(lexer->capture_buff, lexer->read_buff);
	}
	++lexer->location.column;
}

static inline bool
lip_lexer_is_separator(char ch)
{
	return isspace(ch) || ch == ')' || ch == '(' || ch == ';' || ch == '"';
}

static lip_stream_status_t
lip_lexer_error(lip_lexer_t* lexer, lip_lex_error_t error)
{
	lexer->error.code = error;
	lexer->error.location.end = lexer->location;
	--lexer->error.location.end.column;
	lip_lexer_reset_capture(lexer);

	return LIP_STREAM_ERROR;
}

{'fraction_non_alphanumeric': 0.14230833472523802, 'fraction_numerical': 0.002672456990145315, 'mean_word_length': 5.653333333333333, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 113, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6294718', 'n_tokens_mistral': 2710, 'n_tokens_neox': 2526, 'n_words': 367}