<commit_before>#include <gojira/runtime/runtime.h>
#include <gojira/runtime/frame.h>
#include <gojira/runtime/syntax.h>
#include <gojira/runtime/builtin.h>
#include <gojira/runtime/garbage.h>
#include <gojira/parse_debug.h>

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdbool.h>

token_t *eval_loop( stack_frame_t *base, token_t *tokens ){
	stack_frame_t *frame = base;
	token_t *ret = tokens;

	bool running = true;
	bool have_error = false;

	while ( running && !have_error ){
		// Evaluate sub-expressions
		if ( frame->ptr ){
			have_error = eval_frame_subexpr( &frame, base );

		// Evaluation finished, apply function
		} else {
			if ( frame->last ){
				have_error = eval_frame_expr( &frame, base );

			} else {
				running = false;
			}
		}
	}

	ret = base->expr;

	return ret;
}

bool eval_frame_subexpr( stack_frame_t **frame_ret, stack_frame_t *first ){
	bool ret = false;
	stack_frame_t *frame = *frame_ret;
	token_t *move;

	switch ( frame->ptr->type ){
		case TYPE_LIST:
			move = frame->ptr;
			frame->ptr = frame->ptr->next;

			if ( move->down == NULL ){
				printf( "[%s] Error: Empty expression\n", __func__ );

				stack_trace( frame );
				ret = true;
				break;
			}

			frame = *frame_ret = frame_create( frame, move->down );
			break;

		case TYPE_SYMBOL:
			if ( frame->ptr->down ){
				frame_add_token( frame, frame->ptr->down );
				frame->ptr = frame->ptr->next;

			} else {
				// search for symbol in the highest scope, which has the
				// most often used variables
				move = frame_find_var( first, frame->ptr->data );

				// didn't find it, start from top frame
				if ( !move )
					move = frame_find_var( frame, frame->ptr->data );

				if ( move ){
					frame_add_token( frame, move );

					if ( move->type != TYPE_SYNTAX ){
						frame->ptr = frame->ptr->next;

					} else {
						for ( frame->ptr = frame->ptr->next; frame->ptr; frame->ptr = frame->ptr->next )
							frame_add_token( frame, frame->ptr );
					}

				} else {
					printf( "[%s] Error: undefined variable \"%s\"\n",
							__func__, (char *)frame->ptr->data );

					stack_trace( frame );
					ret = true;
				}
			}

			break;

		case TYPE_LAMBDA:
			for ( ; frame->ptr; frame->ptr = frame->ptr->next )
				frame_add_token( frame, frame->ptr );

			break;

		case TYPE_DEF_SYNTAX:
			frame_add_token( frame, frame->ptr );
			frame_add_token( frame, frame->ptr->next );
			frame->ptr = frame->ptr->next->next;
			break;

		case TYPE_SYNTAX_RULES:
			move = calloc( 1, sizeof( token_t ));
			move->type = TYPE_SYNTAX;
			move->down = frame->ptr;

			frame_add_token_noclone( frame, ext_proc_token( builtin_return_first ));
			frame_add_token( frame, move );

			frame->ptr = NULL;
			break;

		case TYPE_IF:
			move = expand_if_expr( frame, frame->ptr );

			if ( move ){
				frame_add_token_noclone( frame, move );
				frame->ptr = move->next;

			} else {
				ret = true;
			}

			break;

		case TYPE_QUOTED_TOKEN:
			frame_add_token( frame, frame->ptr->down );
			frame->ptr = frame->ptr->next;
			break;<commit_msg>Fixing memory leak in syntax definitions<commit_after>#include <gojira/runtime/runtime.h>
#include <gojira/runtime/frame.h>
#include <gojira/runtime/syntax.h>
#include <gojira/runtime/builtin.h>
#include <gojira/runtime/garbage.h>
#include <gojira/parse_debug.h>

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdbool.h>

token_t *eval_loop( stack_frame_t *base, token_t *tokens ){
	stack_frame_t *frame = base;
	token_t *ret = tokens;

	bool running = true;
	bool have_error = false;

	while ( running && !have_error ){
		// Evaluate sub-expressions
		if ( frame->ptr ){
			have_error = eval_frame_subexpr( &frame, base );

		// Evaluation finished, apply function
		} else {
			if ( frame->last ){
				have_error = eval_frame_expr( &frame, base );

			} else {
				running = false;
			}
		}
	}

	ret = base->expr;

	return ret;
}

bool eval_frame_subexpr( stack_frame_t **frame_ret, stack_frame_t *first ){
	bool ret = false;
	stack_frame_t *frame = *frame_ret;
	token_t *move;

	switch ( frame->ptr->type ){
		case TYPE_LIST:
			move = frame->ptr;
			frame->ptr = frame->ptr->next;

			if ( move->down == NULL ){
				printf( "[%s] Error: Empty expression\n", __func__ );

				stack_trace( frame );
				ret = true;
				break;
			}

			frame = *frame_ret = frame_create( frame, move->down );
			break;

		case TYPE_SYMBOL:
			if ( frame->ptr->down ){
				frame_add_token( frame, frame->ptr->down );
				frame->ptr = frame->ptr->next;

			} else {
				// search for symbol in the highest scope, which has the
				// most often used variables
				move = frame_find_var( first, frame->ptr->data );

				// didn't find it, start from top frame
				if ( !move )
					move = frame_find_var( frame, frame->ptr->data );

				if ( move ){
					frame_add_token( frame, move );

					if ( move->type != TYPE_SYNTAX ){
						frame->ptr = frame->ptr->next;

					} else {
						for ( frame->ptr = frame->ptr->next; frame->ptr; frame->ptr = frame->ptr->next )
							frame_add_token( frame, frame->ptr );
					}

				} else {
					printf( "[%s] Error: undefined variable \"%s\"\n",
							__func__, (char *)frame->ptr->data );

					stack_trace( frame );
					ret = true;
				}
			}

			break;

		case TYPE_LAMBDA:
			for ( ; frame->ptr; frame->ptr = frame->ptr->next )
				frame_add_token( frame, frame->ptr );

			break;

		case TYPE_DEF_SYNTAX:
			frame_add_token( frame, frame->ptr );
			frame_add_token( frame, frame->ptr->next );
			frame->ptr = frame->ptr->next->next;
			break;

		case TYPE_SYNTAX_RULES:
			move = calloc( 1, sizeof( token_t ));
			move->type = TYPE_SYNTAX;
			move->down = clone_tokens( frame->ptr );

			gc_unmark( move );
			frame_add_token_noclone( frame, ext_proc_token( builtin_return_first ));
			frame_add_token_noclone( frame, move );

			frame->ptr = NULL;
			break;

		case TYPE_IF:
			move = expand_if_expr( frame, frame->ptr );

			if ( move ){
				frame_add_token_noclone( frame, move );
				frame->ptr = move->next;

			} else {
				ret = true;
			}

			break;

		case TYPE_QUOTED_TOKEN:
			frame_add_token( frame, frame->ptr->down );
			frame->ptr = frame->ptr->next;
			break;
{'fraction_non_alphanumeric': 0.14830988193433609, 'fraction_numerical': 0.00032346757237586933, 'mean_word_length': 2.9489144316730522, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 143, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2059794', 'n_tokens_mistral': 2666, 'n_tokens_neox': 2192, 'n_words': 511}