<commit_before>#include <string.h>

#include "lily_interp.h"
#include "lily_ast.h"
#include "lily_lexer.h"
#include "lily_symtab.h"
#include "lily_impl.h"
#include "lily_emitter.h"
#include "lily_debug.h"

typedef struct {
    int depth;
    int *num_expected;
    lily_ast_pool *ast_pool;
    lily_ast **saved_trees;
    lily_ast *current_tree;
    int num_args;
    lily_reg_data *reg_state;
} parser_data;

static void enter_parenth(parser_data *pr_data, int args_needed)
{
    pr_data->num_expected[pr_data->depth] = pr_data->num_args + args_needed;
    pr_data->depth++;
    pr_data->current_tree = NULL;
}

static parser_data *init_parser_data(void)
{
    parser_data *pr_data = lily_impl_malloc(sizeof(parser_data));
    pr_data->saved_trees = lily_impl_malloc(sizeof(lily_ast *) * 32);
    pr_data->num_expected = lily_impl_malloc(sizeof(int) * 32);
    pr_data->ast_pool = lily_ast_init_pool(32);
    pr_data->depth = 0;
    pr_data->num_args = 0;
    pr_data->current_tree = NULL;

    pr_data->reg_state = lily_init_reg_data();
    return pr_data;
}

static void free_parser_data(parser_data *pr_data)
{
    lily_ast_free_pool(pr_data->ast_pool);
    lily_free_reg_data(pr_data->reg_state);
    free(pr_data->saved_trees);
    free(pr_data->num_expected);
    free(pr_data);
}

static void parse_expr_value(lily_interp *interp, parser_data *pr_data)
{
    lily_token *token = interp->lex_data->token;

    if (token->tok_type == tk_word) {
        lily_symbol *sym;
        sym = lily_st_find_symbol(interp->symtab, token->word_buffer);

        if (sym == NULL)
            lily_impl_fatal("Variable '%s' is undefined.\n",
                            token->word_buffer);

        if (sym->callable) {
            /* New trees will get saved to the args section of this tree
                when they are done. */
            lily_ast *ast = lily_ast_init_call(pr_data->ast_pool, sym);
            pr_data->saved_trees[pr_data->depth] = ast;

            enter_parenth(pr_data, sym->num_args);

            lily_lexer(interp->lex_data);
            if (token->tok_type != tk_left_parenth)
                lily_impl_fatal("Expected '(' after function name.\n");

            lily_lexer(interp->lex_data);
            /* This handles the first value of the function. */
            parse_expr_value(interp, pr_data);
        }
        else {
            lily_symbol *sym = lily_st_new_var_sym(interp, token->word_buffer);<commit_msg>Parser: Don't init var within parse_expr_value.<commit_after>#include <string.h>

#include "lily_interp.h"
#include "lily_ast.h"
#include "lily_lexer.h"
#include "lily_symtab.h"
#include "lily_impl.h"
#include "lily_emitter.h"
#include "lily_debug.h"

typedef struct {
    int depth;
    int *num_expected;
    lily_ast_pool *ast_pool;
    lily_ast **saved_trees;
    lily_ast *current_tree;
    int num_args;
    lily_reg_data *reg_state;
} parser_data;

static void enter_parenth(parser_data *pr_data, int args_needed)
{
    pr_data->num_expected[pr_data->depth] = pr_data->num_args + args_needed;
    pr_data->depth++;
    pr_data->current_tree = NULL;
}

static parser_data *init_parser_data(void)
{
    parser_data *pr_data = lily_impl_malloc(sizeof(parser_data));
    pr_data->saved_trees = lily_impl_malloc(sizeof(lily_ast *) * 32);
    pr_data->num_expected = lily_impl_malloc(sizeof(int) * 32);
    pr_data->ast_pool = lily_ast_init_pool(32);
    pr_data->depth = 0;
    pr_data->num_args = 0;
    pr_data->current_tree = NULL;

    pr_data->reg_state = lily_init_reg_data();
    return pr_data;
}

static void free_parser_data(parser_data *pr_data)
{
    lily_ast_free_pool(pr_data->ast_pool);
    lily_free_reg_data(pr_data->reg_state);
    free(pr_data->saved_trees);
    free(pr_data->num_expected);
    free(pr_data);
}

static void parse_expr_value(lily_interp *interp, parser_data *pr_data)
{
    lily_token *token = interp->lex_data->token;

    if (token->tok_type == tk_word) {
        lily_symbol *sym;
        sym = lily_st_find_symbol(interp->symtab, token->word_buffer);

        if (sym == NULL)
            lily_impl_fatal("Variable '%s' is undefined.\n",
                            token->word_buffer);

        if (sym->callable) {
            /* New trees will get saved to the args section of this tree
                when they are done. */
            lily_ast *ast = lily_ast_init_call(pr_data->ast_pool, sym);
            pr_data->saved_trees[pr_data->depth] = ast;

            enter_parenth(pr_data, sym->num_args);

            lily_lexer(interp->lex_data);
            if (token->tok_type != tk_left_parenth)
                lily_impl_fatal("Expected '(' after function name.\n");

            lily_lexer(interp->lex_data);
            /* This handles the first value of the function. */
            parse_expr_value(interp, pr_data);
        }
        else {
{'fraction_non_alphanumeric': 0.11714876033057851, 'fraction_numerical': 0.003305785123966942, 'mean_word_length': 3.0852320675105487, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7409577', 'n_tokens_mistral': 1788, 'n_tokens_neox': 1738, 'n_words': 306}