<commit_before>use std::collections::HashMap;
use std::fmt;

use crate::ast;
use crate::lexer;
use crate::token;

// For debug visualization I migth potentially use this approach
// https://users.rust-lang.org/t/is-it-possible-to-implement-debug-for-fn-type/14824

// Greeting to the master of functinal Rust - mighty @raventid
type PrefixParseFnAlias = Fn(&Parser) -> token::Expression + 'static;

struct PrefixParseFn(Box<PrefixParseFnAlias>);
impl fmt::Debug for PrefixParseFn {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", "prefix_parse_fn")
    }
}

type InfixParseFnAlias = Fn(&Parser, token::Expression) -> token::Expression + 'static;

struct InfixParseFn(Box<InfixParseFnAlias>);
impl fmt::Debug for InfixParseFn {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", "infix_parse_fn")
    }
}

#[derive(Debug)]
struct LambdaParsers {
    pub prefix_parse_fns: HashMap<token::TokenType, PrefixParseFn>,
    pub infix_parse_fns: HashMap<token::TokenType, InfixParseFn>,
}

impl LambdaParsers {
    fn register_parsers(&mut self) {
        self.register_prefix(
            token::IDENT.to_string(),
            Box::new(|parser| {
                token::Expression::Identifier(token::Identifier {
                    token: parser.current_token.clone(),
                    value: parser.current_token.literal.clone(),
                })

            })
        );
    }

    fn register_prefix(&mut self, token_type: token::TokenType, f: Box<PrefixParseFnAlias>) {
        self.prefix_parse_fns.insert(token_type, PrefixParseFn(f));
    }

    fn register_infix(&mut self, token_type: token::TokenType, f: Box<InfixParseFnAlias>) {
        self.infix_parse_fns.insert(token_type, InfixParseFn(f));
    }

    fn parse_identifier(parser: &Parser) -> token::Expression {
        token::Expression::Identifier(token::Identifier {
            token: parser.current_token.clone(),
            value: parser.current_token.literal.clone(),
        })
    }
}
<commit_msg>parser support for integer literal<commit_after>use std::collections::HashMap;
use std::fmt;

use crate::ast;
use crate::lexer;
use crate::token;

// For debug visualization I migth potentially use this approach
// https://users.rust-lang.org/t/is-it-possible-to-implement-debug-for-fn-type/14824

// Greeting to the master of functinal Rust - mighty @raventid
type PrefixParseFnAlias = Fn(&Parser) -> token::Expression + 'static;

struct PrefixParseFn(Box<PrefixParseFnAlias>);
impl fmt::Debug for PrefixParseFn {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", "prefix_parse_fn")
    }
}

type InfixParseFnAlias = Fn(&Parser, token::Expression) -> token::Expression + 'static;

struct InfixParseFn(Box<InfixParseFnAlias>);
impl fmt::Debug for InfixParseFn {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", "infix_parse_fn")
    }
}

#[derive(Debug)]
struct LambdaParsers {
    pub prefix_parse_fns: HashMap<token::TokenType, PrefixParseFn>,
    pub infix_parse_fns: HashMap<token::TokenType, InfixParseFn>,
}

impl LambdaParsers {
    fn register_parsers(&mut self) {
        self.register_prefix(
            token::IDENT.to_string(),
            Box::new(|parser| {
                token::Expression::Identifier(token::Identifier {
                    token: parser.current_token.clone(),
                    value: parser.current_token.literal.clone(),
                })

            })
        );


        self.register_prefix(
            token::INT.to_string(),
            Box::new(Self::parse_int_literal)
        )
    }

    fn register_prefix(&mut self, token_type: token::TokenType, f: Box<PrefixParseFnAlias>) {
        self.prefix_parse_fns.insert(token_type, PrefixParseFn(f));
    }

    fn register_infix(&mut self, token_type: token::TokenType, f: Box<InfixParseFnAlias>) {
        self.infix_parse_fns.insert(token_type, InfixParseFn(f));
    }

    fn parse_identifier(parser: &Parser) -> token::Expression {
        token::Expression::Identifier(token::Identifier {
            token: parser.current_token.clone(),
            value: parser.current_token.literal.clone(),
        })
    }

    fn parse_int_literal(parser: &Parser) -> token::Expression {
        let to_be_integer = parser.current_token.literal.clone();

        // TODO: This extremly bad
        // Lambda parsers should bubble errors to parser.
        // Parser should handle them gracefully.
        let integer = to_be_integer.parse::<i32>().unwrap();

        token::Expression::IntegerLiteral(token::IntegerLiteral {
            token: parser.current_token.clone(),
            value: integer,
        })
    }
}

{'fraction_non_alphanumeric': 0.13441998306519898, 'fraction_numerical': 0.002540220152413209, 'mean_word_length': 3.148375768217735, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5344310', 'n_tokens_mistral': 1463, 'n_tokens_neox': 1439, 'n_words': 276}