<commit_before>macro_rules! push {
    ($s:ident, $t:expr) => {{
        // println!("parser push: {:?}", $t);
        Ok($s.vm.borrow_mut().alloc($t))
    }}
}

mod error;

use std::rc::Rc;
use std::cell::RefCell;
use core::*;
use lexer::*;
use self::error::Error;

const UNIT_INDX: usize = 5;

pub struct Parser<'a> {
    lexer: Lexer<'a, 'a>,
    vm: Rc<RefCell<Heliqs>>,
    // state: State,
    current_token: RefCell<CurrentToken>,
    // aexp_tokens: Vec<Vec<Tokn>>,
}

#[derive(Eq, PartialEq, Clone, Debug)]
enum CurrentToken {
    SOT, // Start Of Tokens
    Has(Tokn),
    EOT, // End Of Tokens
}

impl<'a> Parser<'a> {
    pub fn new(lexer: Lexer<'a, 'a>, vm: Rc<RefCell<Heliqs>>) -> Self {
        Parser {
            lexer: lexer,
            vm: vm,
            // state: State::Aexp,
            current_token: RefCell::new(CurrentToken::SOT),
            // aexp_tokens: vec![vec![]],
        }
    }

    // Unitは常に1つにする(index=0)
    fn add_unit(&mut self) {
        let _unit = self.vm.borrow_mut().alloc(Epiq::Unit);
    }

    fn add_prim(&mut self, name: &str) {
        let prim = self.vm.borrow_mut().alloc(Epiq::Prim(name.to_string()));
        self.vm.borrow_mut().define(name, prim);
    }

    pub fn parse(&mut self) {

        self.add_prim("decr");
        self.add_prim("ltoreq");
        self.add_prim("eq");
        self.add_prim("plus");
        self.add_prim("minus");

        self.add_unit();

        self.consume_token();

        let _ = self.parse_aexp();
    }

    fn consume_token(&mut self) {
        let res = self.lexer.tokenize();
        match res {
            TokenizeResult::Ok(t) => {
                let mut token = self.current_token.borrow_mut();
                *token = CurrentToken::Has(t);
            },
            TokenizeResult::Err(_e) => {},
            TokenizeResult::EOF => {
                let mut token = self.current_token.borrow_mut();
                *token = CurrentToken::EOT;
            },
        }
    }

    fn parse_aexp(&mut self) -> Result<usize, Error> {
        let res = self.current_token.borrow().clone();

        match res {
            CurrentToken::Has(Tokn::Atsm) => {
                self.consume_token();
                self.parse_resolve()
            },

            CurrentToken::Has(Tokn::Sgqt) => {
                self.consume_token();
                self.parse_otag(Tokn::Sgqt)
            },


            CurrentToken::Has(Tokn::Pipe) => {
                self.consume_token();
                self.parse_otag(Tokn::Pipe)
            },
            CurrentToken::Has(Tokn::Crrt) => {
                self.consume_token();
                self.parse_otag(Tokn::Crrt)
            },
            CurrentToken::Has(Tokn::Smcl) => {
                self.consume_token();
                // push!(self, Epiq::Unit)
                Ok(5)
            }
            CurrentToken::Has(Tokn::Lbkt) => {
                self.consume_token();
                self.parse_list()<commit_msg>fix using magic number<commit_after>macro_rules! push {
    ($s:ident, $t:expr) => {{
        // println!("parser push: {:?}", $t);
        Ok($s.vm.borrow_mut().alloc($t))
    }}
}

mod error;

use std::rc::Rc;
use std::cell::RefCell;
use core::*;
use lexer::*;
use self::error::Error;

const UNIT_INDX: usize = 5;

pub struct Parser<'a> {
    lexer: Lexer<'a, 'a>,
    vm: Rc<RefCell<Heliqs>>,
    // state: State,
    current_token: RefCell<CurrentToken>,
    // aexp_tokens: Vec<Vec<Tokn>>,
}

#[derive(Eq, PartialEq, Clone, Debug)]
enum CurrentToken {
    SOT, // Start Of Tokens
    Has(Tokn),
    EOT, // End Of Tokens
}

impl<'a> Parser<'a> {
    pub fn new(lexer: Lexer<'a, 'a>, vm: Rc<RefCell<Heliqs>>) -> Self {
        Parser {
            lexer: lexer,
            vm: vm,
            // state: State::Aexp,
            current_token: RefCell::new(CurrentToken::SOT),
            // aexp_tokens: vec![vec![]],
        }
    }

    // Unitは常に1つにする(index=0)
    fn add_unit(&mut self) {
        let _unit = self.vm.borrow_mut().alloc(Epiq::Unit);
    }

    fn add_prim(&mut self, name: &str) {
        let prim = self.vm.borrow_mut().alloc(Epiq::Prim(name.to_string()));
        self.vm.borrow_mut().define(name, prim);
    }

    pub fn parse(&mut self) {

        self.add_prim("decr");
        self.add_prim("ltoreq");
        self.add_prim("eq");
        self.add_prim("plus");
        self.add_prim("minus");

        self.add_unit();

        self.consume_token();

        let _ = self.parse_aexp();
    }

    fn consume_token(&mut self) {
        let res = self.lexer.tokenize();
        match res {
            TokenizeResult::Ok(t) => {
                let mut token = self.current_token.borrow_mut();
                *token = CurrentToken::Has(t);
            },
            TokenizeResult::Err(_e) => {},
            TokenizeResult::EOF => {
                let mut token = self.current_token.borrow_mut();
                *token = CurrentToken::EOT;
            },
        }
    }

    fn parse_aexp(&mut self) -> Result<usize, Error> {
        let res = self.current_token.borrow().clone();

        match res {
            CurrentToken::Has(Tokn::Atsm) => {
                self.consume_token();
                self.parse_resolve()
            },

            CurrentToken::Has(Tokn::Sgqt) => {
                self.consume_token();
                self.parse_otag(Tokn::Sgqt)
            },


            CurrentToken::Has(Tokn::Pipe) => {
                self.consume_token();
                self.parse_otag(Tokn::Pipe)
            },
            CurrentToken::Has(Tokn::Crrt) => {
                self.consume_token();
                self.parse_otag(Tokn::Crrt)
            },
            CurrentToken::Has(Tokn::Smcl) => {
                self.consume_token();
                Ok(UNIT_INDX)
            }
            CurrentToken::Has(Tokn::Lbkt) => {
                self.consume_token();
                self.parse_list()
{'fraction_non_alphanumeric': 0.16055200269269607, 'fraction_numerical': 0.0011780545270952542, 'mean_word_length': 1.768048439683279, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3259555', 'n_tokens_mistral': 2039, 'n_tokens_neox': 1973, 'n_words': 314}