<commit_before>use super::Expr;
use super::Atom;
use super::token::{Token, Tokenizer};

pub struct Parser<'a> {
    current_token: Option<Token<'a>>,
    tokenizer: Tokenizer<'a>,
}

impl<'a> Parser<'a> {
    pub fn new(s: &'a str) -> Parser<'a> {
        Parser {
            current_token: None,
            tokenizer: Tokenizer::new(s, true),
        }
    }

    // skips the current token
    fn skip(&mut self) {
        assert!(self.current_token.is_some());
        self.current_token = None;
    }

    // returns the current token.
    fn token(&mut self) -> Option<Token<'a>> {
        if let Some(token) = self.current_token {
            return Some(token);
        }
        assert!(self.current_token.is_none());
        self.current_token = self.tokenizer.next();
        self.current_token
    }

    pub fn at_end(&mut self) -> bool {
        self.token().is_none()
    }

    // Returns either the successfully parsed expression, or
    // the invalid token which was not expected.
    pub fn parse_expr(&mut self) -> Result<Expr, Token<'a>> {
        let current = self.token().unwrap();
        match current {
            Token::Str(s) => {
                self.skip();
                Ok(Expr::from(s))
            }
            Token::UInt(u) => {
                self.skip();
                Ok(Expr::from(Atom::UInt(u)))
            }
            Token::SInt(s) => {
                self.skip();
                Ok(Expr::from(Atom::SInt(s)))
            }
            Token::Float(s) => {
                self.skip();
                Ok(Expr::from(Atom::Float(s)))
            }
            Token::OpenBrace => {
                self.skip();
                let mut exprs = vec![];
                loop {
                    if let Some(Token::CloseBrace) = self.token() {
                        self.skip();
                        break;
                    }
                    match self.parse_expr() {
                        Ok(ex) => exprs.push(ex),
                        Err(tok) => return Err(tok),
                    }
                }
                Ok(Expr::Tuple(exprs))
            }
            tok => Err(tok),
        }
    }
}

#[test]
fn test_parse_expr() {
    let mut p = Parser::new("123.45");
    assert_eq!(Ok(Expr::from(123.45)), p.parse_expr());
    assert_eq!(true, p.at_end());

    let mut p = Parser::new("123");
    assert_eq!(Ok(Expr::from(123usize)), p.parse_expr());
    assert_eq!(true, p.at_end());

    let mut p = Parser::new("(123)");
    assert_eq!(Ok(Expr::from((123usize,))), p.parse_expr());
    assert_eq!(true, p.at_end());

    let mut p = Parser::new("(test 123)   ");
    assert_eq!(Ok(Expr::from(("test", 123usize))), p.parse_expr());
    assert_eq!(true, p.at_end());

    let mut p = Parser::new("(test 123)   123");
    assert_eq!(Ok(Expr::from(("test", 123usize))), p.parse_expr());
    assert_eq!(false, p.at_end());
}<commit_msg>Refactor the parser so that it does not require to peek.<commit_after>use super::Expr;
use super::Atom;
use super::token::{Token, Tokenizer};

pub struct Parser<'a> {
    tokenizer: Tokenizer<'a>,
}

#[derive(Debug, PartialEq)]
pub enum ParseError<'a> {
    UnexpectedToken(Token<'a>),
    UnexpectedEnd,
}

impl<'a> Parser<'a> {
    pub fn new(s: &'a str) -> Parser<'a> {
        Parser { tokenizer: Tokenizer::new(s, true) }
    }

    pub fn at_end(&mut self) -> bool {
        self.tokenizer.next().is_none()
    }

    // Returns either the successfully parsed expression, or
    // the invalid token which was not expected.
    pub fn parse_expr(&mut self) -> Result<Expr, ParseError<'a>> {
        match self.tokenizer.next() {
            None => Err(ParseError::UnexpectedEnd),
            Some(tok) => {
                match tok {
                    Token::Str(s) => Ok(Expr::from(s)),
                    Token::UInt(u) => Ok(Expr::from(Atom::UInt(u))),
                    Token::SInt(s) => Ok(Expr::from(Atom::SInt(s))),
                    Token::Float(s) => Ok(Expr::from(Atom::Float(s))),
                    Token::OpenBrace => {
                        let mut exprs = vec![];
                        loop {
                            match self.parse_expr() {
                                Ok(ex) => exprs.push(ex),
                                Err(ParseError::UnexpectedToken(Token::CloseBrace)) => break,
                                Err(err) => return Err(err),
                            }
                        }
                        Ok(Expr::Tuple(exprs))
                    }
                    tok => Err(ParseError::UnexpectedToken(tok)),
                }
            }
        }
    }
}

#[test]
fn test_parse_expr() {
    let mut p = Parser::new("123.45");
    assert_eq!(Ok(Expr::from(123.45)), p.parse_expr());
    assert_eq!(true, p.at_end());

    let mut p = Parser::new("123");
    assert_eq!(Ok(Expr::from(123usize)), p.parse_expr());
    assert_eq!(true, p.at_end());

    let mut p = Parser::new("(123)");
    assert_eq!(Ok(Expr::from((123usize,))), p.parse_expr());
    assert_eq!(true, p.at_end());

    let mut p = Parser::new("(test 123)   ");
    assert_eq!(Ok(Expr::from(("test", 123usize))), p.parse_expr());
    assert_eq!(true, p.at_end());

    let mut p = Parser::new("(test 123)   123");
    assert_eq!(Ok(Expr::from(("test", 123usize))), p.parse_expr());
    assert_eq!(false, p.at_end());

    let mut p = Parser::new("   ");
    assert_eq!(Err(ParseError::UnexpectedEnd), p.parse_expr());

    let mut p = Parser::new("(abc    ");
    assert_eq!(Err(ParseError::UnexpectedEnd), p.parse_expr());

    let mut p = Parser::new("(abc  }  ");
    assert_eq!(Err(ParseError::UnexpectedToken(Token::CloseCurly)),
               p.parse_expr());
}
{'fraction_non_alphanumeric': 0.17207678883071553, 'fraction_numerical': 0.012914485165794066, 'mean_word_length': 1.8813474107591754, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2826949', 'n_tokens_mistral': 1889, 'n_tokens_neox': 1797, 'n_words': 336}