<commit_before>pub mod ast;
pub mod lexer;
pub mod token;

pub struct Parser {
	pub current: token::Token,
	pub lexer: lexer::Lexer,
	pub peeked: Option<token::Token>,
}

impl Parser {
	pub fn new(input: &str) -> Parser {
		let l = lexer::Lexer::new(input);
		let p = Parser {
			current: token::EOF,
			peeked: None,
			lexer: l
		};
//		p.next_token();
		p
	}

	pub fn parse(&mut self) -> Box<ast::Node> {
		self.expr(1)
	}

	pub fn expr(&mut self, prec: uint) -> Box<ast::Node> {
		let mut lhs = self.atom();
		//println!("expr {}", self.current);		
		let mut rhs;
		loop {
			let curr = self.peek_token();
			//println!("{}", curr);
			if token::is_eof(curr) {
				//println!("breaking out of expr loop");
				break;
			}
			let info_tuple = curr.info();
			if info_tuple.is_none() {
				break;
			}
			let (op_prec, op_assoc) = info_tuple.unwrap();
			if op_prec < prec {
				break;
			}
			self.next_token();
			match op_assoc {
				0 => {
					rhs = self.expr(op_prec + 1);
				}
				_  => {
					rhs = self.expr(op_prec)		
				}
			}
			lhs = self.op(curr, lhs, rhs);

		}
		//println!("expr ending");
		return lhs;
    	//box ast::Add {
        //	left: box ast::Num { num: 1.0 } as Box<ast::Node>,
        //	right: box ast::Num { num: 2.0 } as Box<ast::Node>,
    	//} as Box<ast::Node> 
	}

	pub fn atom(&mut self) -> Box<ast::Node> {
		
		match self.peek_token() {
			token::LPAREN => {
				//println!("looking for subexpr");
				self.expect('(');
				let e = self.expr(1);
				self.expect(')');
				//println!("found subexpr");
				e
			}
			token::NUMBER(val) => {
				//println!("found number {}", val);
				self.next_token();
				box ast::Num { num: val } as Box<ast::Node> 
			}
			a => {
				fail!("unrecognized atom: {}", a);
				box ast::Num { num: 0f64} as Box<ast::Node>
			}
		}
	}

	pub fn op(&self, op: token::Token, lhs: Box<ast::Node>, rhs: Box<ast::Node>)
			-> Box<ast::Node> {
		match op {
			token::ADD => {<commit_msg>slightly better parser behavior on empty lines<commit_after>pub mod ast;
pub mod lexer;
pub mod token;

pub struct Parser {
	pub current: token::Token,
	pub lexer: lexer::Lexer,
	pub peeked: Option<token::Token>,
}

impl Parser {
	pub fn new(input: &str) -> Parser {
		let l = lexer::Lexer::new(input);
		let p = Parser {
			current: token::EOF,
			peeked: None,
			lexer: l
		};
//		p.next_token();
		p
	}

	pub fn parse(&mut self) -> Box<ast::Node> {
		self.expr(1)
	}

	pub fn expr(&mut self, prec: uint) -> Box<ast::Node> {
		let mut lhs = self.atom();
		//println!("expr {}", self.current);		
		let mut rhs;
		loop {
			let curr = self.peek_token();
			//println!("{}", curr);
			if token::is_eof(curr) {
				//println!("breaking out of expr loop");
				break;
			}
			let info_tuple = curr.info();
			if info_tuple.is_none() {
				break;
			}
			let (op_prec, op_assoc) = info_tuple.unwrap();
			if op_prec < prec {
				break;
			}
			self.next_token();
			match op_assoc {
				0 => {
					rhs = self.expr(op_prec + 1);
				}
				_  => {
					rhs = self.expr(op_prec)		
				}
			}
			lhs = self.op(curr, lhs, rhs);

		}
		//println!("expr ending");
		return lhs;
    	//box ast::Add {
        //	left: box ast::Num { num: 1.0 } as Box<ast::Node>,
        //	right: box ast::Num { num: 2.0 } as Box<ast::Node>,
    	//} as Box<ast::Node> 
	}

	pub fn atom(&mut self) -> Box<ast::Node> {
		
		match self.peek_token() {
			token::EOF => {return box ast::Num {num: 0f64} as Box<ast::Node>;}
			token::LPAREN => {
				//println!("looking for subexpr");
				self.expect('(');
				let e = self.expr(1);
				self.expect(')');
				//println!("found subexpr");
				e
			}
			token::NUMBER(val) => {
				//println!("found number {}", val);
				self.next_token();
				box ast::Num { num: val } as Box<ast::Node> 
			}
			a => {
				fail!("unrecognized atom: {}", a);
				box ast::Num { num: 0f64} as Box<ast::Node>
			}
		}
	}

	pub fn op(&self, op: token::Token, lhs: Box<ast::Node>, rhs: Box<ast::Node>)
			-> Box<ast::Node> {
		match op {
			token::ADD => {
{'fraction_non_alphanumeric': 0.19425717852684146, 'fraction_numerical': 0.006242197253433208, 'mean_word_length': 2.5735950044603033, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5129114', 'n_tokens_mistral': 1858, 'n_tokens_neox': 1578, 'n_words': 374}