<commit_before>use std::collections::HashMap;
use lexer::Tokens;
use error::ParsingError;
use constants::KEYWORDS;

// for keyword in KEYWORDS {
//     if keyword.to_string() == variable.to_lowercase() {
//         return Err(LexerError::KeywordError(variable));
//     }
// }

pub type Variables = HashMap<String, String>;
pub type Operators = HashMap<&'static str, Operator>;

// Refs: stackoverflow.com/questions/930486/what-is-associativity-of-operators-and-why-is-it-important
pub enum Associativity {
    LeftToRight,
    RightToLeft,
}

pub struct Operator(u8, Associativity);

/// Operator Precedence
/// | Precedence | Operator | Symbol | Associativity |
/// |:----------:|:--------:|:------:|---------------|
/// |1           |Brackets  |()      |Left-to-Right  |
/// |2           |NOT       |~       |Right-to-Left  |
/// |3           |Shift     |<< >>   |Left-to-Right  |
/// |4           |Equality  |==      |Left-to-Right  |
/// |5           |AND       |&       |Left-to-Right  |
/// |6           |XOR       |^       |Left-to-Right  |
/// |7           |OR        ||       |Left-to-Right  |
/// |8           |Assignment|=       |Right-to-Left  |
impl Operator {
    pub fn new(precedence: u8, associativity: Associativity) -> Operator {
        Operator(precedence, associativity)
    }
}

pub struct Parser {
    operators: Operators, // output: TokenList,
    variables: Variables,
}

impl Parser {
    pub fn new() -> Parser {
        Parser {
            operators: Operators::new(),
            variables: Variables::new(), // output: TokenList,
        }
    }

    pub fn init(&mut self) {
        self.operators.insert("(", Operator::new(1, Associativity::LeftToRight));
        self.operators.insert(")", Operator::new(1, Associativity::LeftToRight));
        self.operators.insert("~", Operator::new(2, Associativity::RightToLeft));
        self.operators.insert(">>", Operator::new(3, Associativity::LeftToRight));
        self.operators.insert("<<", Operator::new(3, Associativity::LeftToRight));
        self.operators.insert("==", Operator::new(4, Associativity::LeftToRight));
        self.operators.insert("&", Operator::new(5, Associativity::LeftToRight));
        self.operators.insert("^", Operator::new(6, Associativity::LeftToRight));
        self.operators.insert("|", Operator::new(7, Associativity::LeftToRight));
        self.operators.insert("=", Operator::new(8, Associativity::RightToLeft));
    }

    pub fn parse(&mut self, tokens: Tokens) -> Result<Tokens, ParsingError> {
        let mut operator_stack = Tokens::with_capacity(10);
        Ok((tokens))
    }
}<commit_msg>More refactoring and better identification of the start position of characters in the Lexer<commit_after>use std::collections::HashMap;
use lexer::Tokens;
use error::ParsingError;
// use constants::KEYWORDS;

// for keyword in KEYWORDS {
//     if keyword.to_string() == variable.to_lowercase() {
//         return Err(LexerError::KeywordError(variable));
//     }
// }

pub type Variables = HashMap<String, String>;
pub type Operators = HashMap<&'static str, Operator>;

// Refs: stackoverflow.com/questions/930486/what-is-associativity-of-operators-and-why-is-it-important
pub enum Associativity {
    LeftToRight,
    RightToLeft,
}

pub struct Operator(u8, Associativity);

/// Operator Precedence
/// | Precedence | Operator | Symbol | Associativity |
/// |:----------:|:--------:|:------:|---------------|
/// |1           |Brackets  |()      |Left-to-Right  |
/// |2           |NOT       |~       |Right-to-Left  |
/// |3           |Shift     |<< >>   |Left-to-Right  |
/// |4           |Equality  |==      |Left-to-Right  |
/// |5           |AND       |&       |Left-to-Right  |
/// |6           |XOR       |^       |Left-to-Right  |
/// |7           |OR        ||       |Left-to-Right  |
/// |8           |Assignment|=       |Right-to-Left  |
impl Operator {
    pub fn new(precedence: u8, associativity: Associativity) -> Operator {
        Operator(precedence, associativity)
    }
}

pub struct Parser {
    operators: Operators, /* output: TokenList,
                           * variables: Variables, */
}

impl Parser {
    pub fn new() -> Parser {
        Parser {
            operators: Operators::new(), // variables: Variables::new(), // output: TokenList,
        }
    }

    pub fn init(&mut self) {
        self.operators.insert("(", Operator::new(1, Associativity::LeftToRight));
        self.operators.insert(")", Operator::new(1, Associativity::LeftToRight));
        self.operators.insert("~", Operator::new(2, Associativity::RightToLeft));
        self.operators.insert(">>", Operator::new(3, Associativity::LeftToRight));
        self.operators.insert("<<", Operator::new(3, Associativity::LeftToRight));
        self.operators.insert("==", Operator::new(4, Associativity::LeftToRight));
        self.operators.insert("&", Operator::new(5, Associativity::LeftToRight));
        self.operators.insert("^", Operator::new(6, Associativity::LeftToRight));
        self.operators.insert("|", Operator::new(7, Associativity::LeftToRight));
        self.operators.insert("=", Operator::new(8, Associativity::RightToLeft));
    }

    pub fn parse(&mut self, tokens: Tokens) -> Result<Tokens, ParsingError> {
        // let mut operator_stack = Tokens::with_capacity(10);
        Ok((tokens))
    }
}
{'fraction_non_alphanumeric': 0.18112005993631766, 'fraction_numerical': 0.010488855590934631, 'mean_word_length': 2.5317460317460316, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1145442', 'n_tokens_mistral': 1683, 'n_tokens_neox': 1621, 'n_words': 311}