<commit_before>use traits::*;

#[derive(PartialEq, Clone, Debug, Copy)]
pub enum BFTokenType {
    IncrementPtr(usize),      // >
    DecrementPtr(usize),      // <
    IncrementData(usize),     // +
    DecrementData(usize),     // -
    Output,                   // .
    Input,                    // ,
    
    // These store their corresponding tokens
    //  aka LoopStart stores the position of
    //  the matching LoopEnd and vice versa.
    LoopStart(usize),         // [
    LoopEnd(usize),           // ]

    // Optimised instructions
    SetCurrent(i8),           // Sets the current cell to value
    AddCurrentUp(usize),      // Adds current cell to cell in [current + value]
    AddCurrentDown(usize)     // Adds current cell to cell in [current - value]
}


#[derive(Clone, Debug, Copy)]
pub struct BFToken {
    pub token_type : BFTokenType,

    // Used for matching brackets
    pub pos : usize
}

pub struct BFLexer {
}

impl Lexer<Vec<BFToken>> for BFLexer {

    fn parse(input_string:String) -> LexResult<Vec<BFToken>> {
        // Import enum -> Allows for using enum values without
        //  BFTokenType:: prefix
        use self::BFTokenType::*;

        // Create empty vector
        let mut tokens : Vec<BFToken> = vec![];

        // Store the index of the token in the vector
        let mut pos : usize = 0;

        // This stores the positions of open brackets
        // This is used to match up bracket pairs 
        //  so that they can jump to each other in O(1)
        let mut loop_stack : Vec<usize> = vec![];

        // Store the previous 5 tokens, these are used
        //  for optimisations.
        let mut last_tokens = [BFToken { pos : 0, token_type : Input }; 5];

        // Loop through each character
        for character in input_string.chars() {
            // Match our character to a TokenType.
            let token_type = match character {
                '>' => {
                    // Batch up all the '>' tokens
                    let mut new_x = 1;
                    if last_tokens[0].pos > 0 {
                        if let IncrementPtr(x) = last_tokens[0].token_type {
                            tokens.pop();
                            pos-=1;
                            new_x += x;
                        }
                    }
                    IncrementPtr(new_x)
                },
                '<' => {
                    let mut new_x = 1;
                    if last_tokens[0].pos > 0 {
                        if let DecrementPtr(x) = last_tokens[0].token_type {
                            tokens.pop();
                            pos-=1;                             
                            new_x += x;                           
                        }
                    }
                    DecrementPtr(new_x)
                },
                '+' =>  {
                    let mut new_x = 1;
                    if last_tokens[0].pos > 0 {
                        if let IncrementData(x) = last_tokens[0].token_type {
                            tokens.pop();
                            pos-=1;
                            new_x += x;
                        }
                    }
                    IncrementData(new_x)
                },
                '-' => {
                    let mut new_x = 1;
                    if last_tokens[0].pos > 0 {
                        if let DecrementData(x) = last_tokens[0].token_type {
                            tokens.pop();
                            pos-=1;
                            new_x += x;
                        }
                    }
                    DecrementData(new_x)
                },
                '.' => Output,
                ',' => Input,
                '[' => {
                    // Push this pos onto stack
                    loop_stack.push(pos);

                    // Store temp value of 0 for now - will be updated
                    //  once we know matching ]
                    LoopStart(0)
                },
                ']' => {

                    // Get matching bracket.
                    let index = loop_stack.pop().expect("Bracket mismatch");
                    let mut ret_token : BFTokenType = LoopEnd(index);      

                    // Update it's data position to this ] token
                    tokens[index].token_type = LoopStart(pos);

                    match last_tokens[0].token_type {
                        // This currently checks for [-] or [+] and replaces 
                        //  those with a set current cell to 0 instruction.
                        IncrementData(_) 
                        | DecrementData(_) => {
                            if let LoopStart(_) = last_tokens[1].token_type {
                                ret_token = SetCurrent(0);
                                tokens.pop();
                                tokens.pop();
                                pos-=2;
                            }
                        },

                        //  Optimisation for [-<+>] or [->+<] pattern.
                        DecrementPtr(x)
                        | IncrementPtr(x) => {
                            if let IncrementData(a) = last_tokens[1].token_type {
                                match last_tokens[2].token_type {
                                    IncrementPtr(y)
                                    | DecrementPtr(y) => {
                                        if x == y {
                                            if let DecrementData(b) = last_tokens[3].token_type  {
                                                if a == b {
                                                    if let LoopStart(_) = last_tokens[4].token_type {
                                                        if last_tokens[0].token_type == DecrementPtr(x) {
                                                            ret_token = AddCurrentUp(x);
                                                        } else {
                                                            ret_token = AddCurrentDown(x);
                                                        }
                                                        tokens.pop();
                                                        tokens.pop();
                                                        tokens.pop();<commit_msg>Added more comments to cell addition optimisation<commit_after>use traits::*;

#[derive(PartialEq, Clone, Debug, Copy)]
pub enum BFTokenType {
    IncrementPtr(usize),      // >
    DecrementPtr(usize),      // <
    IncrementData(usize),     // +
    DecrementData(usize),     // -
    Output,                   // .
    Input,                    // ,
    
    // These store their corresponding tokens
    //  aka LoopStart stores the position of
    //  the matching LoopEnd and vice versa.
    LoopStart(usize),         // [
    LoopEnd(usize),           // ]

    // Optimised instructions
    SetCurrent(i8),           // Sets the current cell to value
    AddCurrentUp(usize),      // Adds current cell to cell in [current + value]
    AddCurrentDown(usize)     // Adds current cell to cell in [current - value]
}


#[derive(Clone, Debug, Copy)]
pub struct BFToken {
    pub token_type : BFTokenType,

    // Used for matching brackets
    pub pos : usize
}

pub struct BFLexer {
}

impl Lexer<Vec<BFToken>> for BFLexer {

    fn parse(input_string:String) -> LexResult<Vec<BFToken>> {
        // Import enum -> Allows for using enum values without
        //  BFTokenType:: prefix
        use self::BFTokenType::*;

        // Create empty vector
        let mut tokens : Vec<BFToken> = vec![];

        // Store the index of the token in the vector
        let mut pos : usize = 0;

        // This stores the positions of open brackets
        // This is used to match up bracket pairs 
        //  so that they can jump to each other in O(1)
        let mut loop_stack : Vec<usize> = vec![];

        // Store the previous 5 tokens, these are used
        //  for optimisations.
        let mut last_tokens = [BFToken { pos : 0, token_type : Input }; 5];

        // Loop through each character
        for character in input_string.chars() {
            // Match our character to a TokenType.
            let token_type = match character {
                '>' => {
                    // Batch up all the '>' tokens
                    let mut new_x = 1;
                    if last_tokens[0].pos > 0 {
                        if let IncrementPtr(x) = last_tokens[0].token_type {
                            tokens.pop();
                            pos-=1;
                            new_x += x;
                        }
                    }
                    IncrementPtr(new_x)
                },
                '<' => {
                    let mut new_x = 1;
                    if last_tokens[0].pos > 0 {
                        if let DecrementPtr(x) = last_tokens[0].token_type {
                            tokens.pop();
                            pos-=1;                             
                            new_x += x;                           
                        }
                    }
                    DecrementPtr(new_x)
                },
                '+' =>  {
                    let mut new_x = 1;
                    if last_tokens[0].pos > 0 {
                        if let IncrementData(x) = last_tokens[0].token_type {
                            tokens.pop();
                            pos-=1;
                            new_x += x;
                        }
                    }
                    IncrementData(new_x)
                },
                '-' => {
                    let mut new_x = 1;
                    if last_tokens[0].pos > 0 {
                        if let DecrementData(x) = last_tokens[0].token_type {
                            tokens.pop();
                            pos-=1;
                            new_x += x;
                        }
                    }
                    DecrementData(new_x)
                },
                '.' => Output,
                ',' => Input,
                '[' => {
                    // Push this pos onto stack
                    loop_stack.push(pos);

                    // Store temp value of 0 for now - will be updated
                    //  once we know matching ]
                    LoopStart(0)
                },
                ']' => {

                    // Get matching bracket.
                    let index = loop_stack.pop().expect("Bracket mismatch");
                    let mut ret_token : BFTokenType = LoopEnd(index);      

                    // Update it's data position to this ] token
                    tokens[index].token_type = LoopStart(pos);

                    match last_tokens[0].token_type {
                        // This currently checks for [-] or [+] and replaces 
                        //  those with a set current cell to 0 instruction.
                        IncrementData(_) 
                        | DecrementData(_) => {
                            if let LoopStart(_) = last_tokens[1].token_type {
                                ret_token = SetCurrent(0);
                                tokens.pop();
                                tokens.pop();
                                pos-=2;
                            }
                        },

                        // Optimisation for [-<+>] or [->+<] pattern.
                        // These loops will add the current cell value 
                        //  to the cell value offset by the amount of
                        //  ptr increments '>' and '<'.
                        //
                        // Examples:
                        //
                        //  [->+<] adds mem[current] to mem[current+1]
                        //
                        //  [->>>>+<<<<<] adds mem[current] to mem[current+4]
                        //
                        // Note that due to the combination optimisation
                        //  this only needs to check for [->+<] or [-<+>]
                        DecrementPtr(x)
                        | IncrementPtr(x) => {
                            if let IncrementData(a) = last_tokens[1].token_type {
                                match last_tokens[2].token_type {
                                    IncrementPtr(y)
                                    | DecrementPtr(y) => {
                                        if x == y {
                                            if let DecrementData(b) = last_tokens[3].token_type  {
                                                if a == b {
                                                    if let LoopStart(_) = last_tokens[4].token_type {
                                                        if last_tokens[0].token_type == DecrementPtr(x) {
                                                            ret_token = AddCurrentUp(x);
                                                        } else {
                                                            ret_token = AddCurrentDown(x);
                                                        }
                                                        tokens.pop();
                                                        tokens.pop();
                                                        tokens.pop();
{'fraction_non_alphanumeric': 0.08788829151732377, 'fraction_numerical': 0.005824372759856631, 'mean_word_length': 0.7982008592910849, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 57, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2619266', 'n_tokens_mistral': 3203, 'n_tokens_neox': 2957, 'n_words': 786}