<commit_before>//! The `ast` module contains a lot of useful functionality
//! to create and walk through the ast (abstract syntaxtree)

use std::fmt::{Debug, Display, Formatter, Result, Write};
use std::collections::HashSet;
use std::iter::Scan;

use config::Config;
use frontend::expressionparser;
use frontend::lexer::Token;
use frontend::lexer::Token::{TokMacroIf, TokMacroElseIf, TokExpression, TokPassage};

use ::utils::extensions::{Constructor, ConstructorExt};
//==============================
// ast

pub struct ASTBuilder {
    path: Vec<usize>,
    is_in_if_expression: bool
}

pub struct AST {
    pub passages: Vec<ASTNode>,
}

/// the parser use these ASTOperations to create the ast
pub enum ASTOperation {
    AddPassage(Token),
    AddChild(Token),
    ChildDown(Token),
    ChildUp(Token),
    Up,
    UpChild(Token),
    UpChildDown(Token),
    UpSpecial,
}

impl ASTBuilder {
    fn new() -> ASTBuilder {
        ASTBuilder {
            path: Vec::new(),
            is_in_if_expression: false
        }
    }

    pub fn build<I: Iterator<Item=ASTOperation>>(cfg: &Config, ops: I)
        -> Scan<Constructor<I, ASTBuilder, ASTNode, fn(&mut ASTBuilder, &mut Option<ASTNode>, ASTOperation) -> Option<ASTNode>>, &Config, fn(&mut &Config, ASTNode) -> Option<ASTNode>>
    {
        ops.construct_state(ASTBuilder::new(),
        {
            fn construct(builder: &mut ASTBuilder, passage: &mut Option<ASTNode>, op: ASTOperation) -> Option<ASTNode>
            {
                builder.operation(passage, op)
            }
            construct as fn(&mut ASTBuilder, &mut Option<ASTNode>, ASTOperation) -> Option<ASTNode> //necessary cast is a known bug
        }).scan(cfg,
            {
                fn scan(cfg: &mut &Config, x: ASTNode) -> Option<ASTNode>
                {
                    x.clone().parse_expressions(*cfg);
                    Some(x)
                }
                scan
            }
        )
    }

    /// calls the matching function to a given ASTOperation
    pub fn operation(&mut self, current_passage: &mut Option<ASTNode>, op: ASTOperation) -> Option<ASTNode> {
        use self::ASTOperation::*;
        match op {
            AddPassage(passage) => self.add_passage(passage),
            AddChild(child) => self.add_child(current_passage, child),
            ChildDown(child) => self.child_down(current_passage, child),
            ChildUp(child) => self.child_up(current_passage, child),
            Up => self.up(),
            UpChild(child) => self.up_child(current_passage, child),
            UpChildDown(child) => self.up_child_down(current_passage, child),
            UpSpecial => self.up_special(),
        }
    }

    /// counts the childs of the path in the asts
    pub fn count_childs(&self, current_passage: &mut Option<ASTNode>, path: Vec<usize>) -> usize {
        if let Some(_) = path.first() {
            current_passage.as_ref().unwrap().count_childs(path.to_vec())
        } else {
            0
        }
    }

    /// adds a passage to the path in the ast
    pub fn add_passage(&mut self, token: Token) -> Option<ASTNode> {
        self.path.clear();
        Some(ASTNode::Passage(NodePassage { category: token, childs: Vec::new() }))
    }

    /// adds a child to the path in the ast
    pub fn add_child(&mut self, current_passage_opt: &mut Option<ASTNode>, token: Token) -> Option<ASTNode> {
        if let Some(ref mut current_passage) = current_passage_opt.as_mut() {
            current_passage.add_child(self.path.to_vec(), token);
            None
        } else {
            Some(ASTNode::Default(NodeDefault { category: token, childs: Vec::new() }))
        }
    }

    /// adds a child an goees one child down
    pub fn child_down(&mut self, current_passage: &mut Option<ASTNode>, token: Token) -> Option<ASTNode> {
        //
        if token.clone().is_same_token(&TokMacroIf { location: (0, 0) }) ||
           token.clone().is_same_token(&TokMacroElseIf { location: (0, 0) }) {
            self.is_in_if_expression = true;
        }

        let ast_count_childs = current_passage.as_ref().unwrap().count_childs(self.path.to_vec());
        let result = self.add_child(current_passage, token);
        self.path.push(ast_count_childs);
        result
    }

    /// adds one child and goes down. adds snd child and goes down.
    pub fn two_childs_down(&mut self, current_passage: &mut Option<ASTNode>, child1: Token, child2: Token) -> Option<ASTNode> {
        let result = self.child_down(current_passage, child1);
        self.child_down(current_passage, child2);
        result
    }

    /// goes one lvl up
    pub fn up(&mut self) -> Option<ASTNode> {
        self.path.pop();
        None
    }

    /// special up of the if-expression
    pub fn up_special(&mut self) -> Option<ASTNode> {
        if !self.is_in_if_expression {
            self.path.pop();
        } else {
            self.is_in_if_expression = false;
        }
        None
    }

    /// adds a child and goes one lvl up
    pub fn child_up(&mut self, current_passage: &mut Option<ASTNode>, token: Token) -> Option<ASTNode> {
        let result = self.add_child(current_passage, token);
        self.up();
        result<commit_msg>Fix two_child_down (still not working)<commit_after>//! The `ast` module contains a lot of useful functionality
//! to create and walk through the ast (abstract syntaxtree)

use std::fmt::{Debug, Display, Formatter, Result, Write};
use std::collections::HashSet;
use std::iter::Scan;

use config::Config;
use frontend::expressionparser;
use frontend::lexer::Token;
use frontend::lexer::Token::{TokMacroIf, TokMacroElseIf, TokExpression, TokPassage};

use ::utils::extensions::{Constructor, ConstructorExt};
//==============================
// ast

pub struct ASTBuilder {
    path: Vec<usize>,
    is_in_if_expression: bool
}

pub struct AST {
    pub passages: Vec<ASTNode>,
}

/// the parser use these ASTOperations to create the ast
pub enum ASTOperation {
    AddPassage(Token),
    AddChild(Token),
    ChildDown(Token),
    ChildUp(Token),
    Up,
    UpChild(Token),
    UpChildDown(Token),
    UpSpecial,
}

impl ASTBuilder {
    fn new() -> ASTBuilder {
        ASTBuilder {
            path: Vec::new(),
            is_in_if_expression: false
        }
    }

    pub fn build<I: Iterator<Item=ASTOperation>>(cfg: &Config, ops: I)
        -> Scan<Constructor<I, ASTBuilder, ASTNode, fn(&mut ASTBuilder, &mut Option<ASTNode>, ASTOperation) -> Option<ASTNode>>, &Config, fn(&mut &Config, ASTNode) -> Option<ASTNode>>
    {
        ops.construct_state(ASTBuilder::new(),
        {
            fn construct(builder: &mut ASTBuilder, passage: &mut Option<ASTNode>, op: ASTOperation) -> Option<ASTNode>
            {
                builder.operation(passage, op)
            }
            construct as fn(&mut ASTBuilder, &mut Option<ASTNode>, ASTOperation) -> Option<ASTNode> //necessary cast is a known bug
        }).scan(cfg,
            {
                fn scan(cfg: &mut &Config, x: ASTNode) -> Option<ASTNode>
                {
                    x.clone().parse_expressions(*cfg);
                    Some(x)
                }
                scan
            }
        )
    }

    /// calls the matching function to a given ASTOperation
    pub fn operation(&mut self, current_passage: &mut Option<ASTNode>, op: ASTOperation) -> Option<ASTNode> {
        use self::ASTOperation::*;
        match op {
            AddPassage(passage) => self.add_passage(passage),
            AddChild(child) => self.add_child(current_passage, child),
            ChildDown(child) => self.child_down(current_passage, child),
            ChildUp(child) => self.child_up(current_passage, child),
            Up => self.up(),
            UpChild(child) => self.up_child(current_passage, child),
            UpChildDown(child) => self.up_child_down(current_passage, child),
            UpSpecial => self.up_special(),
        }
    }

    /// counts the childs of the path in the asts
    pub fn count_childs(&self, current_passage: &mut Option<ASTNode>, path: Vec<usize>) -> usize {
        if let Some(_) = path.first() {
            current_passage.as_ref().unwrap().count_childs(path.to_vec())
        } else {
            0
        }
    }

    /// adds a passage to the path in the ast
    pub fn add_passage(&mut self, token: Token) -> Option<ASTNode> {
        self.path.clear();
        Some(ASTNode::Passage(NodePassage { category: token, childs: Vec::new() }))
    }

    /// adds a child to the path in the ast
    pub fn add_child(&mut self, current_passage_opt: &mut Option<ASTNode>, token: Token) -> Option<ASTNode> {
        if let Some(ref mut current_passage) = current_passage_opt.as_mut() {
            current_passage.add_child(self.path.to_vec(), token);
            None
        } else {
            Some(ASTNode::Default(NodeDefault { category: token, childs: Vec::new() }))
        }
    }

    /// adds a child an goees one child down
    pub fn child_down(&mut self, current_passage: &mut Option<ASTNode>, token: Token) -> Option<ASTNode> {
        //
        if token.clone().is_same_token(&TokMacroIf { location: (0, 0) }) ||
           token.clone().is_same_token(&TokMacroElseIf { location: (0, 0) }) {
            self.is_in_if_expression = true;
        }

        let ast_count_childs = current_passage.as_ref().unwrap().count_childs(self.path.to_vec());
        let result = self.add_child(current_passage, token);
        self.path.push(ast_count_childs);
        result
    }

    /// adds one child and goes down. adds snd child and goes down.
    pub fn two_childs_down(&mut self, current_passage: &mut Option<ASTNode>, child1: Token, child2: Token) -> Option<ASTNode> {
        match self.child_down(current_passage, child1) {
            Some(new_passage) => {
                self.child_down(&mut Some(new_passage.clone()), child2);
                Some(new_passage)
            },
            None => {
                self.child_down(current_passage, child2)
            }
        }

    }

    /// goes one lvl up
    pub fn up(&mut self) -> Option<ASTNode> {
        self.path.pop();
        None
    }

    /// special up of the if-expression
    pub fn up_special(&mut self) -> Option<ASTNode> {
        if !self.is_in_if_expression {
            self.path.pop();
        } else {
            self.is_in_if_expression = false;
        }
        None
    }

    /// adds a child and goes one lvl up
    pub fn child_up(&mut self, current_passage: &mut Option<ASTNode>, token: Token) -> Option<ASTNode> {
        let result = self.add_child(current_passage, token);
        self.up();
        result
{'fraction_non_alphanumeric': 0.13023909985935303, 'fraction_numerical': 0.0017815283638068448, 'mean_word_length': 2.415305795709254, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1751574', 'n_tokens_mistral': 3289, 'n_tokens_neox': 3202, 'n_words': 789}