<commit_before>use std::any::Any;
use std::collections::{HashMap, VecDeque};

use lexer::{Item, ItemType, Lexer};
use node::*;

pub type Func<'a> = &'a Fn(Option<Box<Any>>) -> Option<Box<Any>>;

pub struct Tree<'a> {
    name: String,
    id: TreeId,
    parse_name: String,
    root: Option<ListNode>,
    text: String,
    funcs: HashMap<String, Func<'a>>,
    lex: Option<Lexer>,
    token: VecDeque<Item>,
    peek_count: usize,
    vars: Vec<String>,
    tree_ids: HashMap<TreeId, String>,
    tree_set: HashMap<String, Tree<'a>>,
    line: usize,
}

impl<'a, 'b> Tree<'a> {
    fn new(name: String, id: TreeId, funcs: HashMap<String, Func<'a>>) -> Tree<'a> {
        Tree {
            name,
            id,
            parse_name: String::default(),
            root: None,
            text: String::default(),
            funcs,
            lex: None,
            token: VecDeque::new(),
            peek_count: 0,
            vars: vec![],
            tree_ids: HashMap::new(),
            tree_set: HashMap::new(),
            line: 0,
        }
    }
    fn clone_new(t: &Tree) -> Tree<'b> {
        Tree {
            name: t.name.clone(),
            id: t.id.clone(),
            parse_name: t.parse_name.clone(),
            root: t.root.clone(),
            text: t.text.clone(),
            funcs: HashMap::new(),
            lex: None,
            token: VecDeque::new(),
            peek_count: 0,
            vars: vec![],
            tree_ids: HashMap::new(),
            tree_set: HashMap::new(),
            line: 0,
        }
    }
}

pub fn parse<'a>(name: String,
                 text: String,
                 funcs: HashMap<String, Func<'a>>)
                 -> HashMap<String, Tree<'a>> {
    let tree_ids = HashMap::new();
    tree_ids
}

impl<'a> Tree<'a> {
    fn next_from_lex(&mut self) -> Option<Item> {
        match &mut self.lex {
            &mut Some(ref mut l) => l.next(),
            &mut None => None,
        }
    }

    fn backup(&mut self, t: Item) {
        self.token.push_back(t);
        self.peek_count = 1;
    }

    fn backup2(&mut self, t0: Item, t1: Item) {
        self.token.push_back(t0);
        self.token.push_back(t1);
        self.peek_count = 2;
    }

    fn backup3(&mut self, t0: Item, t1: Item, t2: Item) {
        self.token.push_back(t0);
        self.token.push_back(t1);
        self.token.push_back(t2);
        self.peek_count = 3;
    }

    fn next_non_space(&mut self) -> Option<Item> {
        self.skip_while(|c| c.typ == ItemType::ItemSpace).next()
    }

    fn peek_non_space(&mut self) -> Option<&Item> {
        if let Some(t) = self.next_non_space() {
            self.backup(t);
            return self.token.front();
        }
        None
    }

    fn error_context(&mut self, n: Nodes) -> (String, String) {
        let pos = n.pos();
        let tree_id = n.tree();
        let tree = if tree_id == 0 && self.tree_ids.contains_key(&tree_id) {
            self.tree_by_id(tree_id).unwrap()
        } else {
            self
        };
        let text = &tree.text[0..pos];
        let byte_num = match text.rfind('\n') {
            Some(i) => pos - (i + 1),
            None => pos,
        };
        let line_num = text.chars().filter(|c| *c == '\n').count();
        let context = n.to_string();
        (format!("{}:{}:{}", tree.parse_name, line_num, byte_num), context)
    }

    fn start_parse(&mut self,
                   funcs: HashMap<String, Func<'a>>,
                   lex: Lexer,
                   tree_ids: HashMap<TreeId, String>,
                   tree_set: HashMap<String, Tree<'a>>) {
        self.root = None;
        self.lex = Some(lex);
        self.vars = vec!["$".to_owned()];
        self.funcs = funcs;
        self.tree_ids = tree_ids;
        self.tree_set = tree_set;
    }

    fn stop_parse(&mut self) {
        self.lex = None;
        self.vars = vec![];
        self.funcs = HashMap::new();
        self.tree_ids = HashMap::new();
        self.tree_set = HashMap::new();
    }

    fn parse(&mut self,
             text: String,
             tree_ids: HashMap<TreeId, String>,
             tree_set: HashMap<String, Tree<'a>>,
             funcs: HashMap<String, Func<'a>>)
             -> Result<(), String> {
        self.parse_name = self.name.clone();
        let lex_name = self.name.clone();
        self.start_parse(funcs,
                         Lexer::new(&lex_name, text.clone()),
                         tree_ids,
                         tree_set);
        self.text = text;
        //self.do_parse();
        self.stop_parse();
        Ok(())
    }

    fn tree_by_id(&self, id: TreeId) -> Option<&Tree<'a>> {
        self.tree_ids
            .get(&id)
            .and_then(|name| self.tree_set.get(name))
    }
    fn add_tree(&mut self, name: &str, t: Tree<'a>) {
        self.tree_ids.insert(t.id, name.to_owned());
        self.tree_set.insert(name.to_owned(), t);
    }

    fn error(&mut self, msg: String) -> Result<(), String> {
        self.root = None;
        Err(format!("template: {}:{}:{}", self.parse_name, self.line, msg))
    }

    fn add_to_tree_set(mut tree: Tree<'a>,
                       mut tree_set: HashMap<String, Tree<'a>>)
                       -> Result<(), String> {
        if let Some(t) = tree_set.get(&tree.name) {
            if let Some(ref r) = t.root {
                match r.is_empty_tree() {
                    Err(e) => return Err(e),
                    Ok(false) => {
                        let err = format!("template multiple definitions of template {}",
                                          &tree.name);
                        return tree.error(err);
                    }
                    Ok(true) => {}
                }
            }
        }
        tree_set.insert(tree.name.clone(), tree);
        Ok(())
    }
}

impl<'a> Iterator for Tree<'a> {
    type Item = Item;
    fn next(&mut self) -> Option<Item> {
        let item = if self.peek_count > 0 {
            self.peek_count -= 1;
            self.token.pop_front()
        } else {
            self.next_from_lex()
        };
        match item {
            Some(item) => {
                self.line = item.line;
                Some(item)
            },
            _ => None
        }
    }
}

#[cfg(test)]
mod tests_mocked {
    use super::*;
    use lexer::ItemType;

    /*
       ItemText
       ItemLeftDelim<commit_msg>start on parse<commit_after>use std::any::Any;
use std::collections::{HashMap, VecDeque};

use lexer::{Item, ItemType, Lexer};
use node::*;

pub type Func<'a> = &'a Fn(Option<Box<Any>>) -> Option<Box<Any>>;

pub struct Tree<'a> {
    name: String,
    id: TreeId,
    parse_name: String,
    root: Option<ListNode>,
    text: String,
    funcs: HashMap<String, Func<'a>>,
    lex: Option<Lexer>,
    token: VecDeque<Item>,
    peek_count: usize,
    vars: Vec<String>,
    tree_ids: HashMap<TreeId, String>,
    tree_set: HashMap<String, Tree<'a>>,
    line: usize,
}

impl<'a, 'b> Tree<'a> {
    fn new(name: String, id: TreeId, funcs: HashMap<String, Func<'a>>) -> Tree<'a> {
        Tree {
            name,
            id,
            parse_name: String::default(),
            root: None,
            text: String::default(),
            funcs,
            lex: None,
            token: VecDeque::new(),
            peek_count: 0,
            vars: vec![],
            tree_ids: HashMap::new(),
            tree_set: HashMap::new(),
            line: 0,
        }
    }
    fn clone_new(t: &Tree) -> Tree<'b> {
        Tree {
            name: t.name.clone(),
            id: t.id.clone(),
            parse_name: t.parse_name.clone(),
            root: t.root.clone(),
            text: t.text.clone(),
            funcs: HashMap::new(),
            lex: None,
            token: VecDeque::new(),
            peek_count: 0,
            vars: vec![],
            tree_ids: HashMap::new(),
            tree_set: HashMap::new(),
            line: 0,
        }
    }
}

pub fn parse<'a>(name: String,
                 text: String,
                 funcs: HashMap<String, Func<'a>>)
                 -> HashMap<String, Tree<'a>> {
    let tree_ids = HashMap::new();
    tree_ids
}

impl<'a> Tree<'a> {
    fn next_from_lex(&mut self) -> Option<Item> {
        match &mut self.lex {
            &mut Some(ref mut l) => l.next(),
            &mut None => None,
        }
    }

    fn backup(&mut self, t: Item) {
        self.token.push_back(t);
        self.peek_count = 1;
    }

    fn backup2(&mut self, t0: Item, t1: Item) {
        self.token.push_back(t0);
        self.token.push_back(t1);
        self.peek_count = 2;
    }

    fn backup3(&mut self, t0: Item, t1: Item, t2: Item) {
        self.token.push_back(t0);
        self.token.push_back(t1);
        self.token.push_back(t2);
        self.peek_count = 3;
    }

    fn next_non_space(&mut self) -> Option<Item> {
        self.skip_while(|c| c.typ == ItemType::ItemSpace).next()
    }

    fn peek_non_space(&mut self) -> Option<&Item> {
        if let Some(t) = self.next_non_space() {
            self.backup(t);
            return self.token.front();
        }
        None
    }

    fn peek(&mut self) -> Option<&Item> {
        if let Some(t) = self.next() {
            self.backup(t);
            return self.token.front();
        }
        None
    }

    fn error_context(&mut self, n: Nodes) -> (String, String) {
        let pos = n.pos();
        let tree_id = n.tree();
        let tree = if tree_id == 0 && self.tree_ids.contains_key(&tree_id) {
            self.tree_by_id(tree_id).unwrap()
        } else {
            self
        };
        let text = &tree.text[0..pos];
        let byte_num = match text.rfind('\n') {
            Some(i) => pos - (i + 1),
            None => pos,
        };
        let line_num = text.chars().filter(|c| *c == '\n').count();
        let context = n.to_string();
        (format!("{}:{}:{}", tree.parse_name, line_num, byte_num), context)
    }

    fn start_parse(&mut self,
                   funcs: HashMap<String, Func<'a>>,
                   lex: Lexer,
                   tree_ids: HashMap<TreeId, String>,
                   tree_set: HashMap<String, Tree<'a>>) {
        self.root = None;
        self.lex = Some(lex);
        self.vars = vec!["$".to_owned()];
        self.funcs = funcs;
        self.tree_ids = tree_ids;
        self.tree_set = tree_set;
    }

    fn stop_parse(&mut self) {
        self.lex = None;
        self.vars = vec![];
        self.funcs = HashMap::new();
        self.tree_ids = HashMap::new();
        self.tree_set = HashMap::new();
    }

    fn parse_tree(tree: &mut Tree<'a>,
             text: String,
             tree_ids: HashMap<TreeId, String>,
             tree_set: HashMap<String, Tree<'a>>,
             funcs: HashMap<String, Func<'a>>)
             -> Result<(), String> {
        tree.parse_name = tree.name.clone();
        let lex_name = tree.name.clone();
        tree.start_parse(funcs,
                         Lexer::new(&lex_name, text.clone()),
                         tree_ids,
                         tree_set);
        tree.text = text;
        tree.parse();
        tree.stop_parse();
        Ok(())
    }

    fn tree_by_id(&self, id: TreeId) -> Option<&Tree<'a>> {
        self.tree_ids
            .get(&id)
            .and_then(|name| self.tree_set.get(name))
    }
    fn add_tree(&mut self, name: &str, t: Tree<'a>) {
        self.tree_ids.insert(t.id, name.to_owned());
        self.tree_set.insert(name.to_owned(), t);
    }

    fn error(&mut self, msg: String) -> Result<(), String> {
        self.root = None;
        Err(format!("template: {}:{}:{}", self.parse_name, self.line, msg))
    }

    fn add_to_tree_set(mut tree: Tree<'a>,
                       mut tree_set: HashMap<String, Tree<'a>>)
                       -> Result<(), String> {
        if let Some(t) = tree_set.get(&tree.name) {
            if let Some(ref r) = t.root {
                match r.is_empty_tree() {
                    Err(e) => return Err(e),
                    Ok(false) => {
                        let err = format!("template multiple definitions of template {}",
                                          &tree.name);
                        return tree.error(err);
                    }
                    Ok(true) => {}
                }
            }
        }
        tree_set.insert(tree.name.clone(), tree);
        Ok(())
    }

    fn parse(&mut self) -> Result<(), String> {
        let id = self.id;
        let n = self.next();
        if n.is_none() {
            return self.error(format!("unable to peek for tree {}", id));
        }
        let t = n.unwrap();
        self.root = Some(ListNode::new(id, t.pos));
        let mut typ = t.typ;
        while typ == ItemType::ItemEOF {
            if typ == ItemType::ItemLeftDelim  {
                let nos = self.next_non_space();
                // lots missing
            }
        }
        Ok(())
    }
}

impl<'a> Iterator for Tree<'a> {
    type Item = Item;
    fn next(&mut self) -> Option<Item> {
        let item = if self.peek_count > 0 {
            self.peek_count -= 1;
            self.token.pop_front()
        } else {
            self.next_from_lex()
        };
        match item {
            Some(item) => {
                self.line = item.line;
                Some(item)
            },
            _ => None
        }
    }
}

#[cfg(test)]
mod tests_mocked {
    use super::*;
    use lexer::ItemType;

    /*
       ItemText
       ItemLeftDelim
{'fraction_non_alphanumeric': 0.15390296137656007, 'fraction_numerical': 0.0035447898973487925, 'mean_word_length': 1.5392837052315769, 'pattern_counts': {'":': 0, '<': 112, '<?xml version=': 0, '>': 160, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2167592', 'n_tokens_mistral': 4419, 'n_tokens_neox': 4376, 'n_words': 910}