<commit_before>use std::fs::File;
use std::io::prelude::*;
use std::mem;
use std::str;
use regex::Regex;

use lex::{lex, Token};

lazy_static! {
    static ref PAT_IS_WHITESPACE: Regex = Regex::new(r#"^\s+$"#).expect("Failed to compile whitespace regex");
}


#[derive(Debug)]
pub enum Node {
    Owned(String),
    Children(Vec<Node>),
}

impl Node {
    pub fn new_children(value: Vec<Node>) -> Node {
        Node::Children(value)
    }

    pub fn new_string<S: Into<String>>(value: S) -> Node {
        Node::Owned(value.into())
    }

    #[allow(dead_code)]
    pub fn print(&self, indent: usize) {
        match self {
            &Node::Owned(ref s) => {
                println!("{:indent$}{:?}", "", s, indent = indent);
            }
            &Node::Children(ref children) => {
                println!("{:indent$}\\", "", indent = indent);
                for child in children {
                    child.print(indent + 2);
                }
            }
        }
    }
}

enum StackRequest {
    None,
    Pop(u8),
    Push(Box<TokenHandler>),
}

trait TokenHandler {
    fn handle_token(&mut self, token: &Token) -> StackRequest;
    fn finish(&mut self) -> Node;
    fn push(&mut self, node: Node);
    fn name(&self) -> &'static str;
}

struct StateRocket {
    root: Vec<Node>,
    buffer: Vec<String>,
}

impl StateRocket {
    fn new() -> StateRocket {
        StateRocket {
            root: vec![],
            buffer: vec![],
        }
    }

    fn ensure_string<'a>(&'a mut self) -> &'a mut String {
        if self.buffer.is_empty() {
            self.buffer.push(String::with_capacity(2));
        }

        self.buffer.last_mut().unwrap()
    }
}

impl TokenHandler for StateRocket {
    fn handle_token(&mut self, token: &Token) -> StackRequest {
        match token {
            &Token::Text(s) => {
                self.buffer.push(s.to_owned());
            }
            &Token::Character(c) => {
                self.ensure_string().push(c);
            }
            &Token::String(s) => {
                self.buffer.push(s.to_owned());
            }
            &Token::StartBlock => {
                if !self.buffer.is_empty() {
                    self.root.push(Node::new_string(self.buffer.concat()));
                    self.buffer.clear();
                }<commit_msg>Each rocket should open up a new concat node<commit_after>use std::fs::File;
use std::io::prelude::*;
use std::mem;
use std::str;
use regex::Regex;

use lex::{lex, Token};

lazy_static! {
    static ref PAT_IS_WHITESPACE: Regex = Regex::new(r#"^\s+$"#).expect("Failed to compile whitespace regex");
}


#[derive(Debug)]
pub enum Node {
    Owned(String),
    Children(Vec<Node>),
}

impl Node {
    pub fn new_children(value: Vec<Node>) -> Node {
        Node::Children(value)
    }

    pub fn new_string<S: Into<String>>(value: S) -> Node {
        Node::Owned(value.into())
    }

    #[allow(dead_code)]
    pub fn print(&self, indent: usize) {
        match self {
            &Node::Owned(ref s) => {
                println!("{:indent$}{:?}", "", s, indent = indent);
            }
            &Node::Children(ref children) => {
                println!("{:indent$}\\", "", indent = indent);
                for child in children {
                    child.print(indent + 2);
                }
            }
        }
    }
}

enum StackRequest {
    None,
    Pop(u8),
    Push(Box<TokenHandler>),
}

trait TokenHandler {
    fn handle_token(&mut self, token: &Token) -> StackRequest;
    fn finish(&mut self) -> Node;
    fn push(&mut self, node: Node);
    fn name(&self) -> &'static str;
}

struct StateRocket {
    root: Vec<Node>,
    buffer: Vec<String>,
}

impl StateRocket {
    fn new() -> StateRocket {
        StateRocket {
            root: vec![Node::new_string("concat")],
            buffer: vec![],
        }
    }

    fn ensure_string<'a>(&'a mut self) -> &'a mut String {
        if self.buffer.is_empty() {
            self.buffer.push(String::with_capacity(2));
        }

        self.buffer.last_mut().unwrap()
    }
}

impl TokenHandler for StateRocket {
    fn handle_token(&mut self, token: &Token) -> StackRequest {
        match token {
            &Token::Text(s) => {
                self.buffer.push(s.to_owned());
            }
            &Token::Character(c) => {
                self.ensure_string().push(c);
            }
            &Token::String(s) => {
                self.buffer.push(s.to_owned());
            }
            &Token::StartBlock => {
                if !self.buffer.is_empty() {
                    self.root.push(Node::new_string(self.buffer.concat()));
                    self.buffer.clear();
                }
{'fraction_non_alphanumeric': 0.16209317166560305, 'fraction_numerical': 0.0012763241863433313, 'mean_word_length': 1.8828939301042305, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '449691', 'n_tokens_mistral': 1509, 'n_tokens_neox': 1476, 'n_words': 298}