<commit_before>use ast::{Node,TransformedNode,CompileErr};
use ast::transform::{Transform, TransformResult};
use scope::{ScopeData};
use std::rc::Rc;
use std::cell::RefCell;
use arena::TypedArena;
use std::io::{Write, Read};
use tok;
use std::fs;
use filter;

/// Top level statements, can be Blocks of instructions
/// like Show/Hide/Mixin or single top-level statements,
/// e.g. variable definitions and imports
#[derive(Debug, Clone)]
pub enum Block<'a> {
    Show(Vec<&'a Node<'a>>),
    Hide(Vec<&'a Node<'a>>),
    Import(String),
}

#[derive(Debug, Clone, PartialEq)]
pub enum PlainBlock<'a> {
    Show(Vec<&'a TransformedNode<'a>>),
    Hide(Vec<&'a TransformedNode<'a>>),
}

impl <'a> Transform<'a> for Block<'a> {
    fn transform(&'a self, parent_scope: Rc<RefCell<ScopeData<'a>>>, transformed_arena: &'a TypedArena<TransformedNode<'a>>, ast_arena: &'a TypedArena<Node<'a>> )
            -> Result<Option<&'a TransformedNode<'a>>, CompileErr> {
        let block_scope = Rc::new(RefCell::new(ScopeData::new(Some(parent_scope.clone()))));

        // collect transformed statements from lines in this block
        let mut t_statements : Vec<&TransformedNode> = Vec::new();
        match self {
            &Block::Show(ref statements) | &Block::Hide(ref statements) => {
                for statement in statements {
                    if let Some(t_statement) = statement.transform(block_scope.clone(), transformed_arena, ast_arena)? {
                        match *t_statement {
                            TransformedNode::ExpandedNodes(ref resolved_stmts) => {
                                for stmt in resolved_stmts {
                                    // replace existing same statement if possible
                                    if let Some(index) = t_statements.iter().position(|other| *other == *stmt) {
                                        t_statements[index] = stmt;
                                    } else {
                                        t_statements.push(stmt);
                                    }
                                }
                            },
                            _ => {
                                // replace existing same statement if possible
                                if let Some(index) = t_statements.iter().position(|other| *other == t_statement) {
                                    t_statements[index] = t_statement;
                                } else {
                                    t_statements.push(t_statement);
                                }
                            }
                        }
                    }
                }
            }
            &Block::Import(ref path) => {
                let mut file = fs::File::open(path)?;
                let mut contents = String::new();
                file.read_to_string(&mut contents)?;

                let tokens = Box::new(tok::tokenize(&contents));
                {
                    match filter::parse_Filter(&ast_arena, tokens.into_iter()) {
                        Ok(&Node::Filter(ref filter)) => {
                            if let Some(&TransformedNode::Root(ref nodes)) =
                                    filter.transform_begin(&ast_arena, parent_scope.clone()).unwrap() {<commit_msg>formatting<commit_after>use ast::{Node,TransformedNode,CompileErr};
use ast::transform::{Transform, TransformResult};
use scope::{ScopeData};
use std::rc::Rc;
use std::cell::RefCell;
use arena::TypedArena;
use std::io::{Write, Read};
use tok;
use std::fs;
use filter;

/// Top level statements, can be Blocks of instructions
/// like Show/Hide/Mixin or single top-level statements,
/// e.g. variable definitions and imports
#[derive(Debug, Clone)]
pub enum Block<'a> {
    Show(Vec<&'a Node<'a>>),
    Hide(Vec<&'a Node<'a>>),
    Import(String),
}

#[derive(Debug, Clone, PartialEq)]
pub enum PlainBlock<'a> {
    Show(Vec<&'a TransformedNode<'a>>),
    Hide(Vec<&'a TransformedNode<'a>>),
}

impl <'a> Transform<'a> for Block<'a> {
    fn transform(&'a self,
                 parent_scope: Rc<RefCell<ScopeData<'a>>>,
                 transformed_arena: &'a TypedArena<TransformedNode<'a>>,
                 ast_arena: &'a TypedArena<Node<'a>>)
            -> Result<Option<&'a TransformedNode<'a>>, CompileErr> {
        let block_scope = Rc::new(RefCell::new(ScopeData::new(Some(parent_scope.clone()))));

        // collect transformed statements from lines in this block
        let mut t_statements : Vec<&TransformedNode> = Vec::new();
        match self {
            &Block::Show(ref statements) | &Block::Hide(ref statements) => {
                for statement in statements {
                    if let Some(t_statement) = statement.transform(block_scope.clone(), transformed_arena, ast_arena)? {
                        match *t_statement {
                            TransformedNode::ExpandedNodes(ref resolved_stmts) => {
                                for stmt in resolved_stmts {
                                    // always add condition statements
                                    if let TransformedNode::ConditionStmt(_) = **stmt {
                                        t_statements.push(stmt);
                                    } else if let Some(index) = t_statements.iter().position(|other| *other == *stmt) {
                                        // replace existing same statement if possible
                                        t_statements[index] = stmt;
                                    } else {
                                        t_statements.push(stmt);
                                    }
                                }
                            },
                            _ => {
                                // always add condition statements
                                if let &TransformedNode::ConditionStmt(_) = t_statement {
                                    t_statements.push(t_statement);
                                } else if let Some(index) = t_statements.iter().position(|other| *other == t_statement) {
                                    // replace existing same statement if possible
                                    t_statements[index] = t_statement;
                                } else {
                                    t_statements.push(t_statement);
                                }
                            }
                        }
                    }
                }
            }
            &Block::Import(ref path) => {
                let mut file = fs::File::open(path)?;
                let mut contents = String::new();
                file.read_to_string(&mut contents)?;

                let tokens = Box::new(tok::tokenize(&contents));
                {
                    match filter::parse_Filter(&ast_arena, tokens.into_iter()) {
                        Ok(&Node::Filter(ref filter)) => {
                            if let Some(&TransformedNode::Root(ref nodes)) =
                                    filter.transform_begin(&ast_arena, parent_scope.clone()).unwrap() {
{'fraction_non_alphanumeric': 0.11808690744920994, 'fraction_numerical': 0.0, 'mean_word_length': 1.453790238836968, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2435979', 'n_tokens_mistral': 1812, 'n_tokens_neox': 1773, 'n_words': 376}