<commit_before>#![recursion_limit = "80"]

#[macro_use]
extern crate pest;
extern crate regex;

use pest::prelude::*;
use std::fs::File;
use std::io::{self, BufWriter, Write};

pub use error::ParseError;
pub use name::Name;
pub use path::Path;
pub use template::Template;

pub mod ruby;
mod error;
mod name;
mod path;
mod template;

/// Defines the source code output behavior for compiler backends. The main
/// compiler driver treats the result of each backend identically.
pub trait Compile {
    /// Writes the final translated source code to an output buffer.
    fn emit(&self, buf: &mut Write) -> io::Result<()>;

    /// Saves the translated source code to a file.
    fn write<P>(&self, output: P) -> io::Result<()>
    where
        P: AsRef<std::path::Path>,
    {
        File::create(output)
            .map(|file| BufWriter::new(file))
            .and_then(|mut buf| self.emit(&mut buf))
    }
}

#[derive(Debug, PartialEq)]
pub struct Block {
    statements: Vec<Statement>,
}

impl Block {
    fn new(statements: Vec<Statement>) -> Self {
        Block { statements: statements }
    }

    fn empty() -> Self {
        Self::new(vec![])
    }

    /// Adds the statement as the first element in the block, combining it
    /// with a previous content statement if possible.
    fn prepend(&mut self, mut statement: Statement) {
        let leader = match self.statements.get_mut(0) {
            Some(first) => {
                if statement.merge(first) {
                    *first = statement;
                    None
                } else {
                    Some(statement)
                }
            }
            None => Some(statement),
        };

        if let Some(stmt) = leader {
            self.statements.insert(0, stmt);
        }
    }

    /// Adds the statement as the final element in the block, combining it with
    /// a previous content statement if possible.
    fn append(&mut self, statement: Statement) {
        let trailer = match self.statements.pop() {
            Some(mut last) => {
                if last.merge(&statement) {
                    last
                } else {
                    self.statements.push(last);
                    statement
                }
            }
            None => statement,
        };
        self.statements.push(trailer);
    }
}

#[derive(Debug, PartialEq)]
pub enum Statement {
    Program(Block),
    Section(Path, Block),
    Inverted(Path, Block),
    Variable(Path),
    Html(Path),
    Partial(String, Option<String>),
    Content(String),
    Comment(String),
}

impl Statement {
    /// Parses the Mustache text into a Statement AST.
    pub fn parse(template: &str) -> Result<Statement, ParseError> {
        let mut parser = Rdp::new(StringInput::new(template));
        if parser.program() && parser.end() {
            Ok(parser.tree())
        } else {
            let (_, position) = parser.expected();
            Err(ParseError::UnexpectedToken(position))
        }
    }

    /// Visits each node in the tree collecting the names of partials
    /// referenced by the template.
    pub fn partials<'a>(&'a self) -> Vec<&'a String> {
        match *self {
            Statement::Program(ref block) => {
                block
                    .statements
                    .iter()
                    .flat_map(|stmt| stmt.partials())
                    .collect()
            }
            Statement::Section(_, ref block) => {
                block
                    .statements
                    .iter()
                    .flat_map(|stmt| stmt.partials())
                    .collect()
            }
            Statement::Inverted(_, ref block) => {
                block
                    .statements
                    .iter()
                    .flat_map(|stmt| stmt.partials())
                    .collect()
            }
            Statement::Partial(ref name, _) => vec![name],
            _ => Vec::new(),
        }
    }

    /// Combines adjacent content statements into a single statement.
    ///
    /// Returns true if the statements were merged.
    fn merge(&mut self, statement: &Statement) -> bool {
        match *self {
            Statement::Content(ref mut left) => {
                match *statement {
                    Statement::Content(ref right) => {
                        left.push_str(right);
                        true
                    }
                    _ => false,
                }
            }
            _ => false,
        }
    }
}
<commit_msg>Share section and inverted match blocks<commit_after>#![recursion_limit = "80"]

#[macro_use]
extern crate pest;
extern crate regex;

use pest::prelude::*;
use std::fs::File;
use std::io::{self, BufWriter, Write};

pub use error::ParseError;
pub use name::Name;
pub use path::Path;
pub use template::Template;

pub mod ruby;
mod error;
mod name;
mod path;
mod template;

/// Defines the source code output behavior for compiler backends. The main
/// compiler driver treats the result of each backend identically.
pub trait Compile {
    /// Writes the final translated source code to an output buffer.
    fn emit(&self, buf: &mut Write) -> io::Result<()>;

    /// Saves the translated source code to a file.
    fn write<P>(&self, output: P) -> io::Result<()>
    where
        P: AsRef<std::path::Path>,
    {
        File::create(output)
            .map(|file| BufWriter::new(file))
            .and_then(|mut buf| self.emit(&mut buf))
    }
}

#[derive(Debug, PartialEq)]
pub struct Block {
    statements: Vec<Statement>,
}

impl Block {
    fn new(statements: Vec<Statement>) -> Self {
        Block { statements: statements }
    }

    fn empty() -> Self {
        Self::new(vec![])
    }

    /// Adds the statement as the first element in the block, combining it
    /// with a previous content statement if possible.
    fn prepend(&mut self, mut statement: Statement) {
        let leader = match self.statements.get_mut(0) {
            Some(first) => {
                if statement.merge(first) {
                    *first = statement;
                    None
                } else {
                    Some(statement)
                }
            }
            None => Some(statement),
        };

        if let Some(stmt) = leader {
            self.statements.insert(0, stmt);
        }
    }

    /// Adds the statement as the final element in the block, combining it with
    /// a previous content statement if possible.
    fn append(&mut self, statement: Statement) {
        let trailer = match self.statements.pop() {
            Some(mut last) => {
                if last.merge(&statement) {
                    last
                } else {
                    self.statements.push(last);
                    statement
                }
            }
            None => statement,
        };
        self.statements.push(trailer);
    }
}

#[derive(Debug, PartialEq)]
pub enum Statement {
    Program(Block),
    Section(Path, Block),
    Inverted(Path, Block),
    Variable(Path),
    Html(Path),
    Partial(String, Option<String>),
    Content(String),
    Comment(String),
}

impl Statement {
    /// Parses the Mustache text into a Statement AST.
    pub fn parse(template: &str) -> Result<Statement, ParseError> {
        let mut parser = Rdp::new(StringInput::new(template));
        if parser.program() && parser.end() {
            Ok(parser.tree())
        } else {
            let (_, position) = parser.expected();
            Err(ParseError::UnexpectedToken(position))
        }
    }

    /// Visits each node in the tree collecting the names of partials
    /// referenced by the template.
    pub fn partials<'a>(&'a self) -> Vec<&'a String> {
        match *self {
            Statement::Program(ref block) => {
                block
                    .statements
                    .iter()
                    .flat_map(|stmt| stmt.partials())
                    .collect()
            }
            Statement::Section(_, ref block) |
            Statement::Inverted(_, ref block) => {
                block
                    .statements
                    .iter()
                    .flat_map(|stmt| stmt.partials())
                    .collect()
            }
            Statement::Partial(ref name, _) => vec![name],
            _ => Vec::new(),
        }
    }

    /// Combines adjacent content statements into a single statement.
    ///
    /// Returns true if the statements were merged.
    fn merge(&mut self, statement: &Statement) -> bool {
        match *self {
            Statement::Content(ref mut left) => {
                match *statement {
                    Statement::Content(ref right) => {
                        left.push_str(right);
                        true
                    }
                    _ => false,
                }
            }
            _ => false,
        }
    }
}

{'fraction_non_alphanumeric': 0.11900493052442851, 'fraction_numerical': 0.0008964589870013447, 'mean_word_length': 1.571305099394987, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '700969', 'n_tokens_mistral': 2425, 'n_tokens_neox': 2311, 'n_words': 675}