<commit_before>/*
 * This file is part of Twig (ported to Rust).
 *
 * For the copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

/**
 * Represents a Token
 *
 * @author <NAME> <<EMAIL>>
 */

/////////////
// exports //
/////////////

pub mod stream;
pub use self::stream::Stream;


#[allow(dead_code)]
#[derive(Debug, PartialEq)]
pub enum Token {
    Eof,
    Text(String),
    BlockStart,
    VarStart,
    BlockEnd,
    VarEnd,
    Name(String),
    Number(String),
    String(String),
    Operator(String),
    Punctuation(String),
    InterpolationStart,
    InterpolationEnd,
}

#[allow(dead_code)]
#[derive(Debug, PartialEq)]
pub enum Type {
    Eof                = -1,
    Text               = 0,
    BlockStart         = 1,
    VarStart           = 2,
    BlockEnd           = 3,
    VarEnd             = 4,
    Name               = 5,
    Number             = 6,
    String             = 7,
    Operator           = 8,
    Punctuation        = 9,
    InterpolationStart = 10,
    InterpolationEnd   = 11,
}

#[allow(dead_code)]
#[allow(unused_variables)]
impl Token {
    pub fn get_value<'a>(&'a self) -> Option<&'a str> {
        match *self {
            Token::Eof => None,
            Token::Text(ref x) => Some(x),
            Token::BlockStart => None,
            Token::VarStart => None,
            Token::BlockEnd => None,
            Token::VarEnd => None,
            Token::Name(ref x) => Some(x),
            Token::Number(ref x) => Some(x),
            Token::String(ref x) => Some(x),
            Token::Operator(ref x) => Some(x),
            Token::Punctuation(ref x) => Some(x),
            Token::InterpolationStart => None,
            Token::InterpolationEnd => None,
        }
    }

    pub fn get_type(&self) -> Type {
        match *self {
            Token::Eof => Type::Eof,
            Token::Text(_) => Type::Text,
            Token::BlockStart => Type::BlockStart,
            Token::VarStart => Type::VarStart,
            Token::BlockEnd => Type::BlockEnd,
            Token::VarEnd => Type::VarEnd,
            Token::Name(_) => Type::Name,
            Token::Number(_) => Type::Number,
            Token::String(_) => Type::String,
            Token::Operator(_) => Type::Operator,
            Token::Punctuation(_) => Type::Punctuation,
            Token::InterpolationStart => Type::InterpolationStart,<commit_msg>Token::Number split into Token::IntegerNumber vs. Token::FloatingNumber<commit_after>/*
 * This file is part of Twig (ported to Rust).
 *
 * For the copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

/**
 * Represents a Token
 *
 * @author <NAME> <<EMAIL>>
 */

/////////////
// exports //
/////////////

pub mod stream;
pub use self::stream::Stream;


#[allow(dead_code)]
#[derive(Debug, PartialEq)]
pub enum Token {
    Eof,
    Text(String),
    BlockStart,
    VarStart,
    BlockEnd,
    VarEnd,
    Name(String),
    IntegerNumber(u64), // orig. Number
    FloatingNumber(f64), // orig. Number
    String(String),
    Operator(String),
    Punctuation(String),
    InterpolationStart,
    InterpolationEnd,
}

#[allow(dead_code)]
#[derive(Debug, PartialEq)]
pub enum Type {
    Eof                = -1,
    Text               = 0,
    BlockStart         = 1,
    VarStart           = 2,
    BlockEnd           = 3,
    VarEnd             = 4,
    Name               = 5,
    Number             = 6, // Floating or Integer
    String             = 7,
    Operator           = 8,
    Punctuation        = 9,
    InterpolationStart = 10,
    InterpolationEnd   = 11,
}

#[allow(dead_code)]
#[allow(unused_variables)]
impl Token {
    // Because of Number Types we need to return `String` copys instead of `&'a str`
    pub fn get_value<'a>(&'a self) -> Option<String> {
        match *self {
            Token::Eof => None,
            Token::Text(ref x) => Some(x.to_string()),
            Token::BlockStart => None,
            Token::VarStart => None,
            Token::BlockEnd => None,
            Token::VarEnd => None,
            Token::Name(ref x) => Some(x.to_string()),
            Token::IntegerNumber(ref x) => Some(x.to_string()),
            Token::FloatingNumber(ref x) => Some(x.to_string()),
            Token::String(ref x) => Some(x.to_string()),
            Token::Operator(ref x) => Some(x.to_string()),
            Token::Punctuation(ref x) => Some(x.to_string()),
            Token::InterpolationStart => None,
            Token::InterpolationEnd => None,
        }
    }

    pub fn get_type(&self) -> Type {
        match *self {
            Token::Eof => Type::Eof,
            Token::Text(_) => Type::Text,
            Token::BlockStart => Type::BlockStart,
            Token::VarStart => Type::VarStart,
            Token::BlockEnd => Type::BlockEnd,
            Token::VarEnd => Type::VarEnd,
            Token::Name(_) => Type::Name,
            Token::IntegerNumber(_) => Type::Number,
            Token::FloatingNumber(_) => Type::Number,
            Token::String(_) => Type::String,
            Token::Operator(_) => Type::Operator,
            Token::Punctuation(_) => Type::Punctuation,
            Token::InterpolationStart => Type::InterpolationStart,
{'fraction_non_alphanumeric': 0.15074711556648382, 'fraction_numerical': 0.006430868167202572, 'mean_word_length': 1.9508928571428572, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3182426', 'n_tokens_mistral': 1574, 'n_tokens_neox': 1566, 'n_words': 362}