<commit_before>use std::fmt;

#[derive(Debug, PartialEq, Clone)]
pub struct Lexeme {
    pub token: Token,
    pub offset: Option<usize>,
}

impl Lexeme {
    pub fn combine(&self, other: &Lexeme) -> Lexeme {
        let offset = match self.offset {
            Some(o) => Some(o),
            None => other.offset,
        };
        Lexeme {
            token: self.token.combine(&other.token),
            offset: offset,
        }
    }
}

#[derive(Debug, PartialEq, Clone)]
pub enum Token {
    Plus,
    Minus,
    Star,
    Slash,
    Percent,
    LeftParen,
    RightParen,
    Comma,
    Colon,
    Semicolon,
    LeftCurlyBrace,
    RightCurlyBrace,

    String(String),
    StringLiteral(String),

    Number(f32, Option<String>),

    Comment(String),
}

impl Token {
    pub fn combine(&self, other: &Token) -> Token {
        match (self, other) {
            (&Token::String(ref my_str), &Token::String(ref other_str)) => {
                if my_str.len() > 0 {
                    Token::String(format!("{} {}", my_str, other_str))
                } else {
                    Token::String(other_str.clone())
                }
            },
            (s, o) => panic!("Don't know how to combine {:?} and {:?}", s, o),
        }
    }
}

impl fmt::Display for Token {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Token::Plus => write!(f, "+"),
            Token::Minus => write!(f, "-"),
            Token::Star => write!(f, "*"),
            Token::Slash => write!(f, "/"),
            Token::Percent => write!(f, "%"),
            Token::LeftParen => write!(f, "("),
            Token::RightParen => write!(f, ")"),
            Token::Comma => write!(f, ","),
            Token::Colon => write!(f, ":"),
            Token::Semicolon => write!(f, ";"),
            Token::LeftCurlyBrace => write!(f, "{{"),
            Token::RightCurlyBrace => write!(f, "}}"),

            Token::String(ref i) => write!(f, "{}", i),
            Token::StringLiteral(ref i) => write!(f, "{}", i),
            Token::Number(i, Some(ref u)) => write!(f, "{}{}", i, u),
            Token::Number(i, None) => write!(f, "{}", i),
            Token::Comment(ref i) => write!(f, "{}", i),
        }
    }
}

impl Token {
    pub fn from_char(c: char) -> Option<Token> {
        let r = match c {
            '+' => Token::Plus,
            '-' => Token::Minus,
            '*' => Token::Star,
            '/' => Token::Slash,
            '%' => Token::Percent,
            '(' => Token::LeftParen,
            ')' => Token::RightParen,
            ',' => Token::Comma,
            ':' => Token::Colon,
            ';' => Token::Semicolon,
            '{' => Token::LeftCurlyBrace,
            '}' => Token::RightCurlyBrace,
            _   => return None,
        };
        Some(r)
    }
}
<commit_msg>Combine tokens by stringing them together with a space<commit_after>use std::fmt;

#[derive(Debug, PartialEq, Clone)]
pub struct Lexeme {
    pub token: Token,
    pub offset: Option<usize>,
}

impl Lexeme {
    pub fn combine(&self, other: &Lexeme) -> Lexeme {
        let offset = match self.offset {
            Some(o) => Some(o),
            None => other.offset,
        };
        Lexeme {
            token: self.token.combine(&other.token),
            offset: offset,
        }
    }
}

#[derive(Debug, PartialEq, Clone)]
pub enum Token {
    Plus,
    Minus,
    Star,
    Slash,
    Percent,
    LeftParen,
    RightParen,
    Comma,
    Colon,
    Semicolon,
    LeftCurlyBrace,
    RightCurlyBrace,

    String(String),
    StringLiteral(String),

    Number(f32, Option<String>),

    Comment(String),
}

impl Token {
    pub fn combine(&self, other: &Token) -> Token {
        Token::String(format!("{} {}", self, other))
    }
}

impl fmt::Display for Token {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Token::Plus => write!(f, "+"),
            Token::Minus => write!(f, "-"),
            Token::Star => write!(f, "*"),
            Token::Slash => write!(f, "/"),
            Token::Percent => write!(f, "%"),
            Token::LeftParen => write!(f, "("),
            Token::RightParen => write!(f, ")"),
            Token::Comma => write!(f, ","),
            Token::Colon => write!(f, ":"),
            Token::Semicolon => write!(f, ";"),
            Token::LeftCurlyBrace => write!(f, "{{"),
            Token::RightCurlyBrace => write!(f, "}}"),

            Token::String(ref i) => write!(f, "{}", i),
            Token::StringLiteral(ref i) => write!(f, "{}", i),
            Token::Number(i, Some(ref u)) => write!(f, "{}{}", i, u),
            Token::Number(i, None) => write!(f, "{}", i),
            Token::Comment(ref i) => write!(f, "{}", i),
        }
    }
}

impl Token {
    pub fn from_char(c: char) -> Option<Token> {
        let r = match c {
            '+' => Token::Plus,
            '-' => Token::Minus,
            '*' => Token::Star,
            '/' => Token::Slash,
            '%' => Token::Percent,
            '(' => Token::LeftParen,
            ')' => Token::RightParen,
            ',' => Token::Comma,
            ':' => Token::Colon,
            ';' => Token::Semicolon,
            '{' => Token::LeftCurlyBrace,
            '}' => Token::RightCurlyBrace,
            _   => return None,
        };
        Some(r)
    }
}

{'fraction_non_alphanumeric': 0.1949404761904762, 'fraction_numerical': 0.0009300595238095238, 'mean_word_length': 1.8122384937238494, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 84, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '563483', 'n_tokens_mistral': 1718, 'n_tokens_neox': 1713, 'n_words': 336}