<commit_before>use syntax::core::keywords::Keywords;
use syntax::core::tokens;
use syntax::core::tokens::Token;
use syntax::core::punctuation;
use syntax::core::punctuation::Punctuations;

pub trait Tokenizer {
    fn get_tok(&mut self) -> Token;
}

// A Lexer that keeps track of the current line and column position
// as well as the position in the char input stream.
pub struct Lexer {
    /*****

    pub status of line and column number deprecated, please use get_tok_line(), get_tok_column()

    ******/
    pub line_number: uint,
    pub column_number: uint,
    buffer_pos: uint,
    line_start: uint,
    input: String
}

impl Tokenizer for Lexer {
    // Parse the file where it left off and return the next token
    fn get_tok(&mut self) -> Token {
        // ToDo: More lexing!
        loop {
            if self.eof() {
                return tokens::EOF;
            }

            self.consume_whitespace();

            return match self.next_char() {
                // Find Keywords and Identifiers
                a if a.is_alphabetic() || a == '_' => self.consume_identifier(),

                e if self.eof() => tokens::EOF,

                // Find ints, floats, hex, and bin numeric values
                n if n.is_digit() => self.consume_numeric(),

                // Count tabs: \n\t*
                '\n' => self.consume_tabs(),

                // Error: Found tabs without preceeding newline
                '\t' => {
                    self.consume_char();

                    tokens::Error("Found an out of place tab.".to_string())
                },

                // Find single char punctuations
                '(' | ')' | '[' | ']' | '{' | '}' |
                '.' | ',' | ':' | '<' | '~' | '=' => {
                    // Dumb: cant do self.single_punc_token(self.consume_char()) due to borrowing
                    // But the following works.
                    let ch = self.consume_char();

                    self.single_punc_token(ch)
                },

                // Find multi-char(+=, -=, ..) punctuations or not
                '+' | '-' | '*' | '/' | '%' => {
                    let ch = self.consume_char();

                    match self.next_char() {
                        '=' => {
                            let ch2 = self.consume_char();

                            self.multi_punc_token([ch, ch2])<commit_msg>Added missing > char<commit_after>use syntax::core::keywords::Keywords;
use syntax::core::tokens;
use syntax::core::tokens::Token;
use syntax::core::punctuation;
use syntax::core::punctuation::Punctuations;

pub trait Tokenizer {
    fn get_tok(&mut self) -> Token;
}

// A Lexer that keeps track of the current line and column position
// as well as the position in the char input stream.
pub struct Lexer {
    /*****

    pub status of line and column number deprecated, please use get_tok_line(), get_tok_column()

    ******/
    pub line_number: uint,
    pub column_number: uint,
    buffer_pos: uint,
    line_start: uint,
    input: String
}

impl Tokenizer for Lexer {
    // Parse the file where it left off and return the next token
    fn get_tok(&mut self) -> Token {
        // ToDo: More lexing!
        loop {
            if self.eof() {
                return tokens::EOF;
            }

            self.consume_whitespace();

            return match self.next_char() {
                // Find Keywords and Identifiers
                a if a.is_alphabetic() || a == '_' => self.consume_identifier(),

                e if self.eof() => tokens::EOF,

                // Find ints, floats, hex, and bin numeric values
                n if n.is_digit() => self.consume_numeric(),

                // Count tabs: \n\t*
                '\n' => self.consume_tabs(),

                // Error: Found tabs without preceeding newline
                '\t' => {
                    self.consume_char();

                    tokens::Error("Found an out of place tab.".to_string())
                },

                // Find single char punctuations
                '(' | ')' | '[' | ']' | '{' | '}' |
                '.' | ',' | ':' | '<' | '>' | '~' | '=' => {
                    // Dumb: cant do self.single_punc_token(self.consume_char()) due to borrowing
                    // But the following works.
                    let ch = self.consume_char();

                    self.single_punc_token(ch)
                },

                // Find multi-char(+=, -=, ..) punctuations or not
                '+' | '-' | '*' | '/' | '%' => {
                    let ch = self.consume_char();

                    match self.next_char() {
                        '=' => {
                            let ch2 = self.consume_char();

                            self.multi_punc_token([ch, ch2])
{'fraction_non_alphanumeric': 0.13515771882180908, 'fraction_numerical': 0.0008355964069354502, 'mean_word_length': 1.498956158663883, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7088196', 'n_tokens_mistral': 1401, 'n_tokens_neox': 1296, 'n_words': 353}