<commit_before>mod lexer {
    extern mod core;

    enum TokenType {
        COMMAND,
        EXIT,
        PIPE,
        EOF,
    }

    pub struct Token {
        text: ~str,
        ttype: TokenType,
    }

    pub struct Lexer {
        input: ~str,
        mut currentChar: char,
        mut index: int,
    }
}<commit_msg>Lexing stuff<commit_after>mod lexer {
    extern mod core;


    enum TokenType {
        COMMAND,
        ARGUMENT,
        EXIT,
        PIPE,
        SEMICOLON,
        EOF,
    }

    pub struct Token {
        text: ~str,
        ttype: TokenType,
    }

    impl to_str::ToStr for Token {
        fn to_str(&self) -> ~str {
            copy self.text
        }
    }

    pub struct Lexer {
        input: ~str,
        currentChar: char,
        index: uint,
        eof: bool,
    }

    impl Lexer {

        pub fn consume(&mut self) {
            self.index += 1;
            if (self.index >= str::len(self.input)) {
                self.eof = true;
            } else {
                self.currentChar = str::char_at(self.input, self.index);
            }
        }

        pub fn whitespace(&mut self) {
            while (char::is_whitespace(self.currentChar) && !self.eof) {
                self.consume();
            }
        }

        pub fn command(&mut self) -> Token {
            let start = self.index;
            while (char::is_alphanumeric(self.currentChar) && !self.eof) {
                self.consume();
            }
            Token {text: str::slice(self.input, start, self.index).to_owned(), ttype: COMMAND}
        }

        pub fn next_token(&mut self) -> Token {
            while !self.eof {
                match self.currentChar {
                    ' ' | '\n' | '\r' | '\t' => {
                        self.whitespace();
                        loop;
                    }
                    ';' => {
                        self.consume();
                        return Token {text: ~";", ttype: SEMICOLON};
                    }
                    '|' => {
                        self.consume();
                        return Token {text: ~"|", ttype: PIPE};
                    }
                    _ => {
                        return self.command();
                    }
                }
            }
            return Token {text: ~" ", ttype: EOF};
        }

    }
    pub fn main() {
        let mut lex = Lexer { input: ~"te;st st2ring|", index: 0, currentChar: 't', eof: false };
        let mut t = lex.next_token();
        loop {
            match t.ttype {
                EOF => return,
                _   => {
                    io::println(t.to_str());
                    t = lex.next_token();
                }
            }

        }
    }
}
{'fraction_non_alphanumeric': 0.11764705882352941, 'fraction_numerical': 0.0010893246187363835, 'mean_word_length': 0.9949312092686459, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3162418', 'n_tokens_mistral': 832, 'n_tokens_neox': 770, 'n_words': 175}