<commit_before>use serde_json::*;

use super::super::lex::*;

///
/// Creates a lexing tool for the scripting language
///
pub fn create_lex_script_tool() -> StringLexingTool {
    // Parse the lexer
    let script_json = from_str::<Vec<LexToolSymbol>>(include_str!("syntax_lexer.json")).unwrap();

    // The name isn't used here, but define it anyway
    let lex_defn = LexToolInput { 
        new_tool_name:  String::from("lex-script"),
        symbols:        script_json
    };

    // Create the lexing tool with this definition
    StringLexingTool::from_lex_tool_input(&lex_defn)
}

#[cfg(test)]
mod test {
    use std::error::Error;
    use super::*;

    #[test]
    fn can_parse_syntax_json() {
        let script_json = from_str::<Value>(include_str!("syntax_lexer.json"));

        if script_json.is_err() {
            println!("{:?}", script_json);
            println!("{:?}", script_json.unwrap_err().description());

            assert!(false);
        }
    }

    #[test]
    fn json_can_be_deserialized() {
        let script_json = from_str::<Vec<LexToolSymbol>>(include_str!("syntax_lexer.json"));

        if script_json.is_err() {
            println!("{:?}", script_json);
        }

        script_json.unwrap();
    }

    #[test]
    fn can_create_tool() {
        let _tool = create_lex_script_tool();
    }
}<commit_msg>Add a bunch of tests for the lexer<commit_after>use serde_json::*;

use super::super::lex::*;

///
/// Creates a lexing tool for the scripting language
///
pub fn create_lex_script_tool() -> StringLexingTool {
    // Parse the lexer
    let script_json = from_str::<Vec<LexToolSymbol>>(include_str!("syntax_lexer.json")).unwrap();

    // The name isn't used here, but define it anyway
    let lex_defn = LexToolInput { 
        new_tool_name:  String::from("lex-script"),
        symbols:        script_json
    };

    // Create the lexing tool with this definition
    StringLexingTool::from_lex_tool_input(&lex_defn)
}

#[cfg(test)]
mod test {
    use std::error::Error;
    use super::*;

    fn lex_tokens(input: &str) -> Vec<String> {
        let lex_tool = create_lex_script_tool();

        lex_tool
            .lex(input)
            .iter()
            .map(|x| x.token.clone())
            .collect()
    }

    #[test]
    fn can_parse_syntax_json() {
        let script_json = from_str::<Value>(include_str!("syntax_lexer.json"));

        if script_json.is_err() {
            println!("{:?}", script_json);
            println!("{:?}", script_json.unwrap_err().description());

            assert!(false);
        }
    }

    #[test]
    fn json_can_be_deserialized() {
        let script_json = from_str::<Vec<LexToolSymbol>>(include_str!("syntax_lexer.json"));

        if script_json.is_err() {
            println!("{:?}", script_json);
        }

        script_json.unwrap();
    }

    #[test]
    fn can_create_tool() {
        let _tool = create_lex_script_tool();
    }

    #[test]
    fn can_lex_identifier() {
        assert!(lex_tokens("something") == vec![ String::from("Identifier") ]);
    }

    #[test]
    fn can_lex_let_keyword() {
        assert!(lex_tokens("let") == vec![ String::from("let") ]);
    }

    #[test]
    fn can_lex_whitespace() {
        assert!(lex_tokens(" ") == vec![ String::from("Whitespace") ]);
    }

    #[test]
    fn can_lex_plus_symbol() {
        assert!(lex_tokens("+") == vec![ String::from("+") ]);
    }

    #[test]
    fn can_lex_dot_symbol() {
        assert!(lex_tokens(".") == vec![ String::from(".") ]);
    }

    #[test]
    fn can_lex_newline() {
        assert!(lex_tokens("\n") == vec![ String::from("Newline") ]);
    }

    #[test]
    fn can_lex_simple_number() {
        assert!(lex_tokens("1") == vec![ String::from("Number") ]);
    }

    #[test]
    fn can_lex_longer_number() {
        assert!(lex_tokens("123") == vec![ String::from("Number") ]);
    }

    #[test]
    fn can_lex_decimal_number() {
        assert!(lex_tokens("1.21") == vec![ String::from("Number") ]);
    }

    #[test]
    fn can_lex_decimal_with_exponent() {
        assert!(lex_tokens("1.2e10") == vec![ String::from("Number") ]);
    }

    #[test]
    fn can_lex_integer_with_exponent() {
        assert!(lex_tokens("1e10") == vec![ String::from("Number") ]);
    }

    #[test]
    fn can_lex_decimal_number_beginning_with_dot() {
        assert!(lex_tokens(".21") == vec![ String::from("Number") ]);
    }
}
{'fraction_non_alphanumeric': 0.17155756207674944, 'fraction_numerical': 0.0036117381489841984, 'mean_word_length': 2.69557964970809, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4014274', 'n_tokens_mistral': 1522, 'n_tokens_neox': 1490, 'n_words': 258}