<commit_before>extern crate chamkho;

#[cfg(test)]
mod tests {
    use chamkho::dict::Dict;
    use chamkho::wordcut::Wordcut;
    use chamkho::graph::TextRange;
    
    #[test]
    fn test_default_dict() {
        let dict = Dict::load_default();
        assert!(dict.is_ok());
        assert!(dict.unwrap().r() > 10000);
    }
    
    #[test]
    fn test_segment_into_ranges() {
        let dict = Dict::load_default();
        let wordcut = Wordcut::new(dict.unwrap());
        let ranges = wordcut.segment(&"กากกา".to_string());
        let expected = vec![TextRange{s:0,e:3}, TextRange{s:3,e:5}];
        assert_eq!(ranges, expected)
    }

    #[test]
    fn test_segment_into_strings() {
        let dict = Dict::load_default();
        let wordcut = Wordcut::new(dict.unwrap());
        let words = wordcut.segment_into_strings(&"กากกา".to_string());
        let expected = vec!["กาก".to_string(), "กา".to_string()];
        assert_eq!(words, expected)
    }

    #[test]
    fn test_segment_into_string_with_delimiter() {
        let dict = Dict::load_default();
        let wordcut = Wordcut::new(dict.unwrap());
        let segmented_string = wordcut.put_delimiters(&"กากกา".to_string(), "|");
        let expected = "กาก|กา";
        assert_eq!(segmented_string, expected)
    }


}
<commit_msg>polish<commit_after>extern crate chamkho;

#[cfg(test)]
mod tests {
    use chamkho::dict::Dict;
    use chamkho::wordcut::Wordcut;
    use chamkho::graph::TextRange;
    
    #[test]
    fn test_default_dict() {
        let dict = Dict::load_default();
        assert!(dict.is_ok());
        assert!(dict.unwrap().r() > 10000);
    }
    
    #[test]
    fn test_segment_into_ranges() {
        let dict = Dict::load_default();
        let wordcut = Wordcut::new(dict.unwrap());
        let ranges = wordcut.segment(&"กากกา".to_string());
        let expected = vec![TextRange{s:0,e:3}, TextRange{s:3,e:5}];
        assert_eq!(ranges, expected)
    }

    #[test]
    fn test_segment_into_strings() {
        let dict = Dict::load_default();
        let wordcut = Wordcut::new(dict.unwrap());
        let words = wordcut.segment_into_strings(&"กากกา".to_string());
        let expected = vec!["กาก".to_string(), "กา".to_string()];
        assert_eq!(words, expected)
    }

    #[test]
    fn test_segment_into_string_with_delimiter() {
        let dict = Dict::load_default();
        let wordcut = Wordcut::new(dict.unwrap());
        let segmented_string = wordcut.put_delimiters(&"กากกา".to_string(), "|");
        let expected = "กาก|กา";
        assert_eq!(segmented_string, expected)
    }
}

{'fraction_non_alphanumeric': 0.16021464162514373, 'fraction_numerical': 0.006899195093905711, 'mean_word_length': 2.936651583710407, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2708911', 'n_tokens_mistral': 922, 'n_tokens_neox': 912, 'n_words': 148}