<commit_before>pub mod sentiment {
    use std::collections::HashMap;
    use std::fs::File;
    use std::io::prelude::*;
    use std::io::{self, BufReader};
    use std::path::Path;

    pub struct Analizer {
        wordlist: HashMap<String, i8>
    }

    impl Analizer {
        pub fn analyze(self, s: &str) -> f32 {
            let tokens = s.split(" ");
            let word_count = tokens.clone().count();

            let score = tokens
                .map(|word| word.to_string().to_lowercase())
                .map(|word| match self.wordlist.get(&word) {
                    Some(word_score) => *word_score,
                    None => 0 as i8,
                })
                .fold(0, |sum, curr| sum+curr );

            return score as f32 / word_count as f32;
        }
    }

    pub fn new(path: String) -> Analizer {
        let mut wordlist = HashMap::new();

        let mut f = match File::open(&path) {
            Err(why) => panic!("couldn't open {}: {}", path, why),
            Ok(f) => f,
        };

        let f = BufReader::new(f);

        for line in f.lines() {
            let l = line.unwrap();
            let parts = l.split("\t").collect::<Vec<&str>>();

            let word = parts[0].to_string();
            let score = parts[1].parse::<i8>().unwrap();

            wordlist.insert(word, score);
        }

        return Analizer {
            wordlist: wordlist
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn sentiment_test() {
        /*
           analyze("I am happy"); //Score: 3, Comparative: 1
           analyze("I am so happy"); //Score: 6, Comparative: 1.5
           analyze("I am extremely happy"); //Score: 12, Comparative: 3
           analyze("I am really sad"); //Score: -4, Comparative: -1
*/
        let mut analizer = sentiment::new("/Users/coornail/rust/ircbot/src/wordlist.txt".to_string());
        assert!(analizer.analyze("Hey you worthless scumbag") == -1.5)
    }
}
<commit_msg>Clean up sentiment.rs<commit_after>pub mod sentiment {
    use std::collections::HashMap;
    use std::fs::File;
    use std::io::prelude::*;
    use std::io::BufReader;

    pub struct Analizer {
        wordlist: HashMap<String, i8>
    }

    impl Analizer {
        pub fn analyze(&self, s: &str) -> f32 {
            let tokens = s.split(" ");
            let word_count = tokens.clone().count();

            let score = tokens
                .map(|word| word.to_string().to_lowercase())
                .map(|word| match self.wordlist.get(&word) {
                    Some(word_score) => *word_score,
                    None => 0 as i8,
                })
                .fold(0, |sum, curr| sum+curr );

            return score as f32 / word_count as f32;
        }
    }

    pub fn new(path: String) -> Analizer {
        let mut wordlist = HashMap::new();

        let f = match File::open(&path) {
            Err(why) => panic!("couldn't open {}: {}", path, why),
            Ok(f) => f,
        };

        let f = BufReader::new(f);

        for line in f.lines() {
            let l = line.unwrap();
            let parts = l.split("\t").collect::<Vec<&str>>();

            let word = parts[0].to_string();
            let score = parts[1].parse::<i8>().unwrap();

            wordlist.insert(word, score);
        }

        return Analizer {
            wordlist: wordlist
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn sentiment_test() {
        let analizer = sentiment::new("/Users/coornail/rust/ircbot/src/wordlist.txt".to_string());
        assert!(analizer.analyze("Hey you worthless scumbag") == -1.5);
        assert!(analizer.analyze("I am happy") == 1 as f32);
        assert!(analizer.analyze("I am so happy") == 0.75);
        assert!(analizer.analyze("I am extremely happy") == 0.75 as f32);
        assert!(analizer.analyze("I am really sad") == -0.5 as f32);
    }
}

{'fraction_non_alphanumeric': 0.15944982170147734, 'fraction_numerical': 0.014009169638308712, 'mean_word_length': 1.7811614730878187, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5169696', 'n_tokens_mistral': 1257, 'n_tokens_neox': 1213, 'n_words': 297}