<commit_before>//! Lorem ipsum generator.
//!
//! This crate contains functions for generating pseudo-Latin lorem
//! ipsum placeholder text. The traditional lorem ipsum text start
//! like this:
//!
//! > Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
//! > eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut
//! > enim ad minim veniam, quis nostrud exercitation ullamco laboris
//! > nisi ut aliquip ex ea commodo consequat. [...]
//!
//! This text is in the [`LOREM_IPSUM`] constant. Random text looking
//! like the above can be generated using the [`lipsum`] function.
//! This function allows you to generate as much text as desired and
//! each invocation will generate different text. This is done using a
//! [Markov chain] based on both the [`LOREM_IPSUM`] and
//! [`LIBER_PRIMUS`] texts. The latter constant holds the full text of
//! the first book of a work by Cicero, of which the lorem ipsum text
//! is a scrambled subset.
//!
//! The random looking text is generatd using a Markov chain of order
//! two, which simply means that the next word is based on the
//! previous two words in the input texts. The Markov chain can be
//! used with other input texts by creating an instance of
//! [`MarkovChain`] and calling its [`learn`] method.
//!
//! [`LOREM_IPSUM`]: constant.LOREM_IPSUM.html
//! [`LIBER_PRIMUS`]: constant.LIBER_PRIMUS.html
//! [`lipsum`]: fn.lipsum.html
//! [`MarkovChain`]: struct.MarkovChain.html
//! [`learn`]: struct.MarkovChain.html#method.learn
//! [Markov chain]: https://en.wikipedia.org/wiki/Markov_chain

extern crate rand;

#[macro_use]
extern crate lazy_static;

use std::collections::HashMap;
use rand::Rng;

/// A bigram is simply two consecutive words.
pub type Bigram<'a> = (&'a str, &'a str);

/// Simple order two Markov chain implementation.
///
/// The [Markov chain] is a chain of order two, which means that it
/// will use the previous two words (a bigram) when predicting the
/// next word. This is normally enough to generate random text that
/// looks somewhat plausible. The implementation is based on
/// [Generating arbitrary text with Markov chains in Rust][blog post].
///
/// [Markov chain]: https://en.wikipedia.org/wiki/Markov_chain
/// [blog post]: https://blakewilliams.me/posts/generating-arbitrary-text-with-markov-chains-in-rust
#[derive(Default)]
pub struct MarkovChain<'a> {
    pub map: HashMap<Bigram<'a>, Vec<&'a str>>,
}

impl<'a> MarkovChain<'a> {
    /// Create a new Markov chain. It will use a default thread-local
    /// random number generator.
    pub fn new() -> MarkovChain<'a> {
        MarkovChain { map: HashMap::new() }
    }

    /// Add new text to the Markov chain. This can be called several
    /// times to build up the chain.
    ///
    /// # Examples
    ///
    /// ```
    /// use lipsum::MarkovChain;
    ///
    /// let mut chain = MarkovChain::new();
    /// chain.learn("red green blue");
    /// assert_eq!(chain.map[&("red", "green")], vec!["blue"]);
    ///
    /// chain.learn("red green yellow");
    /// assert_eq!(chain.map[&("red", "green")], vec!["blue", "yellow"]);
    /// ```
    pub fn learn(&mut self, sentence: &'a str) {
        let words = sentence.split_whitespace().collect::<Vec<&str>>();
        for window in words.windows(3) {
            let (a, b, c) = (window[0], window[1], window[2]);
            self.map.entry((a, b)).or_insert(vec![]).push(c);
        }
    }

    /// Generate `n` words worth of lorem ipsum text. The text will
    /// start from a random point in the Markov chain.
    pub fn generate(&self, n: usize) -> String {
        if self.map.is_empty() {
            // The learn method has not been called.
            return String::new();
        }

        let mut rng = rand::thread_rng();
        let keys = self.map.keys().collect::<Vec<&(&str, &str)>>();
        self.generate_from(n, **rng.choose(&keys).unwrap())
    }

    /// Generate `n` words worth of lorem ipsum text. The text will
    /// start from the given bigram.
    pub fn generate_from(&self, n: usize, from: Bigram<'a>) -> String {
        let mut rng = rand::thread_rng(); // make part of struct
        let keys = self.map.keys().collect::<Vec<&Bigram>>();
        let (mut a, mut b) = from;
        let mut sentence = String::from(a) + " " + b;

        for _ in 2..n {
            while !self.map.contains_key(&(a, b)) {
                let new_key = **rng.choose(&keys).unwrap();
                a = new_key.0;
                b = new_key.1;
            }

            let next_words = &self.map[&(a, b)];
            let c = rng.choose(next_words).unwrap();<commit_msg>generate: add example to docstring<commit_after>//! Lorem ipsum generator.
//!
//! This crate contains functions for generating pseudo-Latin lorem
//! ipsum placeholder text. The traditional lorem ipsum text start
//! like this:
//!
//! > Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
//! > eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut
//! > enim ad minim veniam, quis nostrud exercitation ullamco laboris
//! > nisi ut aliquip ex ea commodo consequat. [...]
//!
//! This text is in the [`LOREM_IPSUM`] constant. Random text looking
//! like the above can be generated using the [`lipsum`] function.
//! This function allows you to generate as much text as desired and
//! each invocation will generate different text. This is done using a
//! [Markov chain] based on both the [`LOREM_IPSUM`] and
//! [`LIBER_PRIMUS`] texts. The latter constant holds the full text of
//! the first book of a work by Cicero, of which the lorem ipsum text
//! is a scrambled subset.
//!
//! The random looking text is generatd using a Markov chain of order
//! two, which simply means that the next word is based on the
//! previous two words in the input texts. The Markov chain can be
//! used with other input texts by creating an instance of
//! [`MarkovChain`] and calling its [`learn`] method.
//!
//! [`LOREM_IPSUM`]: constant.LOREM_IPSUM.html
//! [`LIBER_PRIMUS`]: constant.LIBER_PRIMUS.html
//! [`lipsum`]: fn.lipsum.html
//! [`MarkovChain`]: struct.MarkovChain.html
//! [`learn`]: struct.MarkovChain.html#method.learn
//! [Markov chain]: https://en.wikipedia.org/wiki/Markov_chain

extern crate rand;

#[macro_use]
extern crate lazy_static;

use std::collections::HashMap;
use rand::Rng;

/// A bigram is simply two consecutive words.
pub type Bigram<'a> = (&'a str, &'a str);

/// Simple order two Markov chain implementation.
///
/// The [Markov chain] is a chain of order two, which means that it
/// will use the previous two words (a bigram) when predicting the
/// next word. This is normally enough to generate random text that
/// looks somewhat plausible. The implementation is based on
/// [Generating arbitrary text with Markov chains in Rust][blog post].
///
/// [Markov chain]: https://en.wikipedia.org/wiki/Markov_chain
/// [blog post]: https://blakewilliams.me/posts/generating-arbitrary-text-with-markov-chains-in-rust
#[derive(Default)]
pub struct MarkovChain<'a> {
    pub map: HashMap<Bigram<'a>, Vec<&'a str>>,
}

impl<'a> MarkovChain<'a> {
    /// Create a new Markov chain. It will use a default thread-local
    /// random number generator.
    pub fn new() -> MarkovChain<'a> {
        MarkovChain { map: HashMap::new() }
    }

    /// Add new text to the Markov chain. This can be called several
    /// times to build up the chain.
    ///
    /// # Examples
    ///
    /// ```
    /// use lipsum::MarkovChain;
    ///
    /// let mut chain = MarkovChain::new();
    /// chain.learn("red green blue");
    /// assert_eq!(chain.map[&("red", "green")], vec!["blue"]);
    ///
    /// chain.learn("red green yellow");
    /// assert_eq!(chain.map[&("red", "green")], vec!["blue", "yellow"]);
    /// ```
    pub fn learn(&mut self, sentence: &'a str) {
        let words = sentence.split_whitespace().collect::<Vec<&str>>();
        for window in words.windows(3) {
            let (a, b, c) = (window[0], window[1], window[2]);
            self.map.entry((a, b)).or_insert(vec![]).push(c);
        }
    }

    /// Generate `n` words worth of lorem ipsum text. The text will
    /// start from a random point in the Markov chain.
    ///
    /// See [`generate_from`] if you want to control the starting
    /// point for the generated text.
    ///
    /// # Examples
    ///
    /// Generating the sounds of a grandfather clock:
    ///
    /// ```
    /// use lipsum::MarkovChain;
    ///
    /// let mut chain = MarkovChain::new();
    /// chain.learn("Tick, Tock, Tick, Tock, Ding! Tick, Tock, Ding! Ding!");
    /// println!("{}", chain.generate(15));
    /// ```
    ///
    /// The output looks like this:
    ///
    /// > Ding! Tick, Tock, Tick, Tock, Ding! Ding! Tock, Ding! Tick,
    /// > Tock, Tick, Tock, Tick, Tock
    ///
    /// [`generate_from`]: struct.MarkovChain.html#method.generate_from
    pub fn generate(&self, n: usize) -> String {
        if self.map.is_empty() {
            // The learn method has not been called.
            return String::new();
        }

        let mut rng = rand::thread_rng();
        let keys = self.map.keys().collect::<Vec<&(&str, &str)>>();
        self.generate_from(n, **rng.choose(&keys).unwrap())
    }

    /// Generate `n` words worth of lorem ipsum text. The text will
    /// start from the given bigram.
    pub fn generate_from(&self, n: usize, from: Bigram<'a>) -> String {
        let mut rng = rand::thread_rng(); // make part of struct
        let keys = self.map.keys().collect::<Vec<&Bigram>>();
        let (mut a, mut b) = from;
        let mut sentence = String::from(a) + " " + b;

        for _ in 2..n {
            while !self.map.contains_key(&(a, b)) {
                let new_key = **rng.choose(&keys).unwrap();
                a = new_key.0;
                b = new_key.1;
            }

            let next_words = &self.map[&(a, b)];
            let c = rng.choose(next_words).unwrap();
{'fraction_non_alphanumeric': 0.15826471178951595, 'fraction_numerical': 0.0016067483430407712, 'mean_word_length': 2.587536023054755, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 49, 'https://': 6, 'lorem ipsum': 8, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '852055', 'n_tokens_mistral': 3189, 'n_tokens_neox': 3073, 'n_words': 1060}