<commit_before>use std::f64;
use std::iter;
use std::slice;

use num::traits::{cast, NumCast};
use itertools::Itertools;

use bio::stats::logprobs::{self, LogProb};


#[derive(Debug, Clone)]
pub struct CDF<T: PartialOrd> {
    inner: Vec<(T, LogProb)>
}


impl<T: PartialOrd> CDF<T> {
    /// Create CDF from given probability mass function (PMF). The PMF may contain duplicate values
    /// the probabilities of which are summed during generation of the CDF.
    pub fn from_pmf(mut entries: Vec<(T, LogProb)>) -> Self {
        entries.sort_by(|&(ref a, _), &(ref b, _)| a.partial_cmp(b).unwrap());
        let mut entries = entries.into_iter();
        let mut inner = vec![entries.next().unwrap()];
        for mut e in entries {
            let p = logprobs::add(inner.last().unwrap().1, e.1);
            if inner.last().unwrap().0 == e.0 {
                //println!("equal");
                inner.last_mut().unwrap().1 = p;
            }
            else {
                //println!("unequal");
                e.1 = p;
                inner.push(e);
            }
        }
        let mut cdf = CDF {
            inner: inner
        };

        if relative_eq!(cdf.total_prob(), 0.0) && cdf.total_prob() > 0.0 {
            cdf.inner.last_mut().unwrap().1 = 0.0;
        }

        cdf
    }

    /// Create CDF from iterator. This can be used to replace the values of a CDF.
    pub fn from_cdf<I: Iterator<Item = (T, LogProb)>>(entries: I) -> Self {
        CDF { inner: entries.collect_vec() }
    }

    pub fn sample(mut self, n: usize) -> Self {
        if self.inner.len() <= n {
            self
        }
        else {
            let s = self.inner.len() / (n - 1);
            let last = self.inner.pop().unwrap();
            let mut inner = self.inner.into_iter().step(s).collect_vec();
            inner.push(last);
            CDF {
                inner: inner
            }
        }
    }

    pub fn iter(&self) -> slice::Iter<(T, f64)>{
        self.inner.iter()
    }

    pub fn iter_pmf<'a>(&'a self) -> CDFPMFIter<'a, T> {
        fn cdf_to_pmf<'a, G>(last_prob: &mut LogProb, e: &'a (G, LogProb)) -> Option<(&'a G, LogProb)> {
            let &(ref value, cdf_prob) = e;
            let prob = logprobs::sub(cdf_prob, *last_prob);
            *last_prob = cdf_prob;
            Some((value, prob))
        }
        self.inner.iter().scan(f64::NEG_INFINITY, cdf_to_pmf)
    }

    /// Get cumulative probability for a given value. If the value is not present,
    /// return the probability of the previous value.
    pub fn get(&self, value: &T) -> Option<LogProb> {
        if self.inner.is_empty() {
            None
        }
        else {<commit_msg>Fix potential numerical instability issue.<commit_after>use std::f64;
use std::iter;
use std::slice;

use num::traits::{cast, NumCast};
use itertools::Itertools;

use bio::stats::logprobs::{self, LogProb};


#[derive(Debug, Clone)]
pub struct CDF<T: PartialOrd> {
    inner: Vec<(T, LogProb)>
}


impl<T: PartialOrd> CDF<T> {
    /// Create CDF from given probability mass function (PMF). The PMF may contain duplicate values
    /// the probabilities of which are summed during generation of the CDF.
    pub fn from_pmf(mut entries: Vec<(T, LogProb)>) -> Self {
        entries.sort_by(|&(ref a, _), &(ref b, _)| a.partial_cmp(b).unwrap());
        let mut entries = entries.into_iter();
        let mut inner = vec![entries.next().unwrap()];
        for mut e in entries {
            let p = logprobs::add(inner.last().unwrap().1, e.1);
            if inner.last().unwrap().0 == e.0 {
                inner.last_mut().unwrap().1 = p;
            }
            else {
                e.1 = p;
                inner.push(e);
            }
        }
        let mut cdf = CDF {
            inner: inner
        };

        if relative_eq!(cdf.total_prob(), 0.0) && cdf.total_prob() > 0.0 {
            cdf.inner.last_mut().unwrap().1 = 0.0;
        }

        cdf
    }

    /// Create CDF from iterator. This can be used to replace the values of a CDF.
    pub fn from_cdf<I: Iterator<Item = (T, LogProb)>>(entries: I) -> Self {
        CDF { inner: entries.collect_vec() }
    }

    pub fn sample(mut self, n: usize) -> Self {
        if self.inner.len() <= n {
            self
        }
        else {
            let s = self.inner.len() / (n - 1);
            let last = self.inner.pop().unwrap();
            let mut inner = self.inner.into_iter().step(s).collect_vec();
            inner.push(last);
            CDF {
                inner: inner
            }
        }
    }

    pub fn iter(&self) -> slice::Iter<(T, f64)>{
        self.inner.iter()
    }

    pub fn iter_pmf<'a>(&'a self) -> CDFPMFIter<'a, T> {
        fn cdf_to_pmf<'a, G>(last_prob: &mut LogProb, e: &'a (G, LogProb)) -> Option<(&'a G, LogProb)> {
            let &(ref value, cdf_prob) = e;
            // last_prob can't be bigger than cdf_prob. If it is
            // this must be to instability, hence we can set prob to -inf, i.e. 0.
            let prob = if cdf_prob < *last_prob { f64::NEG_INFINITY } else { logprobs::sub(cdf_prob, *last_prob) };
            *last_prob = cdf_prob;
            Some((value, prob))
        }
        self.inner.iter().scan(f64::NEG_INFINITY, cdf_to_pmf)
    }

    /// Get cumulative probability for a given value. If the value is not present,
    /// return the probability of the previous value.
    pub fn get(&self, value: &T) -> Option<LogProb> {
        if self.inner.is_empty() {
            None
        }
        else {
{'fraction_non_alphanumeric': 0.1493962876193909, 'fraction_numerical': 0.007749143989908091, 'mean_word_length': 1.8374233128834356, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2594648', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1832, 'n_words': 486}