<commit_before>//! This module contains the underlying data generation and shrinking
//! mechanism. The main type is the `InfoPool`, which represents a pool of
//! random bytes that can be observed via the `InfoTap` object (obtained via
//! `InfoPool#tap`).
//!
//! Also manages the shrinking process (see [`minimize`](fn.minimize.html)).

use std::fmt;
use hex_slice::AsHex;
use rand::{random, Rng, XorShiftRng};
use std::cmp::min;

/// A pool of data that we can draw upon to generate other types of data.
#[derive(Clone, PartialEq)]
pub struct InfoPool {
    data: Vec<u8>,
}

/// A handle to an info Pool that we can draw newly generated bytes from.
pub struct InfoTap<'a> {
    data: &'a mut Vec<u8>,
    rng: XorShiftRng,
}

/// A handle to an info Pool that we can draw replayed bytes from, and zero after.
#[derive(Clone)]
pub struct InfoReplay<'a> {
    data: &'a [u8],
    off: usize,
}


impl fmt::Debug for InfoPool {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("InfoPool")
            .field("data", &format_args!("{:x}", self.data.as_hex()))
            .finish()
    }
}

/// The reasons why drawing data from a pool can fail.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum DataError {
    /// Not enough data to generate a value
    PoolExhausted,
    /// One of our combinators said that we should not test this value.
    SkipItem,
}


impl InfoPool {
    /// Create an `InfoPool` with a given vector of bytes. (Mostly used for
    /// testing).
    pub fn of_vec(data: Vec<u8>) -> Self {
        InfoPool { data: data }
    }

    /// Create an `InfoPool` with a `size` length vector of random bytes
    /// using the generator `rng`. (Mostly used for testing).
    pub fn new() -> Self {
        Self { data: Vec::new() }
    }


    /// Allows access to the underlying buffer.
    pub fn buffer(&self) -> &[u8] {
        &*self.data
    }

    fn tap_with(&mut self, rng: XorShiftRng) -> InfoTap {
        InfoTap {
            data: &mut self.data,
            rng: rng,
        }
    }
    /// Creates a tap that allows drawing new information from this pool.
    pub fn tap(&mut self) -> InfoTap {
        self.tap_with(random::<XorShiftRng>())
    }

    /// Creates a tap that allows drawing new information from this pool.
    pub fn replay(&self) -> InfoReplay {
        InfoReplay {
            data: &*self.data,
            off: 0,
        }
    }
}

impl<'a> InfoTap<'a> {
    /// Consumes the next byte from this tap. Returns `Ok(x)` if successful,
    /// or `Err(DataError::PoolExhausted)` if we have reached the end.
    pub fn next_byte(&mut self) -> u8 {
        let next = self.rng.gen::<u8>();
        self.data.push(next);
        next
    }
}


impl<'a> Iterator for InfoTap<'a> {
    type Item = u8;
    fn next(&mut self) -> Option<u8> {
        Some(self.next_byte())
    }
}

impl<'a> InfoReplay<'a> {
    /// Consumes the next byte from this tap. Returns `Ok(x)` if successful,
    /// or `Err(DataError::PoolExhausted)` if we have reached the end.
    pub fn next_byte(&mut self) -> u8 {
        let res = self.data.get(self.off).cloned();
        self.off += 1;
        res.unwrap_or(0)
    }
}


impl<'a> Iterator for InfoReplay<'a> {
    type Item = u8;
    fn next(&mut self) -> Option<u8> {
        Some(self.next_byte())
    }
}

fn minimize_via_removal<F: Fn(InfoReplay) -> bool>(
    p: &InfoPool,
    candidate: &mut InfoPool,
    pred: &F,
) -> Option<InfoPool> {
    // First shrink tactic: item removal
    trace!("minimizing by removal: {:?}", p);
    if p.data.len() == 0 {
        return None;
    }

    let max_pow = 0usize.count_zeros();
    let pow = max_pow - p.data.len().leading_zeros();
    for granularity in 0..pow {
        let width = 1 << (pow - granularity);
        for chunk in 0..(1 << granularity) {
            let start = min(chunk * width, p.data.len());
            let end = min(start + width, p.data.len());
            if start == end {
                break;
            }

            candidate.data.clear();
            candidate.data.extend(&p.data[0..start]);
            candidate.data.extend(&p.data[end..]);

            let test = pred(candidate.replay());
            trace!(
                "removed {},{}: {:?}; test result {}",
                start,
                end,
                candidate,
                test
            );
            if test {
                if let Some(res) = minimize(&candidate, pred) {
                    trace!("Returning shrunk: {:?}", res);
                    return Some(res);
                } else {
                    trace!("Returning original: {:?}", candidate);
                    return Some(candidate.clone());
                }
            }
        }
    }
    None
}

fn minimize_via_scalar_shrink<F: Fn(InfoReplay) -> bool>(
    p: &InfoPool,
    candidate: &mut InfoPool,
    pred: &F,
) -> Option<InfoPool> {
    // Second shrink tactic: make values smaller
    trace!("minimizing by scalar shrink: {:?}", p);
    for i in 0..p.data.len() {
        candidate.clone_from(&p);

        let max_pow = 0u8.count_zeros();
        let pow = max_pow - p.data[i].leading_zeros();
        for bitoff in 0..pow {
            candidate.data[i] = p.data[i] - (p.data[i] >> bitoff);
            trace!(
                "shrunk item -(bitoff:{}) {} {}->{}: {:?}",
                bitoff,
                i,
                p.data[i],
                candidate.data[i],
                candidate
            );

            if candidate.buffer() == p.buffer() {
                trace!("No change");
                continue;
            }

            let test = pred(candidate.replay());
            trace!("test result {}", test);
            if test {
                if let Some(res) = minimize(&candidate, pred) {
                    trace!("Returning shrunk: {:?}", res);
                    return Some(res);
                } else {
                    trace!("Returning original: {:?}", candidate);
                    return Some(candidate.clone());
                }
            }
        }
    }

    None
}

/// Try to find the smallest pool `p` such that the predicate `pred` returns
/// true. Given that our [generators](../generators/index.html) tend to
/// generate smaller outputs from smaller inputs, by minimizing the source
/// pool we can find the smallest value that provokes a failure.
///
/// If we do not find any smaller pool that satisifies `pred`; we return
/// `None`.
///
/// Currently, we have two heuristics for shrinking: removing slices, and
/// reducing individual values.
///
/// Removing slices tries to remove as much of the pool as it can whilst still
/// having the predicate hold. At present, we do this by removing each half
/// and testing, then each quarter, eighths, and so on. Eventually, we plan to
/// track the regions of data that each generator draws from, and use that to
/// optimise the shrinking process.
///
/// Reducing individual values basically goes through each position in the
/// pool, and then tries reducing it to zero, then half, thn three quarters,
/// seven eighths, and so on.
pub fn minimize<F: Fn(InfoReplay) -> bool>(p: &InfoPool, pred: &F) -> Option<InfoPool> {
    let mut candidate = p.clone();
    if let Some(res) = minimize_via_removal(p, &mut candidate, pred) {
        return Some(res);
    }

    if let Some(res) = minimize_via_scalar_shrink(p, &mut candidate, pred) {
        return Some(res);
    }

    trace!("Nothing smaller found than {:?}", p);
    None
}

#[cfg(test)]
mod tests {
    extern crate env_logger;
    use super::*;

    #[test]
    fn should_take_each_item_in_pool() {
        let p = InfoPool::of_vec(vec![0, 1, 2, 3]);
        let mut t = p.replay();
        assert_eq!(t.next_byte(), 0);
        assert_eq!(t.next_byte(), 1);
        assert_eq!(t.next_byte(), 2);
        assert_eq!(t.next_byte(), 3);
        assert_eq!(t.next_byte(), 0);
    }

    #[test]
    fn should_generate_random_data() {
        let trials = 1024usize;
        let mut vals = 0;
        let mut p = InfoPool::new();
        let mut t = p.tap_with(::rand::XorShiftRng::new_unseeded());
        let error = 4;
        for _ in 0..trials {
            vals += t.next_byte() as usize;
        }
        let mean = vals / trials;
        let expected = 128;
        assert!(
            (expected - error) < mean && (expected + error) > mean,
            "Expected {} trials to be ({}+/-{}); got {}",
            trials,
            expected,
            error,
            mean
        )
    }

    #[test]
    fn should_allow_restarting_read() {
        let mut p = InfoPool::new();
        let mut v0 = Vec::new();
        {
            let mut t = p.tap();
            for _ in 0..4 {
                v0.push(t.next_byte())
            }
        }

        let mut t = p.replay();
        let mut v1 = Vec::new();
        for _ in 0..4 {
            v1.push(t.next_byte())
        }

        assert_eq!(v0, v1)
    }

    #[test]
    fn should_allow_borrowing_buffer() {
        let p = InfoPool::of_vec(vec![1]);
        assert_eq!(p.buffer(), &[1]);
    }

    #[test]
    fn tap_can_act_as_iterator() {
        let buf = vec![4, 3, 2, 1];
        let p = InfoPool::of_vec(buf.clone());
        let _: &Iterator<Item = u8> = &p.replay();

        assert_eq!(p.replay().take(4).collect::<Vec<_>>(), buf)
    }

    #[test]
    fn replay_can_act_as_iterator() {
        let buf = vec![4, 3, 2, 1];
        let p = InfoPool::of_vec(buf.clone());
        let _: &Iterator<Item = u8> = &p.replay();

        assert_eq!(p.replay().take(4).collect::<Vec<_>>(), buf)
    }

    #[test]
    fn minimiser_should_minimise_to_empty() {
        let p = InfoPool::of_vec(vec![1]);
        let min = minimize(&p, &|_| true);

        assert_eq!(min.as_ref().map(|p| p.buffer()), Some([].as_ref()))
    }

    #[test]
    fn minimiser_should_minimise_to_minimum_given_size() {
        env_logger::init().unwrap_or(());
        let p = InfoPool::of_vec(vec![1; 4]);
        let min = minimize(&p, &|t| t.take(16).filter(|&v| v > 0).count() > 1)
            .expect("some smaller pool");

        assert_eq!(min.buffer(), &[1, 1])
    }

    #[test]
    fn minimiser_should_minimise_scalar_values() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|v| v >= 3)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[3])
    }
    #[test]
    fn minimiser_should_minimise_scalar_values_to_empty() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|_| true)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[] as &[u8])
    }

    #[test]
    fn minimiser_should_minimise_scalar_values_by_search() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|v| v >= 13)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[13])
    }
    #[test]
    fn minimiser_should_minimise_scalar_values_accounting_for_overflow() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|v| v >= 251)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[251])
    }
}
<commit_msg>Refactor removal shrinker to use iterators internally.<commit_after>//! This module contains the underlying data generation and shrinking
//! mechanism. The main type is the `InfoPool`, which represents a pool of
//! random bytes that can be observed via the `InfoTap` object (obtained via
//! `InfoPool#tap`).
//!
//! Also manages the shrinking process (see [`minimize`](fn.minimize.html)).

use std::fmt;
use hex_slice::AsHex;
use rand::{random, Rng, XorShiftRng};
use std::cmp::min;

/// A pool of data that we can draw upon to generate other types of data.
#[derive(Clone, PartialEq)]
pub struct InfoPool {
    data: Vec<u8>,
}

/// A handle to an info Pool that we can draw newly generated bytes from.
pub struct InfoTap<'a> {
    data: &'a mut Vec<u8>,
    rng: XorShiftRng,
}

/// A handle to an info Pool that we can draw replayed bytes from, and zero after.
#[derive(Clone)]
pub struct InfoReplay<'a> {
    data: &'a [u8],
    off: usize,
}


impl fmt::Debug for InfoPool {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("InfoPool")
            .field("data", &format_args!("{:x}", self.data.as_hex()))
            .finish()
    }
}

/// The reasons why drawing data from a pool can fail.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum DataError {
    /// Not enough data to generate a value
    PoolExhausted,
    /// One of our combinators said that we should not test this value.
    SkipItem,
}


impl InfoPool {
    /// Create an `InfoPool` with a given vector of bytes. (Mostly used for
    /// testing).
    pub fn of_vec(data: Vec<u8>) -> Self {
        InfoPool { data: data }
    }

    /// Create an `InfoPool` with a `size` length vector of random bytes
    /// using the generator `rng`. (Mostly used for testing).
    pub fn new() -> Self {
        Self { data: Vec::new() }
    }


    /// Allows access to the underlying buffer.
    pub fn buffer(&self) -> &[u8] {
        &*self.data
    }

    fn tap_with(&mut self, rng: XorShiftRng) -> InfoTap {
        InfoTap {
            data: &mut self.data,
            rng: rng,
        }
    }
    /// Creates a tap that allows drawing new information from this pool.
    pub fn tap(&mut self) -> InfoTap {
        self.tap_with(random::<XorShiftRng>())
    }

    /// Creates a tap that allows drawing new information from this pool.
    pub fn replay(&self) -> InfoReplay {
        InfoReplay {
            data: &*self.data,
            off: 0,
        }
    }
}

impl<'a> InfoTap<'a> {
    /// Consumes the next byte from this tap. Returns `Ok(x)` if successful,
    /// or `Err(DataError::PoolExhausted)` if we have reached the end.
    pub fn next_byte(&mut self) -> u8 {
        let next = self.rng.gen::<u8>();
        self.data.push(next);
        next
    }
}


impl<'a> Iterator for InfoTap<'a> {
    type Item = u8;
    fn next(&mut self) -> Option<u8> {
        Some(self.next_byte())
    }
}

impl<'a> InfoReplay<'a> {
    /// Consumes the next byte from this tap. Returns `Ok(x)` if successful,
    /// or `Err(DataError::PoolExhausted)` if we have reached the end.
    pub fn next_byte(&mut self) -> u8 {
        let res = self.data.get(self.off).cloned();
        self.off += 1;
        res.unwrap_or(0)
    }
}


impl<'a> Iterator for InfoReplay<'a> {
    type Item = u8;
    fn next(&mut self) -> Option<u8> {
        Some(self.next_byte())
    }
}

/// Iterates over a series of shrunk pools. If we imagine that our buffer has
/// a sz of (1 << (log2sz-1)) < sz ≤ (1 << log2sz), then where:
/// f(x) = 1<<(log2sz-x) we want to cut out
/// chunks of:
/// ```
/// 0..f(0),
/// 0..f(1), f(1)..2f(1),
/// 0..f(2), f(2)..2f(2), 2f(2)..3f(2), 3f(2)..4f(2),
/// ```
///
/// In other words, we remove the whole lot, then first half, second half,
/// first quarter, second quarter, etc.
#[derive(Debug)]
struct RemovalShrinker {
    seed: InfoPool,
    log2sz: usize,
    level: usize,
    chunk: usize,
}

impl RemovalShrinker {
    fn new(seed: InfoPool) -> Self {
        let max_pow = 0usize.count_zeros();
        let pow = max_pow - seed.data.len().leading_zeros();
        RemovalShrinker {
            seed,
            log2sz: pow as usize,
            // Ranges from 0..self.log2sz
            level: 0,
            // Ranges from 0..(1<<self.level)
            chunk: 0,
        }
    }
}

impl Iterator for RemovalShrinker {
    type Item = InfoPool;
    fn next(&mut self) -> Option<Self::Item> {
        loop {
            trace!("RemovalShrinker#next: {:?}", self);
            if self.level > self.log2sz {
                return None;
            }

            let granularity = self.log2sz - self.level;
            let width = 1 << granularity;
            let chunk = self.chunk;

            let start = chunk * width;
            let end = start + width;

            if start >= self.seed.data.len() {
                trace!(
                    "Out of slice ({},{}) >= {}",
                    start,
                    end,
                    self.seed.data.len()
                );
                self.chunk = 0;
                self.level += 1;
                continue;
            } else {
                self.chunk += 1;
            }
            let start = min(start, self.seed.data.len());
            let end = min(end, self.seed.data.len());

            let mut candidate = InfoPool::new();
            candidate.data.clear();
            candidate.data.extend(&self.seed.data[0..start]);
            candidate.data.extend(&self.seed.data[end..]);
            trace!("removed {},{} -> {:?}", start, end, candidate);


            return Some(candidate);
        }
    }
}

fn minimize_via_removal<F: Fn(InfoReplay) -> bool>(p: &InfoPool, pred: &F) -> Option<InfoPool> {
    let res = RemovalShrinker::new(p.clone())
        .filter(|c| {
            let test = pred(c.replay());
            trace!("test result: {:?} <= {:?}", test, c);
            test
        })
        .next();

    return res.map(|candidate| {
        trace!("Re-Shrinking: {:?}", candidate);
        return minimize(&candidate, pred).unwrap_or(candidate);
    });
}

fn minimize_via_scalar_shrink<F: Fn(InfoReplay) -> bool>(
    p: &InfoPool,
    candidate: &mut InfoPool,
    pred: &F,
) -> Option<InfoPool> {
    // Second shrink tactic: make values smaller
    trace!("minimizing by scalar shrink: {:?}", p);
    for i in 0..p.data.len() {
        candidate.clone_from(&p);

        let max_pow = 0u8.count_zeros();
        let pow = max_pow - p.data[i].leading_zeros();
        for bitoff in 0..pow {
            candidate.data[i] = p.data[i] - (p.data[i] >> bitoff);
            trace!(
                "shrunk item -(bitoff:{}) {} {}->{}: {:?}",
                bitoff,
                i,
                p.data[i],
                candidate.data[i],
                candidate
            );

            if candidate.buffer() == p.buffer() {
                trace!("No change");
                continue;
            }

            let test = pred(candidate.replay());
            trace!("test result {}", test);
            if test {
                if let Some(res) = minimize(&candidate, pred) {
                    trace!("Returning shrunk: {:?}", res);
                    return Some(res);
                } else {
                    trace!("Returning original: {:?}", candidate);
                    return Some(candidate.clone());
                }
            }
        }
    }

    None
}

/// Try to find the smallest pool `p` such that the predicate `pred` returns
/// true. Given that our [generators](../generators/index.html) tend to
/// generate smaller outputs from smaller inputs, by minimizing the source
/// pool we can find the smallest value that provokes a failure.
///
/// If we do not find any smaller pool that satisifies `pred`; we return
/// `None`.
///
/// Currently, we have two heuristics for shrinking: removing slices, and
/// reducing individual values.
///
/// Removing slices tries to remove as much of the pool as it can whilst still
/// having the predicate hold. At present, we do this by removing each half
/// and testing, then each quarter, eighths, and so on. Eventually, we plan to
/// track the regions of data that each generator draws from, and use that to
/// optimise the shrinking process.
///
/// Reducing individual values basically goes through each position in the
/// pool, and then tries reducing it to zero, then half, thn three quarters,
/// seven eighths, and so on.
pub fn minimize<F: Fn(InfoReplay) -> bool>(p: &InfoPool, pred: &F) -> Option<InfoPool> {
    if let Some(res) = minimize_via_removal(p, pred) {
        return Some(res);
    }

    let mut candidate = p.clone();
    if let Some(res) = minimize_via_scalar_shrink(p, &mut candidate, pred) {
        return Some(res);
    }

    trace!("Nothing smaller found than {:?}", p);
    None
}

#[cfg(test)]
mod tests {
    extern crate env_logger;
    use super::*;

    #[test]
    fn should_take_each_item_in_pool() {
        let p = InfoPool::of_vec(vec![0, 1, 2, 3]);
        let mut t = p.replay();
        assert_eq!(t.next_byte(), 0);
        assert_eq!(t.next_byte(), 1);
        assert_eq!(t.next_byte(), 2);
        assert_eq!(t.next_byte(), 3);
        assert_eq!(t.next_byte(), 0);
    }

    #[test]
    fn should_generate_random_data() {
        let trials = 1024usize;
        let mut vals = 0;
        let mut p = InfoPool::new();
        let mut t = p.tap_with(::rand::XorShiftRng::new_unseeded());
        let error = 4;
        for _ in 0..trials {
            vals += t.next_byte() as usize;
        }
        let mean = vals / trials;
        let expected = 128;
        assert!(
            (expected - error) < mean && (expected + error) > mean,
            "Expected {} trials to be ({}+/-{}); got {}",
            trials,
            expected,
            error,
            mean
        )
    }

    #[test]
    fn should_allow_restarting_read() {
        let mut p = InfoPool::new();
        let mut v0 = Vec::new();
        {
            let mut t = p.tap();
            for _ in 0..4 {
                v0.push(t.next_byte())
            }
        }

        let mut t = p.replay();
        let mut v1 = Vec::new();
        for _ in 0..4 {
            v1.push(t.next_byte())
        }

        assert_eq!(v0, v1)
    }

    #[test]
    fn should_allow_borrowing_buffer() {
        let p = InfoPool::of_vec(vec![1]);
        assert_eq!(p.buffer(), &[1]);
    }

    #[test]
    fn tap_can_act_as_iterator() {
        let buf = vec![4, 3, 2, 1];
        let p = InfoPool::of_vec(buf.clone());
        let _: &Iterator<Item = u8> = &p.replay();

        assert_eq!(p.replay().take(4).collect::<Vec<_>>(), buf)
    }

    #[test]
    fn replay_can_act_as_iterator() {
        let buf = vec![4, 3, 2, 1];
        let p = InfoPool::of_vec(buf.clone());
        let _: &Iterator<Item = u8> = &p.replay();

        assert_eq!(p.replay().take(4).collect::<Vec<_>>(), buf)
    }

    #[test]
    fn minimiser_should_minimise_to_empty() {
        let p = InfoPool::of_vec(vec![1]);
        let min = minimize(&p, &|_| true);

        assert_eq!(min.as_ref().map(|p| p.buffer()), Some([].as_ref()))
    }

    #[test]
    fn minimiser_should_minimise_to_minimum_given_size() {
        env_logger::init().unwrap_or(());
        let p = InfoPool::of_vec(vec![1; 4]);
        let min = minimize(&p, &|t| t.take(16).filter(|&v| v > 0).count() > 1)
            .expect("some smaller pool");

        assert_eq!(min.buffer(), &[1, 1])
    }

    #[test]
    fn minimiser_should_minimise_scalar_values() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|v| v >= 3)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[3])
    }
    #[test]
    fn minimiser_should_minimise_scalar_values_to_empty() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|_| true)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[] as &[u8])
    }

    #[test]
    fn minimiser_should_minimise_scalar_values_by_search() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|v| v >= 13)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[13])
    }
    #[test]
    fn minimiser_should_minimise_scalar_values_accounting_for_overflow() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| t.take(16).any(|v| v >= 251)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[251])
    }
}

{'fraction_non_alphanumeric': 0.15357502517623364, 'fraction_numerical': 0.011161463578382007, 'mean_word_length': 2.0046646495209277, 'pattern_counts': {'":': 0, '<': 80, '<?xml version=': 0, '>': 120, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6218516', 'n_tokens_mistral': 7915, 'n_tokens_neox': 7648, 'n_words': 2126}