<commit_before>#![cfg(feature = "quickcheck")]

//! The purpose of these tests is to cover corner cases of iterators
//! and adaptors.
//!
//! In particular we test the tedious size_hint and exact size correctness.

#[macro_use] extern crate itertools;

extern crate quickcheck;

use std::default::Default;

use quickcheck as qc;
use std::ops::Range;
use itertools::Itertools;
use itertools::{
    Zip,
    Stride,
    EitherOrBoth,
};

/// Our base iterator that we can impl Arbitrary for
///
/// NOTE: Iter is tricky and is not fused, to help catch bugs.
/// At the end it will return None once, then return Some(0),
/// then return None again.
#[derive(Clone, Debug)]
struct Iter<T>(Range<T>, i32); // with fuse/done flag

impl<T> Iter<T>
{
    fn new(it: Range<T>) -> Self
    {
        Iter(it, 0)
    }
}

impl<T> Iterator for Iter<T> where Range<T>: Iterator,
    <Range<T> as Iterator>::Item: Default,
{
    type Item = <Range<T> as Iterator>::Item;

    fn next(&mut self) -> Option<Self::Item>
    {
        let elt = self.0.next();
        if elt.is_none() {
            self.1 += 1;
            // check fuse flag
            if self.1 == 2 {
                return Some(Default::default())
            }
        }
        elt
    }

    fn size_hint(&self) -> (usize, Option<usize>)
    {
        self.0.size_hint()
    }
}

impl<T> DoubleEndedIterator for Iter<T> where Range<T>: DoubleEndedIterator,
    <Range<T> as Iterator>::Item: Default,
{
    fn next_back(&mut self) -> Option<Self::Item> { self.0.next_back() }
}

impl<T> ExactSizeIterator for Iter<T> where Range<T>: ExactSizeIterator,
    <Range<T> as Iterator>::Item: Default,
{ }

impl<T> qc::Arbitrary for Iter<T> where T: qc::Arbitrary
{
    fn arbitrary<G: qc::Gen>(g: &mut G) -> Self
    {
        Iter::new(T::arbitrary(g)..T::arbitrary(g))
    }

    fn shrink(&self) -> Box<Iterator<Item=Iter<T>>>
    {
        let r = self.0.clone();
        Box::new(
            r.start.shrink().flat_map(move |x| {
                r.end.shrink().map(move |y| (x.clone(), y))
            })
            .map(|(a, b)| Iter::new(a..b))
        )
    }
}

fn correct_size_hint<I: Iterator>(mut it: I) -> bool {
    // record size hint at each iteration
    let initial_hint = it.size_hint();
    let mut hints = Vec::with_capacity(initial_hint.0 + 1);
    hints.push(initial_hint);
    while let Some(_) = it.next() {
        hints.push(it.size_hint())
    }

    let mut true_count = hints.len(); // start off +1 too much

    // check all the size hints
    for &(low, hi) in &hints {
        true_count -= 1;
        if low > true_count ||
            (hi.is_some() && hi.unwrap() < true_count)
        {
            println!("True size: {:?}, size hint: {:?}", true_count, (low, hi));
            //println!("All hints: {:?}", hints);
            return false
        }
    }
    true
}

fn exact_size<I: ExactSizeIterator>(mut it: I) -> bool {
    // check every iteration
    let (mut low, mut hi) = it.size_hint();
    if Some(low) != hi { return false; }
    while let Some(_) = it.next() {
        let (xlow, xhi) = it.size_hint();
        if low != xlow + 1 { return false; }
        low = xlow;
        hi = xhi;
        if Some(low) != hi { return false; }
    }
    let (low, hi) = it.size_hint();
    low == 0 && hi == Some(0)
}

/*
 * NOTE: Range<i8> is broken!
 * (all signed ranges are)
#[quickcheck]
fn size_range_i8(a: Iter<i8>) -> bool {
    exact_size(a)
}

#[quickcheck]
fn size_range_i16(a: Iter<i16>) -> bool {
    exact_size(a)
}

#[quickcheck]
fn size_range_u8(a: Iter<u8>) -> bool {
    exact_size(a)
}
 */

macro_rules! quickcheck {
    ($name:ident(1), $func:item) => {
        #[test]
        fn $name() {
            $func
            quickcheck::quickcheck(prop as fn(_) -> _);
        }
    };
    ($name:ident(2), $func:item) => {
        #[test]
        fn $name() {
            $func
            quickcheck::quickcheck(prop as fn(_, _) -> _);
        }
    };
    ($name:ident(3), $func:item) => {
        #[test]
        fn $name() {
            $func
            quickcheck::quickcheck(prop as fn(_, _, _) -> _);
        }
    };
    ($name:ident(4), $func:item) => {
        #[test]
        fn $name() {
            $func
            quickcheck::quickcheck(prop as fn(_, _, _, _) -> _);
        }
    };
}

quickcheck! {
    size_stride(2),
    fn prop(data: Vec<u8>, mut stride: isize) -> bool {
        if stride == 0 {
            stride += 1; // never zero
        }
        exact_size(Stride::from_slice(&data, stride))
    }
}

quickcheck! {
    equal_stride(2),
    fn prop(data: Vec<u8>, mut stride: i8) -> bool {
        if stride == 0 {
            // never zero
            stride += 1;
        }
        if stride > 0 {
            itertools::equal(Stride::from_slice(&data, stride as isize),
                             data.iter().step(stride as usize))
        } else {
            itertools::equal(Stride::from_slice(&data, stride as isize),
                             data.iter().rev().step(-stride as usize))
        }
    }
}

quickcheck! {
    size_product(2),
    fn prop(a: Iter<u16>, b: Iter<u16>) -> bool {
        correct_size_hint(a.cartesian_product(b))
    }
}

quickcheck! {
    size_product3(3),
    fn prop(a: Iter<u16>, b: Iter<u16>, c: Iter<u16>) -> bool {
        correct_size_hint(iproduct!(a, b, c))
    }
}

quickcheck! {
    size_step(2),
    fn prop(a: Iter<i16>, mut s: usize) -> bool {
        if s == 0 {
            s += 1; // never zero
        }
        let filt = a.clone().dedup();
        correct_size_hint(filt.step(s)) &&
            exact_size(a.step(s))
    }
}

quickcheck! {
    size_multipeek(2),
    fn prop(a: Iter<u16>, s: u8) -> bool {
        let mut it = a.multipeek();
        // peek a few times
        for _ in 0..s {
            it.peek();
        }
        exact_size(it)
    }
}

quickcheck! {
    equal_merge(2),
    fn prop(a: Vec<i16>, b: Vec<i16>) -> bool {
        let mut sa = a.clone();
        let mut sb = b.clone();
        sa.sort();
        sb.sort();
        let mut merged = sa.clone();
        merged.extend(sb.iter().cloned());
        merged.sort();
        itertools::equal(&merged, sa.iter().merge(&sb))
    }

}

quickcheck! {
    size_merge(2),
    fn prop(a: Iter<u16>, b: Iter<u16>) -> bool {
        correct_size_hint(a.merge(b))
    }
}

quickcheck! {
    size_zip(3),
    fn prop(a: Iter<i16>, b: Iter<i16>, c: Iter<i16>) -> bool {
        let filt = a.clone().dedup();
        correct_size_hint(Zip::new((filt, b.clone(), c.clone()))) &&
            exact_size(Zip::new((a, b, c)))
    }
}

quickcheck! {
    size_zip_rc(2),
    fn prop(a: Iter<i16>, b: Iter<i16>) -> bool {
        let rc = a.clone().into_rc();
        correct_size_hint(Zip::new((&rc, &rc, b)))
    }
}

quickcheck! {
    equal_kmerge(3),
    fn prop(a: Vec<i16>, b: Vec<i16>, c: Vec<i16>) -> bool {
        use itertools::free::kmerge;
        let mut sa = a.clone();
        let mut sb = b.clone();
        let mut sc = c.clone();
        sa.sort();
        sb.sort();
        sc.sort();
        let mut merged = sa.clone();
        merged.extend(sb.iter().cloned());
        merged.extend(sc.iter().cloned());
        merged.sort();
        itertools::equal(merged.into_iter(), kmerge(vec![sa, sb, sc]))
    }
}

quickcheck! {
    size_kmerge(3),
    fn prop(a: Vec<i16>, b: Vec<i16>, c: Vec<i16>) -> bool {
        use itertools::free::kmerge;
        correct_size_hint(kmerge(vec![a, b, c]))
    }
}

quickcheck! {
    size_zip_longest(2),
    fn prop(a: Iter<i16>, b: Iter<i16>) -> bool {
        let filt = a.clone().dedup();
        let filt2 = b.clone().dedup();
        correct_size_hint(filt.zip_longest(b.clone())) &&
        correct_size_hint(a.clone().zip_longest(filt2)) &&
            exact_size(a.zip_longest(b))<commit_msg>Use `Iter` for kmerge quickcheck<commit_after>#![cfg(feature = "quickcheck")]

//! The purpose of these tests is to cover corner cases of iterators
//! and adaptors.
//!
//! In particular we test the tedious size_hint and exact size correctness.

#[macro_use] extern crate itertools;

extern crate quickcheck;

use std::default::Default;

use quickcheck as qc;
use std::ops::Range;
use itertools::Itertools;
use itertools::{
    Zip,
    Stride,
    EitherOrBoth,
};

/// Our base iterator that we can impl Arbitrary for
///
/// NOTE: Iter is tricky and is not fused, to help catch bugs.
/// At the end it will return None once, then return Some(0),
/// then return None again.
#[derive(Clone, Debug)]
struct Iter<T>(Range<T>, i32); // with fuse/done flag

impl<T> Iter<T>
{
    fn new(it: Range<T>) -> Self
    {
        Iter(it, 0)
    }
}

impl<T> Iterator for Iter<T> where Range<T>: Iterator,
    <Range<T> as Iterator>::Item: Default,
{
    type Item = <Range<T> as Iterator>::Item;

    fn next(&mut self) -> Option<Self::Item>
    {
        let elt = self.0.next();
        if elt.is_none() {
            self.1 += 1;
            // check fuse flag
            if self.1 == 2 {
                return Some(Default::default())
            }
        }
        elt
    }

    fn size_hint(&self) -> (usize, Option<usize>)
    {
        self.0.size_hint()
    }
}

impl<T> DoubleEndedIterator for Iter<T> where Range<T>: DoubleEndedIterator,
    <Range<T> as Iterator>::Item: Default,
{
    fn next_back(&mut self) -> Option<Self::Item> { self.0.next_back() }
}

impl<T> ExactSizeIterator for Iter<T> where Range<T>: ExactSizeIterator,
    <Range<T> as Iterator>::Item: Default,
{ }

impl<T> qc::Arbitrary for Iter<T> where T: qc::Arbitrary
{
    fn arbitrary<G: qc::Gen>(g: &mut G) -> Self
    {
        Iter::new(T::arbitrary(g)..T::arbitrary(g))
    }

    fn shrink(&self) -> Box<Iterator<Item=Iter<T>>>
    {
        let r = self.0.clone();
        Box::new(
            r.start.shrink().flat_map(move |x| {
                r.end.shrink().map(move |y| (x.clone(), y))
            })
            .map(|(a, b)| Iter::new(a..b))
        )
    }
}

fn correct_size_hint<I: Iterator>(mut it: I) -> bool {
    // record size hint at each iteration
    let initial_hint = it.size_hint();
    let mut hints = Vec::with_capacity(initial_hint.0 + 1);
    hints.push(initial_hint);
    while let Some(_) = it.next() {
        hints.push(it.size_hint())
    }

    let mut true_count = hints.len(); // start off +1 too much

    // check all the size hints
    for &(low, hi) in &hints {
        true_count -= 1;
        if low > true_count ||
            (hi.is_some() && hi.unwrap() < true_count)
        {
            println!("True size: {:?}, size hint: {:?}", true_count, (low, hi));
            //println!("All hints: {:?}", hints);
            return false
        }
    }
    true
}

fn exact_size<I: ExactSizeIterator>(mut it: I) -> bool {
    // check every iteration
    let (mut low, mut hi) = it.size_hint();
    if Some(low) != hi { return false; }
    while let Some(_) = it.next() {
        let (xlow, xhi) = it.size_hint();
        if low != xlow + 1 { return false; }
        low = xlow;
        hi = xhi;
        if Some(low) != hi { return false; }
    }
    let (low, hi) = it.size_hint();
    low == 0 && hi == Some(0)
}

/*
 * NOTE: Range<i8> is broken!
 * (all signed ranges are)
#[quickcheck]
fn size_range_i8(a: Iter<i8>) -> bool {
    exact_size(a)
}

#[quickcheck]
fn size_range_i16(a: Iter<i16>) -> bool {
    exact_size(a)
}

#[quickcheck]
fn size_range_u8(a: Iter<u8>) -> bool {
    exact_size(a)
}
 */

macro_rules! quickcheck {
    ($name:ident(1), $func:item) => {
        #[test]
        fn $name() {
            $func
            quickcheck::quickcheck(prop as fn(_) -> _);
        }
    };
    ($name:ident(2), $func:item) => {
        #[test]
        fn $name() {
            $func
            quickcheck::quickcheck(prop as fn(_, _) -> _);
        }
    };
    ($name:ident(3), $func:item) => {
        #[test]
        fn $name() {
            $func
            quickcheck::quickcheck(prop as fn(_, _, _) -> _);
        }
    };
    ($name:ident(4), $func:item) => {
        #[test]
        fn $name() {
            $func
            quickcheck::quickcheck(prop as fn(_, _, _, _) -> _);
        }
    };
}

quickcheck! {
    size_stride(2),
    fn prop(data: Vec<u8>, mut stride: isize) -> bool {
        if stride == 0 {
            stride += 1; // never zero
        }
        exact_size(Stride::from_slice(&data, stride))
    }
}

quickcheck! {
    equal_stride(2),
    fn prop(data: Vec<u8>, mut stride: i8) -> bool {
        if stride == 0 {
            // never zero
            stride += 1;
        }
        if stride > 0 {
            itertools::equal(Stride::from_slice(&data, stride as isize),
                             data.iter().step(stride as usize))
        } else {
            itertools::equal(Stride::from_slice(&data, stride as isize),
                             data.iter().rev().step(-stride as usize))
        }
    }
}

quickcheck! {
    size_product(2),
    fn prop(a: Iter<u16>, b: Iter<u16>) -> bool {
        correct_size_hint(a.cartesian_product(b))
    }
}

quickcheck! {
    size_product3(3),
    fn prop(a: Iter<u16>, b: Iter<u16>, c: Iter<u16>) -> bool {
        correct_size_hint(iproduct!(a, b, c))
    }
}

quickcheck! {
    size_step(2),
    fn prop(a: Iter<i16>, mut s: usize) -> bool {
        if s == 0 {
            s += 1; // never zero
        }
        let filt = a.clone().dedup();
        correct_size_hint(filt.step(s)) &&
            exact_size(a.step(s))
    }
}

quickcheck! {
    size_multipeek(2),
    fn prop(a: Iter<u16>, s: u8) -> bool {
        let mut it = a.multipeek();
        // peek a few times
        for _ in 0..s {
            it.peek();
        }
        exact_size(it)
    }
}

quickcheck! {
    equal_merge(2),
    fn prop(a: Vec<i16>, b: Vec<i16>) -> bool {
        let mut sa = a.clone();
        let mut sb = b.clone();
        sa.sort();
        sb.sort();
        let mut merged = sa.clone();
        merged.extend(sb.iter().cloned());
        merged.sort();
        itertools::equal(&merged, sa.iter().merge(&sb))
    }

}

quickcheck! {
    size_merge(2),
    fn prop(a: Iter<u16>, b: Iter<u16>) -> bool {
        correct_size_hint(a.merge(b))
    }
}

quickcheck! {
    size_zip(3),
    fn prop(a: Iter<i16>, b: Iter<i16>, c: Iter<i16>) -> bool {
        let filt = a.clone().dedup();
        correct_size_hint(Zip::new((filt, b.clone(), c.clone()))) &&
            exact_size(Zip::new((a, b, c)))
    }
}

quickcheck! {
    size_zip_rc(2),
    fn prop(a: Iter<i16>, b: Iter<i16>) -> bool {
        let rc = a.clone().into_rc();
        correct_size_hint(Zip::new((&rc, &rc, b)))
    }
}

quickcheck! {
    equal_kmerge(3),
    fn prop(a: Vec<i16>, b: Vec<i16>, c: Vec<i16>) -> bool {
        use itertools::free::kmerge;
        let mut sa = a.clone();
        let mut sb = b.clone();
        let mut sc = c.clone();
        sa.sort();
        sb.sort();
        sc.sort();
        let mut merged = sa.clone();
        merged.extend(sb.iter().cloned());
        merged.extend(sc.iter().cloned());
        merged.sort();
        itertools::equal(merged.into_iter(), kmerge(vec![sa, sb, sc]))
    }
}

quickcheck! {
    size_kmerge(3),
    fn prop(a: Iter<i16>, b: Iter<i16>, c: Iter<i16>) -> bool {
        use itertools::free::kmerge;
        correct_size_hint(kmerge(vec![a, b, c]))
    }
}

quickcheck! {
    size_zip_longest(2),
    fn prop(a: Iter<i16>, b: Iter<i16>) -> bool {
        let filt = a.clone().dedup();
        let filt2 = b.clone().dedup();
        correct_size_hint(filt.zip_longest(b.clone())) &&
        correct_size_hint(a.clone().zip_longest(filt2)) &&
            exact_size(a.zip_longest(b))
{'fraction_non_alphanumeric': 0.17252396166134185, 'fraction_numerical': 0.013801916932907348, 'mean_word_length': 2.0586281024037523, 'pattern_counts': {'":': 0, '<': 131, '<?xml version=': 0, '>': 197, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3046286', 'n_tokens_mistral': 5695, 'n_tokens_neox': 5574, 'n_words': 1271}