<commit_before>use crate::ProgressBar;
use rayon::iter::{
    plumbing::Consumer, plumbing::Folder, plumbing::UnindexedConsumer, IndexedParallelIterator,
    ParallelIterator,
};
use std::convert::TryInto;
use std::sync::{Arc, Mutex};

pub struct ParProgressBarIter<T> {
    it: T,
    progress: Arc<Mutex<ProgressBar>>,
}

/// Wraps a Rayon parallel iterator.
///
/// See [`ProgressIterator`](trait.ProgressIterator.html) for method
/// documentation.
pub trait ParallelProgressIterator
where
    Self: Sized,
{
    /// Wrap an iterator with a custom progress bar.
    fn progress_with(self, progress: ProgressBar) -> ParProgressBarIter<Self>;

    /// Wrap an iterator with an explicit element count.
    fn progress_count(self, len: u64) -> ParProgressBarIter<Self> {
        self.progress_with(ProgressBar::new(len))
    }
}

pub trait IndexedParallelProgressIterator
where
    Self: Sized,
{
    fn progress_with(self, progress: ProgressBar) -> ParProgressBarIter<Self>;

    fn progress_count(self, len: u64) -> ParProgressBarIter<Self> {
        self.progress_with(ProgressBar::new(len))
    }

    fn progress(self) -> ParProgressBarIter<Self> {
        self.progress_count(0)
    }
}

impl<S: Send, T: ParallelIterator<Item = S>> ParallelProgressIterator for T {
    fn progress_with(self, progress: ProgressBar) -> ParProgressBarIter<Self> {
        ParProgressBarIter {
            it: self,
            progress: Arc::new(Mutex::new(progress)),
        }
    }
}

impl<S: Send, T: IndexedParallelIterator<Item = S>> IndexedParallelProgressIterator for T {
    fn progress_with(self, progress: ProgressBar) -> ParProgressBarIter<Self> {
        ParProgressBarIter {
            it: self,
            progress: Arc::new(Mutex::new(progress)),
        }
    }

    fn progress(self) -> ParProgressBarIter<Self> {
        let len = self.len().try_into().unwrap();
        IndexedParallelProgressIterator::progress_count(self, len)
    }
}

struct ProgressConsumer<C> {
    base: C,
    progress: Arc<Mutex<ProgressBar>>,
}

impl<C> ProgressConsumer<C> {
    fn new(base: C, progress: Arc<Mutex<ProgressBar>>) -> Self {
        ProgressConsumer { base, progress }
    }
}

impl<T, C: Consumer<T>> Consumer<T> for ProgressConsumer<C> {
    type Folder = ProgressFolder<C::Folder>;
    type Reducer = C::Reducer;
    type Result = C::Result;

    fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) {
        let (left, right, reducer) = self.base.split_at(index);
        (
            ProgressConsumer::new(left, self.progress.clone()),
            ProgressConsumer::new(right, self.progress.clone()),
            reducer,
        )
    }

    fn into_folder(self) -> Self::Folder {
        ProgressFolder {
            base: self.base.into_folder(),
            progress: self.progress.clone(),
        }
    }

    fn full(&self) -> bool {
        self.base.full()
    }
}

impl<T, C: UnindexedConsumer<T>> UnindexedConsumer<T> for ProgressConsumer<C> {
    fn split_off_left(&self) -> Self {
        ProgressConsumer::new(self.base.split_off_left(), self.progress.clone())
    }

    fn to_reducer(&self) -> Self::Reducer {
        self.base.to_reducer()
    }
}

struct ProgressFolder<C> {
    base: C,
    progress: Arc<Mutex<ProgressBar>>,
}

impl<T, C: Folder<T>> Folder<T> for ProgressFolder<C> {
    type Result = C::Result;

    fn consume(self, item: T) -> Self {
        self.progress.lock().unwrap().inc(1);
        ProgressFolder {
            base: self.base.consume(item),
            progress: self.progress,
        }
    }

    fn complete(self) -> C::Result {
        self.base.complete()
    }

    fn full(&self) -> bool {
        self.base.full()
    }
}

impl<S: Send, T: ParallelIterator<Item = S>> ParallelIterator for ParProgressBarIter<T> {
    type Item = S;

    fn drive_unindexed<C: UnindexedConsumer<Self::Item>>(self, consumer: C) -> C::Result {
        let consumer1 = ProgressConsumer::new(consumer, self.progress.clone());
        self.it.drive_unindexed(consumer1)<commit_msg>Remove useless Mutex from ParProgressBarIter<commit_after>use crate::ProgressBar;
use rayon::iter::{
    plumbing::Consumer, plumbing::Folder, plumbing::UnindexedConsumer, IndexedParallelIterator,
    ParallelIterator,
};
use std::convert::TryInto;

pub struct ParProgressBarIter<T> {
    it: T,
    progress: ProgressBar,
}

/// Wraps a Rayon parallel iterator.
///
/// See [`ProgressIterator`](trait.ProgressIterator.html) for method
/// documentation.
pub trait ParallelProgressIterator
where
    Self: Sized,
{
    /// Wrap an iterator with a custom progress bar.
    fn progress_with(self, progress: ProgressBar) -> ParProgressBarIter<Self>;

    /// Wrap an iterator with an explicit element count.
    fn progress_count(self, len: u64) -> ParProgressBarIter<Self> {
        self.progress_with(ProgressBar::new(len))
    }
}

pub trait IndexedParallelProgressIterator
where
    Self: Sized,
{
    fn progress_with(self, progress: ProgressBar) -> ParProgressBarIter<Self>;

    fn progress_count(self, len: u64) -> ParProgressBarIter<Self> {
        self.progress_with(ProgressBar::new(len))
    }

    fn progress(self) -> ParProgressBarIter<Self> {
        self.progress_count(0)
    }
}

impl<S: Send, T: ParallelIterator<Item = S>> ParallelProgressIterator for T {
    fn progress_with(self, progress: ProgressBar) -> ParProgressBarIter<Self> {
        ParProgressBarIter { it: self, progress }
    }
}

impl<S: Send, T: IndexedParallelIterator<Item = S>> IndexedParallelProgressIterator for T {
    fn progress_with(self, progress: ProgressBar) -> ParProgressBarIter<Self> {
        ParProgressBarIter { it: self, progress }
    }

    fn progress(self) -> ParProgressBarIter<Self> {
        let len = self.len().try_into().unwrap();
        IndexedParallelProgressIterator::progress_count(self, len)
    }
}

struct ProgressConsumer<C> {
    base: C,
    progress: ProgressBar,
}

impl<C> ProgressConsumer<C> {
    fn new(base: C, progress: ProgressBar) -> Self {
        ProgressConsumer { base, progress }
    }
}

impl<T, C: Consumer<T>> Consumer<T> for ProgressConsumer<C> {
    type Folder = ProgressFolder<C::Folder>;
    type Reducer = C::Reducer;
    type Result = C::Result;

    fn split_at(self, index: usize) -> (Self, Self, Self::Reducer) {
        let (left, right, reducer) = self.base.split_at(index);
        (
            ProgressConsumer::new(left, self.progress.clone()),
            ProgressConsumer::new(right, self.progress),
            reducer,
        )
    }

    fn into_folder(self) -> Self::Folder {
        ProgressFolder {
            base: self.base.into_folder(),
            progress: self.progress,
        }
    }

    fn full(&self) -> bool {
        self.base.full()
    }
}

impl<T, C: UnindexedConsumer<T>> UnindexedConsumer<T> for ProgressConsumer<C> {
    fn split_off_left(&self) -> Self {
        ProgressConsumer::new(self.base.split_off_left(), self.progress.clone())
    }

    fn to_reducer(&self) -> Self::Reducer {
        self.base.to_reducer()
    }
}

struct ProgressFolder<C> {
    base: C,
    progress: ProgressBar,
}

impl<T, C: Folder<T>> Folder<T> for ProgressFolder<C> {
    type Result = C::Result;

    fn consume(self, item: T) -> Self {
        self.progress.inc(1);
        ProgressFolder {
            base: self.base.consume(item),
            progress: self.progress,
        }
    }

    fn complete(self) -> C::Result {
        self.base.complete()
    }

    fn full(&self) -> bool {
        self.base.full()
    }
}

impl<S: Send, T: ParallelIterator<Item = S>> ParallelIterator for ParProgressBarIter<T> {
    type Item = S;

    fn drive_unindexed<C: UnindexedConsumer<Self::Item>>(self, consumer: C) -> C::Result {
        let consumer1 = ProgressConsumer::new(consumer, self.progress.clone());
        self.it.drive_unindexed(consumer1)
{'fraction_non_alphanumeric': 0.1334012219959267, 'fraction_numerical': 0.002036659877800407, 'mean_word_length': 3.0709844559585493, 'pattern_counts': {'":': 0, '<': 81, '<?xml version=': 0, '>': 117, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1818045', 'n_tokens_mistral': 2478, 'n_tokens_neox': 2454, 'n_words': 520}