<commit_before>use alloc::arc::Arc;
use collections::vec_deque::VecDeque;
use io::{AsyncRead,Promise};
use kobj::KObjRef;
use prelude::*;
use spin::Mutex;
use syscall::Result;

struct FlatMapShared {
    queue: Mutex<VecDeque<u8>>,
    buf_len: usize,
    collect: Box<Fn(Vec<u8>) -> Vec<u8>>,
    finished: Box<Fn(&mut Vec<u8>) -> Option<Vec<u8>>>
}

/// An adaptor which will modify the data produced by a reader.
pub struct FlatMap {
    shared: Arc<FlatMapShared>,
    input: KObjRef<AsyncRead>
}

fn read_async_inner(shared: Arc<FlatMapShared>, input: KObjRef<AsyncRead>, mut buf: Vec<u8>, len: usize) -> Promise<Result<Vec<u8>>> {
    {
        let mut queue = lock!(shared.queue);
        while buf.len() < len {
            if let Some(b) = queue.pop_front() {
                buf.push(b);
            } else {
                break;
            }
        }

        if let Some(remainder) = (shared.finished)(&mut buf) {
            queue.extend(remainder);
            return Promise::resolved(Ok(buf));
        }
        else if buf.len() >= len {
            return Promise::resolved(Ok(buf));
        }
    }

    input
        .read_async(vec![0; shared.buf_len])
        .then(move |result| {
            if let Ok(data) = result {
                let output = (shared.collect)(data);
                lock!(shared.queue).extend(output);
                read_async_inner(shared, input, buf, len)
            }
            else {
                Promise::resolved(result)
            }
        })
    .unwrap()<commit_msg>Return to the caller when any bytes are available<commit_after>use alloc::arc::Arc;
use collections::vec_deque::VecDeque;
use io::{AsyncRead,Promise};
use kobj::KObjRef;
use prelude::*;
use spin::Mutex;
use syscall::Result;

struct FlatMapShared {
    queue: Mutex<VecDeque<u8>>,
    buf_len: usize,
    collect: Box<Fn(Vec<u8>) -> Vec<u8>>,
    finished: Box<Fn(&mut Vec<u8>) -> Option<Vec<u8>>>
}

/// An adaptor which will modify the data produced by a reader.
pub struct FlatMap {
    shared: Arc<FlatMapShared>,
    input: KObjRef<AsyncRead>
}

fn read_async_inner(shared: Arc<FlatMapShared>, input: KObjRef<AsyncRead>, mut buf: Vec<u8>, len: usize) -> Promise<Result<Vec<u8>>> {
    {
        let mut queue = lock!(shared.queue);
        while buf.len() < len {
            if let Some(b) = queue.pop_front() {
                buf.push(b);
            } else {
                break;
            }
        }

        if let Some(remainder) = (shared.finished)(&mut buf) {
            queue.extend(remainder);
            return Promise::resolved(Ok(buf));
        }
        else if buf.len() > 0 || len == 0 {
            return Promise::resolved(Ok(buf));
        }
    }

    input
        .read_async(vec![0; shared.buf_len])
        .then(move |result| {
            if let Ok(data) = result {
                let output = (shared.collect)(data);
                lock!(shared.queue).extend(output);
                read_async_inner(shared, input, buf, len)
            }
            else {
                Promise::resolved(result)
            }
        })
    .unwrap()
{'fraction_non_alphanumeric': 0.14157303370786517, 'fraction_numerical': 0.005778491171749599, 'mean_word_length': 2.082096933728981, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7189079', 'n_tokens_mistral': 1001, 'n_tokens_neox': 980, 'n_words': 217}