<commit_before>#![deny(missing_docs)]

use varint::consume_single_varint;
use stream::*;
use std::io::Read;

/// A consumer for a byte stream
pub struct ByteConsumer<'a> {
    read: &'a mut Read,
    type_: StreamType,
}

impl<'a> ByteConsumer<'a> {
    /// Return a ByteConsumer from for single messages, varint or leb128-delimited
    pub fn new(read: &'a mut Read, type_: StreamType) -> ByteConsumer {
        ByteConsumer { read, type_ }
    }
}

impl<'a> Iterator for ByteConsumer<'a> {
    type Item = Vec<u8>;

    fn next(&mut self) -> Option<Vec<u8>> {
        match self.type_ {
            StreamType::Leb128 | StreamType::Varint => consume_single_varint(self.read),
            StreamType::Single => {
                let ret: Option<Vec<u8>>;
                let mut buf = Vec::new();
                match self.read.read_to_end(&mut buf) {
                    Ok(x) => {
                        if x > 0 {
                            ret = Some(buf);
                        } else {
                            ret = None
                        }
                    }
                    Err(_) => ret = None,
                }
                ret
            }
        }
    }
}<commit_msg>Make ByteConsumer own its reader<commit_after>#![deny(missing_docs)]

use varint::consume_single_varint;
use stream::*;
use std::io::Read;

/// A consumer for a byte stream
pub struct ByteConsumer<T: Read> {
    read: T,
    type_: StreamType,
}

impl <T: Read> ByteConsumer<T> {
    /// Return a ByteConsumer from for single messages, varint or leb128-delimited
    pub fn new(read: T, type_: StreamType) -> ByteConsumer<T> {
        ByteConsumer { read, type_ }
    }
}

impl <T: Read> Iterator for ByteConsumer<T> {
    type Item = Vec<u8>;

    fn next(&mut self) -> Option<Vec<u8>> {
        match self.type_ {
            StreamType::Leb128 | StreamType::Varint => consume_single_varint(&mut self.read),
            StreamType::Single => {
                let ret: Option<Vec<u8>>;
                let mut buf = Vec::new();
                match self.read.read_to_end(&mut buf) {
                    Ok(x) => {
                        if x > 0 {
                            ret = Some(buf);
                        } else {
                            ret = None
                        }
                    }
                    Err(_) => ret = None,
                }
                ret
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.11863711001642036, 'fraction_numerical': 0.008210180623973728, 'mean_word_length': 1.410484668644906, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4210743', 'n_tokens_mistral': 736, 'n_tokens_neox': 699, 'n_words': 178}