<commit_before>use std::cmp;
use std::io::Read;
use std::marker::PhantomData;

use byteorder::{ReadBytesExt, ByteOrder};
use serde_crate as serde;
use serde_crate::de::value::ValueDeserializer;
use serde_crate::de::Error as DeError;
use ::SizeLimit;
use super::{Result, Error, ErrorKind};

const BLOCK_SIZE: usize = 65536;

/// A Deserializer that reads bytes from a buffer.
///
/// This struct should rarely be used.
/// In most cases, prefer the `decode_from` function.
///
/// ```rust,ignore
/// let d = Deserializer::new(&mut some_reader, SizeLimit::new());
/// serde::Deserialize::deserialize(&mut deserializer);
/// let bytes_read = d.bytes_read();
/// ```
pub struct Deserializer<R, E: ByteOrder> {
    reader: R,
    size_limit: SizeLimit,
    read: u64,
    _phantom: PhantomData<E>,
}

impl<R: Read, E: ByteOrder> Deserializer<R, E> {
    pub fn new(r: R, size_limit: SizeLimit) -> Deserializer<R, E> {
        Deserializer {
            reader: r,
            size_limit: size_limit,
            read: 0,
            _phantom: PhantomData
        }
    }

    /// Returns the number of bytes read from the contained Reader.
    pub fn bytes_read(&self) -> u64 {
        self.read
    }

    fn read_bytes(&mut self, count: u64) -> Result<()> {
        self.read += count;
        match self.size_limit {
            SizeLimit::Infinite => Ok(()),
            SizeLimit::Bounded(x) if self.read <= x => Ok(()),
            SizeLimit::Bounded(_) => Err(ErrorKind::SizeLimit.into())
        }
    }

    fn read_type<T>(&mut self) -> Result<()> {
        use std::mem::size_of;
        self.read_bytes(size_of::<T>() as u64)
    }

    fn read_vec(&mut self) -> Result<Vec<u8>> {
        let len = try!(serde::Deserialize::deserialize(&mut *self));
        try!(self.read_bytes(len));

        let mut result = Vec::new();
        let mut len = len as usize;
        let mut off = 0;
        while len > 0 {
            let reserve = cmp::min(len, BLOCK_SIZE);
            unsafe {
                result.reserve(reserve);
                result.set_len(off + reserve);
            }
            try!(self.reader.read_exact(&mut result[off..]));
            len -= reserve;
            off += reserve;
        }<commit_msg>call read_bytes iteratively inside of read_vec (#138)<commit_after>use std::cmp;
use std::io::Read;
use std::marker::PhantomData;

use byteorder::{ReadBytesExt, ByteOrder};
use serde_crate as serde;
use serde_crate::de::value::ValueDeserializer;
use serde_crate::de::Error as DeError;
use ::SizeLimit;
use super::{Result, Error, ErrorKind};

const BLOCK_SIZE: usize = 65536;

/// A Deserializer that reads bytes from a buffer.
///
/// This struct should rarely be used.
/// In most cases, prefer the `decode_from` function.
///
/// ```rust,ignore
/// let d = Deserializer::new(&mut some_reader, SizeLimit::new());
/// serde::Deserialize::deserialize(&mut deserializer);
/// let bytes_read = d.bytes_read();
/// ```
pub struct Deserializer<R, E: ByteOrder> {
    reader: R,
    size_limit: SizeLimit,
    read: u64,
    _phantom: PhantomData<E>,
}

impl<R: Read, E: ByteOrder> Deserializer<R, E> {
    pub fn new(r: R, size_limit: SizeLimit) -> Deserializer<R, E> {
        Deserializer {
            reader: r,
            size_limit: size_limit,
            read: 0,
            _phantom: PhantomData
        }
    }

    /// Returns the number of bytes read from the contained Reader.
    pub fn bytes_read(&self) -> u64 {
        self.read
    }

    fn read_bytes(&mut self, count: u64) -> Result<()> {
        self.read += count;
        match self.size_limit {
            SizeLimit::Infinite => Ok(()),
            SizeLimit::Bounded(x) if self.read <= x => Ok(()),
            SizeLimit::Bounded(_) => Err(ErrorKind::SizeLimit.into())
        }
    }

    fn read_type<T>(&mut self) -> Result<()> {
        use std::mem::size_of;
        self.read_bytes(size_of::<T>() as u64)
    }

    fn read_vec(&mut self) -> Result<Vec<u8>> {
        let mut len: usize = try!(serde::Deserialize::deserialize(&mut *self));

        let mut result = Vec::new();
        let mut off = 0;
        while len > 0 {
            let reserve = cmp::min(len, BLOCK_SIZE);
            try!(self.read_bytes(reserve as u64));
            unsafe {
                result.reserve(reserve);
                result.set_len(off + reserve);
            }
            try!(self.reader.read_exact(&mut result[off..]));
            len -= reserve;
            off += reserve;
        }
{'fraction_non_alphanumeric': 0.15048975957257346, 'fraction_numerical': 0.008682101513802316, 'mean_word_length': 2.3883861236802413, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7419312', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1430, 'n_words': 364}