<commit_before>use std::io::prelude::*;
use std::io;
use rustyline::Editor;
use rustyline::error::ReadlineError as RLError;
use combine::{StreamOnce};
use error::*;

#[derive(Debug)]
pub struct Buffer {
    buffer: String,
    prompt: String,
    rl: Editor<()>,
}

impl Buffer {
    fn new<S>(prompt: S) -> Self
    where
        S: Into<String>
    {
        Buffer {
            buffer: String::with_capacity(128),
            prompt: prompt.into(),
            rl: Editor::new(),
        }
    }
}

impl Read for Buffer {
    fn read(&mut self, mut buf: &mut [u8]) -> io::Result<usize> {
        if self.buffer.is_empty() {
            match self.rl.readline(&self.prompt) {
                Ok(line) => {
                    self.rl.add_history_entry(&line);
                    self.buffer.push_str(&line);
                },
                Err(RLError::Io(err)) => return Err(err),
                Err(RLError::Interrupted) => return Err(io::ErrorKind::Interrupted.into()),
                Err(RLError::Eof) => return Ok(0),
                Err(RLError::Errno(num)) => return Err(io::Error::from_raw_os_error(num)),
                Err(err) => return Err(io::ErrorKind::Other.into()),
            }
        }

        // if buf > buffer, drain entire buffer
        if buf.len() >= self.buffer.len() {
            let result = buf.write(self.buffer.as_bytes());
            self.buffer.clear();
            result
        } else {
            let bytes = self.buffer.drain(0..buf.len()).collect::<String>();
            buf.write(bytes.into_bytes().as_slice())
        }
    }
}

impl BufRead for Buffer {
    fn fill_buf(&mut self) -> io::Result<&[u8]> {

    }

    fn consume(&mut self, amt: usize) {

    }
}<commit_msg>Separate buffer into readline, buffer (generic over Read)<commit_after>use std::io;
use std::io::prelude::*;
use rustyline;
use rustyline::error::ReadlineError as RLError;
use combine;
use error::*;

pub struct Readline {
    rl: rustyline::Editor<()>,
    prompt: String,
    buffer: Vec<u8>,
}

impl Readline {
    fn new<S>(prompt: &str) -> Self {
        Readline {
            rl: rustyline::Editor::new(),
            prompt: prompt.to_owned(),
            buffer: Vec::with_capacity(128),
        }
    }
}

impl Read for Readline {
    fn read(&mut self, mut buf: &mut [u8]) -> io::Result<usize> {
        if self.buffer.is_empty() {
            match self.rl.readline(&self.prompt) {
                Ok(line) => {
                    self.rl.add_history_entry(&line);
                    self.buffer.write(line.as_bytes());
                },
                Err(RLError::Io(err)) => return Err(err),
                Err(RLError::Interrupted) => return Err(io::ErrorKind::Interrupted.into()),
                Err(RLError::Eof) => return Ok(0),
                Err(RLError::Errno(num)) => return Err(io::Error::from_raw_os_error(num)),
                Err(err) => return Err(io::ErrorKind::Other.into()),
            }
        }

        // if buf > buffer, drain entire buffer
        if buf.len() >= self.buffer.len() {
            let result = buf.write(&self.buffer);
            self.buffer.clear();
            result
        } else {
            let drained = self.buffer.drain(0..buf.len()).collect::<Vec<_>>();
            buf.write(&drained)
        }
    }
}

impl io::BufRead for Readline {
    fn fill_buf(&mut self) -> io::Result<&[u8]> {
        unimplemented!()
    }

    fn consume(&mut self, amt: usize) {
        unimplemented!()
    }
}

#[derive(Clone, Debug)]
pub struct StringStream<R: io::Read> {
    src: R,
    buffer: String,
}

impl<R: io::Read> StringStream<R> {
    fn new(src: R) -> Self {
        StringStream {
            src: src,
            buffer: String::with_capacity(128),
        }
    }
}

impl<R: io::Read> combine::StreamOnce for StringStream<R> {
    type Item = char;
    type Range = str;
    type Position usize;
    fn uncons(&mut self) -> Result<Self::Item, Error<Self::Item, Self::Range>>;
    fn position(&self) -> Self::Position;
}
{'fraction_non_alphanumeric': 0.1604968944099379, 'fraction_numerical': 0.0044720496894409935, 'mean_word_length': 2.0639269406392695, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '53327', 'n_tokens_mistral': 1318, 'n_tokens_neox': 1268, 'n_words': 262}