<commit_before>use std::io;
use std::result;
use tracker::errors::{Result, ErrorKind};

pub(super) struct Reader {
    data: Vec<u8>,
    idx: usize,
    state: ReadState,
}

enum ReadState {
    ParsingHeaders,
    ParsingResponse,
}

enum ReadRes {
    Done,
    Again,
    Empty,
}

impl ReadState {
    fn ready(&self) -> bool {
        match *self {
            ReadState::ParsingHeaders => false,
            ReadState::ParsingResponse => true,
        }
    }
}

impl Reader {
    pub fn new() -> Reader {
        Reader {
            data: Vec::with_capacity(75),
            idx: 0,
            state: ReadState::ParsingHeaders,
        }
    }

    pub fn readable<R: io::Read>(&mut self, conn: &mut R) -> Result<bool> {
        while let ReadRes::Again = self.read(conn)? { }
        Ok(self.state.ready())
    }

    fn read<R: io::Read>(&mut self, conn: &mut R) -> Result<ReadRes> {
        match conn.read(&mut self.data[self.idx..]) {
            Ok(0) if self.state.ready() => Ok(ReadRes::Done),
            Ok(0) => Err(ErrorKind::EOF.into()),
            Ok(v) => {
                if self.state.ready() {
                    self.idx += v;
                    if self.idx == self.data.len() {
                        self.data.resize(self.idx + 30, 0);
                    }
                } else {
                    for i in 0..v {
                        if self.data[i..i+4] == b"\r\n\r\n"[..] {
                            self.data = self.data.split_off(i+4);
                            self.state = ReadState::ParsingResponse;
                            break;
                        }
                    }
                }
                Ok(ReadRes::Again)
            }
            Err(e) => {
                if e.kind() == io::ErrorKind::WouldBlock {
                    Ok(ReadRes::Empty)
                } else {
                    return Err(ErrorKind::IO.into());
                }
            }
        }
    }

    pub fn consume(self) -> Vec<u8> {
        self.data
    }
}<commit_msg>Add tests to http parser, fix bugs<commit_after>use std::io;
use tracker::errors::{Result, ErrorKind};

pub(super) struct Reader {
    data: Vec<u8>,
    idx: usize,
    state: ReadState,
}

enum ReadState {
    ParsingHeaders,
    ParsingResponse,
}

enum ReadRes {
    Done,
    Again,
    Empty,
}

impl ReadState {
    fn ready(&self) -> bool {
        match *self {
            ReadState::ParsingHeaders => false,
            ReadState::ParsingResponse => true,
        }
    }
}

impl Reader {
    pub fn new() -> Reader {
        Reader {
            data: vec![0; 50],
            idx: 0,
            state: ReadState::ParsingHeaders,
        }
    }

    pub fn readable<R: io::Read>(&mut self, conn: &mut R) -> Result<bool> {
        while let ReadRes::Again = self.read(conn)? { }
        Ok(self.state.ready())
    }

    fn read<R: io::Read>(&mut self, conn: &mut R) -> Result<ReadRes> {
        match conn.read(&mut self.data[self.idx..]) {
            Ok(0) if self.state.ready() => Ok(ReadRes::Done),
            Ok(0) => Err(ErrorKind::EOF.into()),
            Ok(v) => {
                if self.state.ready() {
                    self.idx += v;
                } else {
                    for i in 0..(v-3) {
                        if self.data[i..i+4] == b"\r\n\r\n"[..] {
                            self.data = self.data.split_off(i+4);
                            self.idx = v - (i + 4);
                            self.state = ReadState::ParsingResponse;
                            break;
                        }
                    }
                }
                if self.idx == self.data.len() {
                    self.data.resize(self.idx + 30, 0);
                }
                Ok(ReadRes::Again)
            }
            Err(e) => {
                if e.kind() == io::ErrorKind::WouldBlock {
                    Ok(ReadRes::Empty)
                } else {
                    return Err(ErrorKind::IO.into());
                }
            }
        }
    }

    pub fn consume(mut self) -> Vec<u8> {
        self.data.truncate(self.idx);
        self.data
    }
}

#[cfg(test)]
mod tests {
    use super::Reader;
    use std::io::Cursor;

    #[test]
    fn test_empty_resp() {
        let mut r = Reader::new();
        let data = "SomeHeader: Foo\r\nConnection: Close\r\n\r\n";
        let mut c = Cursor::new(data);
        assert_eq!(r.readable(&mut c).unwrap(), true);
        assert_eq!(r.consume(), Vec::<u8>::new());
    }

    #[test]
    fn test_premature_resp() {
        let mut r = Reader::new();
        let data = "SomeHeader: Foo\r\nConnection: C";
        let mut c = Cursor::new(data);
        assert_eq!(r.readable(&mut c).is_err(), true);
    }

    #[test]
    fn test_valid_resp() {
        let mut r = Reader::new();
        let data = "SomeHeader: Foo\r\nConnection: Close\r\n\r\nhello world spam";
        let mut c = Cursor::new(data);
        assert_eq!(r.readable(&mut c).unwrap(), true);
        assert_eq!(r.consume(), b"hello world spam");
    }
}
{'fraction_non_alphanumeric': 0.14833860759493672, 'fraction_numerical': 0.005933544303797469, 'mean_word_length': 1.352093023255814, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1331061', 'n_tokens_mistral': 1628, 'n_tokens_neox': 1545, 'n_words': 319}