<commit_before>//
//   Copyright 2016 <NAME>
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! # Symbol reader
//!
//! A symbol reader reads one symbol at a time from a source
//!

use std::slice::Iter;

///
/// A symbol reader reads one symbol at a time from a source
///
pub trait SymbolReader<Symbol> {
    ///
    /// Reads the next symbol from this reader
    ///
    fn next_symbol(&mut self) -> Option<Symbol>;
}

///
/// Trait that can be implemented by things that can construct a symbol reader
///
pub trait SymbolSource<'a, Symbol> {
    /// The type of symbol reader that will be generated by this call
    type SymbolReader: SymbolReader<Symbol>+'a;

    /// Returns a new object that can read the symbols from this one
    fn read_symbols(self) -> Self::SymbolReader;
}

impl<'a, Symbol: Clone+'a> SymbolSource<'a, Symbol> for &'a Vec<Symbol> {
    type SymbolReader = Iter<'a, Symbol>;

    fn read_symbols(self) -> Self::SymbolReader {
        self.iter()
    }
}

impl<'a, Symbol: Clone+'a> SymbolReader<Symbol> for Iter<'a, Symbol> {
    fn next_symbol(&mut self) -> Option<Symbol> {
        if let Some(sym) = self.next() {
            Some(sym.clone())
        } else {
            None
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn can_read_from_vec() {
        let source      = vec![1, 2, 3];
        let mut reader  = source.read_symbols();

        assert!(reader.next_symbol() == Some(1));
        assert!(reader.next_symbol() == Some(2));
        assert!(reader.next_symbol() == Some(3));
        assert!(reader.next_symbol() == Nonecar);
    }
}<commit_msg>Some work on making symbol readers to work with streams<commit_after>//
//   Copyright 2016 <NAME>
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! # Symbol reader
//!
//! A symbol reader reads one symbol at a time from a source
//!

use std::slice::Iter;
use std::io::*;

///
/// A symbol reader reads one symbol at a time from a source
///
pub trait SymbolReader<Symbol> {
    ///
    /// Reads the next symbol from this reader
    ///
    fn next_symbol(&mut self) -> Option<Symbol>;
}

///
/// Trait that can be implemented by things that can construct a symbol reader
///
pub trait SymbolSource<'a, Symbol> {
    /// The type of symbol reader that will be generated by this call
    type SymbolReader: SymbolReader<Symbol>+'a;

    /// Returns a new object that can read the symbols from this one
    fn read_symbols(self) -> Self::SymbolReader;
}

impl<'a, Symbol: Clone+'a> SymbolSource<'a, Symbol> for &'a Vec<Symbol> {
    type SymbolReader = Iter<'a, Symbol>;

    fn read_symbols(self) -> Self::SymbolReader {
        self.iter()
    }
}

impl<'a, Symbol: Clone+'a> SymbolReader<Symbol> for Iter<'a, Symbol> {
    fn next_symbol(&mut self) -> Option<Symbol> {
        if let Some(sym) = self.next() {
            Some(sym.clone())
        } else {
            None
        }
    }
}

// Can read from streams

/*
impl<'a> SymbolSource<'a, u8> for Read {
    type SymbolReader = ByteReader<Self>;

    fn read_symbols(self) -> Self::SymbolReader {
        ByteReader::new(self.bytes())
    }
}
*/

pub struct ByteReader<Reader: Read> {
    bytes: Bytes<Reader>
}

impl<Reader: Read> ByteReader<Reader> {
    pub fn new(bytes: Bytes<Reader>) -> ByteReader<Reader> {
        ByteReader { bytes: bytes }
    }
}

impl<Reader: Read> SymbolReader<u8> for ByteReader<Reader> {
    fn next_symbol(&mut self) -> Option<u8> {
        if let Some(Ok(sym)) = self.bytes.next() {
            Some(sym.clone())
        } else {
            None
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn can_read_from_vec() {
        let source      = vec![1, 2, 3];
        let mut reader  = source.read_symbols();

        assert!(reader.next_symbol() == Some(1));
        assert!(reader.next_symbol() == Some(2));
        assert!(reader.next_symbol() == Some(3));
        assert!(reader.next_symbol() == Nonecar);
    }
}
{'fraction_non_alphanumeric': 0.135515873015873, 'fraction_numerical': 0.006150793650793651, 'mean_word_length': 2.585348506401138, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2445272', 'n_tokens_mistral': 1518, 'n_tokens_neox': 1513, 'n_words': 520}