<commit_before>// Copyright 2014-2016 <NAME>.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! FastQ reading and writing.
//!
//! # Example
//!
//! ```
//! use std::io;
//! use bio::io::fastq;
//! let reader = fastq::Reader::new(io::stdin());
//! ```

use std::io;
use std::io::prelude::*;
use std::ascii::AsciiExt;
use std::fs;
use std::fmt;
use std::path::Path;
use std::convert::AsRef;

use utils::TextSlice;

/// A FastQ reader.
pub struct Reader<R: io::Read> {
    reader: io::BufReader<R>,
    sep_line: String,
}


impl Reader<fs::File> {
    /// Read from a given file.
    pub fn from_file<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        fs::File::open(path).map(Reader::new)
    }
}


impl<R: io::Read> Reader<R> {
    /// Read from a given `io::Read`.
    pub fn new(reader: R) -> Self {
        Reader {
            reader: io::BufReader::new(reader),
            sep_line: String::new(),
        }
    }

    /// Read into a given record.
    /// Returns an error if the record in incomplete or syntax is violated.
    /// The content of the record can be checked via the record object.
    pub fn read(&mut self, record: &mut Record) -> io::Result<()> {
        record.clear();
        try!(self.reader.read_line(&mut record.header));

        if !record.header.is_empty() {
            if !record.header.starts_with('@') {
                return Err(io::Error::new(io::ErrorKind::Other, "Expected @ at record start."));
            }
            try!(self.reader.read_line(&mut record.seq));
            try!(self.reader.read_line(&mut self.sep_line));
            try!(self.reader.read_line(&mut record.qual));
            if record.qual.is_empty() {
                return Err(io::Error::new(io::ErrorKind::Other,
                                          "Incomplete record. Each FastQ record has to consist \
                                           of 4 lines: header, sequence, separator and \
                                           qualities."));
            }
        }

        Ok(())
    }

    /// Return an iterator over the records of this FastQ file.
    pub fn records(self) -> Records<R> {
        Records { reader: self }
    }
}


/// A FastQ record.
#[derive(Debug, Clone, Default)]
pub struct Record {
    header: String,
    seq: String,
    qual: String,
}


impl Record {
    /// Create a new, empty FastQ record.
    pub fn new() -> Self {
        Record {
            header: String::new(),
            seq: String::new(),
            qual: String::new(),
        }
    }

    /// Check if record is empty.
    pub fn is_empty(&self) -> bool {
        self.header.is_empty() && self.seq.is_empty() && self.qual.is_empty()
    }

    /// Check validity of FastQ record.
    pub fn check(&self) -> Result<(), &str> {
        if self.id().is_none() {
            return Err("Expecting id for FastQ record.");
        }
        if !self.seq.is_ascii() {
            return Err("Non-ascii character found in sequence.");
        }
        if !self.qual.is_ascii() {
            return Err("Non-ascii character found in qualities.");
        }
        if self.seq().len() != self.qual().len() {
            return Err("Unequal length of sequence an qualities.");
        }

        Ok(())
    }

    /// Return the id of the record.
    pub fn id(&self) -> Option<&str> {
        self.header[1..].trim_right().splitn(2, ' ').nth(0)
    }

    /// Return descriptions if present.
    pub fn desc(&self) -> Option<&str> {
        self.header[1..].trim_right().splitn(2, ' ').nth(1)
    }

    /// Return the sequence of the record.
    pub fn seq(&self) -> TextSlice {
        self.seq.trim_right().as_bytes()
    }

    /// Return the base qualities of the record.
    pub fn qual(&self) -> &[u8] {
        self.qual.trim_right().as_bytes()
    }

    /// Clear the record.
    fn clear(&mut self) {
        self.header.clear();
        self.seq.clear();
        self.qual.clear();
    }
}


impl fmt::Display for Record {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "@{}\n{}\n+\n{}", self.header, self.seq, self.qual)
    }
}


/// An iterator over the records of a FastQ file.
pub struct Records<R: io::Read> {
    reader: Reader<R>,
}


impl<R: io::Read> Iterator for Records<R> {
    type Item = io::Result<Record>;

    fn next(&mut self) -> Option<io::Result<Record>> {
        let mut record = Record::new();
        match self.reader.read(&mut record) {
            Ok(()) if record.is_empty() => None,
            Ok(()) => Some(Ok(record)),
            Err(err) => Some(Err(err)),
        }<commit_msg>Test refactor of fastq parser<commit_after>// Copyright 2014-2016 <NAME>.
// Licensed under the MIT license (http://opensource.org/licenses/MIT)
// This file may not be copied, modified, or distributed
// except according to those terms.

//! FastQ reading and writing.
//!
//! # Example
//!
//! ```
//! use std::io;
//! use bio::io::fastq;
//! let reader = fastq::Reader::new(io::stdin());
//! ```

use std::io;
use std::io::prelude::*;
use std::ascii::AsciiExt;
use std::fs;
use std::fmt;
use std::path::Path;
use std::convert::AsRef;

use utils::TextSlice;

/// A FastQ reader.
pub struct Reader<R: io::Read> {
    reader: io::BufReader<R>,
    sep_line: String,
}


impl Reader<fs::File> {
    /// Read from a given file.
    pub fn from_file<P: AsRef<Path>>(path: P) -> io::Result<Self> {
        fs::File::open(path).map(Reader::new)
    }
}


impl<R: io::Read> Reader<R> {
    /// Read from a given `io::Read`.
    pub fn new(reader: R) -> Self {
        Reader {
            reader: io::BufReader::new(reader),
            sep_line: String::new(),
        }
    }

    /// Read into a given record.
    /// Returns an error if the record in incomplete or syntax is violated.
    /// The content of the record can be checked via the record object.
    pub fn read(&mut self, record: &mut Record) -> io::Result<()> {
        record.clear();

        let mut header = String::new();
        try!(self.reader.read_line(&mut header));

        if !header.is_empty() {
            if !header.starts_with('@') {
                return Err(io::Error::new(io::ErrorKind::Other, "Expected @ at record start."));
            }
            record.id = header[1..]
                .trim_right()
                .splitn(2, ' ')
                .nth(0)
                .unwrap_or_default()
                .to_owned();
            record.desc = header[1..]
                .trim_right()
                .splitn(2, ' ')
                .nth(1)
                .unwrap_or_default()
                .to_owned();
            try!(self.reader.read_line(&mut record.seq));
            try!(self.reader.read_line(&mut self.sep_line));
            try!(self.reader.read_line(&mut record.qual));
            if record.qual.is_empty() {
                return Err(io::Error::new(io::ErrorKind::Other,
                                          "Incomplete record. Each FastQ record has to consist \
                                           of 4 lines: header, sequence, separator and \
                                           qualities."));
            }
        }

        Ok(())
    }

    /// Return an iterator over the records of this FastQ file.
    pub fn records(self) -> Records<R> {
        Records { reader: self }
    }
}


/// A FastQ record.
#[derive(Debug, Clone, Default)]
pub struct Record {
    pub id: String,
    pub desc: String,
    pub seq: String,
    pub qual: String,
}


impl Record {
    /// Create a new, empty FastQ record.
    pub fn new() -> Self {
        Record {
            id: String::new(),
            desc: String::new(),
            seq: String::new(),
            qual: String::new(),
        }
    }

    /// Create a new FastQ record from given attributes.
    pub fn from_attrs(id: &str, desc: Option<&str>, seq: TextSlice, qual: &[u8]) -> Self {
        let desc = match desc {
            Some(desc) => desc.to_owned(),
            _ => String::new(),
        };
        Record {
            id: id.to_owned(),
            desc: desc,
            seq: String::from_utf8(seq.to_vec()).unwrap(),
            qual: String::from_utf8(qual.to_vec()).unwrap(),
        }
    }

    /// Check if record is empty.
    pub fn is_empty(&self) -> bool {
        self.id.is_empty() && self.desc.is_empty() && self.seq.is_empty() && self.qual.is_empty()
    }

    /// Check validity of FastQ record.
    pub fn check(&self) -> Result<(), &str> {
        if self.id().is_none() {
            return Err("Expecting id for FastQ record.");
        }
        if !self.seq.is_ascii() {
            return Err("Non-ascii character found in sequence.");
        }
        if !self.qual.is_ascii() {
            return Err("Non-ascii character found in qualities.");
        }
        if self.seq().len() != self.qual().len() {
            return Err("Unequal length of sequence an qualities.");
        }

        Ok(())
    }

    /// Return the id of the record.
    pub fn id(&self) -> Option<&str> {
        Some(&self.id)
        // self.header[1..].trim_right().splitn(2, ' ').nth(0)
    }

    /// Return descriptions if present.
    pub fn desc(&self) -> Option<&str> {
        Some(&self.desc)
        // self.header[1..].trim_right().splitn(2, ' ').nth(1)
    }

    /// Return the sequence of the record.
    pub fn seq(&self) -> TextSlice {
        self.seq.trim_right().as_bytes()
    }

    /// Return the base qualities of the record.
    pub fn qual(&self) -> &[u8] {
        self.qual.trim_right().as_bytes()
    }

    /// Clear the record.
    fn clear(&mut self) {
        self.id.clear();
        self.desc.clear();
        self.seq.clear();
        self.qual.clear();
    }
}


impl fmt::Display for Record {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f,
               "@{} {}\n{}\n+\n{}",
               self.id,
               self.desc,
               self.seq,
               self.qual)
    }
}


/// An iterator over the records of a FastQ file.
pub struct Records<R: io::Read> {
    reader: Reader<R>,
}


impl<R: io::Read> Iterator for Records<R> {
    type Item = io::Result<Record>;

    fn next(&mut self) -> Option<io::Result<Record>> {
        let mut record = Record::new();
        match self.reader.read(&mut record) {
            Ok(()) if record.is_empty() => None,
            Ok(()) => Some(Ok(record)),
            Err(err) => Some(Err(err)),
        }
{'fraction_non_alphanumeric': 0.16417628144505392, 'fraction_numerical': 0.0038774352184603746, 'mean_word_length': 1.8519417475728155, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6462197', 'n_tokens_mistral': 3313, 'n_tokens_neox': 3206, 'n_words': 854}