<commit_before>#!/usr/bin/env python
r"""
Iterators (:mod:`skbio.core.iterator`)
======================================

.. currentmodule:: skbio.core.iterator

This module provides standardized iterators. The benefit of using these
iterators for things like sequence files is that the type of the file, and
any file format details are abstracted out to the developer. In this mannor,
the developer does not need to worry about whether they're operating on
FASTA or FASTQ, and any differences in the returns from their respective
parsers.

Classes
-------

.. autosummary::
   :toctree: generated/

   SequenceIterator
   FastaIterator
   FastqIterator

See Also
--------

skbio.core.factory.sequence

Examples
--------

>>> from StringIO import StringIO
>>> from skbio.core.iterator import FastaIterator, FastqIterator

In the first example, we're going to construct a FASTA iterator that is also
paired with quality scores.

>>> seqs = StringIO(">seq1\n"
...                 "ATGC\n"
...                 ">seq2\n"
...                 "TTGGCC\n")
>>> qual = StringIO(">seq1\n"
...                 "10 20 30 40\n"
...                 ">seq2\n"
...                 "1 2 3 4 5 6\n")
>>> it = FastaIterator(seq=[seqs], qual=[qual])
>>> for record in it:
...     print record['Sequence']
...     print record['Qual']
ATGC
[10 20 30 40]
TTGGCC
[1 2 3 4 5 6]

In the next example, we're going to iterate over multiple FASTQ files at once.

>>> seqs1 = StringIO("@seq1\n"
...                  "ATGC\n"
...                  "+\n"
...                  "hhhh\n")
>>> seqs2 = StringIO("@seq2\n"
...                 "AATTGGCC\n"
...                 ">seq2\n"
...                 "abcdefgh\n")
>>> it = FastqIterator(seq=[seqs1, seqs2])
>>> for record in it:
...     print record['Sequence']
...     print record['Qual']
ATGC
[40 40 40 40]
AATTGGCC
[33 34 35 36 37 38 39 40]

Finally, we can apply arbitrary transforms to the sequences during iteratation.

>>> seqs1 = StringIO("@seq1\n"
...                  "ATGC\n"
...                  "+\n"
...                  "hhhh\n")
>>> seqs2 = StringIO("@seq2\n"
...                 "AATTGGCC\n"
...                 ">seq2\n"
...                 "abcdefgh\n")
>>> def rev_f(st):
...     st['Sequence'] = st['Sequence'][::-1]
...     st['Qual'] = st['Qual'][::-1] if st['Qual'] is not None else None
>>> it = FastqIterator(seq=[seqs1, seqs2], transform=rev_f)
>>> for record in it:
...     print record['Sequence']
...     print record['Qual']
CGTA
[40 40 40 40]
CCGGTTAA
[40 39 38 37 36 35 34 33]
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2013, The BiPy Developers.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

from itertools import chain, izip

from skbio.parse.sequences import parse_fasta, parse_fastq, parse_qual
from skbio.core.workflow import Workflow, not_none, method, requires


def _has_qual(item):
    """Return True if it appears that there is qual data"""
    return (item['QualID'] is not None) and (item['Qual'] is not None)


class SequenceIterator(Workflow):
    """Provide a standard API for interacting with sequence data

    Provide a common interface for iterating over sequence data, including
    support for quality scores and transforms.

    A transform method is a function that takes the state dict and modifies it
    in place. For instance, to reverse sequences, you could pass in the
    following function:<commit_msg>pep8 issue<commit_after>#!/usr/bin/env python
r"""
Iterators (:mod:`skbio.core.iterator`)
======================================

.. currentmodule:: skbio.core.iterator

This module provides standardized iterators. The benefit of using these
iterators for things like sequence files is that the type of the file, and
any file format details are abstracted out to the developer. In this mannor,
the developer does not need to worry about whether they're operating on
FASTA or FASTQ, and any differences in the returns from their respective
parsers.

Classes
-------

.. autosummary::
   :toctree: generated/

   SequenceIterator
   FastaIterator
   FastqIterator

See Also
--------

skbio.core.factory.sequence

Examples
--------

>>> from StringIO import StringIO
>>> from skbio.core.iterator import FastaIterator, FastqIterator

In the first example, we're going to construct a FASTA iterator that is also
paired with quality scores.

>>> seqs = StringIO(">seq1\n"
...                 "ATGC\n"
...                 ">seq2\n"
...                 "TTGGCC\n")
>>> qual = StringIO(">seq1\n"
...                 "10 20 30 40\n"
...                 ">seq2\n"
...                 "1 2 3 4 5 6\n")
>>> it = FastaIterator(seq=[seqs], qual=[qual])
>>> for record in it:
...     print record['Sequence']
...     print record['Qual']
ATGC
[10 20 30 40]
TTGGCC
[1 2 3 4 5 6]

In the next example, we're going to iterate over multiple FASTQ files at once.

>>> seqs1 = StringIO("@seq1\n"
...                  "ATGC\n"
...                  "+\n"
...                  "hhhh\n")
>>> seqs2 = StringIO("@seq2\n"
...                 "AATTGGCC\n"
...                 ">seq2\n"
...                 "abcdefgh\n")
>>> it = FastqIterator(seq=[seqs1, seqs2])
>>> for record in it:
...     print record['Sequence']
...     print record['Qual']
ATGC
[40 40 40 40]
AATTGGCC
[33 34 35 36 37 38 39 40]

Finally, we can apply arbitrary transforms to the sequences during iteratation.

>>> seqs1 = StringIO("@seq1\n"
...                  "ATGC\n"
...                  "+\n"
...                  "hhhh\n")
>>> seqs2 = StringIO("@seq2\n"
...                 "AATTGGCC\n"
...                 ">seq2\n"
...                 "abcdefgh\n")
>>> def rev_f(st):
...     st['Sequence'] = st['Sequence'][::-1]
...     st['Qual'] = st['Qual'][::-1] if st['Qual'] is not None else None
>>> it = FastqIterator(seq=[seqs1, seqs2], transform=rev_f)
>>> for record in it:
...     print record['Sequence']
...     print record['Qual']
CGTA
[40 40 40 40]
CCGGTTAA
[40 39 38 37 36 35 34 33]
"""
# ----------------------------------------------------------------------------
# Copyright (c) 2013, The BiPy Developers.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

from itertools import chain, izip

from skbio.parse.sequences import parse_fasta, parse_fastq, parse_qual
from skbio.core.workflow import Workflow, not_none, method, requires


def _has_qual(item):
    """Return True if it appears that there is qual data"""
    return (item['QualID'] is not None) and (item['Qual'] is not None)


class SequenceIterator(Workflow):
    """Provide a standard API for interacting with sequence data

    Provide a common interface for iterating over sequence data, including
    support for quality scores and transforms.

    A transform method is a function that takes the state dict and modifies it
    in place. For instance, to reverse sequences, you could pass in the
    following function:
{'fraction_non_alphanumeric': 0.18066122561903444, 'fraction_numerical': 0.027804675612117857, 'mean_word_length': 2.1380208333333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 105, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3270237', 'n_tokens_mistral': 2292, 'n_tokens_neox': 2071, 'n_words': 758}