<commit_before>#!/usr/bin/python
# yac = yet another clipper
# v 1.2.1 - 23-08-2014 - Support FastQ output
# v 1.1.0 - 23-08-2014 - argparse implementation
# Usage yac.py  $input $output $adapter_to_clip $min $max $Nmode
# <NAME> <<EMAIL>>

import sys
import string
import argparse
from itertools import islice


def Parser():
    the_parser = argparse.ArgumentParser()
    the_parser.add_argument(
        '--input', action="store", nargs='+', help="input fastq files")
    the_parser.add_argument(
        '--output', action="store", type=str, help="output, clipped fasta file")
    the_parser.add_argument(
        '--output_format', action="store", type=str, help="output format, fasta or fastq")
    the_parser.add_argument(
        '--adapter_to_clip', action="store", type=str, help="adapter sequence to clip")
    the_parser.add_argument(
        '--min', action="store", type=int, help="minimal size of clipped sequence to keep")
    the_parser.add_argument(
        '--max', action="store", type=int, help="maximal size of clipped sequence to keep")
    the_parser.add_argument('--Nmode', action="store", type=str, choices=[
                            "accept", "reject"], help="accept or reject sequences with N for clipping")
    args = the_parser.parse_args()
    args.adapter_to_clip = args.adapter_to_clip.upper()
    return args


class Clip:

    def __init__(self, inputfile, outputfile, output_format, adapter, minsize, maxsize, Nmode):
        self.inputfile = inputfile
        self.outputfile = outputfile
        self.output_format = output_format
        self.adapter = adapter
        self.minsize = int(minsize)
        self.maxsize = int(maxsize)
        self.Nmode = Nmode

        def motives(sequence):
            '''return a list of motives for perfect (6nt) or imperfect (7nt with one mismatch) search on import string module'''
            sequencevariants = [
                sequence[0:6]]  # initializes the list with the 6mer perfect match
            dicsubst = {"A": "TGCN", "T": "AGCN", "G": "TACN", "C": "GATN"}
            for pos in enumerate(sequence[:6]):
                for subst in dicsubst[pos[1]]:
                    sequencevariants.append(
                        sequence[:pos[0]] + subst + sequence[pos[0] + 1:7])
            return sequencevariants
        self.adaptmotifs = motives(self.adapter)

    def scanadapt(self, adaptmotives=[], sequence="", qscore=""):
        '''scans sequence for adapter motives'''
        match_position = sequence.rfind(adaptmotives[0])
        if match_position != -1:
            return sequence[:match_position], qscore[:match_position]
        for motif in adaptmotives[1:]:
            match_position = sequence.rfind(motif)
            if match_position != -1:
                return sequence[:match_position], qscore[:match_position]
        return sequence, qscore

    def write_output(self, id, read, qscore, output):
        if self.output_format == "fasta":
            block = ">{0}\n{1}\n".format(id, read)
        else:
            block = "@HWI-{0}\n{1}\n+\n{2}\n".format(id, read, qscore)
        output.write(block)

    def handle_io(self):
        '''Open input file, pass read sequence and read qscore to clipping function.
        Pass clipped read and qscore to output function.'''
        id = 0
        output = open(self.outputfile, "w")
        with open(self.inputfile, "r") as input:
            block_gen = islice(input, 1, None, 2)
            for i, line in enumerate(block_gen):
                if i % 2:
                    qscore = line.rstrip()
                else:
                    read = line.rstrip()
                    continue
                trimmed_read, trimmed_qscore = self.scanadapt(
                    self.adaptmotifs, read, qscore)
                if self.minsize <= len(trimmed_read) <= self.maxsize:
                    if (self.Nmode == "reject") and ("N" in trimmed_read):
                        continue
                    id += 1<commit_msg>preserve "output merge fucntionality" by changing filemode to "a" (was "b").<commit_after>#!/usr/bin/python
# yac = yet another clipper
# v 1.2.1 - 23-08-2014 - Support FastQ output
# v 1.1.0 - 23-08-2014 - argparse implementation
# Usage yac.py  $input $output $adapter_to_clip $min $max $Nmode
# <NAME> <<EMAIL>>

import sys
import string
import argparse
from itertools import islice


def Parser():
    the_parser = argparse.ArgumentParser()
    the_parser.add_argument(
        '--input', action="store", nargs='+', help="input fastq files")
    the_parser.add_argument(
        '--output', action="store", type=str, help="output, clipped fasta file")
    the_parser.add_argument(
        '--output_format', action="store", type=str, help="output format, fasta or fastq")
    the_parser.add_argument(
        '--adapter_to_clip', action="store", type=str, help="adapter sequence to clip")
    the_parser.add_argument(
        '--min', action="store", type=int, help="minimal size of clipped sequence to keep")
    the_parser.add_argument(
        '--max', action="store", type=int, help="maximal size of clipped sequence to keep")
    the_parser.add_argument('--Nmode', action="store", type=str, choices=[
                            "accept", "reject"], help="accept or reject sequences with N for clipping")
    args = the_parser.parse_args()
    args.adapter_to_clip = args.adapter_to_clip.upper()
    return args


class Clip:

    def __init__(self, inputfile, outputfile, output_format, adapter, minsize, maxsize, Nmode):
        self.inputfile = inputfile
        self.outputfile = outputfile
        self.output_format = output_format
        self.adapter = adapter
        self.minsize = int(minsize)
        self.maxsize = int(maxsize)
        self.Nmode = Nmode

        def motives(sequence):
            '''return a list of motives for perfect (6nt) or imperfect (7nt with one mismatch) search on import string module'''
            sequencevariants = [
                sequence[0:6]]  # initializes the list with the 6mer perfect match
            dicsubst = {"A": "TGCN", "T": "AGCN", "G": "TACN", "C": "GATN"}
            for pos in enumerate(sequence[:6]):
                for subst in dicsubst[pos[1]]:
                    sequencevariants.append(
                        sequence[:pos[0]] + subst + sequence[pos[0] + 1:7])
            return sequencevariants
        self.adaptmotifs = motives(self.adapter)

    def scanadapt(self, adaptmotives=[], sequence="", qscore=""):
        '''scans sequence for adapter motives'''
        match_position = sequence.rfind(adaptmotives[0])
        if match_position != -1:
            return sequence[:match_position], qscore[:match_position]
        for motif in adaptmotives[1:]:
            match_position = sequence.rfind(motif)
            if match_position != -1:
                return sequence[:match_position], qscore[:match_position]
        return sequence, qscore

    def write_output(self, id, read, qscore, output):
        if self.output_format == "fasta":
            block = ">{0}\n{1}\n".format(id, read)
        else:
            block = "@HWI-{0}\n{1}\n+\n{2}\n".format(id, read, qscore)
        output.write(block)

    def handle_io(self):
        '''Open input file, pass read sequence and read qscore to clipping function.
        Pass clipped read and qscore to output function.'''
        id = 0
        output = open(self.outputfile, "a")
        with open(self.inputfile, "r") as input:
            block_gen = islice(input, 1, None, 2)
            for i, line in enumerate(block_gen):
                if i % 2:
                    qscore = line.rstrip()
                else:
                    read = line.rstrip()
                    continue
                trimmed_read, trimmed_qscore = self.scanadapt(
                    self.adaptmotifs, read, qscore)
                if self.minsize <= len(trimmed_read) <= self.maxsize:
                    if (self.Nmode == "reject") and ("N" in trimmed_read):
                        continue
                    id += 1
{'fraction_non_alphanumeric': 0.12150924661784783, 'fraction_numerical': 0.01166687352612635, 'mean_word_length': 2.3160493827160495, 'pattern_counts': {'":': 10, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1158139', 'n_tokens_mistral': 2462, 'n_tokens_neox': 2332, 'n_words': 645}