<commit_before>#!/usr/bin/env python3

# Simply Looking At Pairwise Comparisons to Optimally Parition
#
# Modifying to be more general.
#
#usage for testing
#rm sobaseqout*; ./slapchop.py example_sobaseq.fastq sobaseqout --operation "get_sample: input > (?P<sample>[ATCG]{5})(?P<forward_prime>GTCCTCGAGGTCTCT){e<=2}(?P<rest>.*)" -o "get_strain: rest > (?P<strain>[ATCG]{10,22})GCGTACGCTGCAGGT" --filter "sample_length == 5" --write-report --bite-size 10 --processes 1 --output-seq "forward_prime+rest" --output-id "input.id+'_'+sample.seq" ; wc -l sobaseqout*

import re
import multiprocessing 
import itertools
import argparse
import subprocess
import sys
import time
import os.path
from Bio import Seq, SeqRecord, SeqIO
import regex
import json

def reader(input_line_queue,input_fastq,
                    pass_lock,pass_fastq,
                    fail_lock,fail_fastq,
                    report_lock,report_txt,
                    record_lock,record_csv,
                    bite_size,operations_dict):

    while True:

        current_pos = input_line_queue.get()

        if current_pos == "poisonpill":
            input_line_queue.put("poisonpill")
            return(0)

        report_lock.acquire()
        print(multiprocessing.current_process().name+
            " trying to read into memory chunk of size "+
            str(bite_size)+" records.",
            file=open(report_txt,"a"))
        report_lock.release()

        ifqp = open(input_fastq,"r")
        ifqp.seek(current_pos)
        chunk = []
        for i in range(bite_size*4):
            chunk.append(ifqp.readline())
#                try:
#                    anEntry = [next(iterf),next(iterf),next(iterf),next(iterf)]
#                    thisBite.append(anEntry)
#                    i += 1
#                except:
#                    break


        current_pos = ifqp.tell()
        if ifqp.readline() == "":
            input_line_queue.put("poisonpill")
            report_lock.acquire()
            print(multiprocessing.current_process().name+\
                str(multiprocessing.current_process().pid)+
                " is done",file=open(report_txt,"a"))
            report_lock.release()
        else:
            input_line_queue.put(current_pos)

        pass_records = []
        fail_records = []
        report_records = []

        for slice_base in itertools.islice(range(len(chunk)),0,None,4):

            if chunk[slice_base] == "":
                break
    
            (passed, output_record, report_object) = \
                alignChop(chunk[slice(slice_base,(slice_base+4))],
                    operations_dict)

            if passed:
                pass_records.append(output_record)
            else:
                fail_records.append(output_record)

            report_records.append(report_object)

        
        pass_lock.acquire()
        with open(pass_fastq,"a") as f:
            for i in pass_records:
                print(str(i.id)+"\n"+str(i.seq)+"\n"+"+"+"\n"+
                        i.letter_annotations['phred_quality']+"\n",
                    file=f)
        pass_lock.release()

        fail_lock.acquire()
        with open(fail_fastq,"a") as f:
            for i in fail_records:
                print(str(i.id)+"\n"+str(i.seq)+"\n"+"+"+"\n"+
                        i.letter_annotations['phred_quality']+"\n",
                    file=f)
        fail_lock.release()

        record_lock.acquire()
        with open(record_csv,"a") as f:
            for i in report_records:
                print(i,file=f)
        record_lock.release()


def alignChop(record,operations_dict):

    input_record = SeqRecord.SeqRecord(Seq.Seq(record[1].rstrip()),
        id = record[0].rstrip().split(" ")[0])
    input_record.letter_annotations['phred_quality'] = \
        record[3][0:len(record[1].rstrip())]

    scores_holder = dict()
    seq_holder = dict()
    seq_holder['input'] = input_record 

#rewrite as a class ????

    for operation_name, operation in operations_dict.items():

        try:
            if len(seq_holder[operation[0]]) == 0:
                continue
        except:
            continue

        fuzzy_match = regex.search(
            operation[1], # the seq_pattern to match
            str(seq_holder[operation[0]].seq), # the input seq 
            regex.BESTMATCH )

        if fuzzy_match is None:
            continue
        else:
            (scores_holder[operation_name+'_substitutions'],
                scores_holder[operation_name+'_insertions'],
                scores_holder[operation_name+'_deletions']
                ) = fuzzy_match.fuzzy_counts
            for match_name in fuzzy_match.groupdict():
                seq_holder[match_name] = \
                    seq_holder[operation[0]]\
                    [slice(*fuzzy_match.span(match_name))]
                (scores_holder[match_name+'_start'],
                    scores_holder[match_name+'_end']
                    ) = fuzzy_match.span(match_name)
                scores_holder[match_name+'_length'] = \
                    (scores_holder[match_name+'_end'] - 
                        scores_holder[match_name+'_start'])

    evaluated_filters = evaluate_filters(args.filter,scores_holder)


    if not all(evaluated_filters):
        output_record = input_record
        return((False,output_record,
            "\"Failed\","+
            "\""+input_record.id+"\",\""+
                input_record.seq+"\",\""+
                re.sub("\"","\\\"",re.sub(",","\,",json.dumps(scores_holder)))+"\""
            ))
    else:
        output_record = evaluate_output_directives(
            args.output_seq,args.output_id,seq_holder) 
        return((True,output_record,
            "\"Passed\","+
            "\""+output_record.id+"\",\""+
                output_record.seq+"\",\""+
                re.sub("\"","\\\"",re.sub(",","\,",json.dumps(scores_holder)))+"\""
            ))


def evaluate_output_directives(output_seq, output_id, seq_holder):
    locals().update(seq_holder)
    return_record = eval(output_seq)
    return_record.id = eval(output_id)
    return(return_record)


def evaluate_filters(filters,scores_holder):
    locals().update(scores_holder)
    return_object = []
    try:
        for each_filter in filters:
            if eval(each_filter):
                return_object.append(True)
            else:<commit_msg>looks like its sort of working, cleaning up umi refs now<commit_after>#!/usr/bin/env python3

# Simply Looking At Pairwise Comparisons to Optimally Parition
#
# Modifying to be more general.
#
#usage for testing
# rm sobaseqout*; ./slapchop.py example_sobaseq.fastq sobaseqout --operation "get_sample: input > (?P<sample>[ATCG]{5})(?P<forward_prime>GTCCTCGAGGTCTCT){e<=2}(?P<rest>.*)" -o "get_strain: rest > (?P<strain>[ATCG]{10,22})GCGTACGCTGCAGGT" --filter "sample_length == 5" --write-report --bite-size 10 --processes 1 --output-seq "forward_prime+rest" --output-id "input.id+'_'+sample.seq" ; wc -l sobaseqout*

import re
import multiprocessing 
import itertools
import argparse
import subprocess
import sys
import time
import os.path
from Bio import Seq, SeqRecord, SeqIO
import regex
import json

def reader(input_line_queue,input_fastq,
                    pass_lock,pass_fastq,
                    fail_lock,fail_fastq,
                    report_lock,report_txt,
                    record_lock,record_csv,
                    bite_size,operations_dict):

    while True:

        current_pos = input_line_queue.get()

        if current_pos == "poisonpill":
            input_line_queue.put("poisonpill")
            return(0)

        report_lock.acquire()
        print(multiprocessing.current_process().name+
            " trying to read into memory chunk of size "+
            str(bite_size)+" records.",
            file=open(report_txt,"a"))
        report_lock.release()

        ifqp = open(input_fastq,"r")
        ifqp.seek(current_pos)
        chunk = []
        for i in range(bite_size*4):
            chunk.append(ifqp.readline())
#                try:
#                    anEntry = [next(iterf),next(iterf),next(iterf),next(iterf)]
#                    thisBite.append(anEntry)
#                    i += 1
#                except:
#                    break


        current_pos = ifqp.tell()
        if ifqp.readline() == "":
            input_line_queue.put("poisonpill")
            report_lock.acquire()
            print(multiprocessing.current_process().name+\
                str(multiprocessing.current_process().pid)+
                " is done",file=open(report_txt,"a"))
            report_lock.release()
        else:
            input_line_queue.put(current_pos)

        pass_records = []
        fail_records = []
        report_records = []

        for slice_base in itertools.islice(range(len(chunk)),0,None,4):

            if chunk[slice_base] == "":
                break
    
            (passed, output_record, report_object) = \
                alignChop(chunk[slice(slice_base,(slice_base+4))],
                    operations_dict)

            if passed:
                pass_records.append(output_record)
            else:
                fail_records.append(output_record)

            report_records.append(report_object)

        
        pass_lock.acquire()
        with open(pass_fastq,"a") as f:
            for i in pass_records:
                print(str(i.id)+"\n"+str(i.seq)+"\n"+"+"+"\n"+
                        i.letter_annotations['phred_quality']+"\n",
                    file=f)
        pass_lock.release()

        fail_lock.acquire()
        with open(fail_fastq,"a") as f:
            for i in fail_records:
                print(str(i.id)+"\n"+str(i.seq)+"\n"+"+"+"\n"+
                        i.letter_annotations['phred_quality']+"\n",
                    file=f)
        fail_lock.release()

        record_lock.acquire()
        with open(record_csv,"a") as f:
            for i in report_records:
                print(i,file=f)
        record_lock.release()


def alignChop(record,operations_dict):

    input_record = SeqRecord.SeqRecord(Seq.Seq(record[1].rstrip()),
        id = record[0].rstrip().split(" ")[0])
    input_record.letter_annotations['phred_quality'] = \
        record[3][0:len(record[1].rstrip())]

    scores_holder = dict()
    seq_holder = dict()
    seq_holder['input'] = input_record 

#rewrite as a class ????

    for operation_name, operation in operations_dict.items():

        try:
            if len(seq_holder[operation[0]]) == 0:
                continue
        except:
            continue

        fuzzy_match = regex.search(
            operation[1], # the seq_pattern to match
            str(seq_holder[operation[0]].seq), # the input seq 
            regex.BESTMATCH )

        if fuzzy_match is None:
            continue
        else:
            (scores_holder[operation_name+'_substitutions'],
                scores_holder[operation_name+'_insertions'],
                scores_holder[operation_name+'_deletions']
                ) = fuzzy_match.fuzzy_counts
            for match_name in fuzzy_match.groupdict():
                seq_holder[match_name] = \
                    seq_holder[operation[0]]\
                    [slice(*fuzzy_match.span(match_name))]
                (scores_holder[match_name+'_start'],
                    scores_holder[match_name+'_end']
                    ) = fuzzy_match.span(match_name)
                scores_holder[match_name+'_length'] = \
                    (scores_holder[match_name+'_end'] - 
                        scores_holder[match_name+'_start'])

    evaluated_filters = evaluate_filters(args.filter,scores_holder)


    if not all(evaluated_filters):
        output_record = input_record
        return((False,output_record,
            "\"Failed\","+
            "\""+input_record.id+"\",\""+
                input_record.seq+"\",\""+
                re.sub("\"","\\\"",re.sub(",","\,",json.dumps(scores_holder)))+"\""
            ))
    else:
        try:
            output_record = evaluate_output_directives(
                args.output_seq,args.output_id,seq_holder) 
            return((True,output_record,
                "\"Passed\","+
                "\""+output_record.id+"\",\""+
                    output_record.seq+"\",\""+
                    re.sub("\"","\\\"",re.sub(",","\,",json.dumps(scores_holder)))+"\""
                ))
        except:
            output_record = input_record
            return((False,output_record,
                "\"Failed\","+
                "\""+input_record.id+"\",\""+
                    input_record.seq+"\",\""+
                    re.sub("\"","\\\"",re.sub(",","\,",json.dumps(scores_holder)))+"\""
                ))


def evaluate_output_directives(output_seq, output_id, seq_holder):
    locals().update(seq_holder)
    return_record = eval(output_seq)
    return_record.id = eval(output_id)
    return(return_record)


def evaluate_filters(filters,scores_holder):
    locals().update(scores_holder)
    return_object = []
    try:
        for each_filter in filters:
            if eval(each_filter):
                return_object.append(True)
            else:
{'fraction_non_alphanumeric': 0.13055028462998103, 'fraction_numerical': 0.004250474383301708, 'mean_word_length': 1.9509518477043672, 'pattern_counts': {'":': 6, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '285922', 'n_tokens_mistral': 4096, 'n_tokens_neox': 3877, 'n_words': 618}