<commit_before>#!/usr/bin/env python 3
from accessoryFunctions.accessoryFunctions import MetadataObject, GenObject, printtime, make_path, write_to_logfile, \
    run_subprocess
from sipprCommon.objectprep import Objectprep
from sipprCommon.sippingmethods import Sippr
from Bio.Blast.Applications import NcbiblastnCommandline
import Bio.Application
from Bio import SeqIO
from threading import Thread
from subprocess import PIPE
from csv import DictReader
from glob import glob
import operator
import time
import os

__author__ = 'adamkoziol'


class SixteenSBait(Sippr):

    def main(self):
        """

        """
        self.targets()
        self.bait(k=51)
        self.reversebait()
        self.subsample_reads()

    def targets(self):
        """
        Create the GenObject for the analysis type, create the hash file for baiting (if necessary)
        """
        for sample in self.runmetadata:
            if sample.general.bestassemblyfile != 'NA':
                setattr(sample, self.analysistype, GenObject())
                sample[self.analysistype].runanalysis = True
                sample[self.analysistype].targetpath = self.targetpath
                baitpath = os.path.join(self.targetpath, 'bait')
                sample[self.analysistype].baitfile = glob(os.path.join(baitpath, '*.fa'))[0]
                sample[self.analysistype].outputdir = os.path.join(sample.run.outputdirectory, self.analysistype)
                sample[self.analysistype].logout = os.path.join(sample[self.analysistype].outputdir, 'logout.txt')
                sample[self.analysistype].logerr = os.path.join(sample[self.analysistype].outputdir, 'logerr.txt')
                sample[self.analysistype].baitedfastq = os.path.join(sample[self.analysistype].outputdir,
                                                                     '{}_targetMatches.fastq'.format(self.analysistype))
                sample[self.analysistype].complete = False


class SixteenSSipper(Sippr):

    def main(self):
        """

        """
        self.targets()
        self.bait()
        # If desired, use bbduk to bait the target sequences with the previously baited FASTQ files
        if self.revbait:
            self.reversebait()
        # Run the bowtie2 read mapping module
        self.mapping()
        # Use samtools to index the sorted bam file
        self.indexing()
        # Parse the results
        self.parsing()
        # Clear out the large attributes that will difficult to handle objects
        self.clear()
        # Filter out any sequences with cigar features such as internal soft-clipping from the results
        self.clipper()

    def targets(self):
        """
        Using the data from the BLAST analyses, set the targets folder, and create the 'mapping file'. This is the
        genera-specific FASTA file that will be used for all the reference mapping; it replaces the 'bait file' in the
        code
        """
        printtime('Performing analysis with {} targets folder'.format(self.analysistype), self.start,
                  output=self.portallog)
        for sample in self.runmetadata:
            if sample.general.bestassemblyfile != 'NA':
                sample[self.analysistype].targetpath = \
                    os.path.join(self.targetpath, 'genera', sample[self.analysistype].genus, '')
                # There is a relatively strict databasing scheme necessary for the custom targets. Eventually,
                # there will be a helper script to combine individual files into a properly formatted combined file
                try:
                    sample[self.analysistype].mappingfile = glob('{}*.fa'
                                                                 .format(sample[self.analysistype].targetpath))[0]
                # If the fasta file is missing, raise a custom error
                except IndexError as e:
                    # noinspection PyPropertyAccess
                    e.args = ['Cannot find the combined fasta file in {}. Please note that the file must have a '
                              '.fasta extension'.format(sample[self.analysistype].targetpath)]
                    if os.path.isdir(sample[self.analysistype].targetpath):
                        raise
                    else:
                        sample.general.bestassemblyfile = 'NA'


class SixteenS(object):

    def runner(self):
        """
        Run the necessary methods in the correct order
        """
        printtime('Starting {} analysis pipeline'.format(self.analysistype), self.starttime, output=self.portallog)
        if not self.pipeline:
            # If the metadata has been passed from the method script, self.pipeline must still be false in order to
            # get Sippr() to function correctly, but the metadata shouldn't be recreated
            try:
                _ = vars(self.runmetadata)['samples']
            except KeyError:
                # Create the objects to be used in the analyses
                objects = Objectprep(self)
                objects.objectprep()
                self.runmetadata = objects.samples

        else:
            for sample in self.runmetadata.samples:
                setattr(sample, self.analysistype, GenObject())
                sample.run.outputdirectory = sample.general.outputdirectory
        self.threads = int(self.cpus / len(self.runmetadata.samples)) \
            if self.cpus / len(self.runmetadata.samples) > 1 \
            else 1
        # Use a custom sippr method to use the full reference database as bait, and run mirabait against the FASTQ
        # reads - do not perform reference mapping yet
        SixteenSBait(self, self.cutoff)
        # Subsample 1000 reads from the FASTQ files
        self.subsample()
        # Convert the subsampled FASTQ files to FASTA format
        self.fasta()
        # Create BLAST databases if required
        self.makeblastdb()
        # Run BLAST analyses of the subsampled FASTA files against the NCBI 16S reference database
        self.blast()
        # Parse the BLAST results
        self.blastparse()
        # Feed the BLAST results into a modified sippr method to perform reference mapping using the calculated
        # genus of the sample as the mapping file
        SixteenSSipper(self, self.cutoff)
        # Create reports
        self.reporter()

    def subsample(self):
        """
        Subsample 1000 reads from the baited files
        """
        # Create the threads for the analysis
        printtime('Subsampling FASTQ reads', self.starttime, output=self.portallog)
        for _ in range(self.cpus):
            threads = Thread(target=self.subsamplethreads, args=())
            threads.setDaemon(True)
            threads.start()
        for sample in self.runmetadata.samples:
            if sample.general.bestassemblyfile != 'NA':
                # Set the name of the subsampled FASTQ file
                sample[self.analysistype].subsampledfastq = os.path.splitext(sample[self.analysistype].baitedfastq)[0] \
                                                            + '_subsampled.fastq'
                # Set the system call
                sample[self.analysistype].seqtkcall = 'seqtk sample {} 1000 > {}'\
                    .format(sample[self.analysistype].baitedfastq,
                            sample[self.analysistype].subsampledfastq)
                # Add the sample to the queue
                self.samplequeue.put(sample)
        self.samplequeue.join()

    def subsamplethreads(self):
        while True:
            sample = self.samplequeue.get()
            # Check to see if the subsampled FASTQ file has already been created<commit_msg>Changed sixteenS read subsampling to reformat.sh instead of seqtk - seqtk runs forever if previous steps have failed<commit_after>#!/usr/bin/env python 3
from accessoryFunctions.accessoryFunctions import MetadataObject, GenObject, printtime, make_path, write_to_logfile, \
    run_subprocess
from sipprCommon.objectprep import Objectprep
from sipprCommon.sippingmethods import Sippr
from Bio.Blast.Applications import NcbiblastnCommandline
import Bio.Application
from Bio import SeqIO
from threading import Thread
from subprocess import PIPE
from csv import DictReader
from glob import glob
import operator
import time
import os

__author__ = 'adamkoziol'


class SixteenSBait(Sippr):

    def main(self):
        """

        """
        self.targets()
        self.bait(k=51)
        self.reversebait()
        self.subsample_reads()

    def targets(self):
        """
        Create the GenObject for the analysis type, create the hash file for baiting (if necessary)
        """
        for sample in self.runmetadata:
            if sample.general.bestassemblyfile != 'NA':
                setattr(sample, self.analysistype, GenObject())
                sample[self.analysistype].runanalysis = True
                sample[self.analysistype].targetpath = self.targetpath
                baitpath = os.path.join(self.targetpath, 'bait')
                sample[self.analysistype].baitfile = glob(os.path.join(baitpath, '*.fa'))[0]
                sample[self.analysistype].outputdir = os.path.join(sample.run.outputdirectory, self.analysistype)
                sample[self.analysistype].logout = os.path.join(sample[self.analysistype].outputdir, 'logout.txt')
                sample[self.analysistype].logerr = os.path.join(sample[self.analysistype].outputdir, 'logerr.txt')
                sample[self.analysistype].baitedfastq = os.path.join(sample[self.analysistype].outputdir,
                                                                     '{}_targetMatches.fastq'.format(self.analysistype))
                sample[self.analysistype].complete = False


class SixteenSSipper(Sippr):

    def main(self):
        """

        """
        self.targets()
        self.bait()
        # If desired, use bbduk to bait the target sequences with the previously baited FASTQ files
        if self.revbait:
            self.reversebait()
        # Run the bowtie2 read mapping module
        self.mapping()
        # Use samtools to index the sorted bam file
        self.indexing()
        # Parse the results
        self.parsing()
        # Clear out the large attributes that will difficult to handle objects
        self.clear()
        # Filter out any sequences with cigar features such as internal soft-clipping from the results
        self.clipper()

    def targets(self):
        """
        Using the data from the BLAST analyses, set the targets folder, and create the 'mapping file'. This is the
        genera-specific FASTA file that will be used for all the reference mapping; it replaces the 'bait file' in the
        code
        """
        printtime('Performing analysis with {} targets folder'.format(self.analysistype), self.start,
                  output=self.portallog)
        for sample in self.runmetadata:
            if sample.general.bestassemblyfile != 'NA':
                sample[self.analysistype].targetpath = \
                    os.path.join(self.targetpath, 'genera', sample[self.analysistype].genus, '')
                # There is a relatively strict databasing scheme necessary for the custom targets. Eventually,
                # there will be a helper script to combine individual files into a properly formatted combined file
                try:
                    sample[self.analysistype].mappingfile = glob('{}*.fa'
                                                                 .format(sample[self.analysistype].targetpath))[0]
                # If the fasta file is missing, raise a custom error
                except IndexError as e:
                    # noinspection PyPropertyAccess
                    e.args = ['Cannot find the combined fasta file in {}. Please note that the file must have a '
                              '.fasta extension'.format(sample[self.analysistype].targetpath)]
                    if os.path.isdir(sample[self.analysistype].targetpath):
                        raise
                    else:
                        sample.general.bestassemblyfile = 'NA'


class SixteenS(object):

    def runner(self):
        """
        Run the necessary methods in the correct order
        """
        printtime('Starting {} analysis pipeline'.format(self.analysistype), self.starttime, output=self.portallog)
        if not self.pipeline:
            # If the metadata has been passed from the method script, self.pipeline must still be false in order to
            # get Sippr() to function correctly, but the metadata shouldn't be recreated
            try:
                eq = vars(self.runmetadata)['samples']
            except KeyError:
                # Create the objects to be used in the analyses
                objects = Objectprep(self)
                objects.objectprep()
                self.runmetadata = objects.samples

        else:
            for sample in self.runmetadata.samples:
                setattr(sample, self.analysistype, GenObject())
                sample.run.outputdirectory = sample.general.outputdirectory
        self.threads = int(self.cpus / len(self.runmetadata.samples)) \
            if self.cpus / len(self.runmetadata.samples) > 1 \
            else 1
        # Use a custom sippr method to use the full reference database as bait, and run mirabait against the FASTQ
        # reads - do not perform reference mapping yet
        SixteenSBait(self, self.cutoff)
        # Subsample 1000 reads from the FASTQ files
        self.subsample()
        # Convert the subsampled FASTQ files to FASTA format
        self.fasta()
        # Create BLAST databases if required
        self.makeblastdb()
        # Run BLAST analyses of the subsampled FASTA files against the NCBI 16S reference database
        self.blast()
        # Parse the BLAST results
        self.blastparse()
        # Feed the BLAST results into a modified sippr method to perform reference mapping using the calculated
        # genus of the sample as the mapping file
        SixteenSSipper(self, self.cutoff)
        # Create reports
        self.reporter()

    def subsample(self):
        """
        Subsample 1000 reads from the baited files
        """
        # Create the threads for the analysis
        printtime('Subsampling FASTQ reads', self.starttime, output=self.portallog)
        for _ in range(self.cpus):
            threads = Thread(target=self.subsamplethreads, args=())
            threads.setDaemon(True)
            threads.start()
        for sample in self.runmetadata.samples:
            if sample.general.bestassemblyfile != 'NA':
                # Set the name of the subsampled FASTQ file
                sample[self.analysistype].subsampledfastq = os.path.splitext(sample[self.analysistype].baitedfastq)[0] \
                                                            + '_subsampled.fastq'
                # Set the system call
                sample[self.analysistype].seqtkcall = 'reformat.sh in={} out={} samplereadstarget=1000'\
                    .format(sample[self.analysistype].baitedfastq,
                            sample[self.analysistype].subsampledfastq)
                # Add the sample to the queue
                self.samplequeue.put(sample)
        self.samplequeue.join()

    def subsamplethreads(self):
        while True:
            sample = self.samplequeue.get()
            # Check to see if the subsampled FASTQ file has already been created
{'fraction_non_alphanumeric': 0.0788250484183344, 'fraction_numerical': 0.0029696578437701745, 'mean_word_length': 2.0309137155155548, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6507580', 'n_tokens_mistral': 4124, 'n_tokens_neox': 3902, 'n_words': 1187}