<commit_before>import logging, os.path, tempfile, time, shutil
from optparse import OptionParser
from contextlib import contextmanager

from Bio import SeqIO

import util


#TODO: general next steps:

#start figuring out how to supply arguments to the various programs,
#and hash output filenames based on those args so that datestamp cache
#handles different invocations of a program.

#make paths less dependent on ~nathan

BINPATH=os.path.realpath(os.path.join(os.path.dirname(__file__),"../luciano-c/bin"))


def binfile(name) :
    return os.path.join(BINPATH, name)

class GenBankProcessor(object) :
    """Manipulate Genebank files in the pynpact project.

    Genbank files are documented at: http://www.ncbi.nlm.nih.gov/Sitemap/samplerecord.html

    Takes GenBankProcessor(<gbkfile>, **kwargs)

    Where kwargs:

     * force: (default: False) recreate intermediate files even if
       they already exist.
     * logger: (default: None) what logger to use
     * outputdir: (default: directory of genebank file) What directory
       to create intermediate and PS files in.
     * cleanup: (default: True) Should the temporary files (not
       intermediate products) be deleted in case of errors

"""
    #
    gbkfile = None
    force = False
    logger = logging.getLogger('pynpact.main')
    outputdir = None
    cleanup = True


    def __init__(self, gbkfile = None,**kwargs) :
        for k in kwargs : setattr(self,k, kwargs[k])

        if gbkfile :
            self.parse(gbkfile)

    def parse(self,gbkfile) :
        self.gbkfile = gbkfile

        if os.path.exists(gbkfile) :
            self.logger.info("Parsing genbank file: %r", gbkfile)
            #TODO: gbk can have multiple records in which case this
            #will err (i wasn't able to find one that did though)
            self.seqrec = SeqIO.read(gbkfile,"genbank")
        else :
            raise Exception("Asked to parse nonexistant genebank file: %r", gbkfile)

        if not self.outputdir :
            self.outputdir = os.path.dirname(os.path.realpath(self.gbkfile))


    def derivative_filename(self, part) :
        """Build the filename of a derivative product of the gbk
        file. If the derivative file already exists return whether it
        is out of date"""

        if not part[0] == "." :
            part = "." + part

        base = os.path.splitext(os.path.basename(self.gbkfile))[0]
        outfilename = os.path.join(self.outputdir, base + part)

        outofdate = self.force or (not os.path.exists(outfilename)) or \
                    os.path.getmtime(outfilename) < os.path.getmtime(self.gbkfile)

        return (outfilename, outofdate)

    def mkstemp_overwrite(self, destination, **kwargs) :
        kwargs.setdefault('dir', self.outputdir)
        kwargs.setdefault('logger', self.logger)
        kwargs.setdefault('cleanup', self.cleanup)
        return util.mkstemp_overwrite(destination, **kwargs)

    @contextmanager
    def mkdtemp(self,**kwargs) :
        kwargs.setdefault('dir', self.outputdir)
        path = tempfile.mkdtemp(**kwargs)
        try :
            yield path
        finally :
            if self.cleanup :
                self.logger.debug("Cleaning up mkdtemp %r", path)
                shutil.rmtree(path,ignore_errors=True)

    def biopy_extract(self,gene_descriptor="gene") :
        outfilename,generate = self.derivative_filename(".extracted")

        if not generate :
            self.logger.debug("Skipped extraction")
            return outfilename

        with self.mkstemp_overwrite(outfilename) as outfile :
            def print_feature(desc, strand, start, end) :
                location = "%s..%s" % (start +1, end)
                if f.strand == -1 :
                    location = "complement(" + location + ")"
                print >>outfile, "%-11s %s" % (desc, location )

            for f in self.seqrec.features :
                if f.type == 'CDS' :
                    desc = f.qualifiers.get(gene_descriptor)
                    #TODO: original stopped at the first space in the descripton
                    if desc :
                        if f.location_operator=="join" :
                            ex=1
                            for sf in f.sub_features :
                                #TODO: original one reversed the order
                                #here if it was on the complement
                                #strand
                                print_feature(desc[0] + "_E" + str(ex), sf.strand,
                                              sf.location.nofuzzy_start, sf.location.nofuzzy_end)
                                ex += 1
                        else :
                            print_feature(desc[0],f.strand,f.location.nofuzzy_start, f.location.nofuzzy_end)
        return outfilename

    def run_extract(self,gene_descriptor="gene") :
        """Go through the genbank record pulling out gene names and locations
        $ extract MYCGE.gbk 0 gene 0 locus_tag > MYCGE.genes
"""
        return self.biopy_extract(gene_descriptor)

    def run_CG(self) :
        """Do the CG ratio calculations.
$ CG MYCGE.gbk 1 580074 201 51 3 > MYCGE.CG200
"""
        outfilename,generate = self.derivative_filename("CG200")
        if generate :
            with self.mkstemp_overwrite(outfilename) as outfile :
                util.capturedCall([binfile("CG"), self.gbkfile, 1, len(self.seqrec), 201, 51, 3],
                                  stdout=outfile, logger=self.logger)
        return outfilename

    def run_atg(self) :
        # ../atg MYCGE.gbk > atg.txt
        pass

    def run_Allplots(self) :
        #need to generate AllPlots.def
        #/Users/luciano/src/Allplots 0 50000 5 1000 3 > XANCA.S-profiles.001.ps
        # fprintf(stderr,"\nUsage:\n\n  Allplots start interval lines [x-tics period_of_frames]\n");
        # fprintf(stderr,"\nstart                 Genome interval first base.");
        # fprintf(stderr,"\ninterval              Number of bases.\n");
        # fprintf(stderr,"\nlines                 Number of lines on page (one page).\n");
        # fprintf(stderr,"\nx-tics                Number of subdivisions.\n");
        # fprintf(stderr,"\nperiod_of_frame       Number of frames.\n");
<commit_msg>Getting the C version of extract available in pynpact<commit_after>import logging, os.path, tempfile, time, shutil
from optparse import OptionParser
from contextlib import contextmanager

from Bio import SeqIO

import util


#TODO: general next steps:

#start figuring out how to supply arguments to the various programs,
#and hash output filenames based on those args so that datestamp cache
#handles different invocations of a program.

#make paths less dependent on ~nathan

BINPATH=os.path.realpath(os.path.join(os.path.dirname(__file__),"../luciano-c/bin"))


def binfile(name) :
    return os.path.join(BINPATH, name)

class GenBankProcessor(object) :
    """Manipulate Genebank files in the pynpact project.

    Genbank files are documented at: http://www.ncbi.nlm.nih.gov/Sitemap/samplerecord.html

    Takes GenBankProcessor(<gbkfile>, **kwargs)

    Where kwargs:

     * force: (default: False) recreate intermediate files even if
       they already exist.
     * logger: (default: None) what logger to use
     * outputdir: (default: directory of genebank file) What directory
       to create intermediate and PS files in.
     * cleanup: (default: True) Should the temporary files (not
       intermediate products) be deleted in case of errors

"""
    #
    gbkfile = None
    force = False
    logger = logging.getLogger('pynpact.main')
    outputdir = None
    cleanup = True


    def __init__(self, gbkfile = None,**kwargs) :
        for k in kwargs : setattr(self,k, kwargs[k])

        if gbkfile :
            self.parse(gbkfile)

    def parse(self,gbkfile) :
        self.gbkfile = gbkfile

        if os.path.exists(gbkfile) :
            self.logger.info("Parsing genbank file: %r", gbkfile)
            #TODO: gbk can have multiple records in which case this
            #will err (i wasn't able to find one that did though)
            self.seqrec = SeqIO.read(gbkfile,"genbank")
        else :
            raise Exception("Asked to parse nonexistant genebank file: %r", gbkfile)

        if not self.outputdir :
            self.outputdir = os.path.dirname(os.path.realpath(self.gbkfile))


    def derivative_filename(self, part) :
        """Build the filename of a derivative product of the gbk
        file. If the derivative file already exists return whether it
        is out of date"""

        if not part[0] == "." :
            part = "." + part

        base = os.path.splitext(os.path.basename(self.gbkfile))[0]
        outfilename = os.path.join(self.outputdir, base + part)

        outofdate = self.force or (not os.path.exists(outfilename)) or \
                    os.path.getmtime(outfilename) < os.path.getmtime(self.gbkfile)

        return (outfilename, outofdate)

    def mkstemp_overwrite(self, destination, **kwargs) :
        kwargs.setdefault('dir', self.outputdir)
        kwargs.setdefault('logger', self.logger)
        kwargs.setdefault('cleanup', self.cleanup)
        return util.mkstemp_overwrite(destination, **kwargs)

    @contextmanager
    def mkdtemp(self,**kwargs) :
        kwargs.setdefault('dir', self.outputdir)
        path = tempfile.mkdtemp(**kwargs)
        try :
            yield path
        finally :
            if self.cleanup :
                self.logger.debug("Cleaning up mkdtemp %r", path)
                shutil.rmtree(path,ignore_errors=True)

    def biopy_extract(self,gene_descriptor="gene") :
        outfilename,generate = self.derivative_filename(".extracted")

        if not generate :
            self.logger.debug("Skipping extract")
            return outfilename

        with self.mkstemp_overwrite(outfilename) as outfile :
            def print_feature(desc, strand, start, end) :
                location = "%s..%s" % (start +1, end)
                if f.strand == -1 :
                    location = "complement(" + location + ")"
                print >>outfile, "%-11s %s" % (desc, location )

            for f in self.seqrec.features :
                if f.type == 'CDS' :
                    desc = f.qualifiers.get(gene_descriptor)
                    #TODO: original stopped at the first space in the descripton
                    if desc :
                        if f.location_operator=="join" :
                            ex=1
                            for sf in f.sub_features :
                                #TODO: original one reversed the order
                                #here if it was on the complement
                                #strand
                                print_feature(desc[0] + "_E" + str(ex), sf.strand,
                                              sf.location.nofuzzy_start, sf.location.nofuzzy_end)
                                ex += 1
                        else :
                            print_feature(desc[0],f.strand,f.location.nofuzzy_start, f.location.nofuzzy_end)
        return outfilename

    def original_extract(self) :
        outfilename,generate = self.derivative_filename("genes")
        if generate :
            with self.mkstemp_overwrite(outfilename) as outfile :
                util.capturedCall([binfile("extract"), self.gbkfile, 0, "gene", 0, "locus_tag"],
                                  stdout=outfile, logger=self.logger)
        else :
            self.logger.debug("Skipping extract")
        return outfilename


    def run_extract(self) :
        """Go through the genbank record pulling out gene names and locations
        $ extract MYCGE.gbk 0 gene 0 locus_tag > MYCGE.genes
"""
        return self.biopy_extract()

    def run_CG(self) :
        """Do the CG ratio calculations.
$ CG MYCGE.gbk 1 580074 201 51 3 > MYCGE.CG200
"""
        outfilename,generate = self.derivative_filename("CG200")
        if generate :
            with self.mkstemp_overwrite(outfilename) as outfile :
                util.capturedCall([binfile("CG"), self.gbkfile, 1, len(self.seqrec), 201, 51, 3],
                                  stdout=outfile, logger=self.logger)
        return outfilename

    def run_atg(self) :
        # ../atg MYCGE.gbk > atg.txt
        pass

    def run_Allplots(self) :
        #need to generate AllPlots.def
        #/Users/luciano/src/Allplots 0 50000 5 1000 3 > XANCA.S-profiles.001.ps
        # fprintf(stderr,"\nUsage:\n\n  Allplots start interval lines [x-tics period_of_frames]\n");
        # fprintf(stderr,"\nstart                 Genome interval first base.");
        # fprintf(stderr,"\ninterval              Number of bases.\n");
        # fprintf(stderr,"\nlines                 Number of lines on page (one page).\n");
        # fprintf(stderr,"\nx-tics                Number of subdivisions.\n");
        # fprintf(stderr,"\nperiod_of_frame       Number of frames.\n");

{'fraction_non_alphanumeric': 0.0984971098265896, 'fraction_numerical': 0.008323699421965317, 'mean_word_length': 2.0218910107126224, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5782871', 'n_tokens_mistral': 3911, 'n_tokens_neox': 3618, 'n_words': 1003}