<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
Subroutines to aid ALLPATHS-LG assembly.
"""

import os.path as op
import os
import sys
import logging
import numpy as np

from glob import glob
from struct import pack, unpack
from itertools import islice, izip
from optparse import OptionParser

from jcvi.formats.base import BaseFile
from jcvi.assembly.base import FastqNamings, Library
from jcvi.apps.grid import Jobs
from jcvi.apps.base import ActionDispatcher, debug, need_update, sh
debug()


class PairsFile (BaseFile):

    def __init__(self, filename):
        super(PairsFile, self).__init__(filename)

        fp = open(filename, "rb")
        binwrite, = unpack("8s", fp.read(8))
        assert binwrite == "BINWRITE"

        self.version, = unpack("i", fp.read(4))
        assert self.version == 1

        self.nreads, = unpack("Q", fp.read(8))
        self.nlibs, = unpack("Q", fp.read(8))
        self.libstats = []
        self.libnames = []

        for i in xrange(self.nlibs):
            self.libstats.append(unpack("ii", fp.read(8)))

        nlibs, = unpack("Q", fp.read(8))
        assert nlibs == self.nlibs

        for i in xrange(self.nlibs):
            slen, = unpack("i", fp.read(4))
            libname, nul = unpack("{0}sc".format(slen - 1), fp.read(slen))
            self.libnames.append(libname)

        npairs, = unpack("Q", fp.read(8))
        self.r1 = np.fromfile(fp, dtype=np.int64, count=npairs)

        npairs2, = unpack("Q", fp.read(8))
        assert npairs2 == npairs
        self.r2 = np.fromfile(fp, dtype=np.int64, count=npairs)

        npairsl, = unpack("Q", fp.read(8))
        assert npairsl == npairs
        self.libs = np.fromfile(fp, dtype=np.int8, count=npairs)

        assert len(fp.read()) == 0  # EOF
        self.npairs = npairs

    @property
    def header(self):
        from jcvi.utils.cbook import percentage

        s = "Number of paired reads: {0}\n".format(\
                percentage(self.npairs * 2, self.nreads))
        s += "Libraries: {0}\n".format(", ".join(self.libnames))
        s += "LibraryStats: {0}\n".format(self.libstats)
        s += "r1: {0}\n".format(self.r1)
        s += "r2: {0}\n".format(self.r2)
        s += "libs: {0}".format(self.libs)
        return s

    def fixLibraryStats(self, sep, sd):
        libstat = (sep, sd)
        logging.debug("New library stat: {0}".format(libstat))
        self.libstats = [libstat] * self.nlibs

    def write(self, filename):
        fw = open(filename, "wb")
        fw.write(pack("8s", "BINWRITE"))
        fw.write(pack("i", self.version))
        fw.write(pack("Q", self.nreads))
        fw.write(pack("Q", self.nlibs))
        for a, b in self.libstats:
            fw.write(pack("ii", a, b))
        fw.write(pack("Q", self.nlibs))
        for name in self.libnames:
            slen = len(name) + 1
            fw.write(pack("i", slen))
            fw.write(pack("{0}s".format(slen), name))
        fw.write(pack("Q", self.npairs))
        self.r1.tofile(fw)
        fw.write(pack("Q", self.npairs))
        self.r2.tofile(fw)
        fw.write(pack("Q", self.npairs))
        self.libs.tofile(fw)
        logging.debug("New pairs file written to `{0}`.".format(filename))


def main():

    actions = (
        ('prepare', 'prepare ALLPATHS csv files and run script'),
        ('log', 'prepare a log of created files'),
        ('pairs', 'parse ALLPATHS pairs file'),
        ('fastq', 'export ALLPATHS fastb file to fastq'),
        ('fixpairs', 'fix pairs library stats'),
        ('fill', 'run FillFragments on `frag_reads_corr.fastb`'),
        ('automaton', 'run assembly on a folder of paired reads'),
            )
    p = ActionDispatcher(actions)
    p.dispatch(globals())


def automaton(args):
    """
    %prog automaton folder tag

    Run assembly on a folder of paired reads and apply tag (PE-200, PE-500).
    """
    from glob import glob
    from jcvi.utils.iter import grouper
    
    p = OptionParser(automaton.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    folder, tag = args
    os.chdir(folder)
    filelist = glob("*.*")
    print filelist
    for p in grouper(filelist, 2):
        print op.commonprefix(p)


def fastq(args):
    """
    %prog fastq fastbfile

    Export ALLPATHS fastb file to fastq file.
    """
    p = OptionParser(fastq.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    fastbfile, = args
    pf = "j" if "jump" in fastbfile else "f"
<commit_msg>add assembly.allpaths.automaton to streamline assembly in a folder of paired reads<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
Subroutines to aid ALLPATHS-LG assembly.
"""

import os.path as op
import os
import sys
import logging
import numpy as np

from glob import glob
from struct import pack, unpack
from itertools import islice, izip
from optparse import OptionParser

from jcvi.formats.base import BaseFile
from jcvi.assembly.base import FastqNamings, Library
from jcvi.apps.grid import Jobs
from jcvi.apps.base import ActionDispatcher, debug, need_update, sh
debug()


class PairsFile (BaseFile):

    def __init__(self, filename):
        super(PairsFile, self).__init__(filename)

        fp = open(filename, "rb")
        binwrite, = unpack("8s", fp.read(8))
        assert binwrite == "BINWRITE"

        self.version, = unpack("i", fp.read(4))
        assert self.version == 1

        self.nreads, = unpack("Q", fp.read(8))
        self.nlibs, = unpack("Q", fp.read(8))
        self.libstats = []
        self.libnames = []

        for i in xrange(self.nlibs):
            self.libstats.append(unpack("ii", fp.read(8)))

        nlibs, = unpack("Q", fp.read(8))
        assert nlibs == self.nlibs

        for i in xrange(self.nlibs):
            slen, = unpack("i", fp.read(4))
            libname, nul = unpack("{0}sc".format(slen - 1), fp.read(slen))
            self.libnames.append(libname)

        npairs, = unpack("Q", fp.read(8))
        self.r1 = np.fromfile(fp, dtype=np.int64, count=npairs)

        npairs2, = unpack("Q", fp.read(8))
        assert npairs2 == npairs
        self.r2 = np.fromfile(fp, dtype=np.int64, count=npairs)

        npairsl, = unpack("Q", fp.read(8))
        assert npairsl == npairs
        self.libs = np.fromfile(fp, dtype=np.int8, count=npairs)

        assert len(fp.read()) == 0  # EOF
        self.npairs = npairs

    @property
    def header(self):
        from jcvi.utils.cbook import percentage

        s = "Number of paired reads: {0}\n".format(\
                percentage(self.npairs * 2, self.nreads))
        s += "Libraries: {0}\n".format(", ".join(self.libnames))
        s += "LibraryStats: {0}\n".format(self.libstats)
        s += "r1: {0}\n".format(self.r1)
        s += "r2: {0}\n".format(self.r2)
        s += "libs: {0}".format(self.libs)
        return s

    def fixLibraryStats(self, sep, sd):
        libstat = (sep, sd)
        logging.debug("New library stat: {0}".format(libstat))
        self.libstats = [libstat] * self.nlibs

    def write(self, filename):
        fw = open(filename, "wb")
        fw.write(pack("8s", "BINWRITE"))
        fw.write(pack("i", self.version))
        fw.write(pack("Q", self.nreads))
        fw.write(pack("Q", self.nlibs))
        for a, b in self.libstats:
            fw.write(pack("ii", a, b))
        fw.write(pack("Q", self.nlibs))
        for name in self.libnames:
            slen = len(name) + 1
            fw.write(pack("i", slen))
            fw.write(pack("{0}s".format(slen), name))
        fw.write(pack("Q", self.npairs))
        self.r1.tofile(fw)
        fw.write(pack("Q", self.npairs))
        self.r2.tofile(fw)
        fw.write(pack("Q", self.npairs))
        self.libs.tofile(fw)
        logging.debug("New pairs file written to `{0}`.".format(filename))


def main():

    actions = (
        ('prepare', 'prepare ALLPATHS csv files and run script'),
        ('log', 'prepare a log of created files'),
        ('pairs', 'parse ALLPATHS pairs file'),
        ('fastq', 'export ALLPATHS fastb file to fastq'),
        ('fixpairs', 'fix pairs library stats'),
        ('fill', 'run FillFragments on `frag_reads_corr.fastb`'),
        ('automaton', 'run assembly on a folder of paired reads'),
            )
    p = ActionDispatcher(actions)
    p.dispatch(globals())


def assemble_pairs(p, pf, tag):
    """
    Taken one pair of reads and assemble to contigs.fasta.
    """
    from jcvi.apps.base import mkdir

    logging.debug("Work on {0} ({1})".format(pf, ','.join(p)))
    asm = "{0}.contigs.fasta".format(pf)
    if not need_update(p, asm):
        logging.debug("Assembly found: {0}. Skipped.".format(asm))
        return

    mkdir(pf, overwrite=True)
    cwd = os.getcwd()
    os.chdir(pf)
    # Create sym-links for the input files
    for i, f in enumerate(p):
        for t in tag:
            sh("ln -sf ../{0} {1}.{2}.fastq".format(f, t, i + 1))

    prepare([pf] + glob("*.fastq"))
    sh("chmod u+x run.sh")
    sh("./run.sh")
    sh("cp allpaths/ASSEMBLIES/run/final.contigs.fasta ../{0}".format(asm))
    os.chdir(cwd)

    logging.debug("Assembly finished: {0}".format(asm))


def automaton(args):
    """
    %prog automaton folder tag

    Run assembly on a folder of paired reads and apply tag (PE-200, PE-500).
    Allow multiple tags separated by comma, e.g. PE-350,TT-1050
    """
    from jcvi.utils.iter import grouper

    p = OptionParser(automaton.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    folder, tag = args
    tag = tag.split(",")
    filelist = glob(folder + "/*.*")
    for p in grouper(2, filelist):
        pp = [op.basename(x) for x in p]
        pf = op.commonprefix(pp).strip("._-")
        assemble_pairs(p, pf, tag)


def fastq(args):
    """
    %prog fastq fastbfile

    Export ALLPATHS fastb file to fastq file.
    """
    p = OptionParser(fastq.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    fastbfile, = args
    pf = "j" if "jump" in fastbfile else "f"

{'fraction_non_alphanumeric': 0.1538159316373637, 'fraction_numerical': 0.011884883606718397, 'mean_word_length': 2.419073203492277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1161151', 'n_tokens_mistral': 3659, 'n_tokens_neox': 3448, 'n_words': 865}