<commit_before>"""
Stitch together exons from targeted assemblies.

It uses amino acid targets and DNA assemblies from aTRAM.
"""

import os
from os.path import abspath, join, basename
from pathlib import Path
from collections import namedtuple
import csv
from glob import glob
from Bio.SeqIO.FastaIO import SimpleFastaParser
import lib.stitcher_db as db
import lib.log as log
import lib.util as util


CODON_LEN = 3


def stitch(args):
    """Stitch the exons together."""
    log.stitcher_setup(args.log_file)
    iteration = 0

    with util.make_temp_dir(
            where=args.temp_dir,
            prefix='atram_stitcher_',
            keep=args.keep_temp_dir) as temp_dir:
        temp_dir = temp_dir

        cxn = db.connect(temp_dir, 'atram_stitcher')

        create_tables(cxn)

        taxon_names = get_taxa(args)
        insert_reference_genes(args, temp_dir, cxn)
        create_reference_files(cxn)

        # First iteration gets its data from the input
        iteration += 1
        get_contigs_from_fasta(args, temp_dir, cxn, taxon_names, iteration)
        contig_file_write(cxn, iteration)
        run_exonerate(temp_dir, cxn, iteration)

        # Iterations 2-N get their data from the previous stitch
        for i in range(1, args.iterations):
            stitch_everything(args, cxn, iteration)

            iteration += 1
            get_contigs_from_previous_stitch(
                temp_dir, cxn, taxon_names, iteration)
            contig_file_write(cxn, iteration)
            run_exonerate(temp_dir, cxn, iteration)

        # The final stitch is pickier than the others
        stitch_with_gaps(args, cxn, taxon_names, iteration)

        log.info('Writing output')
        output_stitched_genes(args, cxn, taxon_names, iteration)
        output_summary_per_gene(args, cxn, iteration)
        output_summary_per_taxon(args, cxn, iteration)

        log.info('Finished.')


def create_tables(cxn):
    """Create database tables."""
    db.create_reference_genes_table(cxn)
    db.create_exonerate_table(cxn)
    db.create_contigs_table(cxn)
    db.create_stitch_table(cxn)


def get_taxa(args):
    """Insert taxa into the database."""
    log.info('Preparing taxa')

    taxon_names = []

    with open(args.taxa) as taxa:
        for taxon_name in taxa:
            taxon_name = taxon_name.strip()
            taxon_names.append(taxon_name)

    return sorted(taxon_names)


def insert_reference_genes(args, temp_dir, cxn):
    """Prepare reference sequences for exonerate."""
    batch = []

    ref_genes = args.reference_genes

    log.info('Preparing reference genes: {}'.format(ref_genes))

    with open(ref_genes) as ref_in:

        for ref_name, ref_seq in SimpleFastaParser(ref_in):

            ref_name = util.clean_name(ref_name)

            ref_file = abspath(join(
                temp_dir,
                '{}.fasta'.format(ref_name)))

            batch.append({
                'ref_name': ref_name,
                'ref_seq': ref_seq,
                'ref_file': ref_file})

    db.insert_reference_genes(cxn, batch)


def create_reference_files(cxn):
    """Create reference gene fasta files for exonerate."""
    log.info('Preparing reference gene files for exonerate')
    for ref in db.select_reference_genes(cxn):
        with open(ref['ref_file'], 'w') as ref_file:
            util.write_fasta_record(ref_file, ref['ref_name'], ref['ref_seq'])


def get_contigs_from_fasta(args, temp_dir, cxn, taxon_names, iteration):
    """Prepare fasta files for exonerate.

    In this iteration we are getting the contigs from the given fasta files.
    """
    log.info('{} contig insert: {}'.format(
        util.as_word(iteration), args.assemblies_dir))

    batch = []

    ref_names = set(x['ref_name'] for x in db.select_reference_genes(cxn))

    pattern = join(args.assemblies_dir, '*.fasta')
    for contig_path in sorted(glob(pattern)):

        if os.stat(contig_path).st_size == 0:
            continue

        tiebreaker = 0

        with open(contig_path) as contig_old:
            for i, (contig_name, contig_seq) \
                    in enumerate(SimpleFastaParser(contig_old)):

                contig_file = basename(contig_path)
                ref_name, taxon_name = contig_file.split('.')[0:2]

                if ref_name not in ref_names or taxon_name not in taxon_names:
                    continue

                tiebreaker += 1
                contig_name = util.clean_name(contig_name)
                contig_name = '{}_{}'.format(contig_name, tiebreaker)

                contig_file = abspath(join(temp_dir, contig_file))

                batch.append({
                    'ref_name': ref_name,
                    'taxon_name': taxon_name,
                    'contig_name': contig_name,
                    'contig_seq': contig_seq,
                    'contig_file': contig_file,
                    'contig_rec': i,
                    'iteration': iteration})

    db.insert_contigs(cxn, batch)<commit_msg>Change how we parse the contig file name #255<commit_after>"""
Stitch together exons from targeted assemblies.

It uses amino acid targets and DNA assemblies from aTRAM.
"""

import os
from os.path import abspath, join, basename
from pathlib import Path
from collections import namedtuple
import csv
from glob import glob
from Bio.SeqIO.FastaIO import SimpleFastaParser
import lib.stitcher_db as db
import lib.log as log
import lib.util as util


CODON_LEN = 3


def stitch(args):
    """Stitch the exons together."""
    log.stitcher_setup(args.log_file)
    iteration = 0

    with util.make_temp_dir(
            where=args.temp_dir,
            prefix='atram_stitcher_',
            keep=args.keep_temp_dir) as temp_dir:
        temp_dir = temp_dir

        cxn = db.connect(temp_dir, 'atram_stitcher')

        create_tables(cxn)

        taxon_names = get_taxa(args)
        insert_reference_genes(args, temp_dir, cxn)
        create_reference_files(cxn)

        # First iteration gets its data from the input
        iteration += 1
        get_contigs_from_fasta(args, temp_dir, cxn, taxon_names, iteration)
        contig_file_write(cxn, iteration)
        run_exonerate(temp_dir, cxn, iteration)

        # Iterations 2-N get their data from the previous stitch
        for i in range(1, args.iterations):
            stitch_everything(args, cxn, iteration)

            iteration += 1
            get_contigs_from_previous_stitch(
                temp_dir, cxn, taxon_names, iteration)
            contig_file_write(cxn, iteration)
            run_exonerate(temp_dir, cxn, iteration)

        # The final stitch is pickier than the others
        stitch_with_gaps(args, cxn, taxon_names, iteration)

        log.info('Writing output')
        output_stitched_genes(args, cxn, taxon_names, iteration)
        output_summary_per_gene(args, cxn, iteration)
        output_summary_per_taxon(args, cxn, iteration)

        log.info('Finished.')


def create_tables(cxn):
    """Create database tables."""
    db.create_reference_genes_table(cxn)
    db.create_exonerate_table(cxn)
    db.create_contigs_table(cxn)
    db.create_stitch_table(cxn)


def get_taxa(args):
    """Insert taxa into the database."""
    log.info('Preparing taxa')

    taxon_names = []

    with open(args.taxa) as taxa:
        for taxon_name in taxa:
            taxon_name = taxon_name.strip()
            taxon_names.append(taxon_name)

    return sorted(taxon_names)


def insert_reference_genes(args, temp_dir, cxn):
    """Prepare reference sequences for exonerate."""
    batch = []

    ref_genes = args.reference_genes

    log.info('Preparing reference genes: {}'.format(ref_genes))

    with open(ref_genes) as ref_in:

        for ref_name, ref_seq in SimpleFastaParser(ref_in):

            ref_name = util.clean_name(ref_name)

            ref_file = abspath(join(
                temp_dir,
                '{}.fasta'.format(ref_name)))

            batch.append({
                'ref_name': ref_name,
                'ref_seq': ref_seq,
                'ref_file': ref_file})

    db.insert_reference_genes(cxn, batch)


def create_reference_files(cxn):
    """Create reference gene fasta files for exonerate."""
    log.info('Preparing reference gene files for exonerate')
    for ref in db.select_reference_genes(cxn):
        with open(ref['ref_file'], 'w') as ref_file:
            util.write_fasta_record(ref_file, ref['ref_name'], ref['ref_seq'])


def parse_contig_file_name(ref_names, taxon_names, contig_file):
    """Extract the reference & taxon names from the contig file name."""
    ref_name = [x for x in ref_names if x in contig_file] + [None]
    taxon_name = [x for x in taxon_names if x in contig_file] + [None]
    return ref_name[0], taxon_name[0]


def get_contigs_from_fasta(args, temp_dir, cxn, taxon_names, iteration):
    """Prepare fasta files for exonerate.

    In this iteration we are getting the contigs from the given fasta files.
    """
    log.info('{} contig insert: {}'.format(
        util.as_word(iteration), args.assemblies_dir))

    batch = []

    ref_names = set(x['ref_name'] for x in db.select_reference_genes(cxn))

    pattern = join(args.assemblies_dir, '*.fasta')
    for contig_path in sorted(glob(pattern)):

        if os.stat(contig_path).st_size == 0:
            continue

        tiebreaker = 0

        with open(contig_path) as contig_old:
            for i, (contig_name, contig_seq) \
                    in enumerate(SimpleFastaParser(contig_old)):

                contig_file = basename(contig_path)
                ref_name, taxon_name = parse_contig_file_name(
                        ref_names, taxon_names, contig_file)

                if ref_name not in ref_names or taxon_name not in taxon_names:
                    continue

                tiebreaker += 1
                contig_name = util.clean_name(contig_name)
                contig_name = '{}_{}'.format(contig_name, tiebreaker)

                contig_file = abspath(join(temp_dir, contig_file))

                batch.append({
                    'ref_name': ref_name,
                    'taxon_name': taxon_name,
                    'contig_name': contig_name,
                    'contig_seq': contig_seq,
                    'contig_file': contig_file,
                    'contig_rec': i,
                    'iteration': iteration})

    db.insert_contigs(cxn, batch)
{'fraction_non_alphanumeric': 0.0913022585295531, 'fraction_numerical': 0.002402691013935608, 'mean_word_length': 2.486097152428811, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3058092', 'n_tokens_mistral': 3432, 'n_tokens_neox': 3164, 'n_words': 771}