<commit_before>import re
import os
import os.path
import tempfile
import subprocess
from edge.models import Fragment, Genome
from edge.blast import BLAST_DB, default_genome_db_name
from edge.blast import Blast_Accession
from django.conf import settings


def make_required_dirs(path):
    dirn = os.path.dirname(path)
    try:
        original_umask = os.umask(0)
        os.makedirs(dirn, 0777)
    except:
        pass
    finally:
        os.umask(original_umask)


def fragment_fasta_fn(fragment):
    return '%s/fragment/%s/%s/edge-fragment-%s-nucl.fa' % (settings.NCBI_DATA_DIR,
                                                           fragment.id % 1024,
                                                           (fragment.id >> 10) % 1024,
                                                           fragment.id)


def build_fragment_fasta(fragment):
    fn = fragment_fasta_fn(fragment)
    make_required_dirs(fn)

    if not os.path.isfile(fn):  # have not built this fasta
        print 'building %s' % fn
        # this may take awhile, so do this first, so user interrupt does
        # not create an empty file
        sequence = fragment.indexed_fragment().sequence
        # be really lenient, convert any unknown bp to N
        sequence = re.sub(r'[^agctnAGCTN]', 'n', sequence)
        if fragment.circular is True:
            sequence = sequence + sequence
        f = open(fn, 'w')
        f.write(">gnl|edge|%s %s\n%s\n" %
                (Blast_Accession.make(fragment), fragment.name, sequence))
        f.close()
    return fn


def build_db(fragments, dbname, refresh=True):
    if len(fragments) == 0:
        return None

    if refresh is False and \
       (os.path.isfile(dbname + '.nal') or os.path.isfile(dbname + '.nsq')):
        print 'already built %s' % dbname
        return dbname

    fns = []
    for fragment in fragments:
        fn = build_fragment_fasta(fragment)
        fns.append(fn)

    print 'concat fasta files for %s' % dbname
    with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
        fafile = f.name
        for fn in fns:
            with open(fn) as inf:
                for line in inf:
                    f.write(line)

    print 'building blast db %s' % dbname
    make_required_dirs(dbname)
    cmd = "%s/makeblastdb -in %s -out %s " % (settings.NCBI_BIN_DIR, fafile, dbname)
    cmd += "-title edge -dbtype nucl -parse_seqids -input_type fasta"

    r = subprocess.check_output(cmd.split(' '))
    if 'Adding sequences from FASTA' not in r:
        print r

    os.unlink(fafile)
    return dbname


def build_genome_db(genome, refresh=False):
    fragments = list(genome.fragments.all())
    dbname = build_db(fragments, default_genome_db_name(genome), refresh=refresh)<commit_msg>Add refresh option for build_fragment_fasta<commit_after>import re
import os
import os.path
import tempfile
import subprocess
from edge.models import Fragment, Genome
from edge.blast import BLAST_DB, default_genome_db_name
from edge.blast import Blast_Accession
from django.conf import settings


def make_required_dirs(path):
    dirn = os.path.dirname(path)
    try:
        original_umask = os.umask(0)
        os.makedirs(dirn, 0777)
    except:
        pass
    finally:
        os.umask(original_umask)


def fragment_fasta_fn(fragment):
    return '%s/fragment/%s/%s/edge-fragment-%s-nucl.fa' % (settings.NCBI_DATA_DIR,
                                                           fragment.id % 1024,
                                                           (fragment.id >> 10) % 1024,
                                                           fragment.id)


def build_fragment_fasta(fragment, refresh=False):
    fn = fragment_fasta_fn(fragment)
    make_required_dirs(fn)

    if not os.path.isfile(fn) or refresh is True:  # have not built this fasta
        print 'building %s' % fn
        # this may take awhile, so do this first, so user interrupt does
        # not create an empty file
        sequence = fragment.indexed_fragment().sequence
        # be really lenient, convert any unknown bp to N
        sequence = re.sub(r'[^agctnAGCTN]', 'n', sequence)
        if fragment.circular is True:
            sequence = sequence + sequence
        f = open(fn, 'w')
        f.write(">gnl|edge|%s %s\n%s\n" %
                (Blast_Accession.make(fragment), fragment.name, sequence))
        f.close()
    return fn


def build_db(fragments, dbname, refresh=True):
    if len(fragments) == 0:
        return None

    if refresh is False and \
       (os.path.isfile(dbname + '.nal') or os.path.isfile(dbname + '.nsq')):
        print 'already built %s' % dbname
        return dbname

    fns = []
    for fragment in fragments:
        fn = build_fragment_fasta(fragment, refresh)
        fns.append(fn)

    print 'concat fasta files for %s' % dbname
    with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
        fafile = f.name
        for fn in fns:
            with open(fn) as inf:
                for line in inf:
                    f.write(line)

    print 'building blast db %s' % dbname
    make_required_dirs(dbname)
    cmd = "%s/makeblastdb -in %s -out %s " % (settings.NCBI_BIN_DIR, fafile, dbname)
    cmd += "-title edge -dbtype nucl -parse_seqids -input_type fasta"

    r = subprocess.check_output(cmd.split(' '))
    if 'Adding sequences from FASTA' not in r:
        print r

    os.unlink(fafile)
    return dbname


def build_genome_db(genome, refresh=False):
    fragments = list(genome.fragments.all())
    dbname = build_db(fragments, default_genome_db_name(genome), refresh=refresh)
{'fraction_non_alphanumeric': 0.09834017490630019, 'fraction_numerical': 0.005711226128859539, 'mean_word_length': 2.1786727169597278, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1314208', 'n_tokens_mistral': 1781, 'n_tokens_neox': 1695, 'n_words': 465}