<commit_before>import pysam
import os
from bcbio.utils import file_exists, safe_makedir
from bcbio.provenance import do
from bcbio.distributed.transaction import tx_tmpdir, file_transaction
from sqhelper import logger
import shutil

MAX_EDIT_DISTANCE = 2
MAX_BEST = 1000


def run_seqcluster(data, args):
    out_dir = "seqcluster"
    minl, minf = args.minl, args.minf
    out_dir = os.path.abspath(safe_makedir(out_dir))
    config_file = os.path.join(out_dir, "prepare.conf")
    prepare_dir = os.path.join(out_dir, "prepare")
    prepare_dir = _prepare(data, config_file, prepare_dir, minl, minf)
    fastq_file = os.path.join(prepare_dir, "seqs.fastq")
    bam_file = _align(data, fastq_file, args)
    #quality = _coverage(bam_file, prepare_dir)
    cluster_dir = os.path.join(out_dir, "cluster")
    cluster_dir = _cluster(bam_file, prepare_dir, cluster_dir, args.gtf_file)
    data = _update(data, prepare_dir, bam_file, cluster_dir)
    return data


def _prepare(data, config_file, out_dir, minl, minf):
    with file_transaction(config_file) as tx_out_file:
        with open(tx_out_file, 'w') as out_handle:
            for sample in data:
                fasta = sample['collapse']
                name = sample['sample_id']
                out_handle.write("%s\t%s\n" % (fasta, name))
    cmd = ("seqcluster prepare -c {config_file} -u 40 -l {minl} -e {minf} -o {tx_out_dir}")
    if not file_exists(out_dir):
        with tx_tmpdir() as work_dir:
            tx_out_dir = os.path.join(work_dir, "prepare")
            do.run(cmd.format(**locals()), "seqcluster prepare")
            shutil.move(tx_out_dir, out_dir)
    return out_dir


def _align(data, fastq_file, args):
    work_dir = os.path.join("align")
    work_dir = os.path.abspath(safe_makedir(work_dir))
    out_prefix = os.path.join(work_dir, "seqs_")
    bam_file = star_align(data, args, fastq_file, out_prefix, 1000)<commit_msg>add 10% samples with that sequence<commit_after>import pysam
import os
from bcbio.utils import file_exists, safe_makedir
from bcbio.provenance import do
from bcbio.distributed.transaction import tx_tmpdir, file_transaction
from sqhelper import logger
import shutil

MAX_EDIT_DISTANCE = 2
MAX_BEST = 1000


def run_seqcluster(data, args):
    out_dir = "seqcluster"
    minl, minf = args.minl, args.minf
    out_dir = os.path.abspath(safe_makedir(out_dir))
    config_file = os.path.join(out_dir, "prepare.conf")
    prepare_dir = os.path.join(out_dir, "prepare")
    prepare_dir = _prepare(data, config_file, prepare_dir, minl, minf)
    fastq_file = os.path.join(prepare_dir, "seqs.fastq")
    bam_file = _align(data, fastq_file, args)
    #quality = _coverage(bam_file, prepare_dir)
    cluster_dir = os.path.join(out_dir, "cluster")
    cluster_dir = _cluster(bam_file, prepare_dir, cluster_dir, args.gtf_file)
    data = _update(data, prepare_dir, bam_file, cluster_dir)
    return data


def _prepare(data, config_file, out_dir, minl, minf):
    with file_transaction(config_file) as tx_out_file:
        with open(tx_out_file, 'w') as out_handle:
            for sample in data:
                fasta = sample['collapse']
                name = sample['sample_id']
                out_handle.write("%s\t%s\n" % (fasta, name))
    min_samples = int(round(len(data) / 10))
    cmd = ("seqcluster prepare -c {config_file} -u 40 -l {minl} -e {minf} -o {tx_out_dir} --min-shared {min_samples}")
    if not file_exists(out_dir):
        with tx_tmpdir() as work_dir:
            tx_out_dir = os.path.join(work_dir, "prepare")
            do.run(cmd.format(**locals()), "seqcluster prepare")
            shutil.move(tx_out_dir, out_dir)
    return out_dir


def _align(data, fastq_file, args):
    work_dir = os.path.join("align")
    work_dir = os.path.abspath(safe_makedir(work_dir))
    out_prefix = os.path.join(work_dir, "seqs_")
    bam_file = star_align(data, args, fastq_file, out_prefix, 1000)
{'fraction_non_alphanumeric': 0.11460731644921975, 'fraction_numerical': 0.00665131747249936, 'mean_word_length': 3.6272189349112427, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4287646', 'n_tokens_mistral': 1447, 'n_tokens_neox': 1403, 'n_words': 307}