<commit_before>#!/usr/bin/python2.7
"""
Simulate from a baseline graph made with construct-baseline-hs37d5-ec2.py
Currently needs to be run as follows: scripts/sim-ec2.py
"""
# Doesn't really simply toil-vg sim interface much, but does take care of naming
import os, sys, subprocess, argparse

def parse_args(args):
    parser = argparse.ArgumentParser(description=__doc__, 
        formatter_class=argparse.RawDescriptionHelpFormatter)    
    parser.add_argument("job_store")
    parser.add_argument("out_store")
    parser.add_argument("basename", help="input file prefix.  will look for [basename]_thread_0.xg,"
                        "[basename]_thread_1.xg, [basename].xg")
    parser.add_argument("num_reads", type=int, help="number of read pairs")
    parser.add_argument("--leader", help="name of leader created with create-ec2-leader.sh")    
    parser.add_argument("--config", help="path of config on leader")
    parser.add_argument("--fastq", help="template fastq (default giab nist sample).  use \"None\" to turn off",
                        default="ftp://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/data/NA12878/"
                        "NIST_NA12878_HG001_HiSeq_300x/131219_D00360_005_BH814YADXX/"
                        "Project_RM8398/Sample_U5a/U5a_AGTCAA_L002_R1_007.fastq.gz")    
    parser.add_argument("--sim_opts", help="override simulation options")
    parser.add_argument("--seed", type=int, default=1)
    parser.add_argument("--restart", action="store_true", help="resume toil workflow")
    args = args[1:]        
    return parser.parse_args(args)
options = parse_args(sys.argv)

if options.fastq.lower() == "none":
    options.fastq = None
    default_sim_opts = None
else:
    default_sim_opts = '-p 570 -v 65 -S 4 -i 0.002 -I'
sim_opts = options.sim_opts if options.sim_opts else default_sim_opts
    
def print_num(n):
    """ there must be a package that does this better. oh well"""
    if n >= 1000000000:
        return "{}{}".format(round(float(n) / 1000000000, 2), "B")    
    elif n >= 1000000:
        return "{}{}".format(round(float(n) / 1000000, 2), "M")
    elif n >= 1000:
        return "{}{}".format(round(float(n) / 1000, 2), "K")
    else:
        return str(n)

if not options.job_store.startswith('aws:'):
    options.job_store = 'aws:us-west-2:{}'.format(options.job_store)
if not options.out_store.startswith('aws:'):
    options.out_store = 'aws:us-west-2:{}'.format(options.out_store)<commit_msg>turn off scaling from sim script by default<commit_after>#!/usr/bin/python2.7
"""
Simulate from a baseline graph made with construct-baseline-hs37d5-ec2.py
Currently needs to be run as follows: scripts/sim-ec2.py
"""
# Doesn't really simply toil-vg sim interface much, but does take care of naming
import os, sys, subprocess, argparse

def parse_args(args):
    parser = argparse.ArgumentParser(description=__doc__, 
        formatter_class=argparse.RawDescriptionHelpFormatter)    
    parser.add_argument("job_store")
    parser.add_argument("out_store")
    parser.add_argument("basename", help="input file prefix.  will look for [basename]_thread_0.xg,"
                        "[basename]_thread_1.xg, [basename].xg")
    parser.add_argument("num_reads", type=int, help="number of read pairs")
    parser.add_argument("--leader", help="name of leader created with create-ec2-leader.sh")    
    parser.add_argument("--config", help="path of config on leader")
    parser.add_argument("--fastq", help="template fastq (default giab nist sample).  use \"None\" to turn off",
                        default="ftp://ftp-trace.ncbi.nlm.nih.gov/giab/ftp/data/NA12878/"
                        "NIST_NA12878_HG001_HiSeq_300x/131219_D00360_005_BH814YADXX/"
                        "Project_RM8398/Sample_U5a/U5a_AGTCAA_L002_R1_007.fastq.gz")    
    parser.add_argument("--sim_opts", help="override simulation options")
    parser.add_argument("--seed", type=int, default=1)
    parser.add_argument("--restart", action="store_true", help="resume toil workflow")
    args = args[1:]        
    return parser.parse_args(args)
options = parse_args(sys.argv)

if options.fastq.lower() == "none":
    options.fastq = None
    default_sim_opts = None
else:
    default_sim_opts = '-p 570 -v 65 -i 0.002 -I'
sim_opts = options.sim_opts if options.sim_opts else default_sim_opts
    
def print_num(n):
    """ there must be a package that does this better. oh well"""
    if n >= 1000000000:
        return "{}{}".format(round(float(n) / 1000000000, 2), "B")    
    elif n >= 1000000:
        return "{}{}".format(round(float(n) / 1000000, 2), "M")
    elif n >= 1000:
        return "{}{}".format(round(float(n) / 1000, 2), "K")
    else:
        return str(n)

if not options.job_store.startswith('aws:'):
    options.job_store = 'aws:us-west-2:{}'.format(options.job_store)
if not options.out_store.startswith('aws:'):
    options.out_store = 'aws:us-west-2:{}'.format(options.out_store)
{'fraction_non_alphanumeric': 0.1408308004052685, 'fraction_numerical': 0.046403242147923, 'mean_word_length': 3.4032114183764497, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5749121', 'n_tokens_mistral': 1826, 'n_tokens_neox': 1613, 'n_words': 360}