<jupyter_start><jupyter_text>Scattering and SBG-FS file stream read notebookThis notebook explores setting up tasks for scatter and SBG's file storage setup. This runs multiple samples in a scatter plus batch mode.<jupyter_code>import sevenbridges as sbg
from sevenbridges.errors import SbgError
from sevenbridges.http.error_handlers import *
import re
import datetime
import binpacking
print("SBG library imported.")
print sbg.__version__<jupyter_output>SBG library imported.
0.7.2<jupyter_text>Logging into your account on CGC Use your authentication token to sync up your account<jupyter_code>prof = 'default'
config_file = sbg.Config(profile=prof)
api = sbg.Api(config=config_file,error_handlers=[rate_limit_sleeper,maintenance_sleeper,general_error_sleeper])
print "Api Configured!!"
print "Api Username : ", api.users.me()<jupyter_output>Api Configured!!
Api Username :  <User: username=anellor1><jupyter_text>Finding the project<jupyter_code>my_project = api.projects.get(id='anellor1/omfgene')
for m in my_project.get_members():
    print m
print my_project.billing_group<jupyter_output><Member: username=anellor1>
<Member: username=elehnert>
<Member: username=raunaqm_cgc>
aaf01c96-dc89-4b54-b0e9-46397b3d907d<jupyter_text>Listing bam files in the project<jupyter_code>#Listing all files in a project
files = [f for f in api.files.query(project=my_project,limit=100).all() if f.name.endswith(".bam")]
print len(files)<jupyter_output>11096<jupyter_text>Get the app to run<jupyter_code>app = api.apps.get(id="anellor1/omfgene/omfgene-wrapper")
print app.name
input_port_app = 'input_file'<jupyter_output>omfgene-wrapper<jupyter_text>Set up the number of files per task<jupyter_code>import math
inputs = {}

num_files = len(files)
num_hosts = 10 #instances in workflow
jobs_per_host = 36 #threads in per instance
minutes_per_run = 25 #estimated
runs_per_hour = 300 / minutes_per_run # Setting number of hours to run an task to be a LCD of minutes_per_run
tasks_per_run = runs_per_hour * jobs_per_host * num_hosts
num_runs = int(math.ceil(num_files*1.0 / tasks_per_run))
print num_files,tasks_per_run,num_runs<jupyter_output>11096 4320 3<jupyter_text>Set up draft tasks and perform analysis<jupyter_code>for run_index in range(num_runs): 
    low_bound = run_index * tasks_per_run
    high_bound = min((run_index + 1) * tasks_per_run, num_files)
    #print low_bound,high_bound
    
    input_files = files[low_bound:high_bound]
    
    task_name = "OMFGene task Run:{}, NumFiles:{}, TimeStamp {}".format(run_index+1, high_bound-low_bound, datetime.datetime.now())
    
    inputs[input_port_app] = input_files
    my_task = api.tasks.create(name=task_name, project=my_project, 
                             app=app, inputs=inputs, run=False)
    if my_task.errors:
        print(my_task.errors())
    else: 
        print('Your task %s is ready to go' % my_task.name)
        # Comment off the statement for execution of tasks.
        my_task.run()<jupyter_output>Your task OMFGene task Run:1, NumFiles:4320, TimeStamp 2017-04-11 15:50:08.901118 is ready to go
Your task OMFGene task Run:2, NumFiles:4320, TimeStamp 2017-04-11 15:50:27.317010 is ready to go
Your task OMFGene task Run:3, NumFiles:2456, TimeStamp 2017-04-11 15:50:43.074345 is ready to go
{'fraction_non_alphanumeric': 0.0931943592887799, 'fraction_numerical': 0.0407725321888412, 'mean_word_length': 5.552208835341365, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '416060', 'license': 'MIT', 'n_tokens_mistral': 1244, 'n_tokens_neox': 1146, 'n_words': 280, 'path': 'omfgene_submit.ipynb', 'repo_name': 'nellore/omfgene', 'size': '3262'}