<jupyter_start><jupyter_text>GoalValidation of fragment KDE estimation.* Settings to test:  * Trying differing bandwidths  Setting variables<jupyter_code>workDir = '/home/nick/notebook/SIPSim/dev/Ecoli/'
genomeDir = '/home/nick/notebook/SIPSim/dev/Ecoli/genomes/'<jupyter_output><empty_output><jupyter_text>Init<jupyter_code>import glob
import nestly
from IPython.display import Image, display
import matplotlib.pyplot as plt
%load_ext rpy2.ipython
%matplotlib inline
%%R
library(ggplot2)
library(dplyr)
library(tidyr)
library(gridExtra)<jupyter_output><empty_output><jupyter_text>Setting up nestly<jupyter_code># building tree structure
nest = nestly.Nest()

## varying params
nest.add('bw', 
         ['scott',
          'silverman',
          1, 0.5, 0.1, 0.01, 0.001])

## set params
nest.add('np', [1], create_dir=False)

## input/output files
nest.add('fileName', ['ampFrags'], create_dir=False)
nest.add('genome_index', [os.path.join(genomeDir, 'genome_index.txt')], create_dir=False)
nest.add('genome_dir', [genomeDir], create_dir=False)
nest.add('primers', [os.path.join(workDir, '../', '515F-806R.fna')], create_dir=False)

# building directory tree
buildDir = os.path.join(workDir, 'frag_kde')
nest.build(buildDir)
bashFile = os.path.join(workDir, 'SIPSimRun.sh')
%%writefile $bashFile
#!/bin/bash

# simulating fragments
SIPSim fragments \
    {genome_index} \
    --fp {genome_dir} \
    --fr {primers} \
    --fld skewed-normal,9000,2500,-5 \
    --flr None,None \
    --nf 10000 \
    --np {np} \
    2> {fileName}.log \
    > {fileName}.pkl
    
# converting to kde object
SIPSim fragment_kde \
    {fileName}.pkl \
    --bw {bw} \
    > {fileName}_kde.pkl    
    
# getting bandwidth (for checking on scott * silverman)    
SIPSim KDE_bandwidth {fileName}_kde.pkl > {fileName}_kde_bw.txt
!chmod 775 $bashFile
!cd $workDir; \
    nestrun -j 10 --template-file $bashFile -d frag_kde<jupyter_output>2015-07-12 19:09:55,090 * INFO * Template: ./SIPSimRun.sh
2015-07-12 19:09:55,092 * INFO * [79792] Started ./SIPSimRun.sh in frag_kde/0.5
2015-07-12 19:09:55,095 * INFO * [79793] Started ./SIPSimRun.sh in frag_kde/0.1
2015-07-12 19:09:55,098 * INFO * [79795] Started ./SIPSimRun.sh in frag_kde/0.01
2015-07-12 19:09:55,100 * INFO * [79797] Started ./SIPSimRun.sh in frag_kde/1
2015-07-12 19:09:55,101 * INFO * [79799] Started ./SIPSimRun.sh in frag_kde/scott
2015-07-12 19:09:55,103 * INFO * [79801] Started ./SIPSimRun.sh in frag_kde/0.001
2015-07-12 19:09:55,105 * INFO * [79803] Started ./SIPSimRun.sh in frag_kde/silverman
2015-07-12 19:09:59,098 * INFO * [79803] frag_kde/silverman Finished with 0
2015-07-12 19:09:59,107 * INFO * [79793] frag_kde/0.1 Finished with 0
2015-07-12 19:09:59,110 * INFO * [79795] frag_kde/0.01 Finished with 0
2015-07-12 19:09:59,116 * INFO * [79799] frag_kde/scott Finished with 0
2015-07-12 19:09:59,121 * INFO * [79792] frag_kde/0.5 Finished with 0
2015-07-12 19:[...]<jupyter_text>Plotting<jupyter_code>!cd $workDir; \
    find . -name "*_kde.pkl" | \
    perl -pe 's/\.pkl//' | \
    xargs -I % -P 5 SIPSim KDE_plot %.pkl -o %.png
tmpDir = os.path.join(workDir, 'frag_kde')
pngs = glob.glob(os.path.join(tmpDir, '*/*_kde.png'))
bw = glob.glob(os.path.join(tmpDir, '*/*_kde_bw.txt'))

for x,y in zip(sorted(pngs), sorted(bw)):
    print x 
    !cat $y
    img = Image(x)
    display(img)<jupyter_output>/home/nick/notebook/SIPSim/dev/Ecoli/frag_kde/0.001/ampFrags_kde.png
Ecoli_O157H7	0.001
{'fraction_non_alphanumeric': 0.16066240557815223, 'fraction_numerical': 0.10720511330621732, 'mean_word_length': 3.9826338639652676, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '80096', 'license': 'MIT', 'n_tokens_mistral': 1680, 'n_tokens_neox': 1415, 'n_words': 282, 'path': 'ipynb/bac_genome/Ecoli/kde_sim.ipynb', 'repo_name': 'arischwartz/test', 'size': '3442'}