<jupyter_start><jupyter_text>BSMAP was run on 8 samples (on different machines)<jupyter_code>ls analyses/2016-10-11
for i in ("M2","M3"):
    !python {bsmaploc}methratio.py \
-d ../data/Crassostrea_gigas.GCAz_000297895.1.dna_sm.toplevel.fa \
-u -z -g \
-o methratio_out_{i}.txt \
-s {bsmaploc}samtools \
bsmap_out_{i}.sam \
!curl https://raw.githubusercontent.com/che625/olson-ms-nb/master/scripts/mr3x.awk \
> /Users/sr320/git-repos/sr320.github.io/jupyter/scripts/mr3x.awk

!curl https://raw.githubusercontent.com/che625/olson-ms-nb/master/scripts/mr_gg.awk.sh \
> /Users/sr320/git-repos/sr320.github.io/jupyter/scripts/mr_gg.awk.sh
#first methratio files are converted to filter for CG context, 3x coverage (mr3x.awk), and reformatting (mr_gg.awk.sh).
#due to issue passing variable to awk, simple scripts were used (included in repository)
for i in ("M2","M3"):
    !echo {i}
    !grep "[A-Z][A-Z]CG[A-Z]" <methratio_out_{i}.txt> methratio_out_{i}CG.txt
    !awk -f /Users/sr320/git-repos/sr320.github.io/jupyter/scripts/mr3x.awk methratio_out_{i}CG.txt \
    > mr3x.{i}.txt
    !awk -f /Users/sr320/git-repos/sr320.github.io/jupyter/scripts/mr_gg.awk.sh \
    mr3x.{i}.txt > mkfmt_{i}.txt
#first methratio files are converted to filter for CG context, 3x coverage (mr3x.awk), and reformatting (mr_gg.awk.sh).
#due to issue passing variable to awk, simple scripts were used (included in repository)
for i in ("M2","M3"):
    !echo {i}
    !grep -i "[A-Z][A-Z]CG[A-Z]" <methratio_out_{i}.txt> methratio_out_{i}CGi.txt
    !awk -f /Users/sr320/git-repos/sr320.github.io/jupyter/scripts/mr3x.awk methratio_out_{i}CGi.txt \
    > mr3xi.{i}.txt
    !awk -f /Users/sr320/git-repos/sr320.github.io/jupyter/scripts/mr_gg.awk.sh \
    mr3xi.{i}.txt > mkfmti_{i}.txt
#maybe we need to ignore case
!md5 mkfmt_M2.txt mkfmti_M2.txt | head
#nope
!head -100 mkfmt_M2.txt<jupyter_output>chr.Base	chr	base	strand	coverage	freqC	freqT
scaffold1.14274	scaffold1	14274	F	4	0.00	100.00
scaffold1.14305	scaffold1	14305	F	4	0.00	100.00
scaffold1.15309	scaffold1	15309	F	4	0.00	100.00
scaffold1.15315	scaffold1	15315	F	4	0.00	100.00
scaffold1.15336	scaffold1	15336	F	5	0.00	100.00
scaffold1.16016	scaffold1	16016	F	3	0.00	100.00
scaffold1.17209	scaffold1	17209	F	3	0.00	100.00
scaffold1.19038	scaffold1	19038	F	4	0.00	100.00
scaffold1.19069	scaffold1	19069	F	4	0.00	100.00
scaffold1.19077	scaffold1	19077	F	3	0.00	100.00
scaffold1.19147	scaffold1	19147	F	7	0.00	100.00
scaffold1.19211	scaffold1	19211	F	3	0.00	100.00
scaffold1.19222	scaffold1	19222	F	3	0.00	100.00
scaffold1.19427	scaffold1	19427	F	5	0.00	100.00
scaffold1.19454	scaffold1	19454	F	3	0.00	100.00
scaffold1.19456	scaffold1	19456	F	3	0.00	100.00
scaffold1.19864	scaffold1	19864	F	3	0.00	100.00
scaffold1.19935	scaffold1	19935	F	5	0.00	100.00
scaffold1.19963	scaffold1	19963	F	4	0.00	100.00
scaffold1.20056	scaffo[...]<jupyter_text>Products<jupyter_code>cd /Users/sr320/git-repos/sr320.github.io/jupyter
mkdir analyses
mkdir analyses/$(date +%F)
for i in ("M2","M3"):
    !cp /Volumes/caviar/wd/2016-10-11/mkfmt_{i}.txt analyses/$(date +%F)/mkfmt_{i}.txt
!head analyses/$(date +%F)/*<jupyter_output>==> analyses/2016-10-11/mkfmt_M2.txt <==
chr.Base	chr	base	strand	coverage	freqC	freqT
scaffold1.14274	scaffold1	14274	F	4	0.00	100.00
scaffold1.14305	scaffold1	14305	F	4	0.00	100.00
scaffold1.15309	scaffold1	15309	F	4	0.00	100.00
scaffold1.15315	scaffold1	15315	F	4	0.00	100.00
scaffold1.15336	scaffold1	15336	F	5	0.00	100.00
scaffold1.16016	scaffold1	16016	F	3	0.00	100.00
scaffold1.17209	scaffold1	17209	F	3	0.00	100.00
scaffold1.19038	scaffold1	19038	F	4	0.00	100.00
scaffold1.19069	scaffold1	19069	F	4	0.00	100.00

==> analyses/2016-10-11/mkfmt_M3.txt <==
chr.Base	chr	base	strand	coverage	freqC	freqT
scaffold1.259	scaffold1	259	F	4	100.00	0.00
scaffold1.263	scaffold1	263	F	4	100.00	0.00
scaffold1.267	scaffold1	267	F	4	100.00	0.00
scaffold1.271	scaffold1	271	F	4	100.00	0.00
scaffold1.299	scaffold1	299	F	4	50.00	50.00
scaffold1.10033	scaffold1	10033	F	3	0.00	100.00
scaffold1.10343	scaffold1	10343	F	4	0.00	100.00
scaffold1.10530	scaffold1	10530	F	4	0.00	100.00
scaffol[...]
{'fraction_non_alphanumeric': 0.12097935669707154, 'fraction_numerical': 0.20691310609697552, 'mean_word_length': 3.74061433447099, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '201963', 'license': 'MIT', 'n_tokens_mistral': 2519, 'n_tokens_neox': 2070, 'n_words': 441, 'path': 'jupyter/Olurida/.ipynb_checkpoints/Fidalgo-SIbs-postbsmap-checkpoint.ipynb', 'repo_name': 'sr320/ocattau.github.io', 'size': '4166'}