<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Goal2&nbsp;&nbsp;Var3&nbsp;&nbsp;Init4&nbsp;&nbsp;Load5&nbsp;&nbsp;MG sample subset (n=100)5.1&nbsp;&nbsp;Writing samples file5.2&nbsp;&nbsp;LLMGQC5.3&nbsp;&nbsp;Determining which worked5.4&nbsp;&nbsp;LLMGQC5.5&nbsp;&nbsp;Determining which worked5.6&nbsp;&nbsp;LLMGQC5.6.1&nbsp;&nbsp;Creating genome-sample map table5.6.2&nbsp;&nbsp;Writing table5.6.3&nbsp;&nbsp;Creating features5.7&nbsp;&nbsp;Summary5.8&nbsp;&nbsp;Overlap with PasolliE-20196&nbsp;&nbsp;sessionInfo  Goal* create feature tables for [AlmeidaA et al., 2019 dataset](https://doi.org/10.1038/s41586-019-0965-1)  Var<jupyter_code>work_dir = '/ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/AlmeidaA-2019/'

# checkM results for all MAGs
checkm_res_file = file.path(work_dir, 'mags-gut_qs50_checkm.tab')<jupyter_output><empty_output><jupyter_text>Init<jupyter_code>library(dplyr)
library(tidyr)
library(ggplot2)
set.seed(18734)<jupyter_output>Attaching package: ‘dplyr’

The following objects are masked from ‘package:stats’:

    filter, lag

The following objects are masked from ‘package:base’:

    intersect, setdiff, setequal, union<jupyter_text>Load<jupyter_code>checkm_res = read.delim(checkm_res_file, sep='\t') 
checkm_res %>% nrow %>% print
checkm_res %>% head(n=3)<jupyter_output>[1] 92143<jupyter_text>MG sample subset (n=100)<jupyter_code># getting NCBI accessions
checkm_res = checkm_res %>%
    mutate(acc = gsub('_bin\\.[0-9]+$', '', MAG)) 

checkm_res$acc %>% unique %>% length %>% print
# randomly selecting subset of samples
acc_sub1 = sample(checkm_res$acc, 100)
acc_sub1 %>% length %>% print
acc_sub1 %>% sort %>% head
# number of MAGs corresponding to samples
checkm_res_f = checkm_res %>%
    filter(acc %in% acc_sub1) 

checkm_res_f %>% nrow
# checkm stats
checkm_res %>%
    dplyr::select(Completeness, Contamination, Strain_heterogeneity) %>%
    summary

checkm_res_f %>%
    dplyr::select(Completeness, Contamination, Strain_heterogeneity) %>%
    summary<jupyter_output><empty_output><jupyter_text>Writing samples file<jupyter_code>samples_file = file.path(work_dir, 'samples_n100.txt')
checkm_res_f %>%
    rename('Sample' = acc) %>%
    mutate(Remote = Sample) %>%
    distinct(Sample, Remote) %>%
    mutate(Run = 1, Lane = 1) %>%
    write.table(file=samples_file, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', samples_file, '\n')<jupyter_output><empty_output><jupyter_text>LLMGQC* downloading and QC of selected MG samples ```{bash}(snakemake_dev) @ rick:/ebio/abt3_projects/databases_no-backup/DeepMAsED/bin/llmgqc$ screen -L -S llmgqc-DM ./snakemake_sge.sh /ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/AlmeidaA-2019/LLMGQC/config.yaml cluster.json /ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/AlmeidaA-2019/LLMGQC/SGE_log 20```  Determining which worked<jupyter_code>p = file.path(work_dir, 'LLMGQC', 'final')
samps_success = list.files(P, 'R2_final.fq.gz', recursive=TRUE) %>% dirname
samps_success %>% length %>% print
samps_success %>% head
# randomly selecting subset of new samples
acc_sub2 = sample(checkm_res$acc %>% unique, 100)
acc_sub2 %>% length %>% print
acc_sub2 %>% sort %>% head
# number of MAGs corresponding to samples
checkm_res_f = checkm_res %>%
    filter(acc %in% acc_sub2) 

checkm_res_f %>% nrow
# checking for overlap 
intersect(samps_success, checkm_res_f$acc) %>% print

checkm_res_f = checkm_res_f %>%
    filter(! acc %in% samps_success)

checkm_res_f$acc %>% unique %>% length %>% print
checkm_res_f %>% nrow %>% print
samples_file = file.path(work_dir, 'samples_n99.txt')
checkm_res_f %>%
    rename('Sample' = acc) %>%
    mutate(Remote = Sample) %>%
    distinct(Sample, Remote) %>%
    mutate(Run = 1, Lane = 1) %>%
    write.table(file=samples_file, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', samples_file, '\n')<jupyter_output><empty_output><jupyter_text>LLMGQC ```{bash}(snakemake_dev) @ rick:/ebio/abt3_projects/databases_no-backup/DeepMAsED/bin/llmgqc$ screen -L -S llmgqc-DM ./snakemake_sge.sh /ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/AlmeidaA-2019/LLMGQC/config.yaml cluster.json /ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/AlmeidaA-2019/LLMGQC/SGE_log 20```  Determining which worked<jupyter_code>p = file.path(work_dir, 'LLMGQC', 'final')
samps_success = list.files(P, 'R2_final.fq.gz', recursive=TRUE) %>% dirname
samps_success %>% length %>% print
samps_success %>% head
# subsampling
checkm_res_f = checkm_res %>%
    filter(acc %in% samps_success)
checkm_res_f$acc %>% unique %>% length %>% print
checkm_res_f %>% nrow %>% print
# number of associated MAGs
cat('Number of MAGs:', checkm_res_f$MAG %>% unique %>% length, '\n')
samples_file = file.path(work_dir, 'samples_n143.txt')
checkm_res_f %>%
    rename('Sample' = acc) %>%
    mutate(Remote = Sample) %>%
    distinct(Sample, Remote) %>%
    mutate(Run = 1, Lane = 1) %>%
    write.table(file=samples_file, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', samples_file, '\n')<jupyter_output><empty_output><jupyter_text>LLMGQC ```{bash}(snakemake_dev) @ rick:/ebio/abt3_projects/databases_no-backup/DeepMAsED/bin/llmgqc$ screen -L -S llmgqc-DM ./snakemake_sge.sh /ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/AlmeidaA-2019/LLMGQC/config.yaml cluster.json /ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/AlmeidaA-2019/LLMGQC/SGE_log 20```  Creating genome-sample map table* for mapping metagenome reads to MAGs<jupyter_code>mag_dir = '/ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/AlmeidaA-2019/mags/'

samp_mag_idx = checkm_res_f %>%
    mutate(Fasta = sapply(as.character(MAG), function(x) paste(c(mag_dir, x, '.fa.gz'), collapse=''))) %>%
    rename('Taxon' = MAG,
           'Sample' = acc)

samp_mag_idx %>% nrow %>% print
samp_mag_idx %>% head(n=3)
# getting read locations for all samples
P = '/ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/AlmeidaA-2019/LLMGQC/final/'
R1_files = list.files(P, 'R1_final.fq.gz', recursive=TRUE, full.names=TRUE) 
R2_files = list.files(P, 'R2_final.fq.gz', recursive=TRUE, full.names=TRUE) 

R1_files %>% length %>% print
R1_files %>% head(n=3)

R2_files %>% length %>% print
R2_files %>% head(n=3)

## joining into a df
samp_files = data.frame(Sample = basename(dirname(R1_files)),
                        Read1 = R1_files,
                        Read2 = R2_files)
samp_files %>% nrow %>% print
samp_files %>% head(n=3)
# joining with samp-mag index
samp_mag_idx = samp_mag_idx %>%
    left_join(samp_files, c('Sample')) 

samp_mag_idx %>% nrow %>% print
samp_mag_idx %>% head(n=3)
samp_mag_idx %>% 
    summary<jupyter_output><empty_output><jupyter_text>Writing table<jupyter_code>samp_mag_idx_file = file.path(work_dir, 'samples_n143_w-MAGs.txt')
write.table(samp_mag_idx, samp_mag_idx_file, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', samp_mag_idx_file, '\n')<jupyter_output><empty_output><jupyter_text>Creating features* Using DeepMAsED<jupyter_code>work_dir1 = file.path(work_dir, 'samples_n143_MAG-feats')
work_dir1<jupyter_output><empty_output><jupyter_text>```{bash}(snakemake) @ rick:/ebio/abt3_projects/software/dev/DeepMAsED/DeepMAsED-SM$ screen -L -S DM-MAG ./snakemake_sge.sh /ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/AlmeidaA-2019/samples_n143_MAG-feats/config.yaml cluster.json /ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/AlmeidaA-2019/samples_n143_MAG-feats/SGE_log/ 24[detached from 30285.DM-MAG]```  Summary<jupyter_code>P = file.path(work_dir1, 'map')
feat_files = list.files(P, 'features.tsv.gz', recursive=TRUE)
feat_files %>% length<jupyter_output><empty_output><jupyter_text>sessionInfo<jupyter_code>sessionInfo()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15050466334483198, 'fraction_numerical': 0.027596780375622843, 'mean_word_length': 6.208103130755064, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 129, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '299486', 'license': 'MIT', 'n_tokens_mistral': 3282, 'n_tokens_neox': 3016, 'n_words': 464, 'path': 'notebooks/02_real_datasets/.ipynb_checkpoints/01_AlmeidaA-2019-checkpoint.ipynb', 'repo_name': 'chrisLanderson/DeepMAsED', 'size': '7827'}