<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Goal2&nbsp;&nbsp;Var3&nbsp;&nbsp;Init4&nbsp;&nbsp;Load5&nbsp;&nbsp;Overlapping samples5.1&nbsp;&nbsp;Creating samples file for DeepMAsED5.2&nbsp;&nbsp;CheckM for selected MAGs5.3&nbsp;&nbsp;DeepMAsED5.4&nbsp;&nbsp;Summary6&nbsp;&nbsp;sessionInfo  Goal* create feature tables for [PasolliE et al., 2019 dataset](https://doi.org/10.1016/j.cell.2019.01.001)      Var<jupyter_code>work_dir = '/ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/PasolliE-2019/'

# NCBI accessions for all metagenome samples used
acc_file = file.path(work_dir, 'NCBIaccession_long.txt')

# path to all ~150k MAGs
MAG_dir = file.path(work_dir, 'all_MAGs')

# checkM & other info for all PasolliE MAGs
MAG_info_file = file.path(MAG_dir, 'TableS3.xlsx')

# AlmeidaA-2019 MAGs (random selection)
AlmeidaA_n143_file = '/ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/AlmeidaA-2019/samples_n143_w-MAGs.txt'<jupyter_output><empty_output><jupyter_text>Init<jupyter_code>library(dplyr)
library(tidyr)
library(ggplot2)
library(readxl)
set.seed(18734)
source('/ebio/abt3_projects/software/dev/DeepMAsED/bin/misc_r_functions/init.R')<jupyter_output><empty_output><jupyter_text>Load<jupyter_code># MAG info
MAG_info = readxl::read_excel(MAG_info_file)
colnames(MAG_info) = gsub(' ', '_', colnames(MAG_info))
colnames(MAG_info) = gsub('#', 'num', colnames(MAG_info))

MAG_info %>% nrow %>% print
MAG_info %>% head(n=3)
# NCBI accessions
acc = read.delim(acc_file, sep='\t')
acc %>% nrow %>% print
acc %>% head(n=3)
# MAG fasta files
MAG_fasta = list.files(MAG_dir, '*.fa.gz$', recursive=TRUE, full.names=TRUE) 
MAG_fasta = data.frame(MAG = MAG_fasta %>% basename %>% gsub('\\.fa.gz$', '', .),                       
                       sample = MAG_fasta %>% basename %>% gsub('__bin\\.[0-9]+\\.fa\\.gz$', '', .),
                       fasta = MAG_fasta)
MAG_fasta %>% nrow %>% print
MAG_fasta %>% head(n=3)
# MAGs selected from AlmeidaA analysis
AlmeidaA_n143 = read.delim(AlmeidaA_n143_file, sep='\t') 
AlmeidaA_n143 %>% nrow %>% print
AlmeidaA_n143 %>% head(n=3)<jupyter_output>[1] 1519<jupyter_text>Overlapping samples* Justing using samples overlapping with Almeida-2019  * These samples are already QC'ed<jupyter_code>MAG_fasta = MAG_fasta %>%
    inner_join(acc, c('sample'='sampleID')) 

MAG_fasta %>% nrow %>% print
MAG_fasta %>% head(n=3)
# number of overlapping NCBI accessions
samp_overlap = intersect(AlmeidaA_n143$Sample, MAG_fasta$NCBI_accession) 
samp_overlap %>% length %>% print
# MAGs in Pasolli for overlapping samples
MAG_fasta_j = MAG_fasta %>%
    inner_join(AlmeidaA_n143, c('NCBI_accession'='Sample')) 

# overlapping samples
cat('Number of overlapping samples:', 
    MAG_fasta_j$NCBI_accession %>% unique %>% length, '\n')

# number of MAGs
cat('PasolliE-2019 MAGs:', MAG_fasta_j$MAG %>% unique %>% length, '\n')
cat('Almeida-2019 MAGs:', MAG_fasta_j$Taxon %>% unique %>% length, '\n')

MAG_fasta_j %>% nrow %>% print
MAG_fasta_j %>% head(n=3)<jupyter_output>Warning message:
“Column `NCBI_accession`/`Sample` joining factors with different levels, coercing to character vector”<jupyter_text>Creating samples file for DeepMAsED* Making feature tables for each MAG<jupyter_code>samples_file = file.path(work_dir, 'samples_Almeida-Pasolli.txt')
MAG_fasta_j  %>%
    distinct(MAG, NCBI_accession, fasta, Read1, Read2) %>%
    rename('Taxon' = MAG,
           'Sample' = NCBI_accession,
           'Fasta' = fasta) %>%
    write.table(file=samples_file, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', samples_file, '\n')<jupyter_output>File written: /ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/PasolliE-2019//samples_Almeida-Pasolli.txt<jupyter_text>CheckM for selected MAGs<jupyter_code>MAG_info_f = MAG_info %>%
    semi_join(MAG_fasta_j, c('Genome_Name'='MAG')) 

cat('Number of MAGs:', MAG_info_f %>% nrow, '\n')

cat('#-- checkM Completeness --#\n')
MAG_info_f$Completeness %>% summary %>% print
cat('#-- checkM Contamination --#\n')
MAG_info_f$Contamination %>% summary %>% print
checkM_file = file.path(work_dir, 'samples_Almeida-Pasolli_checkM.txt')
MAG_info_f %>%
    write.table(file=checkM_file, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', checkM_file, '\n')<jupyter_output>File written: /ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/PasolliE-2019//samples_Almeida-Pasolli_checkM.txt<jupyter_text>DeepMAsED* Creating feature tables for each MAG ```(snakemake) @ rick:/ebio/abt3_projects/software/dev/DeepMAsED/DeepMAsED-SM$ ./snakemake_sge.sh /ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/PasolliE-2019/samples_Almeida-Pasolli/config.yaml cluster.json /ebio/abt3_projects/databases_no-backup/DeepMAsED/MAG_datasets/PasolliE-2019/samples_Almeida-Pasolli/SGE_log 20 --dryrun```  Summary<jupyter_code>P = file.path(work_dir, 'samples_Almeida-Pasolli', 'map')
feat_files = list.files(P, 'features.tsv.gz', recursive=TRUE)
feat_files %>% length<jupyter_output><empty_output><jupyter_text>sessionInfo<jupyter_code>sessionInfo()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15150331901600936, 'fraction_numerical': 0.025771183131589222, 'mean_word_length': 6.125173852573018, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 72, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '636026', 'license': 'MIT', 'n_tokens_mistral': 2154, 'n_tokens_neox': 1966, 'n_words': 307, 'path': 'notebooks/02_real_datasets/02_PasolliE-2019.ipynb', 'repo_name': 'chrisLanderson/DeepMAsED', 'size': '5122'}