<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Goal2&nbsp;&nbsp;Var3&nbsp;&nbsp;Init4&nbsp;&nbsp;Load5&nbsp;&nbsp;Summarizing5.1&nbsp;&nbsp;Representatives5.2&nbsp;&nbsp;CheckM6&nbsp;&nbsp;Selecting the best rep per GTDB species6.1&nbsp;&nbsp;Creating unique org. names6.2&nbsp;&nbsp;Summarizing filtered6.3&nbsp;&nbsp;Writing filtered metadata table6.4&nbsp;&nbsp;Filtering by checkM6.4.0.1&nbsp;&nbsp;Writing filtered metadata table7&nbsp;&nbsp;Selecting the best rep per NCBI spec-level taxID7.1&nbsp;&nbsp;Creating unique org. names7.2&nbsp;&nbsp;Summarizing filtered7.2.1&nbsp;&nbsp;CheckM7.2.2&nbsp;&nbsp;Assemblies7.2.3&nbsp;&nbsp;Taxonomy7.2.3.1&nbsp;&nbsp;Species level7.2.3.2&nbsp;&nbsp;Genus level7.3&nbsp;&nbsp;Writing filtered metadata table7.4&nbsp;&nbsp;Filtering by CheckM7.4.0.1&nbsp;&nbsp;Writing filtered metadata table8&nbsp;&nbsp;Creating GTDB-based taxdump8.0.1&nbsp;&nbsp;CMD8.1&nbsp;&nbsp;Adding GTDB taxIDs to metadata8.1.1&nbsp;&nbsp;Writing table9&nbsp;&nbsp;Genome download9.1&nbsp;&nbsp;Writing final table10&nbsp;&nbsp;Struo pipeline run10.1&nbsp;&nbsp;config10.2&nbsp;&nbsp;Run11&nbsp;&nbsp;sessionInfo  Goal* Summarize GTDB metadata (release 89)* The data will be used to select which genomes will be used for the `Struo` pipeline  Var<jupyter_code>work_dir = '/ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB/'
bac_metadata_url = 'https://data.ace.uq.edu.au/public/gtdb/data/releases/release89/89.0/bac120_metadata_r89.tsv'
arc_metadata_url = 'https://data.ace.uq.edu.au/public/gtdb/data/releases/release89/89.0/ar122_metadata_r89.tsv'<jupyter_output><empty_output><jupyter_text>Init<jupyter_code>library(dplyr)
library(tidyr)
library(ggplot2)
library(data.table)

set.seed(8473)<jupyter_output><empty_output><jupyter_text>Load<jupyter_code># reading in metadata
metadata_urls = c(bac_metadata_url, arc_metadata_url)

df = list()
for(url in metadata_urls){
    write(sprintf('Reading in file: %s', url), stderr())
    df[[url]] = fread(url, sep='\t', check.names=TRUE)
}
df = do.call(rbind, df)

df %>% nrow %>% print
df %>% head<jupyter_output><empty_output><jupyter_text>Summarizing  Representatives<jupyter_code># number of GTDB representatives
df %>%
    filter(gtdb_representative == 't') %>%
    nrow
# number of GTDB representatives per species
df_s = df %>%
    filter(gtdb_representative == 't') %>%
    group_by(gtdb_taxonomy) %>%
    summarize(n=n()) %>%
    ungroup() 

df_s$n %>% summary %>% print
# assessing these multi-rep per species genomes
df_s %>%
    filter(n >= 2) %>%
    head(n=30)
# number of GTDB representatives per species
df_s = df %>%
    filter(gtdb_representative == 't') %>%
    group_by(ncbi_species_taxid) %>%
    summarize(n=n()) %>%
    ungroup() 

df_s$n %>% summary %>% print
# number of GTDB representatives per species
df_s = df %>%
    filter(gtdb_representative == 't') %>%
    group_by(ncbi_species_taxid) %>%
    summarize(n=n()) %>%
    ungroup() %>%
    filter(ncbi_species_taxid != 'none')

df_s$n %>% summary %>% print<jupyter_output>Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
  1.000   1.000   1.000   1.213   1.000 184.000<jupyter_text>CheckM<jupyter_code>df$checkm_completeness %>% summary %>% print
df$checkm_contamination %>% summary %>% print
# plotting association
p = df %>%
    filter(checkm_contamination <= 100) %>%
    ggplot(aes(checkm_completeness, checkm_contamination)) +
    geom_point(alpha=0.3) +
    facet_grid(. ~ gtdb_representative) +
    theme_bw()

options(repr.plot.width=6, repr.plot.height=3)
plot(p)<jupyter_output><empty_output><jupyter_text>Selecting the best rep per GTDB species<jupyter_code># filtering out genomes lacking species classifications
df %>%
    filter(gtdb_representative == 't') %>%
    nrow %>% print

df_s = df %>%
    filter(gtdb_representative == 't') %>%
    mutate(SPECIES = gsub('.+;s__', '', gtdb_taxonomy)) %>%
    filter(SPECIES != '') 

df_s %>% nrow %>% print
# filtering out UBA genomes
df_s %>% nrow %>% print
df_s = df_s %>%
    filter(!grepl('^UBA[0-9]+$', accession)) 

df_s %>% nrow
# filtering out genomes lacking NCBI assembly accession
df_s %>% nrow %>% print
df_s = df_s %>%
    filter(ncbi_genbank_assembly_accession != 'none') 

df_s %>% nrow
# ranking based on checkM data
df_s = df_s %>%
    group_by(gtdb_taxonomy) %>%
    mutate(checkm_completeness_rank = min_rank(-checkm_completeness),
           checkm_contamination_rank = min_rank(checkm_contamination),
           summed_rank = min_rank(checkm_completeness_rank + checkm_contamination_rank),
           min_summed_rank = min(summed_rank)) %>%
    ungroup() 

df_s %>% nrow %>% print
df_s %>% head
# assessing
df_s %>%
    dplyr::select(gtdb_taxonomy, checkm_completeness, checkm_contamination,
                  checkm_completeness_rank, checkm_contamination_rank, 
                  summed_rank, min_summed_rank) %>%
    arrange(gtdb_taxonomy) %>%
    head(n=20)
# Selecting best representative
df_s_f = df_s %>%
    filter(summed_rank == min_summed_rank) %>%
    group_by(gtdb_taxonomy) %>%
    sample_n(1) %>%
    ungroup() %>%
    dplyr::select(-checkm_completeness_rank, -checkm_contamination_rank, 
                  -summed_rank, -min_summed_rank)

df_s_f %>% nrow %>% print
df_s_f %>% head<jupyter_output>[1] 24080<jupyter_text>Creating unique org. names* species names are not unique<jupyter_code># duplicate organism names?
df_s_f %>%
    group_by(ncbi_organism_name) %>%
    summarize(n=n()) %>%
    ungroup() %>%
    filter(n>1) %>%
    nrow
# duplicate organism names?
df_s_f %>%
    group_by(accession, ncbi_organism_name) %>%
    summarize(n=n()) %>%
    ungroup() %>%
    filter(n>1) %>%
    nrow
# ncbi_organism_name = ncbi_organism_name
df_s_f = df_s_f %>%
    unite(ncbi_organism_name, accession, ncbi_organism_name, sep='_', remove=FALSE) 

df_s_f %>%
    group_by(ncbi_organism_name) %>%
    summarize(n=n()) %>%
    ungroup() %>%
    filter(n>1) %>%
    nrow %>% print

df_s_f$ncbi_organism_name %>% head(n=10) %>% print<jupyter_output>[1] 0
 [1] "GB_GCA_000405245.1_Candidatus Aenigmarchaeum subterraneum SCGC AAA011-O16"         
 [2] "GB_GCA_002789635.1_Candidatus Aenigmarchaeota archaeon CG_4_9_14_3_um_filter_37_18"
 [3] "GB_GCA_001784635.1_Candidatus Micrarchaeota archaeon RBG_16_49_10"                 
 [4] "GB_GCA_002254545.1_Candidatus Aenigmarchaeota archaeon ex4484_224"                 
 [5] "GB_GCA_002688965.1_archaeon"                                                       
 [6] "GB_GCA_000806115.1_archaeon GW2011_AR5"                                            
 [7] "GB_GCA_001873845.1_Candidatus Altiarchaeum sp. CG2_30_32_3053"                     
 [8] "GB_GCA_002083985.1_Candidatus Altiarchaeales archaeon A3"                          
 [9] "GB_GCA_002841105.1_Candidatus Altiarchaeales archaeon HGW-Altiarchaeales-1"        
[10] "GB_GCA_001742785.1_Candidatus Altiarchaeales archaeon IMC4"<jupyter_text>Summarizing filtered<jupyter_code># number of representatives per species
df_s_f_s = df_s_f %>%
    group_by(gtdb_taxonomy) %>%
    summarize(n=n()) %>%
    ungroup() 

df_s_f_s$n %>% summary %>% print
# checkM stats
df_s_f$checkm_completeness %>% summary %>% print
df_s_f$checkm_contamination %>% summary %>% print
# plotting association
p = df_s_f %>%
    ggplot(aes(checkm_completeness, checkm_contamination, color=mimag_high_quality)) +
    geom_point(alpha=0.3) +
    theme_bw()

options(repr.plot.width=5, repr.plot.height=3)
plot(p)<jupyter_output><empty_output><jupyter_text>Writing filtered metadata table<jupyter_code>F = file.path(work_dir, 'metadata_1per-GTDB-Spec.tsv')
write.table(df_s_f, file=F, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', F, '\n')<jupyter_output>File written: /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB//metadata_1per-GTDB-Spec.tsv<jupyter_text>Filtering by checkM<jupyter_code># all
df_s_f %>% nrow %>% print

# relaxed
df_s_f %>%
    filter(checkm_completeness >= 50,
           checkm_contamination < 10) %>%
    nrow %>% print

# stringent
df_s_f %>%
    filter(checkm_completeness >= 50,
           checkm_contamination < 5) %>%
    nrow %>% print
# stringent 
df_s_ff = df_s_f %>%
    filter(checkm_completeness >= 50,
           checkm_contamination < 5) 

df_s_ff$checkm_completeness %>% summary
df_s_ff$checkm_contamination %>% summary<jupyter_output><empty_output><jupyter_text>Writing filtered metadata table<jupyter_code>F = file.path(work_dir, 'metadata_1per-GTDB-Spec_gte50comp-lt5cont.tsv')
df_s_ff %>%
    write.table(file=F, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', F, '\n')<jupyter_output>File written: /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB//metadata_1per-GTDB-Spec_gte50comp-lt5cont.tsv<jupyter_text>Selecting the best rep per NCBI spec-level taxID* How many species reps versus GTDB?<jupyter_code># number of representatives per NCBI species taxID
df_s = df %>%
    filter(ncbi_species_taxid != 'none') %>%
    group_by(ncbi_species_taxid) %>%
    summarize(n=n()) %>%
    ungroup() 

df_s$n %>% summary %>% print
# number of species taxIDs
df %>%
    filter(gtdb_representative == 't') %>%
    .$ncbi_species_taxid %>% unique %>% length %>% print

# number of NAs/none for species taxIDs
df %>%
    filter(gtdb_representative == 't') %>%
    filter(!is.na(ncbi_species_taxid),
           ncbi_species_taxid != 'none') %>%
    .$ncbi_species_taxid %>% unique %>% length %>% print

df %>% 
   filter(gtdb_representative == 't') %>%
    nrow %>% print

df %>%
    filter(gtdb_representative == 't') %>%
    filter(!is.na(ncbi_species_taxid),
           ncbi_species_taxid != 'none') %>% 
    nrow %>% print
# filtering out genomes lacking species classifications
df %>%
    filter(gtdb_representative == 't') %>%
    nrow %>% print

df_s = df %>%
    filter(gtdb_representative == 't') %>%
    filter(!is.na(ncbi_species_taxid),
           ncbi_species_taxid != 'none') 

df_s %>% nrow %>% print
# filtering out UBA genomes
df_s %>% nrow %>% print
df_s = df_s %>%
    filter(!grepl('^UBA[0-9]+$', accession)) 

df_s %>% nrow
# filtering out genomes lacking NCBI assembly accession
df_s %>% nrow %>% print
df_s = df_s %>%
    filter(ncbi_genbank_assembly_accession != 'none') 

df_s %>% nrow
# ranking based on checkM data
df_s = df_s %>%
    group_by(ncbi_species_taxid) %>%
    mutate(checkm_completeness_rank = min_rank(-checkm_completeness),
           checkm_contamination_rank = min_rank(checkm_contamination),
           summed_rank = min_rank(checkm_completeness_rank + checkm_contamination_rank),
           min_summed_rank = min(summed_rank)) %>%
    ungroup() 

df_s %>% nrow %>% print
df_s %>% head
# assessing
df_s %>%
    dplyr::select(ncbi_species_taxid, checkm_completeness, checkm_contamination,
                  checkm_completeness_rank, checkm_contamination_rank, 
                  summed_rank, min_summed_rank) %>%
    arrange(ncbi_species_taxid) %>%
    head(n=30)
# filtering (randomly selecting 1 if tied for best assembly)
df_s_f = df_s %>%
    filter(summed_rank == min_summed_rank) %>%
    group_by(ncbi_species_taxid) %>%
    sample_n(1) %>%
    ungroup() %>%
    dplyr::select(-checkm_completeness_rank, -checkm_contamination_rank, 
                  -summed_rank, -min_summed_rank)

df_s_f %>% nrow %>% print
df_s_f %>% head<jupyter_output>[1] 19854<jupyter_text>Creating unique org. names<jupyter_code># duplicate organism names?
df_s_f %>%
    group_by(ncbi_organism_name) %>%
    summarize(n=n()) %>%
    ungroup() %>%
    filter(n>1) 
# duplicate organism names?
df_s_f %>%
    group_by(accession, ncbi_organism_name) %>%
    summarize(n=n()) %>%
    ungroup() %>%
    filter(n>1) %>%
    nrow
# ncbi_organism_name = ncbi_organism_name
df_s_f = df_s_f %>%
    unite(ncbi_organism_name, accession, ncbi_organism_name, sep='_', remove=FALSE) 

df_s_f %>%
    group_by(ncbi_organism_name) %>%
    summarize(n=n()) %>%
    ungroup() %>%
    filter(n>1) %>%
    nrow %>% print

df_s_f$ncbi_organism_name %>% head(n=10) %>% print<jupyter_output>[1] 0
 [1] "RS_GCF_000243815.2_Leptospira alexanderi serovar Manhao 3 str. L 60"
 [2] "RS_GCF_000764165.1_Cryobacterium roopkundense"                      
 [3] "RS_GCF_900143645.1_Anaerocolumna xylanovorans DSM 12503"            
 [4] "RS_GCF_900205925.1_Clostridium peptidivorans"                       
 [5] "RS_GCF_900176335.1_Papillibacter cinnamivorans DSM 12816"           
 [6] "GB_GCA_000755365.1_Vibrio sp. C7"                                   
 [7] "RS_GCF_000422585.1_Hugenholtzia roseola DSM 9546"                   
 [8] "RS_GCF_000298175.1_Austwickia chelonae NBRC 105200"                 
 [9] "RS_GCF_000213615.1_Psychrobacter sp. 1501(2011)"                    
[10] "RS_GCF_000203975.2_Leisingera sp. ANG1"<jupyter_text>Summarizing filtered<jupyter_code># number of representatives per species
df_s_f_s = df_s_f %>%
    group_by(ncbi_species_taxid) %>%
    summarize(n=n()) %>%
    ungroup() 

df_s_f_s$n %>% summary %>% print<jupyter_output>Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
      1       1       1       1       1       1<jupyter_text>CheckM<jupyter_code># checkM stats
df_s_f$checkm_completeness %>% summary %>% print
df_s_f$checkm_contamination %>% summary %>% print
# plotting the association between completeness & contamination
p = df_s_f %>%
    ggplot(aes(checkm_completeness, checkm_contamination, color=mimag_high_quality)) +
    geom_point(alpha=0.3) +
    theme_bw()

options(repr.plot.width=5, repr.plot.height=3)
plot(p)<jupyter_output><empty_output><jupyter_text>Assemblies<jupyter_code># plotting the association between completeness & contamination
p = df_s_f %>%
    ggplot(aes(checkm_completeness, contig_count, color=mimag_high_quality)) +
    geom_point(alpha=0.3) +
    theme_bw()

options(repr.plot.width=5, repr.plot.height=3)
plot(p)
# plotting the association between completeness & contamination
p = df_s_f %>%
    ggplot(aes(checkm_completeness, protein_count, color=mimag_high_quality)) +
    geom_point(alpha=0.3) +
    theme_bw()

options(repr.plot.width=5, repr.plot.height=3)
plot(p)
# plotting the association between completeness & contamination
p = df_s_f %>%
    ggplot(aes(checkm_completeness, ssu_count, color=mimag_high_quality)) +
    geom_point(alpha=0.3) +
    theme_bw()

options(repr.plot.width=5, repr.plot.height=3)
plot(p)
df_s_f$ssu_count %>% summary %>% print

p = df_s_f %>%
    ggplot(aes(ssu_count, fill=mimag_high_quality)) +
    geom_histogram(alpha=0.3) +
    scale_x_log10() +
    theme_bw()

options(repr.plot.width=5, repr.plot.height=3)
plot(p)<jupyter_output>Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
  0.000   1.000   1.000   1.664   2.000  54.000<jupyter_text>Taxonomy  Species level<jupyter_code># species level NCBI taxonomy
x = df_s_f %>%
    mutate(SPECIES = gsub('.+;s__', '', ncbi_taxonomy)) %>%
    filter(SPECIES != '') %>%
    .$SPECIES %>% table 

x %>% length %>% print
x %>% sort(decreasing=TRUE) %>% head(n=30)
# species level GTDB taxonomy
x = df_s_f %>%
    mutate(SPECIES = gsub('.+;s__', '', gtdb_taxonomy)) %>%
    filter(SPECIES != '') %>%
    .$SPECIES %>% table 

x %>% length %>% print
x %>% sort(decreasing=TRUE) %>% head(n=30)<jupyter_output>[1] 19854<jupyter_text>Genus level<jupyter_code># genus level NCBI taxonomy
x = df_s_f %>%
    mutate(GENUS = gsub('.+;g__([^;]*);s__.*', '\\1', ncbi_taxonomy)) %>%
    filter(GENUS != '') %>%
    .$GENUS %>% table 

x %>% length %>% print
x %>% sort(decreasing=TRUE) %>% head(n=30)
# genus level GTDB taxonomy
x = df_s_f %>%
    mutate(GENUS = gsub('.+;g__([^;]*);s__.*', '\\1', gtdb_taxonomy)) %>%
    filter(GENUS != '') %>%
    .$GENUS %>% table 

x %>% length %>% print
x %>% sort(decreasing=TRUE) %>% head(n=30)<jupyter_output>[1] 6797<jupyter_text>Writing filtered metadata table<jupyter_code>F = file.path(work_dir, 'metadata_1per-NCBI-Spec.tsv')
write.table(df_s_f, file=F, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', F, '\n')<jupyter_output>File written: /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB//metadata_1per-NCBI-Spec.tsv<jupyter_text>Filtering by CheckM<jupyter_code># all
df_s_f %>% nrow %>% print

# relaxed
df_s_f %>%
    filter(checkm_completeness >= 50,
           checkm_contamination < 10) %>%
    nrow %>% print

# stringent
df_s_f %>%
    filter(checkm_completeness >= 50,
           checkm_contamination < 5) %>%
    nrow %>% print
# stringent 
df_s_ff = df_s_f %>%
    filter(checkm_completeness >= 50,
           checkm_contamination < 5) 

df_s_ff$checkm_completeness %>% summary
df_s_ff$checkm_contamination %>% summary<jupyter_output><empty_output><jupyter_text>Writing filtered metadata table<jupyter_code>F = file.path(work_dir, 'metadata_1per-NCBI-Spec_gte50comp-lt5cont.tsv')
df_s_ff %>%
    write.table(file=F, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', F, '\n')<jupyter_output>File written: /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB//metadata_1per-NCBI-Spec_gte50comp-lt5cont.tsv<jupyter_text>Creating GTDB-based taxdump* creating alternative to NCBI taxdump based on GTDB taxonomy   * needed for LLMGP-DB pipeline and downstream steps (eg., kraken2 & humann2)   * using custom script for creating taxdump  CMD```{bash}(base) @ rick:/ebio/abt3_projects/databases_no-backup/GTDB/release89/taxdump/ebio/abt3_projects/software/dev/gtdb_to_taxdump/gtdb_to_taxdump.py \  https://data.ace.uq.edu.au/public/gtdb/data/releases/release89/89.0/ar122_taxonomy_r89.tsv \  https://data.ace.uq.edu.au/public/gtdb/data/releases/release89/89.0/bac120_taxonomy_r89.tsv \  > taxID_info.tsv```  Adding GTDB taxIDs to metadata<jupyter_code># genome metadata
F = '/ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB/metadata_1per-GTDB-Spec_gte50comp-lt5cont.tsv'
meta = read.delim(F, sep='\t')
meta %>% dim
meta %>% head
# taxdump
F = file.path('/ebio/abt3_projects/databases_no-backup/GTDB/release89/taxdump/',
              'taxID_info.tsv')
taxIDs = read.delim(F, sep='\t') 
taxIDs %>% dim
taxIDs %>% head
# table join
meta = meta %>%
    inner_join(taxIDs %>% dplyr::select(taxID, name) %>% rename('gtdb_taxid' = taxID), 
               c('accession'='name')) 

meta %>% dim
meta %>% head(n=3)<jupyter_output>Warning message:
“Column `accession`/`name` joining factors with different levels, coercing to character vector”<jupyter_text>Writing table<jupyter_code>F = file.path(work_dir, 'metadata_1per-GTDB-Spec_gte50comp-lt5cont_wtaxID.tsv')
meta %>%
    write.table(file=F, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', F, '\n')<jupyter_output>File written: /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB//metadata_1per-GTDB-Spec_gte50comp-lt5cont_wtaxID.tsv<jupyter_text>Genome download ```(llmgp-db) @ rick:/ebio/abt3_projects/software/dev/llmgp-db/tmp$ screen -L -S genome_dl bash -c "../genome_download.R -p 30 -o /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB/genomes/ /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB/metadata_1per-GTDB-Spec_gte50comp-lt5cont_wtaxID.tsv > /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB/genomes/genomes_dl-run1.tsv```<jupyter_code># metadata for all genomes
F = file.path(work_dir, 'metadata_1per-GTDB-Spec_gte50comp-lt5cont_wtaxID.tsv')
df_all = read.delim(F, sep='\t')
df_all %>% dim
# output table that contains metadata + file paths
F = file.path(work_dir, 'genomes', 'genomes_dl-run1.tsv')
df_dl = read.delim(F, sep='\t') 
df_dl %>% dim
# which genomes are missing from the dl table?
df_all %>%
    anti_join(df_dl, c('accession')) %>%
    nrow
# number of genomes lacking genome file
df_dl %>%
    filter(is.na(fasta_file_path)) %>%
    nrow
# filtering
df_dl_f = df_dl %>%
    filter(!is.na(fasta_file_path))

df_dl_f %>% dim<jupyter_output><empty_output><jupyter_text>Writing final table<jupyter_code># final table stats
df_dl_f %>% dim %>% print
df_dl_f %>% head(n=3)
# writing table
F = file.path(work_dir, 'metadata_1per-GTDB-Spec_gte50comp-lt5cont_wtaxID_wPath.tsv')
df_dl_f %>%
    write.table(file=F, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', F, '\n')<jupyter_output>File written: /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB//metadata_1per-GTDB-Spec_gte50comp-lt5cont_wtaxID_wPath.tsv<jupyter_text>Struo pipeline run  config<jupyter_code>system('cat /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB/config.yaml', 
       intern=TRUE) %>% paste(collapse='\n') %>% cat<jupyter_output>#-- I/O --#
# file listing samples and associated data
samples_file: /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB/metadata_1per-GTDB-Spec_gte50comp-lt5cont_wtaxID_wPath.tsv

## column names in samples table
samples_col: 'ncbi_organism_name'
fasta_file_path_col: 'fasta_file_path'
taxID_col: 'gtdb_taxid'
taxonomy_col: 'gtdb_taxonomy'

# output location
output_dir: /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB/

#-- databases to create --#
# Replace "Create" with "Skip" to skip creation of any of these
# Note that braken relies on the kraken2 database
databases:
  kraken2: Create
  bracken: Create
  humann2_bowtie2: Create
  humann2_diamond: Create

#-- keep intermediate files required for re-creating DBs (eg., w/ more genomes) --#
# If "True", the intermediate files are saved to `output_dir`
# Else, the intermediate files are temporarily stored in `temp_folder`
keep_intermediate: True

#-- if custom NCBI taxdump files (or just Skip) --#
names_dmp: /eb[...]<jupyter_text>Run ```(snakemake_dev) @ rick:/ebio/abt3_projects/databases_no-backup/bin/llmgp-db$ screen -L -S llmgp-db-r89 ./snakemake_sge.sh /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB/config.yaml cluster.json /ebio/abt3_projects/databases_no-backup/GTDB/release89/LLMGP-DB/SGE_log 200```  sessionInfo<jupyter_code>sessionInfo()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14279172009525554, 'fraction_numerical': 0.033156255724491664, 'mean_word_length': 4.540979446840903, 'pattern_counts': {'":': 0, '<': 99, '<?xml version=': 0, '>': 371, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '27', 'dir': 'jupyter-structured-clean-dedup', 'id': '126297', 'license': 'MIT', 'n_tokens_mistral': 9185, 'n_tokens_neox': 8274, 'n_words': 1370, 'path': 'notebooks/release89/01_GTDB_metadata_summary.ipynb', 'repo_name': 'leylabmpi/Struo', 'size': '21836'}