<jupyter_start><jupyter_text>Goal: * Download most up-to-date version of NCBI 'complete' genomes  Setting variables<jupyter_code>workDir = '/var/seq_data/ncbi_db/genome/Jan2016/'
proksFile = 'proks_complete.txt'
taxFile = 'proks_complete_tax.txt'<jupyter_output><empty_output><jupyter_text>Init<jupyter_code>import os
%load_ext rpy2.ipython
%load_ext pushnote
%%R
library(ggplot2)
library(dplyr)
library(tidyr)
library(genomes)
if not os.path.isdir(workDir):
    os.makedirs(workDir)

%cd $workDir<jupyter_output>/var/seq_data/ncbi_db/genome/Jan2016<jupyter_text>Loading list of complete prok genomes<jupyter_code>%%R -i workDir -i proksFile 

F = file.path(workDir, proksFile)

df.proks.complete = read.delim(F, sep='\t')

# checking join
df.proks.complete %>% nrow %>% print
df.proks.complete %>% head(n=3)
%%R -i workDir -i taxFile

F = file.path(workDir, taxFile)

df.tax = read.delim(F, sep='\t') %>%
    distinct(taxid)
df.proks.complete = dplyr::inner_join(df.proks.complete, df.tax, c('taxid' = 'taxid'))

# checking join
df.proks.complete %>% nrow %>% print
df.proks.complete %>% nrow %>% print
df.proks.complete %>% head(n=3)<jupyter_output><empty_output><jupyter_text>Just Bacteria<jupyter_code>%%R
df.bac.complete = df.proks.complete %>%
    filter(superkingdom == 'Bacteria')

df.bac.complete %>% nrow<jupyter_output><empty_output><jupyter_text>Phylum representation<jupyter_code>%%R -w 800
df.bac.complete.s = df.bac.complete %>%
    group_by(phylum) %>%
    summarize(n = n()) %>%
    filter(! is.na(n), n > 0)

ggplot(df.bac.complete.s, aes(phylum, n)) +
    geom_bar(stat='identity') +
    scale_y_log10() +
    labs(y = 'Number of genomes') +
    theme_bw() +
    theme(
        text = element_text(size=16),
        axis.text.x = element_text(angle=60, hjust=1)
    )<jupyter_output><empty_output><jupyter_text>removing what are really phage/plasmid genomes<jupyter_code>%%R
cat('Pre-filter:', df.bac.complete %>% nrow, '\n')

to.rm = c("Thermoanaerobacterium saccharolyticum JW/SL-YS485",
          "Streptococcus salivarius 57.I")

df.bac.complete = df.bac.complete %>%
    filter(! name %in% to.rm)

cat('Post-filter:', df.bac.complete %>% nrow, '\n')<jupyter_output><empty_output><jupyter_text>Sequence download<jupyter_code>%%R -i workDir

outFile = file.path(workDir, 'bac_complete.txt')
write.table(df.bac.complete, outFile, sep='\t', quote=FALSE, row.names=FALSE)
!seqDB_tools accession-GI2fasta \
    -a 11 -n 2 -f 12 -header -o bac_complete \
    < bac_complete.txt \
    2> bac_complete.log
%pushnote genome download complete<jupyter_output><empty_output><jupyter_text>Getting list of empty genome files<jupyter_code>fileSizes = !ls -tlc *.fna | perl -pe 's/[ \t]+/ /g' 

outFile = 'empty_genome_files.txt'
with open(outFile, 'wb') as outFH:
    for x in fileSizes:
        xx = x.split(' ')
        if xx[4] == '0':
            xx[-1] = xx[-1].replace('_', ' ').rstrip('.fna')
            outFH.write(xx[-1] + '\n')

# status
!printf 'Number of empty genome files: '     
!wc -l $outFile
!head $outFile<jupyter_output>Number of empty genome files: 13 empty_genome_files.txt
Proteus mirabilis
Pseudomonas chlororaphis subsp aurantiac
Pseudomonas putida BIRD-1
Pseudothermotoga elfii DSM 9442 NBRC 107921
Pseudomonadaceae bacterium B4199
Pseudomonas syringae pv syringae B728
Pseudomonas stutzeri DSM 4166
Pseudomonas sp CCOS 191
Pseudomonas aeruginosa PA1R
Pusillimonas sp T7-7<jupyter_text>Deleting empty files<jupyter_code>fileSizes = !ls -tlc bac_complete/*.fna | perl -pe 's/[ \t]+/ /g' 

for x in fileSizes:
    xx = x.split(' ')
    if float(xx[4]) < 100000.0:
        os.remove(xx[-1])<jupyter_output><empty_output><jupyter_text>Checking output<jupyter_code>genomeDir = os.path.join(workDir, 'bac_complete')
%cd $genomeDir
# number of genomes downloaded
!printf "Number of bacterial genomes: "
!find . -name "*.fna" | wc -l    
# file size
!echo "Genome file size distribution (bytes):"
!ls -tlc *.fna | \
    perl -pe 's/ +/\t/g' | \
    cut -f 5 | NY_misc_perl stats_descriptive
# checking for non-bacterial genomes
!find . -name "*fna" | xargs -P 20 egrep "phage|virus|phage"
# deleting non-bacterial genomes
!rm -f ./Clostridium_perfringens_SM101.fna \
    ./Chlamydophila_pneumoniae_AR39.fna \
    ./Enterococcus_faecalis_62.fna
# number of genomes downloaded
!printf "Number of bacterial genomes: "
!find . -name "*.fna" | wc -l<jupyter_output><empty_output><jupyter_text>Renaming genomes<jupyter_code>genomeDirRn = genomeDir + '_rn'
genomeDirRn
# renameing
!find . -name "*.fna" | \
    SIPSim genome_rename -n 26 --prefix $genomeDirRn -<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14638906961613532, 'fraction_numerical': 0.01973541531121232, 'mean_word_length': 4.4904761904761905, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '202306', 'license': 'MIT', 'n_tokens_mistral': 1954, 'n_tokens_neox': 1774, 'n_words': 369, 'path': 'ipynb/bac_genome/OTU-level_variability/.ipynb_checkpoints/p1_NCBI_complete_genome_download-checkpoint.ipynb', 'repo_name': 'arischwartz/test', 'size': '4611'}