<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Goal2&nbsp;&nbsp;Var3&nbsp;&nbsp;Init4&nbsp;&nbsp;Load5&nbsp;&nbsp;Running ALE6&nbsp;&nbsp;Summarizing results6.1&nbsp;&nbsp;Writing combined ALE results7&nbsp;&nbsp;-- WAITING --8&nbsp;&nbsp;sessionInfo  Goal* run ALE on simulated test dataset for DeepMAsED   * the ground truth results can be used for selecting cutoffs for ALE and assessing accuracy  Var<jupyter_code>work_dir = '/ebio/abt3_projects/databases_no-backup/DeepMAsED/test_runs/n100_r5/ALE/'

# MAGs for genome analysis
asmbl_dir = '/ebio/abt3_projects/databases_no-backup/DeepMAsED/test_runs/n100_r5/assembly/'

# bam file dir (mapping of metagenome reads to corresponding MAGs)
map_dir = '/ebio/abt3_projects/databases_no-backup/DeepMAsED/test_runs/n100_r5/map/'

# ALE executable
ALE_exe = '/ebio/abt3_projects/databases_no-backup/DeepMAsED/bin/ALE/src/ALE'

# params
conda_env = 'base'
threads = 12<jupyter_output><empty_output><jupyter_text>Init<jupyter_code>library(dplyr)
library(tidyr)
library(ggplot2)
library(data.table)
library(doParallel)

library(future)
library(future.batchtools)
library(future.apply)
options(future.wait.interval = 2.0)<jupyter_output>Attaching package: ‘dplyr’

The following objects are masked from ‘package:stats’:

    filter, lag

The following objects are masked from ‘package:base’:

    intersect, setdiff, setequal, union


Attaching package: ‘data.table’

The following objects are masked from ‘package:dplyr’:

    between, first, last

Loading required package: foreach
Loading required package: iterators
Loading required package: parallel

Attaching package: ‘future.apply’

The following object is masked from ‘package:future’:

    future_lapply<jupyter_text>Load<jupyter_code># adding bam files
fasta_files = list.files(asmbl_dir, '*.fasta$', recursive=TRUE, full.names=TRUE)
fasta_files = data.frame(Assembler = fasta_files %>% dirname %>% basename,
                         Rep = fasta_files %>% dirname %>% dirname %>% basename,
                         Fasta = fasta_files)

fasta_files %>% nrow %>% print
fasta_files %>% head(n=3)
# adding bam files
bam_files = list.files(map_dir, '*.bam$', recursive=TRUE, full.names=TRUE)
bam_files = data.frame(Assembler = bam_files %>% basename %>% gsub('\\.bam$', '', .),
                       Rep = bam_files %>% dirname %>% basename,
                       Bam = bam_files)

bam_files %>% nrow %>% print
bam_files %>% head(n=3)
# joining
MAGs = fasta_files %>%
    inner_join(bam_files, c('Assembler', 'Rep'))
MAGs %>% nrow %>% print
MAGs %>% head(n=3)
# ALE output file name
MAGs = MAGs %>%
    mutate(ALE = mapply(function(A, R) file.path(work_dir, R, paste0(A, '.txt')),
                        A=Assembler, R=Rep))
                        
MAGs %>% head(n=3)
# making output directories
ret = MAGs$ALE %>% 
    dirname() %>%
    unique() %>%
    sapply(., function(x) dir.create(x, showWarnings=FALSE))<jupyter_output><empty_output><jupyter_text>Running ALE<jupyter_code># cluster resources
resources = list(h_rt = '00:59:00',
                 h_vmem = '8G',
                 threads = '1',
                 conda.env = 'py3_physeq')     # conda env with batchtools installed
plan(batchtools_sge, resources=resources, workers=30)
# creating commands for calling ALE 
ALE_params = function(x, exe){
    cmd = '{exe} {bam} {fasta} {outfile}'
    cmd = glue::glue(cmd, exe=exe, bam=x[['Bam']], fasta=x[['Fasta']], outfile=x[['ALE']])
    return(cmd)
}

ALE_cmds = apply(MAGs, 1, ALE_params, exe=ALE_exe) %>%
    as.list

ALE_cmds %>% length %>% print
ALE_cmds[[1]]
# general bash job function
bash_job = function(cmd, conda_env, stdout=TRUE, stderr=FALSE){
    # cmd : string; commandline job (eg., 'ls -thlc')
    # conda_env : string; conda environment name
    cmd = sprintf('. ~/.bashrc; conda activate %s; %s', conda_env, cmd)
    cmd = sprintf('-c "%s"', cmd)
    system2('bash', cmd, stdout=stdout, stderr=stderr)
}
# apply function on cluster
job_ret = future_lapply(ALE_cmds, FUN = function(x) bash_job(x, conda_env=conda_env))
job_ret<jupyter_output><empty_output><jupyter_text>Summarizing results<jupyter_code>ALE_res_files = list.files(work_dir, '*.txt$', recursive=TRUE, full.names=TRUE)
ALE_res_files %>% length %>% print
ALE_res_files %>% head
read_ALE_files = function(in_file){
    # input command
    cmd = 'egrep -v "^#" {in_file}'
    cmd = glue::glue(cmd, in_file=in_file)
    # reading in table
    df = fread(cmd, sep=' ', header=FALSE)
    # formatting
    colnames(df) = c('contig', 'position', 'depth', 
                     'ln__depthLike', 'ln__placeLike', 
                     'ln__insertLike', 'ln__kmerLike')
    df$Assembler = in_file %>% basename %>% gsub('\\.txt$', '', .)
    df$Rep = in_file %>% dirname %>% basename
    return(df)
}
doParallel::registerDoParallel(threads)
ALE_res = plyr::llply(as.list(ALE_res_files), read_ALE_files, .parallel=TRUE) %>%
    do.call(rbind, .)

ALE_res %>% nrow %>% print
ALE_res %>% head(n=3)<jupyter_output>[1] 295103567<jupyter_text>Writing combined ALE results<jupyter_code>out_file = file.path(work_dir, 'ALE_results.tsv')
ALE_res %>%
    write.table(file=out_file, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', out_file, '\n')<jupyter_output>File written: /ebio/abt3_projects/databases_no-backup/DeepMAsED/test_runs/n100_r5/ALE//ALE_results.tsv<jupyter_text>sessionInfo<jupyter_code>sessionInfo()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1448529411764706, 'fraction_numerical': 0.01213235294117647, 'mean_word_length': 4.446446446446447, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '193663', 'license': 'MIT', 'n_tokens_mistral': 2054, 'n_tokens_neox': 1945, 'n_words': 391, 'path': 'notebooks/03_stateOfTheArt/02_ALE/02a_ALE_on_sim-test_r5.ipynb', 'repo_name': 'chrisLanderson/DeepMAsED', 'size': '5440'}