# Path: picrust/util.py
# def make_output_dir_for_file(filepath):
#     """Create sub-directories for a new file if they don't already exist"""
#     dirpath = dirname(filepath)
#     if not isdir(dirpath) and not dirpath == '':
#         makedirs(dirpath)
# 
# def file_contains_nulls(file):
#     """Checks given file for null characters. These are sometimes created on SGE clusters when system IO is overloaded."""
# 
#     return '\x00' in open(file,'rb').read()
# 
# Path: picrust/util.py
# def get_picrust_project_dir():
#     """ Returns the top-level PICRUST directory
#     """
#     # Get the full path of util.py
#     current_file_path = abspath(__file__)
#     # Get the directory containing util.py
#     current_dir_path = dirname(current_file_path)
#     # Return the directory containing the directory containing util.py
#     return dirname(current_dir_path)
# 
# def make_output_dir(dirpath, strict=False):
#     """Make an output directory if it doesn't exist
# 
#     Returns the path to the directory
#     dirpath -- a string describing the path to the directory
#     strict -- if True, raise an exception if dir already
#     exists
#     """
#     dirpath = abspath(dirpath)
# 
#     #Check if directory already exists
#     if isdir(dirpath):
#         if strict == True:
#             err_str = "Directory '%s' already exists" % dirpath
#             raise IOError(err_str)
# 
#         return dirpath
#     try:
#         makedirs(dirpath)
#     except IOError,e:
#         err_str = "Could not create directory '%s'. Are permissions set correctly? Got error: '%s'" %e
#         raise IOError(err_str)
# 
#     return dirpath
# 
# Path: picrust/parallel.py
# def submit_jobs(path_to_cluster_jobs, jobs_fp, job_prefix,num_jobs=100,delay=0):
#     """ Submit the jobs to the queue using cluster_jobs.py
#     """
#     cmd = '%s -d %s -n %s -ms %s %s' % (path_to_cluster_jobs, delay, num_jobs, jobs_fp, job_prefix)
#     stdout, stderr, return_value = system_call(cmd)
#     if return_value != 0:
#         msg = "\n\n*** Could not start parallel jobs. \n" +\
#          "Command run was:\n %s\n" % cmd +\
#          "Command returned exit status: %d\n" % return_value +\
#          "Stdout:\n%s\nStderr\n%s\n" % (stdout,stderr)
#         raise RuntimeError, msg
# 
#     # Leave this comments in as they're useful for debugging.
#     # print 'Return value: %d\n' % return_value
#     # print 'STDOUT: %s\n' % stdout
#     # print 'STDERR: %s\n' % stderr
# 
# def wait_for_output_files(files):
#     ''' Function waits until all files exist in the filesystem'''
#     #make a copy of the list
#     waiting_files=list(files)
# 
#     #wait until nothing left in the list
#     while(waiting_files):
#         #wait 30 seconds between each check
#         sleep(30)
#         #check each file and keep ones that don't yet exist
#         waiting_files=filter(lambda x: not exists(x),waiting_files)


from cogent.util.option_parsing import parse_command_line_parameters, make_option
from glob import glob
from picrust.util import make_output_dir_for_file,file_contains_nulls
from cogent.app.util import get_tmp_filename
from picrust.util import get_picrust_project_dir,make_output_dir
from picrust.parallel import submit_jobs, wait_for_output_files
from os.path import join,exists
from os import remove
from re import split


    #get the test datasets to run in the input directory (based on exp_traits files)
    expect_test_files=glob(join(input_dir,'exp_traits--*')) 

    test_datasets={}
    for file_name in expect_test_files:
        test_id=file_name.replace(join(input_dir,'exp_traits--'),'',1)
        #create a dict with the test files as values in the ref list
        test_datasets[test_id]=[ join(input_dir,'test_trait_table--'+test_id),join(input_dir,'test_tree--'+test_id),join(input_dir,'exp_traits--'+test_id)]
    
    created_tmp_files=[]    
    output_files=[]

    #create a tmp file to store the job commands (which we will pass to our parallel script to run)
    jobs_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='jobs_')
    jobs=open(jobs_fp,'w')
    created_tmp_files.append(jobs_fp)

    #get location of scripts we need to run
    asr_script_fp = join(get_picrust_project_dir(),'scripts','ancestral_state_reconstruction.py')
    predict_traits_script_fp = join(get_picrust_project_dir(),'scripts','predict_traits.py')

    #run each test dataset through the pipeline
    for test_id in test_datasets:

        asr_out_fp=join(output_dir,'asr--'+asr_method+'--'+test_id)
        asr_params_out_fp=join(output_dir,'--'.join(['asr',asr_method,'asr_params',test_id]))
        created_tmp_files.append(asr_out_fp)


# Path: picrust/util.py
# def make_output_dir_for_file(filepath):
#     """Create sub-directories for a new file if they don't already exist"""
#     dirpath = dirname(filepath)
#     if not isdir(dirpath) and not dirpath == '':
#         makedirs(dirpath)
# 
# def file_contains_nulls(file):
#     """Checks given file for null characters. These are sometimes created on SGE clusters when system IO is overloaded."""
# 
#     return '\x00' in open(file,'rb').read()
# 
# Path: picrust/util.py
# def get_picrust_project_dir():
#     """ Returns the top-level PICRUST directory
#     """
#     # Get the full path of util.py
#     current_file_path = abspath(__file__)
#     # Get the directory containing util.py
#     current_dir_path = dirname(current_file_path)
#     # Return the directory containing the directory containing util.py
#     return dirname(current_dir_path)
# 
# def make_output_dir(dirpath, strict=False):
#     """Make an output directory if it doesn't exist
# 
#     Returns the path to the directory
#     dirpath -- a string describing the path to the directory
#     strict -- if True, raise an exception if dir already
#     exists
#     """
#     dirpath = abspath(dirpath)
# 
#     #Check if directory already exists
#     if isdir(dirpath):
#         if strict == True:
#             err_str = "Directory '%s' already exists" % dirpath
#             raise IOError(err_str)
# 
#         return dirpath
#     try:
#         makedirs(dirpath)
#     except IOError,e:
#         err_str = "Could not create directory '%s'. Are permissions set correctly? Got error: '%s'" %e
#         raise IOError(err_str)
# 
#     return dirpath
# 
# Path: picrust/parallel.py
# def submit_jobs(path_to_cluster_jobs, jobs_fp, job_prefix,num_jobs=100,delay=0):
#     """ Submit the jobs to the queue using cluster_jobs.py
#     """
#     cmd = '%s -d %s -n %s -ms %s %s' % (path_to_cluster_jobs, delay, num_jobs, jobs_fp, job_prefix)
#     stdout, stderr, return_value = system_call(cmd)
#     if return_value != 0:
#         msg = "\n\n*** Could not start parallel jobs. \n" +\
#          "Command run was:\n %s\n" % cmd +\
#          "Command returned exit status: %d\n" % return_value +\
#          "Stdout:\n%s\nStderr\n%s\n" % (stdout,stderr)
#         raise RuntimeError, msg
# 
#     # Leave this comments in as they're useful for debugging.
#     # print 'Return value: %d\n' % return_value
#     # print 'STDOUT: %s\n' % stdout
#     # print 'STDERR: %s\n' % stderr
# 
# def wait_for_output_files(files):
#     ''' Function waits until all files exist in the filesystem'''
#     #make a copy of the list
#     waiting_files=list(files)
# 
#     #wait until nothing left in the list
#     while(waiting_files):
#         #wait 30 seconds between each check
#         sleep(30)
#         #check each file and keep ones that don't yet exist
#         waiting_files=filter(lambda x: not exists(x),waiting_files)


# Path: scripts/run_genome_evaluations.py
from cogent.util.option_parsing import parse_command_line_parameters, make_option
from glob import glob
from picrust.util import make_output_dir_for_file,file_contains_nulls
from cogent.app.util import get_tmp_filename
from picrust.util import get_picrust_project_dir,make_output_dir
from picrust.parallel import submit_jobs, wait_for_output_files
from os.path import join,exists
from os import remove
from re import split



    #get the test datasets to run in the input directory (based on exp_traits files)
    expect_test_files=glob(join(input_dir,'exp_traits--*')) 

    test_datasets={}
    for file_name in expect_test_files:
        test_id=file_name.replace(join(input_dir,'exp_traits--'),'',1)
        #create a dict with the test files as values in the ref list
        test_datasets[test_id]=[ join(input_dir,'test_trait_table--'+test_id),join(input_dir,'test_tree--'+test_id),join(input_dir,'exp_traits--'+test_id)]
    
    created_tmp_files=[]    
    output_files=[]

    #create a tmp file to store the job commands (which we will pass to our parallel script to run)
    jobs_fp=get_tmp_filename(tmp_dir=tmp_dir,prefix='jobs_')
    jobs=open(jobs_fp,'w')
    created_tmp_files.append(jobs_fp)

    #get location of scripts we need to run
    asr_script_fp = join(get_picrust_project_dir(),'scripts','ancestral_state_reconstruction.py')
    predict_traits_script_fp = join(get_picrust_project_dir(),'scripts','predict_traits.py')

    #run each test dataset through the pipeline
    for test_id in test_datasets:

        asr_out_fp=join(output_dir,'asr--'+asr_method+'--'+test_id)
        asr_params_out_fp=join(output_dir,'--'.join(['asr',asr_method,'asr_params',test_id]))
        created_tmp_files.append(asr_out_fp)


        if opts.check_for_null_files and exists(asr_out_fp) and file_contains_nulls(asr_out_fp):