# Path: picrust/evaluate_test_datasets.py
# def update_pooled_data(obs_table,exp_table,tags,pooled_observations,\
#    pooled_expectations,new_unique_id,verbose=False):
#     """Update pooled data using an observed and expected table, tags, and a dict of pooled_observations"""
# 
#     #Update the global pooled obs,exp tables with the current results
#     #Format results for printing
#     #NOTE: currently merged values actually add value into a single column rather than creating new columns for each
#     for tag in tags:
#         #pooled entries will either be empty or be a .BIOM table
#         if pooled_observations.get(tag,None) is None:
#             pooled_observations[tag] = obs_table
#             pooled_expectations[tag] = exp_table
#         else:
#             #Entry should already be a table.  So we want to update it by merging in
#             #the current result
#             old_obs_table = pooled_observations[tag]
# 
#             pooled_observations[tag]=\
#                 merge_by_column_union(old_obs_table,obs_table,new_unique_id)
# 
#             old_exp_table = pooled_expectations[tag]
#             pooled_expectations[tag]=\
#                 merge_by_column_union(old_exp_table,exp_table,new_unique_id)
# 
#     return pooled_observations,pooled_expectations


from os import listdir
from os.path import join
from cogent.util.option_parsing import parse_command_line_parameters,\
  make_option
from picrust.evaluate_test_datasets import update_pooled_data
from biom import load_table
    #TODO: separate out into a 'get_paired_data_from_dirs' function

    pooled_observations = {}
    pooled_expectations = {}
    pairs = iter_prediction_expectation_pairs(obs_dir_fp,exp_dir_fp,file_name_field_order,file_name_delimiter,verbose=verbose)
    file_number = 0
    for obs_table,exp_table,filename in pairs:
        #print "analyzing filename:",filename
        filename_metadata= get_metadata_from_filename(filename,file_name_field_order,\
          file_name_delimiter,verbose=verbose)

        #base_tag =  '%s\t%s\t' %(filename_metadata['holdout_method'],filename_metadata['prediction_method'])
        #tags = [base_tag+'all_results']
        if 'file_type' in pool_by:
            pool_by.remove('file_type') #we do this manually at the end
        combined_tag = ['all']*len(file_name_field_order.keys())
        for field in file_name_field_order.keys():
          #print combined_tag
          #print file_name_field_order
          idx = file_name_field_order[field]
          #print idx
          if field in pool_by:
              combined_tag[idx] = filename_metadata[field]
        tags=[file_name_delimiter.join(combined_tag)]

        if verbose:
          print "Pooling by:", pool_by
          print "Combined tags:",tags

        pooled_observations,pooled_expectations =\

# Path: picrust/evaluate_test_datasets.py
# def update_pooled_data(obs_table,exp_table,tags,pooled_observations,\
#    pooled_expectations,new_unique_id,verbose=False):
#     """Update pooled data using an observed and expected table, tags, and a dict of pooled_observations"""
# 
#     #Update the global pooled obs,exp tables with the current results
#     #Format results for printing
#     #NOTE: currently merged values actually add value into a single column rather than creating new columns for each
#     for tag in tags:
#         #pooled entries will either be empty or be a .BIOM table
#         if pooled_observations.get(tag,None) is None:
#             pooled_observations[tag] = obs_table
#             pooled_expectations[tag] = exp_table
#         else:
#             #Entry should already be a table.  So we want to update it by merging in
#             #the current result
#             old_obs_table = pooled_observations[tag]
# 
#             pooled_observations[tag]=\
#                 merge_by_column_union(old_obs_table,obs_table,new_unique_id)
# 
#             old_exp_table = pooled_expectations[tag]
#             pooled_expectations[tag]=\
#                 merge_by_column_union(old_exp_table,exp_table,new_unique_id)
# 
#     return pooled_observations,pooled_expectations


# Path: scripts/pool_test_datasets.py
from os import listdir
from os.path import join
from cogent.util.option_parsing import parse_command_line_parameters,\
  make_option
from picrust.evaluate_test_datasets import update_pooled_data
from biom import load_table

    #TODO: separate out into a 'get_paired_data_from_dirs' function

    pooled_observations = {}
    pooled_expectations = {}
    pairs = iter_prediction_expectation_pairs(obs_dir_fp,exp_dir_fp,file_name_field_order,file_name_delimiter,verbose=verbose)
    file_number = 0
    for obs_table,exp_table,filename in pairs:
        #print "analyzing filename:",filename
        filename_metadata= get_metadata_from_filename(filename,file_name_field_order,\
          file_name_delimiter,verbose=verbose)

        #base_tag =  '%s\t%s\t' %(filename_metadata['holdout_method'],filename_metadata['prediction_method'])
        #tags = [base_tag+'all_results']
        if 'file_type' in pool_by:
            pool_by.remove('file_type') #we do this manually at the end
        combined_tag = ['all']*len(file_name_field_order.keys())
        for field in file_name_field_order.keys():
          #print combined_tag
          #print file_name_field_order
          idx = file_name_field_order[field]
          #print idx
          if field in pool_by:
              combined_tag[idx] = filename_metadata[field]
        tags=[file_name_delimiter.join(combined_tag)]

        if verbose:
          print "Pooling by:", pool_by
          print "Combined tags:",tags

        pooled_observations,pooled_expectations =\

        update_pooled_data(obs_table,exp_table,tags,pooled_observations,\