<jupyter_start><jupyter_text>DeepBiome: MetagenomicsAug. 30. 2020@ Youngwon (<EMAIL>)<jupyter_code>import os
import json
import numpy as np
import pandas as pd
import copy
import logging
import sys

import keras.backend as k
import tensorflow as tf

import matplotlib.pyplot as plt
%matplotlib inline

os.environ['CUDA_VISIBLE_DEVICES']=''
from deepbiome.deepbiome import *
if not tf.__version__.startswith('2'): 
    config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))
    k.set_session(tf.Session(config=config))<jupyter_output><empty_output><jupyter_text>Pick Models<jupyter_code>save = False
kfold=5
# kfold=20
network_model_keys = ['optimizer','lr','decay']
architecture_keys = ['weight_decay', 'weight_l1_penalty', #'weight_l2_penalty',
                     'tree_thrd', 'weight_initial',
                     'batch_normalization','drop_out']
network_training_keys = ['batch_size','epochs']

logging.basicConfig(format = '[%(name)-8s|%(levelname)s|%(filename)s:%(lineno)s] %(message)s',
                    level=logging.DEBUG)
log = logging.getLogger()
filenames = 'simulation_metagenomics.Rmd'
models = [
          'realdata_metagenomics/t2d_deep',
          'realdata_metagenomics/t2d_deep_l1',
          'realdata_metagenomics/t2d_deepbiome',
         ]

models_aka = [
          'DNN',
          'DNN+$\ell_1$',
          'DeepBiome',
         ]
num_classes = 1
model_network_info = {}
model_path_info = {}
for model_path in models:
    config_data = configuration.Configurator('%s/config/path_info.cfg' % model_path, log, verbose=False)
    config_data.set_config_map(config_data.get_section_map())
    config_network = configuration.Configurator('%s/config/network_info.cfg' % model_path, log, verbose=False)
    config_network.set_config_map(config_network.get_section_map())

    model_path_info[model_path] = config_data.get_config_map()
    model_network_info[model_path] = config_network.get_config_map()
    
if num_classes == 0: y_names = ['loss','correlation_coefficient']
elif num_classes==1: y_names = ['loss','binary_accuracy','sensitivity','specificity','gmeasure', 'auc']
else: y_names=['loss','categorical_accuracy','precision','recall','f1', 'auc']

if num_classes == 0: measure_index = np.array([0,1])
elif num_classes==1: measure_index = np.array([2,3,4,1,5])
else: measure_index = np.array([1,2,3,4,5])<jupyter_output><empty_output><jupyter_text>Accuracy<jupyter_code>results = []
# log.info('%20s & %s' % ('model', '& '.join(['%s ' % name for name in np.array(y_names)[[measure_index]]])))
print('%10s & %s \\\\\ \hline' % ('model', '& '.join(['%7s &   (sd) ' % name for name in np.array(y_names)[[measure_index]]])))
# for model, aka in zip(models, models_aka):
#     evaluation = np.load('%s/eval.npy' % model)
#     log.info('%20s: %s' % (aka, ''.join(['%10.4f (%10.4f)'%(mean, std) for mean, std in zip(np.mean(evaluation, axis=0),np.std(evaluation, axis=0))])))
#     results.append(np.vstack([np.mean(evaluation, axis=0),np.std(evaluation, axis=0)]).transpose())
for model, aka in zip(models, models_aka):
    train_evaluation = np.load('%s/train_eval.npy' % model)[:,measure_index]
    train_res = '&'.join(['%7.3f & %7.3f'%(mean, std) for mean, std in zip(np.nanmean(train_evaluation, axis=0),np.nanstd(train_evaluation, axis=0))])
    test_evaluation = np.load('%s/test_eval.npy' % model)[:,measure_index]
    test_res = '&'.join(['%7.3f & %7.3f'%(mean, std) for mean, std in zip(np.nanmean(test_evaluation, axis=0),np.nanstd(test_evaluation, axis=0))])
#     log.info('%s & %s & %s \\\\' % (aka, train_res, test_res))
    print('%10s & %s & %s \\\\' % (aka, test_res, train_res))
#     results.append(np.vstack([np.mean(evaluation, axis=0),np.std(evaluation, axis=0)]).transpose())
for model, aka in zip(models, models_aka):
    print('---------------------------------------------------------------------------------------------------------')
    print('%14s test' % aka)
    print('---------------------------------------------------------------------------------------------------------')
    test_evaluation = np.load('%s/test_eval.npy' % model)[:,measure_index]
    print('      %s' % ''.join(['%16s'%s.strip() for s in np.array(y_names)[[measure_index]]]))
    print('Mean: %s' % ''.join(['%16.4f'%v for v in np.mean(test_evaluation, axis=0)]))
    print('Std : %s' % ''.join(['%16.4f'%v for v in np.std(test_evaluation, axis=0)]))<jupyter_output>---------------------------------------------------------------------------------------------------------
           DNN test
---------------------------------------------------------------------------------------------------------
           sensitivity     specificity        gmeasure binary_accuracy             auc
Mean:           0.5777          0.5465          0.3914          0.5318          0.6804
Std :           0.3889          0.3842          0.1192          0.0411          0.0366
---------------------------------------------------------------------------------------------------------
  DNN+$\ell_1$ test
---------------------------------------------------------------------------------------------------------
           sensitivity     specificity        gmeasure binary_accuracy             auc
Mean:           0.7035          0.4968          0.5233          0.5900          0.6808
Std :           0.2619          0.2780          0.0606          0.0402          0.0450
--------------[...]<jupyter_text>Weight estimation of DeepBiom  DNN<jupyter_code>num=0
model_path = models[num]
model_aka = models_aka[num]

config_data = configuration.Configurator('%s/config/path_info.cfg' % model_path, log, verbose=False)
config_data.set_config_map(config_data.get_section_map())
config_network = configuration.Configurator('%s/config/network_info.cfg' % model_path, log, verbose=False)
config_network.set_config_map(config_network.get_section_map())

path_info = config_data.get_config_map()
network_info = config_network.get_config_map()

path_info['data_info']['data_path'] = '/'.join(path_info['data_info']['data_path'].split('/')[2:])
path_info['data_info']['tree_info_path'] = '/'.join(path_info['data_info']['tree_info_path'].split('/')[2:])
try: path_info['data_info']['count_list_path'] = '/'.join(path_info['data_info']['count_list_path'].split('/')[2:])
except: pass
try: path_info['data_info']['count_path'] = '/'.join(path_info['data_info']['count_path'].split('/')[2:])
except: pass
path_info['data_info']['idx_path'] = '/'.join(path_info['data_info']['idx_path'].split('/')[2:])

path_info['model_info']['model_dir'] = './%s/%s'%(model_path,path_info['model_info']['model_dir'])

log.info('%22s : %s' % ('model', model_path))
log.info('%22s : %s' % ('model_aka', model_aka))
for k in architecture_keys:
    log.info('%22s : %s' % (k, network_info['architecture_info'].get(k, None)))
for k in network_model_keys:
    log.info('%22s : %s' % (k, network_info['model_info'].get(k, None)))
for k in network_training_keys:
    log.info('%22s : %s' % (k, network_info['training_info'].get(k, None)))
weight_path = '%s/weight/%s' % (path_info['model_info']['model_dir'], 'weight_0.h5')
trained_weight_list = deepbiome_get_trained_weight(log, network_info, path_info, num_classes=1, weight_path=weight_path,
                                                  tree_level_list=['Species','Genus', 'Family', 'Order', 'Class', 'Phylum'])
log.info(len(trained_weight_list))
log.info(trained_weight_list[0].shape)
trained_weight_list[0]<jupyter_output><empty_output><jupyter_text>DNN + $\ell_1$<jupyter_code>num=1
model_path = models[num]
model_aka = models_aka[num]

config_data = configuration.Configurator('%s/config/path_info.cfg' % model_path, log, verbose=False)
config_data.set_config_map(config_data.get_section_map())
config_network = configuration.Configurator('%s/config/network_info.cfg' % model_path, log, verbose=False)
config_network.set_config_map(config_network.get_section_map())

path_info = config_data.get_config_map()
network_info = config_network.get_config_map()

path_info['data_info']['data_path'] = '/'.join(path_info['data_info']['data_path'].split('/')[2:])
path_info['data_info']['tree_info_path'] = '/'.join(path_info['data_info']['tree_info_path'].split('/')[2:])
try: path_info['data_info']['count_list_path'] = '/'.join(path_info['data_info']['count_list_path'].split('/')[2:])
except: pass
try: path_info['data_info']['count_path'] = '/'.join(path_info['data_info']['count_path'].split('/')[2:])
except: pass
path_info['data_info']['idx_path'] = '/'.join(path_info['data_info']['idx_path'].split('/')[2:])

path_info['model_info']['model_dir'] = './%s/%s'%(model_path,path_info['model_info']['model_dir'])

log.info('%22s : %s' % ('model', model_path))
log.info('%22s : %s' % ('model_aka', model_aka))
for k in architecture_keys:
    log.info('%22s : %s' % (k, network_info['architecture_info'].get(k, None)))
for k in network_model_keys:
    log.info('%22s : %s' % (k, network_info['model_info'].get(k, None)))
for k in network_training_keys:
    log.info('%22s : %s' % (k, network_info['training_info'].get(k, None)))
weight_path = '%s/weight/%s' % (path_info['model_info']['model_dir'], 'weight_0.h5')
trained_weight_list = deepbiome_get_trained_weight(log, network_info, path_info, num_classes=1, weight_path=weight_path,
                                                  tree_level_list=['Species','Genus', 'Family', 'Order', 'Class', 'Phylum'])
log.info(len(trained_weight_list))
log.info(trained_weight_list[0].shape)
trained_weight_list[0]<jupyter_output><empty_output><jupyter_text>DeepBiome<jupyter_code>num=2
model_path = models[num]
model_aka = models_aka[num]

config_data = configuration.Configurator('%s/config/path_info.cfg' % model_path, log, verbose=False)
config_data.set_config_map(config_data.get_section_map())
config_network = configuration.Configurator('%s/config/network_info.cfg' % model_path, log, verbose=False)
config_network.set_config_map(config_network.get_section_map())

path_info = config_data.get_config_map()
network_info = config_network.get_config_map()

path_info['data_info']['data_path'] = '/'.join(path_info['data_info']['data_path'].split('/')[2:])
path_info['data_info']['tree_info_path'] = '/'.join(path_info['data_info']['tree_info_path'].split('/')[2:])
try: path_info['data_info']['count_list_path'] = '/'.join(path_info['data_info']['count_list_path'].split('/')[2:])
except: pass
try: path_info['data_info']['count_path'] = '/'.join(path_info['data_info']['count_path'].split('/')[2:])
except: pass
path_info['data_info']['idx_path'] = '/'.join(path_info['data_info']['idx_path'].split('/')[2:])

path_info['model_info']['model_dir'] = './%s/%s'%(model_path,path_info['model_info']['model_dir'])

log.info('%22s : %s' % ('model', model_path))
log.info('%22s : %s' % ('model_aka', model_aka))
for k in architecture_keys:
    log.info('%22s : %s' % (k, network_info['architecture_info'].get(k, None)))
for k in network_model_keys:
    log.info('%22s : %s' % (k, network_info['model_info'].get(k, None)))
for k in network_training_keys:
    log.info('%22s : %s' % (k, network_info['training_info'].get(k, None)))
weight_path = '%s/weight/%s' % (path_info['model_info']['model_dir'], 'weight_0.h5')
trained_weight_list = deepbiome_get_trained_weight(log, network_info, path_info, num_classes=1, weight_path=weight_path,
                                                  tree_level_list=['Species','Genus', 'Family', 'Order', 'Class', 'Phylum'])
log.info(len(trained_weight_list))
log.info(trained_weight_list[0].shape)
trained_weight_list[0]
tot_trained_weight_list = []
for fold in range(kfold):
    weight_path = '%s/weight/%s' % (path_info['model_info']['model_dir'], 'weight_%d.h5' % fold)
    trained_weight_list = deepbiome_get_trained_weight(log, network_info, path_info, num_classes=1,
                                                       weight_path=weight_path,
                                                       tree_level_list=['Species','Genus', 'Family', 'Order', 'Class', 'Phylum'],
                                                       verbose=False)
    tot_trained_weight_list.append(trained_weight_list)
    
trained_weight_list = []
for i in range(len(tot_trained_weight_list[0])):
    level_weights = tot_trained_weight_list[0][i]
    for j in range(1,len(tot_trained_weight_list)):
        level_weights +=tot_trained_weight_list[j][i]
    level_weights /= len(tot_trained_weight_list)
    trained_weight_list.append(level_weights)
phylum_color = ['lemonchiffon',
                 'lightsteelblue',
                 'moccasin',
                 'darkseagreen',
                 'khaki',
                 'mediumturquoise',
                 'thistle',
                 'tan',
                 'mistyrose',
                 'lavender',
                 'teal',
                 'plum',
                 'cyan',
                 'orange',
                 'lightpink',
                 'violet',
                 'hotpink',
                 'deepskyblue',
#                  'lightpink',
#                  'violet',
#                  'hotpink',
#                  'deepskyblue'
                ]
len(phylum_color)
# img = deepbiome_draw_phylogenetic_tree(log, network_info, path_info, num_classes=1,
#                                      file_name='%%inline', img_w=1000,
#                                      branch_vertical_margin=10, arc_start=0, arc_span=360,
#                                      node_name_on=True, name_fsize=50,
#                                      tree_weight_on=True, tree_weight=trained_weight_list,
#                                      tree_level_list=['Species', 'Genus', 'Family', 'Order', 'Class', 'Phylum'],
#                                      weight_opacity=0.8, weight_max_radios=30,
#                                      phylum_background_color_on=True, phylum_color=phylum_color, phylum_color_legend=True,
#                                      show_covariates=False,
#                                      verbose=True)
# img
img = deepbiome_draw_phylogenetic_tree(log, network_info, path_info, num_classes=1,
                                     file_name='%%inline', img_w=1000,
                                     branch_vertical_margin=10, arc_start=0, arc_span=360,
                                     node_name_on=True, name_fsize=50,
                                     tree_weight_on=True, tree_weight=trained_weight_list[4:],
                                     tree_level_list = ['Class', 'Phylum', 'Disease'],
                                     weight_opacity=0.8, weight_max_radios=30,
                                     phylum_background_color_on=True, phylum_color=phylum_color, phylum_color_legend=True,
                                     show_covariates=False,
                                     verbose=True)
img<jupyter_output>[root    |INFO|readers.py:58] -----------------------------------------------------------------------
[root    |INFO|readers.py:59] Construct Dataset
[root    |INFO|readers.py:60] -----------------------------------------------------------------------
[root    |INFO|readers.py:61] Load data
{'fraction_non_alphanumeric': 0.20846280991735539, 'fraction_numerical': 0.01943801652892562, 'mean_word_length': 2.8119959677419355, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '516498', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 5093, 'n_tokens_neox': 4894, 'n_words': 703, 'path': 'examples/analysis-metagenomics.ipynb', 'repo_name': 'Young-won/deepbiome', 'size': '15138'}