<jupyter_start><jupyter_text>Required LibrariesUsing `pandas` to read and process tabular data and `missingno` to visualize missing data.<jupyter_code>from data import load_data

# loading FISH makers, gene expressions
_, fish, genes, _, _ = load_data()

fish.head()<jupyter_output><empty_output><jupyter_text>Filtering Fish VariablesBelow in fit models to estimate fish variables from gene counts. We consider X fish variables: 13q14, 13q34, 17p13, 1q21, 11p15, 15q15, 19q13, 20q13, 21q22, 3q21, 5q31, 7q22, 9q33, t_11_14_ccnd1, t_12_14_ccnd2, t_14_16_maf, t_14_20_mafb, t_4_14_whsc1, t_6_14_ccnd3, t_8_14_mafa, t_8_14_myc, and hyperdiploid_flag.<jupyter_code>%matplotlib inline

import numpy as np
import missingno as msn

msn.matrix(fish);<jupyter_output><empty_output><jupyter_text>Fitting Models and Complete Missing Values<jupyter_code>from sklearn.metrics import roc_auc_score, log_loss
from sklearn.model_selection import StratifiedKFold

from correlation import select_genes
from evaluation import optimize_threshold
from constants import N_FOLDS, RANDOM_STATE

from optimization import LightGBMOptimizer

from lightgbm import LGBMModel

import pandas as pd
import pickle
import time
import os

result = {c: [] for c in ['experiment', 'marker', 'train_auc', 'valid_auc', 'train_loss', 'valid_loss', 'execution_time']}

# removing fish markers with less than
# five samples in class 1

for m in ['t_12_14_ccnd2', 't_14_20_mafb', 't_6_14_ccnd3', 't_8_14_mafa']:
    if m in fish:
        del fish[m]

for t in fish.columns:
    
    print('*********************************************************************************')
    print(t)
    print('*********************************************************************************\n')
    
    y = fish[t].dropna()
    
    x = genes.join(y, how='inner')[genes.columns]
    
    opt_kf = StratifiedKFold(N_FOLDS, shuffle=True, random_state=RANDOM_STATE)

    for experiment, (opt_train, opt_valid) in enumerate(opt_kf.split(x, y)):
        
        initial_time = time.time()
        
        #################################################################################################
        # Train and Valid Split
        #################################################################################################
        
        x_train, y_train = x.iloc[opt_train,:], y.iloc[opt_train]
        
        x_valid, y_valid = x.iloc[opt_valid,:], y.iloc[opt_valid]

        #################################################################################################
        # Gene Selection
        #################################################################################################
        
        selected_genes = select_genes(x_train, y_train, threshold=0.00002)
        
        x_train, x_valid = x_train[selected_genes[0]], x_valid[selected_genes[0]]
        
        #################################################################################################
        # Export selected genes
        #################################################################################################
        
        filepath = './output/fish/{}/selected_genes_experiment_{}.csv'.format(t, experiment)
        try:
            os.makedirs(os.path.dirname(os.path.abspath(filepath)))
        except FileExistsError:
            pass
        
        df = pd.DataFrame({'gene': selected_genes[0], 'pvalue': selected_genes[1], 'entropy': selected_genes[2]})
        
        df.to_csv(filepath, sep=',', index=False)
        
        del df
        
        del filepath
        
        #################################################################################################
        # Hyper parameters optimization
        #################################################################################################        
        
        training_default_params = {
            'objective': 'binary',
            'metric': 'binary_logloss',
            'n_estimators': 100,
            'objective': 'binary',
            'is_unbalance': False, 
            'extra_trees': True,
            'max_depth': 4,
            'learning_rate': 0.1,
            'min_split_gain': 0.0001,
            'min_child_weight': 0.0001,
            'random_state': RANDOM_STATE}
        
        optimizer_params = {
            'n_folds': 2, 
            'n_calls': 50, 
            'shuffle': True, 
            'early_stopping_rounds': 1,
            'fixed_parameters': training_default_params, 
            'random_state': RANDOM_STATE, 
            'use_gpu': False}
        
        optimizer = LightGBMOptimizer(**optimizer_params)
        
        params = optimizer.optimize(x_train, y_train)

        params = {**params, **training_default_params}
        
        skf = StratifiedKFold(3, shuffle=True, random_state=RANDOM_STATE)

        models = []
        
        for train_index, valid_index in skf.split(x_train, y_train):

            xx_train, yy_train = x_train.values[train_index, :], y_train.values[train_index]
            xx_valid, yy_valid = x_train.values[valid_index, :], y_train.values[valid_index]

            gbm = LGBMModel(**params)

            gbm.fit(xx_train, yy_train,
                    eval_set=[(xx_valid, yy_valid)],
                    early_stopping_rounds=1,
                    verbose=False)
            
            models.append(gbm)
        
        #################################################################################################
        # Predicting
        #################################################################################################        
        
        y_hat_train, y_hat_valid = None, None
        
        for model in models:
            
            y_ = model.predict(x_train)
            y_hat_train = y_ if y_hat_train is None else y_ + y_hat_train
            
            y_ = model.predict(x_valid)
            y_hat_valid = y_ if y_hat_valid is None else y_ + y_hat_valid
        
        y_hat_train /= len(models)
        y_hat_valid /= len(models)
        
        train_auc = roc_auc_score(y_train, y_hat_train)
        valid_auc = roc_auc_score(y_valid, y_hat_valid)
        
        train_loss = log_loss(y_train, y_hat_train)
        valid_loss = log_loss(y_valid, y_hat_valid)
        
        result['experiment'].append(experiment)
        result['marker'].append(t)
        result['train_auc'].append(train_auc)
        result['valid_auc'].append(valid_auc)
        result['train_loss'].append(train_loss)
        result['valid_loss'].append(valid_loss)
        result['execution_time'].append(time.time() - initial_time)
        
        log_message = 'Experiment #{}: '.format(experiment) + 'Train AUC: {}'.format(train_auc) + ' '
        log_message += 'Valid AUC: {}'.format(valid_auc)
        
        print(log_message)
        
result = pd.DataFrame(result)

result.head()
        
result.to_csv('output/fish/metrics.csv', sep=',', index=False)

result.groupby('marker').mean()
result.groupby('marker').mean().mean()
result.groupby('marker').mean().min()
result.groupby('marker').mean().std()
result.groupby('marker').std()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2475275107953754, 'fraction_numerical': 0.017969076473046384, 'mean_word_length': 3.040517726505346, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '426953', 'license': 'Apache-2.0', 'n_tokens_mistral': 2211, 'n_tokens_neox': 2025, 'n_words': 383, 'path': '01_00_fish_from_genes.ipynb', 'repo_name': 'urielcaire/mult', 'size': '7179'}