<jupyter_start><jupyter_text>IntroductionThis notebook will select models using two different methods.  **Method 1**The best VAMP scores at a given lag and fixed number of implied processes. **Method 2**The best joint ranked VAMP and timescale gap.<jupyter_code>vamps = pd.read_hdf('./summaries/vamps_bs_summary.h5')
vampe = pd.read_hdf('./summaries/vampe_bs_summary.h5')

ratios = pd.read_hdf('./summaries/ts_ratio_bs_summary.h5')
hps = pd.read_hdf('./summaries/hp_summary.h5')
chosen_lags = pd.read_hdf('chosen_lag_times.h5')
chosen_ks = pd.read_hdf('chosen_num_dominant.h5')
lags = dict(zip(chosen_lags['protein'], chosen_lags['lag']))
num_its = dict(zip(chosen_ks['protein'], chosen_ks['num_its']))
ratios.head()
def select_rows_by_dict(df, select_dict, selector):
    ix = df.apply(lambda x: x[selector] == select_dict[x['protein']], axis=1)
    return df.loc[ix, :].copy()


vamps = select_rows_by_dict(vamps, lags, 'lag')
ratios = select_rows_by_dict(ratios, lags, 'lag')
vampe = select_rows_by_dict(vampe, lags, 'lag')<jupyter_output><empty_output><jupyter_text>Method 1 Choose hyperparametersWe'll select the best models per feature based on the selected values of $k$ (`num_its`) and lag time. Also include the worst and then split it off at the end.<jupyter_code>df = select_rows_by_dict(vamps, num_its, 'num_its')
df = df.merge(hps.loc[:, ['protein', 'hp_index', 'feature']], on=['protein', 'hp_index'])
df['feature_vamp_rank'] = df.groupby(['protein', 'feature'])['median'].rank(ascending=False)

df['overall_vamp_rank'] = df.groupby(['protein'])['median'].rank(ascending=False)

m1_selection = df.loc[df.feature_vamp_rank == 1.0, :].copy()

grps = df.groupby(['protein'], as_index=False)
worst = grps.apply(lambda g: g[g['overall_vamp_rank']==g['overall_vamp_rank'].max()])
worst.reset_index(inplace=True, drop=True)


m1_selection = pd.concat([m1_selection, worst], axis=0)
m1_selection.drop(labels=['feature'], inplace=True, axis=1)
m1_selection = m1_selection.merge(hps, on=['protein', 'hp_index'], how='left')
m1_selection.sort_values(by=['protein', 'overall_vamp_rank'], inplace=True)


m1_selection.head()
worst.head()<jupyter_output><empty_output><jupyter_text>Choose new num_its<jupyter_code>max_ratio = ratios.groupby(['protein', 'hp_index'], as_index=False)['median'].max()
max_ratio = max_ratio.merge(ratios, on=['protein', 'hp_index'], suffixes=('_ratio_max', '_ratio_value'))
max_ratio = max_ratio.loc[np.abs(max_ratio.median_ratio_max-max_ratio.median_ratio_value)<1e-9, :]
m1_selection = m1_selection.merge(max_ratio, on=['protein', 'hp_index'], how='left')
m1_selection.rename(columns=dict(median='median_vamp', lb_diff_x='lb_diff_vamp', ub_diff_x='ub_diff_vamp', lag_x='chosen_lag', num_its_x='chosen_num_its', num_its_y='new_num_its', 
                                lb_diff_y='lb_diff_ratio', ub_diff_y='ub_diff_ratio'), inplace=True)


# select worst
m3_selection = m1_selection.loc[m1_selection.index.isin(m1_selection.groupby('protein')['overall_vamp_rank'].idxmax()), :]
# Select not worst
m1_selection = m1_selection.loc[~m1_selection.index.isin(m1_selection.groupby('protein')['overall_vamp_rank'].idxmax()), :]


m3_selection.to_hdf('./summaries/m3_model_selection.h5', key='m3_selection')
m1_selection.to_hdf('./summaries/m1_model_selection.h5', key='m1_selection')<jupyter_output><empty_output><jupyter_text>Method 2Best VAMP scores conditional on timescale gap threshold<jupyter_code>threshold = 0.98 # Percentage of timescale gaps to consider. hopefully this give a range of different features. 
df = vamps.merge(ratios, on=['protein', 'hp_index', 'lag', 'num_its'], suffixes=('_vamp', '_gap'))
df = df.merge(hps.loc[:, ['protein', 'hp_index', 'feature']], on=['protein', 'hp_index'])

m2_selection = []
for protein in funcs.PROTEIN_LABELS[:5]+funcs.PROTEIN_LABELS[6:9]:

    q = np.quantile(df.loc[df.protein == protein, 'median_gap'].values, q=threshold)
    best_gap = df.loc[(df.protein==protein) & (df.median_gap > q), :].copy()

    best_gap['feature_vamp_rank'] = best_gap.groupby(['feature'])['median_vamp'].rank(ascending=False)
    best_per_feature = best_gap.loc[best_gap.feature_vamp_rank==1.0, :]
    m2_selection.append(best_per_feature)
    
m2_selection = pd.concat(m2_selection, axis=0)
m2_selection.drop(labels='feature', axis=1,  inplace=True)

m2_selection = m2_selection.merge(hps, on=['protein', 'hp_index'], how='left')
m2_selection.rename(columns={'num_its': 'new_num_its', 'lag': 'chosen_lag'}, inplace=True) # This is to make this consistent with m1 models.  
m2_selection.to_hdf('./summaries/m2_model_selection.h5', key='m2_selection')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16562432607289196, 'fraction_numerical': 0.014448997196463231, 'mean_word_length': 7.685393258426966, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '177310', 'license': 'MIT', 'n_tokens_mistral': 1805, 'n_tokens_neox': 1738, 'n_words': 278, 'path': 'results/2_model_selection.ipynb', 'repo_name': 'RobertArbon/msm_hyperparameter_optimisation', 'size': '4637'}