<jupyter_start><jupyter_text>right now, this compares 9 channel CNN vs. GLMs, in terms of raw performance (cc^2 or cc), on ALL neurons (so no subset stuff yet)<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt
from collections import OrderedDict
from itertools import product
import h5py
import numpy as np
import pandas as pd
from tang_jcompneuro.model_fitting_postprocess import load_model_performance
def check_all():
    subsets_to_check = ('all',
#                         'OT'
                       )
    datasets_to_check = ('MkA_Shape',
#                          'MkE2_Shape'
                        )
    
    models_to_examine = [
        ('cnn', 'b.9'),
        ('cnn', 'b.9_avg'),
        ('cnn', 'b.9_halfsq'),
        ('cnn', 'b.9_avg_halfsq'),
        ('cnn', 'b.9_sq'),
        ('cnn', 'b.9_avg_sq'),
        ('cnn', 'b.9_linear'),
        ('cnn', 'b.9_avg_linear'),
        ('cnn', 'b.9@0.05'),
        ('cnn', 'b.9_avg@0.05'),
        ('cnn', 'b.9_halfsq@0.05'),
        ('cnn', 'b.9_avg_halfsq@0.05'),
        ('cnn', 'b.9_sq@0.05'),
        ('cnn', 'b.9_avg_sq@0.05'),
        ('cnn', 'b.9_linear@0.05'),
        ('cnn', 'b.9_avg_linear@0.05'),
        ('cnn', 'b.9_avg_abs@0.05'),
        ('cnn', 'b.9_abs@0.05'),
        ('cnn', 'b.9@0.005'),
        ('cnn', 'b.9_avg@0.005'),
        ('cnn', 'b.9_halfsq@0.005'),
        ('cnn', 'b.9_avg_halfsq@0.005'),
        ('cnn', 'b.9_sq@0.005'),
        ('cnn', 'b.9_avg_sq@0.005'),
        ('cnn', 'b.9_linear@0.005'),
        ('cnn', 'b.9_avg_linear@0.005'),
        ('cnn', 'b.9_avg_abs@0.005'),
        ('cnn', 'b.9_abs@0.005'),
    ] # TODO should make this a ordered dict.
      # so that it will be easier to collect best val model for each model class.
    
    score_all = []
    for dataset, subset in product(datasets_to_check, subsets_to_check):
        
        print(dataset, subset)
        for (model_type, model_subtype) in models_to_examine: 
            score_new_cc = np.asarray(
                [load_model_performance(dataset, subset, 100, s, model_type, model_subtype)['corr'] for s in range(1)]
            ).mean(axis=0)
            
            score_new_cc2 = np.asarray(
                [load_model_performance(dataset, subset, 100, s, model_type, model_subtype)['corr']**2 for s in range(1)]
            ).mean(axis=0)
            
            if '@' in model_subtype:
                model_subtype_base, hack_ratio = model_subtype.split('@')
                hack_ratio = float(hack_ratio)
            else:
                model_subtype_base= model_subtype
                hack_ratio = 1
            
            score_all.append({
                'dataset': dataset,
                'subset': subset,
                'model': model_type + '_' + model_subtype_base,
                'hack_ratio': hack_ratio,
                'cc': score_new_cc.mean(),
                'cc2': score_new_cc2.mean(),
                'raw_cc': score_new_cc.copy(),
                # later on, I can add neuron subset, etc.
            })
            
    score_all = pd.DataFrame(score_all,
                             columns = ['dataset', 'subset', 'model',
                                        'hack_ratio', 'cc', 'cc2', 'raw_cc']).set_index(['dataset', 'subset', 'model', 'hack_ratio'], verify_integrity=True)
    return score_all
df_all = check_all()
df_all.columns.name = 'metric'
# this is the way to change a multiindex's names.
# df_all.index.rename(['haha','haha2','haha3'], inplace=True)
df_to_process = df_all.unstack('subset').unstack('hack_ratio')
df_to_process = df_to_process.xs('MkA_Shape', level='dataset')
df_to_process
%matplotlib inline
import matplotlib.pyplot as plt
# I will show 3 plots for each model.
# 0.05 vs 1.0 (may not be available)
# 0.05 vs 0.005
# 0.05 vs cnn_b.9 1.0
def vis_stuff():
    baseline_cc = df_to_process.at['cnn_b.9',('raw_cc', 'all', 1.0)]
    #print(baseline_cc.shape, baseline_cc.mean())
    for model_name in df_to_process.index:
        print(model_name)
        cc_0p05 = df_to_process.at[model_name,('raw_cc', 'all', 0.05)]
        cc_0p005 = df_to_process.at[model_name,('raw_cc', 'all', 0.005)]
        cc_base = df_to_process.at[model_name,('raw_cc', 'all', 1.0)]
        assert cc_0p05 is not None
        plt.close('all')
        fig, axes = plt.subplots(1,4,figsize=(20, 6))
        display_sth(axes[0], cc_0p05, cc_base, '@0.05', '@1')
        display_sth(axes[1], cc_0p05, cc_0p005, '@0.05', '@0.005')
        display_sth(axes[2], cc_0p05, baseline_cc, '@0.05', 'baseline')
        display_sth(axes[3], cc_base, baseline_cc, '@1', 'baseline')
        plt.show()
        
from scipy.stats import pearsonr
def display_sth(ax, x, y, xlabel, ylabel):
    if x is None or y is None:
        return
    
    assert x.shape == y.shape == (x.size,)
    ax.scatter(x, y)
    ax.plot([0,1],[0,1],linestyle='--', color='red')
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.set_xlim(0,1)
    ax.set_ylim(0,1)
    cc = pearsonr(x,y)[0]
    ax.set_title('{} {:.3f} - {} {:.3f}, cc {:.2f}'.format(xlabel, x.mean(), ylabel, y.mean(), cc))
    
vis_stuff()<jupyter_output>cnn_b.9
{'fraction_non_alphanumeric': 0.1627997660362644, 'fraction_numerical': 0.04113862351335543, 'mean_word_length': 2.292682926829268, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '575366', 'license': 'Apache-2.0', 'n_tokens_mistral': 2007, 'n_tokens_neox': 1896, 'n_words': 380, 'path': 'results_ipynb/step_1_rough_exploration/model_performance_comparison_1layer_detailed.ipynb', 'repo_name': 'leelabcnbc/tang_jcompneuro_revision', 'size': '5129'}