<jupyter_start><jupyter_text>right now, this compares 9 channel CNN vs. GLMs, in terms of raw performance (cc^2 or cc), on ALL neurons (so no subset stuff yet)<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt
from collections import OrderedDict
from itertools import product
import h5py
import numpy as np
import pandas as pd
from tang_jcompneuro.model_fitting_postprocess import load_data_generic
def check_all(squared, score_col_name):
    models_to_examine = [
        ('cnn', 'b.1'),
        ('cnn', 'b.2'),
        ('cnn', 'b.3'),
        ('cnn', 'b.6'),
        ('cnn', 'b.9'),
        ('glm', 'linear_poisson'),
        ('glm', 'fpower_poisson'),
        ('glm', 'gqm.2_poisson'),
        ('glm', 'gqm.4_poisson'),
        ('glm', 'gqm.8_poisson'),
        ('glm', 'linear_gaussian'),
        ('glm', 'fpower_gaussian'),
        ('glm', 'gqm.2_gaussian'),
        ('glm', 'gqm.4_gaussian'),
        ('glm', 'gqm.8_gaussian'),
    ] # TODO should make this a ordered dict.
      # so that it will be easier to collect best val model for each model class.
    
    return load_data_generic(models_to_examine, load_naive=True, metric='raw', squared=squared,
                             score_col_name=score_col_name, percentages_to_load=(25,50,75,100))
df_all_cc = check_all(squared=False, score_col_name='cc')
df_all_cc2 = check_all(squared=True, score_col_name='cc2')
# this is the way to change a multiindex's names.
# df_all.index.rename(['haha','haha2','haha3'], inplace=True)
df_all = pd.concat([df_all_cc, df_all_cc2], axis=1)
df_all.columns.name = 'metric'
df_all.unstack('subset').unstack('percentage')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14629741119807346, 'fraction_numerical': 0.017459361830222758, 'mean_word_length': 3.8034682080924855, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '286379', 'license': 'Apache-2.0', 'n_tokens_mistral': 615, 'n_tokens_neox': 596, 'n_words': 139, 'path': 'results_ipynb/step_1_rough_exploration/model_performance_comparison_by_train_size.ipynb', 'repo_name': 'leelabcnbc/tang_jcompneuro_revision', 'size': '1661'}