<jupyter_start><jupyter_text>Same Dataset Different Models<jupyter_code>epochs = 101
len_split = 20

nc_95 = defaultdict(list)
nc_90 = defaultdict(list)
for ep in tqdm(range(len_split, epochs, len_split), leave=False):
    idx = 1
    fig = plt.figure(figsize=(15, 5))
    ax1 = fig.add_subplot(111)
    tick_names = []
    ticks = []
    start_ep = ep-len_split
    end_ep = ep + 1
    print('[{}:{}]'.format(start_ep, end_ep))
    table = [['expt', 'max(acc)', 'min(loss)', 'opt(0.95)', 'opt(0.99)']]
    for file, tick_name in tqdm(files.items(), leave=False):
        if not ('' in tick_name):
            continue
        ticks.append(idx)
        tick_names.append(tick_name)
        _,_,acc_test,_, loss_test, grads = pkl.load(open(file, 'rb'))
        acc_test = acc_test[start_ep:end_ep]
        loss_test = loss_test[start_ep:end_ep]
        best_acc = np.max(acc_test)
        best_loss = np.min(loss_test)
        grads = grads[start_ep:end_ep]
        tot_opt_95 = 0
        tot_opt_99 = 0
        tot_epochs = 0
        for layer_num in range(len(grads[0])):
            accum = []
            for epoch_num in range(len(grads)):
                g = grads[epoch_num][layer_num]
                shape = g.shape
                accum.append(g.reshape(1, -1).cpu().numpy())
            stack_g = np.vstack(accum)
            o_95, o_99, o_epochs = estimate_optimal_ncomponents(stack_g.T)
            tot_opt_95 += o_95
            tot_opt_99 += o_99
            tot_epochs += 1
        table.append([tick_name, 
                      '{:.4f}'.format(best_acc), 
                      '{:.4f}'.format(best_loss),
                      '{:.4f}'.format(tot_opt_95/tot_epochs), 
                      '{:.4f}'.format(tot_opt_99/tot_epochs)])
        nc_95[tick_name].append(tot_opt_99/tot_epochs)
        nc_90[tick_name].append(tot_opt_95/tot_epochs)
        plt.bar(idx-0.25, best_acc*100, width = 0.2, color='b')
        plt.bar(idx-0.25, best_loss, width=0.2, color='y')
        plt.bar(idx, len(grads), width=0.2, color='g')
        plt.bar(idx+0.25, tot_opt_95/tot_epochs, width=0.2, color='r')
        plt.bar(idx+0.25, tot_opt_99/tot_epochs, width=0.2, color='r', alpha=0.5)
        idx += 1
#     table = Table(table)
#     print(table.table)
    ax1.set_xticks(ticks)
    ax1.set_xticklabels(tick_names, rotation='vertical')
    ax1.grid()
    plt.show()
    
epochs = 101
len_split = 20

nc_95 = defaultdict(list)
nc_90 = defaultdict(list)
for ep in tqdm(range(len_split, epochs, len_split), leave=False):
    idx = 1
    fig = plt.figure(figsize=(15, 5))
    ax1 = fig.add_subplot(111)
    tick_names = []
    ticks = []
    start_ep = ep-len_split
    end_ep = ep + 1
    print('[{}:{}]'.format(start_ep, end_ep))
    for file, tick_name in tqdm(files.items(), leave=False):
        if 'resnet' not in tick_name:
            continue
        print(tick_name)
        ticks.append(idx)
        tick_names.append(tick_name)
        _,_,acc_test,_, loss_test, grads = pkl.load(open(file, 'rb'))
        acc_test = acc_test[start_ep:end_ep]
        loss_test = loss_test[start_ep:end_ep]
        best_acc = np.max(acc_test)
        best_loss = np.min(loss_test)
        grads = grads[start_ep:end_ep]
        tot_opt_95 = 0
        tot_opt_99 = 0
        tot_epochs = 0
        table = [['shape', 'stack', 'opt(0.90)', 'opt(0.95)']]
        for layer_num in range(len(grads[0])):
            accum = []
            for epoch_num in range(len(grads)):
                g = grads[epoch_num][layer_num]
                shape = g.shape
                accum.append(g.reshape(1, -1).cpu().numpy())
            stack_g = np.vstack(accum)
            o_95, o_99, o_epochs = estimate_optimal_ncomponents(stack_g.T)
            tot_opt_95 += o_95
            tot_opt_99 += o_99
            tot_epochs += 1
            table.append([shape, stack_g.T.shape, o_95, o_99])
        table.append(['avg', 
                      '', 
                      '{:.4f}'.format(tot_opt_95/tot_epochs), 
                      '{:.4f}'.format(tot_opt_99/tot_epochs)])
        nc_95[tick_name].append(tot_opt_99/tot_epochs)
        nc_90[tick_name].append(tot_opt_95/tot_epochs)
        plt.bar(idx-0.25, best_acc*100, width = 0.2, color='b')
        plt.bar(idx-0.25, best_loss, width=0.2, color='y')
        plt.bar(idx, len(grads), width=0.2, color='g')
        plt.bar(idx+0.25, tot_opt_95/tot_epochs, width=0.2, color='r')
        plt.bar(idx+0.25, tot_opt_99/tot_epochs, width=0.2, color='r', alpha=0.5)
        idx += 1
        table = Table(table)
        print(table.table)
    ax1.set_xticks(ticks)
    ax1.set_xticklabels(tick_names, rotation='vertical')
    ax1.grid()
    plt.show()
    
epochs = 101
for file in files:
    fig = plt.figure(figsize=(20, 5))
    ax1 = fig.add_subplot(111)
    print(file)
    _,_,acc_test,_, loss_test, grads = pkl.load(open(file, 'rb'))
    grads = grads[:epochs]
    corr_mat = np.zeros((epochs, epochs))
    num_layers = 0
    components = []
    for layer_num in tqdm(range(len(grads[0])), leave=False):
        num_layers += 1
        accum = []
        for epoch_num in range(len(grads)):
            g = grads[epoch_num][layer_num]
            shape = g.shape
            accum.append(g.reshape(1, -1).cpu().numpy())
        stack_g = np.vstack(accum).T
        pca_g = pca_transform(stack_g)
        components.append(pca_g.shape[1])
        pca_g = normalize(pca_g, axis=0)
#         stack_g, _ = np.linalg.qr(stack_g)
        stack_g = normalize(stack_g, axis=0)
        corr = np.matmul(pca_g.T, stack_g)
        corr_mat[:corr.shape[0], :corr.shape[1]] += corr 
    sns.heatmap(corr_mat[:sum(components)//len(components)]/num_layers, center=0.0, square=True)
    plt.show()
    break<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14791666666666667, 'fraction_numerical': 0.03819444444444445, 'mean_word_length': 2.32237600922722, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '373994', 'license': 'MIT', 'n_tokens_mistral': 2308, 'n_tokens_neox': 2192, 'n_words': 347, 'path': 'LBGM/nb/05. LocalLowRankApproximation.ipynb', 'repo_name': 'sidsrini12/FURL_Sim', 'size': '5760'}