<jupyter_start><jupyter_text>TCAV スコア<jupyter_code>is_bonferroni = False
for path in results_path:
    results = pickle_load(path)
    utils_plot.plot_results(results, num_random_exp=num_random_exp,is_bonferroni=is_bonferroni)<jupyter_output><empty_output><jupyter_text>TCAVスコアの分布を可視化<jupyter_code>dist = utils_plot.get_dist(results)
for path in results_path[:1]:
    results = pickle_load(path)
    for concept in dist:
        if concept != 'random':
            for bottleneck in dist[concept]:
                concept_label = '{} N = {}'.format(concept, len(dist[concept][bottleneck])) 
                random_label = 'random N = {}'.format(len(dist['random'][bottleneck]))
                plt.figure()
                plt.hist(dist[concept][bottleneck], bins=100, alpha=0.3, histtype='stepfilled', color='r',label= concept_label)
                plt.hist(dist['random'][bottleneck], bins=100, alpha=0.3, histtype='stepfilled', color='b',label = random_label)
                plt.legend()
                plt.title(bottleneck)
                plt.show()<jupyter_output><empty_output><jupyter_text>CAVの分離精度を確認<jupyter_code>cav_accuracy = utils_plot.get_cav_accuracy(results)
# conceptごとに
for path in results_path:
    results = pickle_load(path)
    for concept in cav_accuracy:
        if concept != 'random':
            for bottleneck in cav_accuracy[concept]:
                print('---------------------------------')
                print('{}:{} vs random'.format(bottleneck,concept))
                concept_acc = [list(ele.values())[0] for ele in cav_accuracy[concept][bottleneck]]
                random_acc = [list(ele.values())[1] for ele in cav_accuracy[concept][bottleneck]]
                print('CAV concept accuracy : {} ± {}'.format(np.mean(concept_acc),np.std(concept_acc)))
                print('CAV random accuracy : {} ± {}'.format(np.mean(random_acc),np.std(random_acc)))

        else:
            for bottleneck in cav_accuracy[concept]:
                print('---------------------------------')
                print('{}:random vs random'.format(bottleneck))
                acc = [list(ele.values())[2] for ele in cav_accuracy[concept][bottleneck]]
                print('CAV random accuracy : {} ± {}'.format(np.mean(acc),np.std(acc)))
# conceptをまとめる
for path in results_path[:1]:
    results = pickle_load(path)
    for bottleneck in cav_accuracy['random']:
        print('===============================')
        print('bottleneck is {}'.format(bottleneck))
        concept_acc = []
        random_acc = []
        for concept in cav_accuracy:
            if concept != 'random':
                concept_acc.extend([list(ele.values())[0] for ele in cav_accuracy[concept][bottleneck]])
                random_acc.extend([list(ele.values())[1] for ele in cav_accuracy[concept][bottleneck]])

        print('---------------------------------')
        print('concept vs random')
        print('CAV concept accuracy : {} ± {}'.format(np.mean(concept_acc),np.std(concept_acc)))
        print('CAV random accuracy : {} ± {}'.format(np.mean(random_acc),np.std(random_acc)))

        print('---------------------------------')
        print('random vs random'.format(bottleneck))
        acc = [list(ele.values())[2] for ele in cav_accuracy['random'][bottleneck]]
        print('CAV random accuracy : {} ± {}'.format(np.mean(acc),np.std(acc)))<jupyter_output>===============================
bottleneck is mixed3a
---------------------------------
concept vs random
CAV concept accuracy : 0.9433308823529414 ± 0.0657478566591856
CAV random accuracy : 0.974433823529412 ± 0.04605898493229177
---------------------------------
random vs random
CAV random accuracy : 0.4982189239332096 ± 0.08464955106321942
===============================
bottleneck is mixed4a
---------------------------------
concept vs random
CAV concept accuracy : 0.9433308823529414 ± 0.0657478566591856
CAV random accuracy : 0.974433823529412 ± 0.04605898493229177
---------------------------------
random vs random
CAV random accuracy : 0.4982189239332096 ± 0.08464955106321942
===============================
bottleneck is mixed4c
---------------------------------
concept vs random
CAV concept accuracy : 0.9433308823529414 ± 0.0657478566591856
CAV random accuracy : 0.974433823529412 ± 0.04605898493229177
---------------------------------
random vs random
CAV random accuracy : 0.4982[...]
{'fraction_non_alphanumeric': 0.21908449100432703, 'fraction_numerical': 0.06832156684126622, 'mean_word_length': 2.541935483870968, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '320226', 'license': 'Apache-2.0', 'n_tokens_mistral': 1548, 'n_tokens_neox': 1302, 'n_words': 260, 'path': 'show_results.ipynb', 'repo_name': 'munema/tcav', 'size': '4391'}