<jupyter_start><jupyter_text>TCAV スコア<jupyter_code>import importlib
import seaborn as sns
sns.set()

importlib.reload(utils_plot)
for i,results in enumerate(results_all_lst):
    #utils_plot.plot_results(results, num_random_exp=num_random_exp, save_path = root_dir + 'log/' + project_names + '/plot',keyword='color_'+str(i))
    utils_plot.plot_results(results, num_random_exp=num_random_exp)
import importlib
importlib.reload(utils_plot)
for i,results in enumerate(results_all_lst):
    utils_plot.plot_sensitivity_results(results, num_random_exp=num_random_exp, save_path = root_dir + 'log/' + project_names + '/plot',keyword='simi'+str(i))
    #utils_plot.plot_sensitivity_results(results)
random_i_ups = {}
for result in results_all_lst[0]:
    # store random
    if 'random500_' in result['cav_concept']:
        print(result['val_directional_dirs'])
        あ
bottlenecks = ['conv1']
targets = ['mnist-0','mnist-1','mnist-4','mnist-5','mnist-6']
targets = ['mnist-0']

cos_simirarity_score = {}
simirarity_score = {}
for target in targets:
    simirarity_score[target] = {}
    cos_simirarity_score[target] = {}
    for bottleneck in bottlenecks:
        if bottleneck not in simirarity_score[target]:
            simirarity_score[target][bottleneck] = {}
            simirarity_score[target][bottleneck]['mean'] = []
            simirarity_score[target][bottleneck]['std'] = []
            cos_simirarity_score[target][bottleneck] = {}
            cos_simirarity_score[target][bottleneck]['mean'] = []
            cos_simirarity_score[target][bottleneck]['std'] = []
        logit_grad = get_logit_grad(cavs_path, bottleneck, target)
        cav = get_random_cav(random_cav_path, bottleneck)
        s_simi_lst = []
        cos_s_simi_lst = []
        for c in cav:
            for lg in logit_grad:
                cos_s_simi = np.array(cos_sim(lg,c))
                s_simi = np.array(np.dot(lg,c))
                cos_s_simi_lst.append(cos_s_simi)
                s_simi_lst.append(s_simi)
        simirarity_score[target][bottleneck]['mean'].append(np.mean(s_simi_lst))
        simirarity_score[target][bottleneck]['std'].append(np.std(s_simi_lst))
        cos_simirarity_score[target][bottleneck]['mean'].append(np.mean(cos_s_simi_lst))
        cos_simirarity_score[target][bottleneck]['std'].append(np.std(cos_s_simi_lst))
simirarity_score
cos_simirarity_score
plot_concepts = ['Random']
for target in targets:
    save_path = root_dir + 'log/' + project_names + '/plot'
    num_bottlenecks = len(bottlenecks)
    num_concepts = len(plot_concepts)
    bar_width = 0.35
    index = np.arange(num_concepts) * bar_width * (num_bottlenecks + 1)
    # matplotlib
    fig, ax = plt.subplots()
    # draw all bottlenecks individually
    for i, [bn, vals] in enumerate(simirarity_score[target].items()):
        bar = ax.bar(index + i * bar_width, vals['mean'],
            bar_width, yerr=vals['std'], label=bn)
    target_class = target.title()
    ax.set_title('{} cos similarity'.format(target_class))
    ax.set_ylabel('cos similarity')
    # ax.set_xlabel(xlabel_name)
    y_range = 0.005
    ax.set_ylim(-y_range, y_range)
    ax.set_xticks(index + num_bottlenecks * bar_width / 2)
    #plt.xticks(fontsize=8)
    ax.set_xticklabels(plot_concepts)
    ax.legend(loc='upper left',bbox_to_anchor=(1.05, 1))
    fig.tight_layout()
    #plt.savefig(f'{save_path}/{target}:{num_concepts}:cos_similarity.eps')
    plt.show()<jupyter_output><empty_output><jupyter_text>TCAVスコアの分布を可視化<jupyter_code>for results in results_all_lst[:1]:
    dist = get_score_dist(results)
    for concept in dist:
        if concept != 'random':
            for bottleneck in dist[concept]:
                concept_label = '{} N = {}'.format(concept, len(dist[concept][bottleneck])) 
                random_label = 'random N = {}'.format(len(dist['random'][bottleneck]))
                plt.figure()
                plt.hist(dist[concept][bottleneck], bins=100, alpha=0.3, histtype='stepfilled', color='r',label= concept_label)
                plt.hist(dist['random'][bottleneck], bins=100, alpha=0.3, histtype='stepfilled', color='b',label = random_label)
                plt.legend()
                plt.title(bottleneck)
                plt.show()
# random
#for results in results_non_dup_lst[:1]:
for results in results_all_lst[:1]:
    dist = get_score_dist(results)
    for concept in dist:
        if concept == 'random':
            for bottleneck in dist[concept]:
                concept_label = '{} N = {}'.format(concept, len(dist[concept][bottleneck])) 
                random_label = 'random N = {}'.format(len(dist['random'][bottleneck]))
                plt.figure()
                plt.hist(dist['random'][bottleneck], bins=100, alpha=0.3, histtype='stepfilled', color='b',label = random_label)
                plt.legend()
                plt.title(bottleneck)
                plt.show()<jupyter_output><empty_output><jupyter_text>CAVの分離精度を確認<jupyter_code># conceptごとに
for results in results_all_lst[:1]:
    cav_accuracy = utils_plot.get_cav_accuracy(results)
    for concept in cav_accuracy:
        if concept != 'random':
            for bottleneck in cav_accuracy[concept]:
                print('---------------------------------')
                print('{}:{} vs random'.format(bottleneck,concept))
                concept_acc = [list(ele.values())[0] for ele in cav_accuracy[concept][bottleneck]]
                random_acc = [list(ele.values())[1] for ele in cav_accuracy[concept][bottleneck]]
                print('CAV concept accuracy : {} ± {}'.format(np.mean(concept_acc),np.std(concept_acc)))
                print('CAV random accuracy : {} ± {}'.format(np.mean(random_acc),np.std(random_acc)))

        else:
            for bottleneck in cav_accuracy[concept]:
                print('---------------------------------')
                print('{}:random vs random'.format(bottleneck))
                acc = [list(ele.values())[2] for ele in cav_accuracy[concept][bottleneck]]
                print('CAV random accuracy : {} ± {}'.format(np.mean(acc),np.std(acc)))
# conceptをまとめる
for results in results_all_lst[:1]:
    cav_accuracy = utils_plot.get_cav_accuracy(results)
    for bottleneck in cav_accuracy['random']:
        print('===============================')
        print('bottleneck is {}'.format(bottleneck))
        concept_acc = []
        random_acc = []
        for concept in cav_accuracy:
            if concept != 'random':
                concept_acc.extend([list(ele.values())[0] for ele in cav_accuracy[concept][bottleneck]])
                random_acc.extend([list(ele.values())[1] for ele in cav_accuracy[concept][bottleneck]])

        print('---------------------------------')
        print('concept vs random')
        print('CAV concept accuracy : {} ± {}'.format(np.mean(concept_acc),np.std(concept_acc)))
        print('CAV random accuracy : {} ± {}'.format(np.mean(random_acc),np.std(random_acc)))

        print('---------------------------------')
        print('random vs random'.format(bottleneck))
        acc = [list(ele.values())[2] for ele in cav_accuracy['random'][bottleneck]]
        print('CAV random accuracy : {} ± {}'.format(np.mean(acc),np.std(acc)))
bottlenecks = ['conv2']
sim_all = []
for bottleneck in bottlenecks:
    cav = get_random_cav(random_cav_path, bottleneck)
    sim_lst = []
    for v in itertools.combinations(np.arange(len(cav)), 2):
        sim = cos_sim(cav[v[0]],cav[v[1]])
        sim_lst.append(sim)
    sim_all.append(sim_lst)
#plt.title('Cos Similarity in ' + concept +  ' CAV')
plt.hist(sim_all[0], bins=100, alpha=0.3, histtype='stepfilled', color='b',label=bottlenecks[0])
plt.hist(sim_all[1], bins=100, alpha=0.3, histtype='stepfilled', color='r',label=bottlenecks[1])
plt.xlabel('cos_similarity')
plt.ylim(0,50)
plt.legend()
#plt.savefig(root_dir + 'log/' + project_names + '/plot/cav_simi-'+concept)
plt.show()
import seaborn as sns
sns.set()
plt.hist(sim_all[0], bins=100, alpha=0.6, histtype='stepfilled', color='black',label=bottlenecks[0]);
plt.title('cosine similarity in Random CAV')
plt.xlabel('cosine similarity')
plt.ylim(0,600000)
plt.xlim(-1,1)
plt.ylabel('frequency')
plt.savefig(root_dir + 'log/' + project_names + '/plot/cav_simi_hist-random.eps')
plt.show()
random_cav_path<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16335859497173102, 'fraction_numerical': 0.010585829423794057, 'mean_word_length': 3.2922044398554466, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '62552', 'license': 'Apache-2.0', 'n_tokens_mistral': 2961, 'n_tokens_neox': 2796, 'n_words': 452, 'path': 'show_results_plus_append_rand.ipynb', 'repo_name': 'munema/tcav', 'size': '8313'}