<jupyter_start><jupyter_text>Data Visualization 0. Importing Libraries<jupyter_code>import json
import os
import sys
from sklearn import metrics
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from plotly.validators.scatter.marker import SymbolValidator
import numpy as np
import pandas as pd

# base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# sys.path.append(base_path)<jupyter_output><empty_output><jupyter_text>1. Basic Processing Block<jupyter_code>username = "fnx11"

# just for notebook
base_path = f'/home/{username}/thesis/codes/CosDefence/'

# for picking data from the repo
json_folder = os.path.join(base_path, 'json_files/')

# for picking data from results folder
# json_folder = f'/home/{username}/thesis/codes/Results/iter3/'

plots_folder = base_path + 'plots/'

file_p_frac0 = json_folder + 'mnist_C0.1_P0.0_FDRS30_LR0.001_OptAdam.txt'
file_p_frac10 = json_folder + 'mnist_C0.1_P0.1_FDRS30_LR0.001_OptAdam.txt'
file_p_frac20 = json_folder + 'mnist_C0.1_P0.2_FDRS30_LR0.001_OptAdam.txt'
file_p_frac40 = json_folder + 'mnist_C0.1_P0.4_FDRS30_LR0.001_OptAdam.txt'

files_dict = {}
c_fracs = [0.1, 0.2, 0.4, 1.0]
p_fracs = [0.0, 0.1, 0.2, 0.4]
atts = [0, 1]
for c_frac in c_fracs:
    for p_frac in p_fracs:
        for att in atts:
            dict_key = f'file_c{int(c_frac*100)}_p{int(p_frac*100):02}_a{att}'
#             print(dict_key)
            files_dict[dict_key] = json_folder + f'basic_C{c_frac}_P{p_frac}_Att{att}_FDRS400_LR0.001_OptAdam.txt'

def get_poisoned_client_nums(file_name):
    result_data = {}
    with open(file_name) as result_file:
        result_data = json.load(result_file)
    
    return result_data['poisoned_client_sel']

def get_training_losses(file_name):
    result_data = {}
    with open(file_name) as result_file:
        result_data = json.load(result_file)
    
    return result_data['avg_training_losses']

def get_test_data_df(file_name):
    result_data = {}
    with open(file_name) as result_file:
        result_data = json.load(result_file)

    testing_rounds = len(result_data['testing_losses'])
    testing_every = result_data['config']['testing_every']
    round_num = [i*testing_every for i in range(1, testing_rounds+1)]

    test_data_df = pd.DataFrame()
    test_data_df['round_num'] = round_num
    test_data_df['testing_loss'] = result_data['testing_losses']
    test_data_df['total_acc'] = result_data['total_accuracies']
    
    avg_precision = []
    avg_recall = []
    avg_f1score = []
    for j in range(testing_rounds):
        avg_precision.append(result_data['avg_metric_vals'][j][0])
        avg_recall.append(result_data['avg_metric_vals'][j][1])
        avg_f1score.append(result_data['avg_metric_vals'][j][2])
    test_data_df['avg_precision'] = avg_precision
    test_data_df['avg_recall'] = avg_recall
    test_data_df['avg_f1score'] = avg_f1score
    
        
    for i in range(10):
        class_name = f'class_{i}'
        temp_class_acc = []
        temp_class_precision = []
        temp_class_recall = []
        temp_class_f1score = []
        for j in range(testing_rounds):
            temp_class_acc.append(result_data['class_accuracies'][j][i])
            temp_class_precision.append(result_data['class_precisions'][j][i])
            temp_class_recall.append(result_data['class_recalls'][j][i])
            temp_class_f1score.append(result_data['class_f1scores'][j][i])
            
        test_data_df[class_name+'_acc'] = temp_class_acc
        test_data_df[class_name+'_precision'] = temp_class_precision
        test_data_df[class_name+'_recall'] = temp_class_recall
        test_data_df[class_name+'_f1score'] = temp_class_f1score
        
    return test_data_df

def get_final_test_data(file_name):
    result_data = {}
    with open(file_name) as result_file:
        result_data = json.load(result_file)
        
    predictions = result_data['final_test_data']['predictions']
    ground_truths = result_data['final_test_data']['ground_truths']
    total_acc = result_data['final_test_data']['total_acc']
    classes_acc = result_data['final_test_data']['classes_acc']
    testing_loss = result_data['final_test_data']['testing_loss']
    
    return total_acc, classes_acc, predictions, ground_truths, testing_loss

# saving all the data in a dataframe so that we can easily plot it with plottly<jupyter_output><empty_output><jupyter_text>1. Plotting Training Loss of ClientsClients are picked randomly there is no gurantee that all the clients have equal number of training epochs.<jupyter_code>avg_training_losses = get_training_losses(file_p_frac20)
fig = go.Figure()
# for i in range(100):
#     losses = client_training_losses[i][:100]
#     fig.add_trace(go.Scatter(x=np.arange(len(losses)), y=losses,
#                     mode='lines+markers',
#                     name=f'client{i}'))

fig.add_trace(go.Scatter(x=np.arange(len(avg_training_losses)), y=avg_training_losses,
                    mode='lines',
                    name=f'Training Loss'))

fig.update_layout(title='Training Losses',
                   xaxis_title='Epoch',
                   yaxis_title='Loss')
fig.show()<jupyter_output><empty_output><jupyter_text>2. Plotting Testing Loss and AccuraciesTesting and accuracy calculation is done at every 10th round(can be changed).<jupyter_code>data_df = get_test_data_df(file_p_frac20)
data_df.head(40) # we know that we trained for 800 rounds and tested every 10 round.<jupyter_output><empty_output><jupyter_text>A. Comparing All accuracies<jupyter_code># Create traces
fig = go.Figure()
fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['total_acc'],
                    mode='lines+markers',
                    name='total_acc'))
for i in range(10):
    class_name = f'class_{i}_acc'
    fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df[class_name],
                        mode='lines+markers',
                        name=class_name))

fig.update_layout(title='Accuracy Evolution',
                   xaxis_title='Round Num',
                   yaxis_title='Accuracy')
fig.show()<jupyter_output><empty_output><jupyter_text>B. Comparing Accuracy of Flipped classes(Source and Target)<jupyter_code># Create traces
fig = go.Figure()
fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['total_acc'],
                    mode='lines+markers',
                    name='total_acc'))

fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_2_acc'],
                    mode='lines+markers',
                    name='flipped_source'))

fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_9_acc'],
                    mode='lines+markers',
                    name='flipped_target'))

fig.update_layout(title='Accuracy Evolution',
                   xaxis_title='Round Num',
                   yaxis_title='Accuracy')
fig.show()<jupyter_output><empty_output><jupyter_text>C. Testing Loss<jupyter_code># Create traces
fig = go.Figure()
fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['testing_loss'],
                    mode='lines+markers',
                    name='testing_loss'))
fig.update_layout(title='Testing Loss over the Training',
                   xaxis_title='Round Num',
                   yaxis_title='Loss')
fig.show()<jupyter_output><empty_output><jupyter_text>3. Compare Different Poisoned Data ScenariosRead 4 different files with different compare the accuracies and taining losses<jupyter_code>data0_df = get_test_data_df(file_p_frac0)
data10_df = get_test_data_df(file_p_frac10)
data20_df = get_test_data_df(file_p_frac20)
data40_df = get_test_data_df(file_p_frac40)
# all_data_dfs = [data0_df[:20], data10_df[:20], data20_df[:20], data40_df[:20]]
all_data_dfs = [data0_df, data10_df, data20_df, data40_df]<jupyter_output><empty_output><jupyter_text>A. Comparing Total Acc, Flipped Source and Flipped Target in different scenarios<jupyter_code># Create traces
fig = go.Figure()
for i, data_df in enumerate(all_data_dfs):
    
    # to make data40 and plot curves make better sense
    if i==3:
        i += 1

    fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['total_acc'],
                        mode='lines+markers',
                        name=f'total_acc{i*10}'))

    fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_2_acc'],
                        mode='lines+markers',
                        name=f'flipped_source{i*10}'))

    fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_9_acc'],
                        mode='lines+markers',
                        name=f'flipped_target{i*10}'))

fig.update_layout(title='Accuracy Evolution',
                   xaxis_title='Round Num',
                   yaxis_title='Accuracy')
fig.show()<jupyter_output><empty_output><jupyter_text>Separate sub plots<jupyter_code>fig = make_subplots(rows=2, cols=2)

# Top left
fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['total_acc'],
                    mode='lines+markers',
                    name='total_acc0'), row=1, col=1)

fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_2_acc'],
                    mode='lines+markers',
                    name='flipped_source0'), row=1, col=1)

fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_9_acc'],
                    mode='lines+markers',
                    name='flipped_target0'), row=1, col=1)

# Top right
fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['total_acc'],
                    mode='lines+markers',
                    name='total_acc10'), row=1, col=2)

fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_2_acc'],
                    mode='lines+markers',
                    name='flipped_source10'), row=1, col=2)

fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_9_acc'],
                    mode='lines+markers',
                    name='flipped_target10'), row=1, col=2)

# Bottom left
fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['total_acc'],
                    mode='lines+markers',
                    name='total_acc20'), row=2, col=1)

fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_2_acc'],
                    mode='lines+markers',
                    name='flipped_source20'), row=2, col=1)

fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_9_acc'],
                    mode='lines+markers',
                    name='flipped_target20'), row=2, col=1)

# Bottom right
fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['total_acc'],
                    mode='lines+markers',
                    name='total_acc40'), row=2, col=2)

fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_2_acc'],
                    mode='lines+markers',
                    name='flipped_source40'), row=2, col=2)

fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_9_acc'],
                    mode='lines+markers',
                    name='flipped_target40'), row=2, col=2)

fig.update_layout(title='Accuracy Evolution',
                   xaxis_title='Round Num',
                   yaxis_title='Accuracy')
fig.show()<jupyter_output><empty_output><jupyter_text>4. Calculate Precision, Recall and F1-Measure of each class<jupyter_code>total_acc, classes_acc, predictions, ground_truths, testing_loss = get_final_test_data(file_p_frac0)
confusion_matrix = metrics.confusion_matrix(ground_truths, predictions)
clf_report = metrics.classification_report(ground_truths, predictions, digits=3)
print(confusion_matrix)
print(clf_report)
total_acc, classes_acc, predictions, ground_truths, testing_loss = get_final_test_data(file_p_frac10)
confusion_matrix = metrics.confusion_matrix(ground_truths, predictions)
clf_report = metrics.classification_report(ground_truths, predictions, digits=3)
print(confusion_matrix)
print(clf_report)
total_acc, classes_acc, predictions, ground_truths, testing_loss = get_final_test_data(file_p_frac20)
confusion_matrix = metrics.confusion_matrix(ground_truths, predictions)
clf_report = metrics.classification_report(ground_truths, predictions, digits=3)
print(confusion_matrix)
print(clf_report)
total_acc, classes_acc, predictions, ground_truths, testing_loss = get_final_test_data(file_p_frac40)
confusion_matrix = metrics.confusion_matrix(ground_truths, predictions)
clf_report = metrics.classification_report(ground_truths, predictions, digits=3)
print(confusion_matrix)
print(clf_report)<jupyter_output>[[   0    0    0    0    0    0    0    0    0 1000]
 [   0    0    0    0    0    0    0    0    0 1000]
 [   0    0    0    0    0    0    0    0    0 1000]
 [   0    0    0    0    0    0    0    0    0 1000]
 [   0    0    0    0    0    0    0    0    0 1000]
 [   0    0    0    0    0    0    0    0    0 1000]
 [   0    0    0    0    0    0    0    0    0 1000]
 [   0    0    0    0    0    0    0    0    0 1000]
 [   0    0    0    0    0    0    0    0    0 1000]
 [   0    0    0    0    0    0    0    0    0 1000]]
              precision    recall  f1-score   support

           0      0.000     0.000     0.000      1000
           1      0.000     0.000     0.000      1000
           2      0.000     0.000     0.000      1000
           3      0.000     0.000     0.000      1000
           4      0.000     0.000     0.000      1000
           5      0.000     0.000     0.000      1000
           6      0.000     0.000     0.000      1000
           7      0.000     0.000   [...]<jupyter_text>5. Varying c_frac, p_frac and attackc_frac={0.1, 0.2, 0.4, 1.0} and p_frac = {0.0, 0.1, 0.2, 0.4} and attack = {(2, 9), (0, 2)}  bird-> truck, airplane -> bird.  First keeping c_per same and varying p_per, plotting total_acc and source_accc_frac = 0.1, 0.2, 0.4, 1.0c_per = 10, 20, 40, 100<jupyter_code>for k in [10, 20, 40, 100]:
    all_data_dfs = []
    for j in [0, 10, 20, 40]:
        file_name = f'file_c{k:02}_p{j:02}_a0'
        all_data_dfs.append(get_test_data_df(files_dict[file_name]))
    fig = go.Figure()
    for i, data_df in enumerate(all_data_dfs):
        # to make data40 and plot curves make better sense
        if i==3:
            i += 1
            
        i = i*10
#         fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['total_acc'],
#                             mode='lines', line={'dash': 'dash'},
#                             name=f'total_acc_c_per{k:02}_p_per{i:02}'))

        fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['total_acc'],
                            mode='lines+markers',
                            name=f'total_acc_p_per{i:02}'))

        fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_2_acc'],
                            mode='lines+markers', marker_symbol = 'square',
                            name=f'flipped_source_p_per{i:02}'))


    fig.update_layout(title=f'Effect on Accuracy when c_per={k:02} p_per increased from 0 to 40',
                       xaxis_title='Round Num',
                       yaxis_title='Accuracy')
    
    fig.write_image(plots_folder + file_name +".png", width=800)
    fig.show()<jupyter_output><empty_output><jupyter_text>Now let's compare two attack typesfirst we see for same c_per = 20 and p_per = 20, plot att0 and att1<jupyter_code>atts = [0, 1]
all_data_dfs = []
all_data_dfs.append(get_test_data_df(files_dict['file_c20_p20_a0']))
all_data_dfs.append(get_test_data_df(files_dict['file_c20_p20_a1']))

# Create traces
fig = go.Figure()
for att, data_df in zip(atts, all_data_dfs):

    fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['total_acc'],
                        mode='lines+markers',
                        name=f'total_acc_att{att}'))

    fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_2_acc'],
                        mode='lines+markers',
                        name=f'flipped_source_att{att}'))

#     fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['class_9_acc'],
#                         mode='lines+markers',
#                         name=f'flipped_target_att{att}'))

fig.update_layout(title='Accuracy Evolution',
                   xaxis_title='Round Num',
                   yaxis_title='Accuracy')
fig.show()<jupyter_output><empty_output><jupyter_text>Plotting avg training loss and testing losswhen p_per increases for c_per=20<jupyter_code>fig = go.Figure()
for j in [0, 10, 20, 40]:
    file_name = f'file_c20_p{j:02}_a0'
    avg_training_losses =  get_training_losses(files_dict[file_name])
    
    fig.add_trace(go.Scatter(x=np.arange(len(avg_training_losses)), y=avg_training_losses,
                        mode='lines',
                        name=f'c_per_20_p_per_{j:02}'))

fig.update_layout(title='Training Losses',
                   xaxis_title='Fed Round',
                   yaxis_title='Loss')
fig.show()

fig = go.Figure()
for j in [0, 10, 20, 40]:
    file_name = f'file_c20_p{j:02}_a0'
    data_df =  get_test_data_df(files_dict[file_name])
    
    fig.add_trace(go.Scatter(x=data_df['round_num'], y=data_df['testing_loss'],
                    mode='lines+markers',
                    name=f'testing_loss_c20_p{j:02}'))

fig.update_layout(title='Testing Loss over the Training',
                   xaxis_title='Round Num',
                   yaxis_title='Loss')
fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12197226856912721, 'fraction_numerical': 0.036764282837581264, 'mean_word_length': 2.992191088654111, 'pattern_counts': {'":': 0, '<': 56, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '102998', 'license': 'MIT', 'n_tokens_mistral': 6499, 'n_tokens_neox': 6056, 'n_words': 1010, 'path': 'notebooks/data_visualization.ipynb', 'repo_name': 'fnc11/CosDefence', 'size': '17381'}