<jupyter_start><jupyter_text>Evaluation of doLLy in Krutz's benchmark<jupyter_code>%matplotlib inline

import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
import os
from IPython.display import display
from sklearn.metrics import auc
sns.set()

# Graphics
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
mpl.rcParams['figure.figsize'] = (8,6)
mpl.rcParams['font.family'] = 'SF Mono'
mpl.rcParams.update({'font.size': 12})
mpl.rcParams['axes.titlepad'] = 20 
mpl.rcParams['axes.titlesize'] = 12
mpl.rcParams['axes.labelpad'] = 10

# Seaborn settings
sns.set(font_scale = 1.25)
sns.set_style('ticks', {'font.family' : 'Fira Code', 
                        'axes.grid' : True, 
                        'grid.linestyle': 'dotted',
                        'grid.color': '0.8'})
sns.axes_style({'axes.grid' : 'True'})
sns.set_palette(sns.color_palette(['Black']))<jupyter_output><empty_output><jupyter_text>Add results of doLLy to Krutz's benchmark<jupyter_code>PROJECTS = ['apache', 'postgre', 'python']
UIRS = ['0', '10', '20', '30', '40', '50', '60', '70', '80', '90']
OPT_LVLS = ['O0', 'O1', 'O2']
TYPES = ['Type 2', 'type 2', 'Type 3', 'type 3', 'Type 4', 'type 4']
def krutz_format(row, reverse=False):
    filename_0 = os.path.splitext(row['filename'])[0]
    function_0 = row['function'].lower()
    filename_1 = os.path.splitext(row['filename.1'])[0]
    function_1 = row['function.1'].lower()
    if reverse:
        return f"{filename_1}..{function_1}()-{filename_0}..{function_0}()"
    else:
        return f"{filename_0}..{function_0}()-{filename_1}..{function_1}()"


def append_doLLy_to_krutz(krutz_pairs, doLLy_pairs, col_name):
    dolly_pairs_list = doLLy_pairs.apply(krutz_format, axis=1)
    dolly_pairs_list_rev = doLLy_pairs.apply(lambda row: krutz_format(row, reverse=True), axis=1)
    krutz_pairs[col_name] = np.zeros(len(krutz_pairs))
    detected_clones_index = krutz_pairs['pair'].isin(dolly_pairs_list) | krutz_pairs['pair'].isin(dolly_pairs_list_rev)
    krutz_pairs.loc[detected_clones_index, col_name] = 1.0
    return krutz_pairs

def save_krutz_with_doLLy(project, uir_values, opt_levels, krutz_pairs):
    for uir in uir_values:
        for opt_level in opt_levels:
            doLLy_pairs = pd.read_csv(f'doLLy-pairs/{project}-doLLy_{uir}_{opt_level}.csv')
            krutz_pairs = append_doLLy_to_krutz(krutz_pairs, doLLy_pairs, f'doLLy_{uir}_{opt_level}')
    krutz_pairs.to_csv(f'clone-pairs/{project}.csv')
for project in PROJECTS:
    project_pairs = pd.read_csv(f'cloneresults/{project}/{project}.csv')
    save_krutz_with_doLLy(project, UIRS, OPT_LVLS, project_pairs)<jupyter_output><empty_output><jupyter_text>Evaluate performance of each tool including doLLy<jupyter_code>def confusion_matrix(pairs, col, types):
    clones = pairs[pairs['Expert'].isin(types)]
    no_clones = pairs[pairs['Expert'].isin(['No Clone', 'no clone'])]
    tp = clones.loc[clones[col] == 1.0, col].count()
    fp = no_clones.loc[no_clones[col] == 1.0, col].count()
    tn = no_clones.loc[no_clones[col] == 0.0, col].count()
    fn = clones.loc[clones[col] == 0.0, col].count()
    if fn == 0:
        recall = 1.0
    else:
        recall = tp / (tp + fn)
        
    if fp == 0:
        precision = 1.0
    else:
        precision = tp / (tp + fp)
    accuracy = (tp + tn) / (tp + tn + fp + fn)
    return [col, precision, recall, accuracy, tp, fp, tn, fn]



def tool_stats(pairs, tools, clone_types):
    types = [t for t in TYPES if int(t.split()[1]) in clone_types]
    tools_stats_list = []
    for tool in tools:
        tools_stats_list.append(confusion_matrix(pairs, tool, types))
    result = pd.DataFrame(data=tools_stats_list, columns=['tool', 'precision', 'recall', 'accuracy', 'TP', 'FP', 'TN', 'FN']).set_index('tool')
    result['f1'] = compute_f1(result)
    result = result[['precision', 'recall', 'f1', 'TP', 'FP', 'TN', 'FN', 'accuracy']]
    return result.sort_values('f1', ascending=False)


def compute_f1(df):
    return df.apply(lambda row: f1_score(row[0], row[1]), axis=1)


def f1_score(precision, recall):
    if precision == 0 and recall == 0:
        return 0.0
    return 2 * (precision * recall) / (precision + recall)


def roc_plot(stats):
    roc_stats = stats[stats.index.str.contains('doLLy.*O1')].sort_values('tool')
    tpr = roc_stats['recall']
    tpr.at['doLLy_100_01'] = 1.0
    fpr = (roc_stats['FP'] / (roc_stats['FP'] + roc_stats['TN']))
    fpr.at['doLLy_100_01'] = 1.0
    auc_val = auc(fpr, tpr)

    fig = plt.figure(figsize=[8, 8])
    plt.plot(fpr, tpr, lw=2, label='ROC curve (area = %0.2f)' % auc_val)
    plt.plot([0, 1], [0, 1], linestyle='--')
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC of doLLy-O1 on Krutz\'s benchmark')
    plt.xlim([-0.05, 1.0])
    plt.ylim([0.0, 1.05])
    plt.legend(loc="lower right")
    plt.show()
    return fig
def evaluate_project(project, clone_types):
    project_pairs = pd.read_csv(f'clone-pairs/{project}.csv', index_col=0)
    project_tools = project_pairs.columns[5:]
    project_stats = tool_stats(project_pairs, project_tools, clone_types)
    return project_stats<jupyter_output><empty_output><jupyter_text>Current experiment statistics<jupyter_code>apache_pairs = pd.read_csv(f'clone-pairs/apache.csv', index_col= 0)
postgre_pairs = pd.read_csv(f'clone-pairs/postgre.csv', index_col= 0)
python_pairs = pd.read_csv(f'clone-pairs/python.csv', index_col= 0)

# evaluate type 2 clones
for project in PROJECTS:
    project_stats = evaluate_project(project, [2])
    
all_pairs = apache_pairs.append(postgre_pairs, sort=False).append(python_pairs, sort=False)
all_pairs = all_pairs.drop(columns='CCCD-30')
tools = all_pairs.columns[5:]

type2_stats = tool_stats(all_pairs, tools, [2])

# evaluate type 3 clones
for project in ['postgre', 'python']:
    project_stats = evaluate_project(project, [3])
python_postgre_pairs = postgre_pairs.append(python_pairs, sort=False)
type3_stats = tool_stats(python_postgre_pairs, tools, [3])


# evaluate type 4 clones
for project in ['postgre', 'python']:
    project_stats = evaluate_project(project, [4])
python_postgre_pairs = postgre_pairs.append(python_pairs, sort=False)
type4_stats = tool_stats(python_postgre_pairs, tools, [4])


# evaluate all clone types
all_stats = tool_stats(all_pairs, tools, [2,3,4])
all_stats.sort_values('f1', ascending=False).round(3)
# relevant_indices = all_stats.index[~all_stats.index.str.contains('doLLy_[0|10|20|60|70|80|90]')]
# print(all_stats.loc[relevant_indices, ['TP', 'FP', 'TN', 'FN']].round(3).to_latex())<jupyter_output>\begin{tabular}{lrrrr}
\toprule
{} &  TP &   FP &    TN &  FN \\
tool        &     &      &       &     \\
\midrule
doLLy\_40\_O1 &  48 &    4 &  1466 &  18 \\
doLLy\_50\_O1 &  54 &   25 &  1445 &  12 \\
doLLy\_50\_O0 &  56 &   33 &  1437 &  10 \\
doLLy\_40\_O0 &  46 &   18 &  1452 &  20 \\
doLLy\_30\_O0 &  35 &    3 &  1467 &  31 \\
doLLy\_30\_O1 &  33 &    2 &  1468 &  33 \\
doLLy\_40\_O2 &  32 &    3 &  1467 &  34 \\
Nicad-50    &  35 &   17 &  1453 &  31 \\
doLLy\_50\_O2 &  33 &   20 &  1450 &  33 \\
MeCC\_90\_4   &  26 &    4 &  1466 &  40 \\
MeCC\_80\_10  &  27 &    7 &  1463 &  39 \\
doLLy\_30\_O2 &  25 &    2 &  1468 &  41 \\
MeCC\_80\_20  &  24 &    3 &  1467 &  42 \\
SimCad      &  24 &   12 &  1458 &  42 \\
MeCC\_70\_20  &  31 &   37 &  1433 &  35 \\
MeCC\_90\_10  &  17 &    0 &  1470 &  49 \\
MeCC\_90\_20  &  17 &    0 &  1470 &  49 \\
MeCC\_95\_4   &  17 &    0 &  1470 &  49 \\
MeCC\_70\_10  &  35 &   76 &  1394 &  31 \\
MeCC\_95\_10  &  15 &    0 &  1470 &  51 \\
MeCC\_80[...]<jupyter_text>Previous experiment statistics<jupyter_code>apache_pairs = pd.read_csv(f'previous-experiment-pairs/apache.csv', index_col= 0)
postgre_pairs = pd.read_csv(f'previous-experiment-pairs/postgre.csv', index_col= 0)
python_pairs = pd.read_csv(f'previous-experiment-pairs/python.csv', index_col= 0)

# evaluate type 2 clones
for project in PROJECTS:
    project_stats = evaluate_project(project, [2])
    
all_pairs_old = apache_pairs.append(postgre_pairs, sort=False).append(python_pairs, sort=False)
all_pairs_old = all_pairs_old.drop(columns='CCCD-30')
tools_old = all_pairs_old.columns[5:]

type2_stats_old = tool_stats(all_pairs_old, tools_old, [2])

# evaluate type 3 clones
for project in ['postgre', 'python']:
    project_stats = evaluate_project(project, [3])
python_postgre_pairs = postgre_pairs.append(python_pairs, sort=False)
type3_stats_old = tool_stats(python_postgre_pairs, tools_old, [3])


# evaluate type 4 clones
for project in ['postgre', 'python']:
    project_stats = evaluate_project(project, [4])
python_postgre_pairs = postgre_pairs.append(python_pairs, sort=False)
type4_stats_old = tool_stats(python_postgre_pairs, tools_old, [4])


# evaluate all clone types
all_stats_old = tool_stats(all_pairs_old, tools_old, [2,3,4])
all_stats_old.sort_values('f1', ascending=False)<jupyter_output><empty_output><jupyter_text>Difference statistics<jupyter_code>diff = all_stats - all_stats_old
diff_stats = diff[diff.index.str.contains('doLLy')].sort_values('f1', ascending=False)
diff_stats
#diff_stats.to_csv('../../reports/2019-10-15-data/higher-level-diff-krutz.csv')<jupyter_output><empty_output><jupyter_text>Train-test split<jupyter_code>pairs_labelled = all_pairs[all_pairs['Expert'].str.contains('no clone|type', case=False, na=False)]
train = pairs_labelled.sample(frac=0.5, random_state=0)
test = pairs_labelled.drop(train.index)
train_stats = tool_stats(train, tools, [2,3,4]).sort_values('f1', ascending=False)
best_parameters = train_stats[train_stats.index.str.contains('doLLy')].iloc[0].name
print(f'Best performing doLLy instance in training set is: {best_parameters}')
test_stats = tool_stats(test, tools, [2,3,4]).sort_values('f1', ascending=False).round(3)
test_stats

# train_stats.to_csv('../../reports/2019-10-15-data/stats-train-krutz.csv')
# test_stats.to_csv('../../reports/2019-10-15-data/stats-test-krutz.csv')<jupyter_output>Best performing doLLy instance in training set is: doLLy_40_O1<jupyter_text>ROC curve<jupyter_code>fig = roc_plot(all_stats)
# fig.savefig('../../reports/2019-10-15-data/roc-curve-krutz.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15791012769275758, 'fraction_numerical': 0.0478604152451506, 'mean_word_length': 4.218718209562564, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '257425', 'license': 'MIT', 'n_tokens_mistral': 4311, 'n_tokens_neox': 4007, 'n_words': 725, 'path': 'benchmarks/krutz-benchmark/.ipynb_checkpoints/eval-krutz-benchmark-checkpoint.ipynb', 'repo_name': 'stonecauldron/IWSC2020', 'size': '10259'}