<jupyter_start><jupyter_text>20211012-zhan-statisticsEvaluate IPCs, accuracy, and coverage over simpoints for the base prefetchers, for *Zhan's traces*.<jupyter_code>import os
import sys
import glob
from collections import defaultdict

import numpy as np
import pandas as pd
from scipy import stats
import attrdict
import matplotlib.pyplot as plt

from utils.load import load_simpoint_weights, parse_champsim_result_file, load_champsim_base_results, parse_paper_result_file<jupyter_output><empty_output><jupyter_text>--- User parameters.You can look at the simpoint-weights or the data to get the trace_id for the trace you want.<jupyter_code>simpoints_dir = 'data/simpoint-weights/'
results_dir = 'data/zhan-prefetch1-fixedcache-250m/'
max_instruction_num = 250   # in millions
min_instruction_interval = 0 # in millions (this does not affect IPC results much)
verbose = True<jupyter_output><empty_output><jupyter_text>--- Load and preprocess results.- `ml_prefetch_sim.py eval --results-dir `<jupyter_code>stats = pd.read_csv(results_dir + 'stats.csv')
stats.Trace = stats.Trace.map(lambda t: t.split('.')[0].split('_')[0])
#stats
# Add Offline UAC metrics to the table
diffs = pd.read_csv(results_dir + 'diff.csv')
diffs.Trace = diffs.Trace.map(lambda t: t.split('_')[0])
diffs['Offline_UAC'] = (1 - diffs['diff12']) * 100.
diffs.loc[diffs['diff12'] == 0, 'Offline_UAC'] = 0. # Set to 0 for BO traces that don't do any prefetches (e.g. mcf)
offline_uacs = diffs[['Trace', 'Baseline', 'Offline_UAC']]
offline_uacs = offline_uacs[offline_uacs['Baseline'].isin(['sisb', 'bo'])]
diffs
stats = pd.merge(stats, offline_uacs, how='outer')
stats<jupyter_output><empty_output><jupyter_text>--- Plotting / table helpers**TODO** Break out into a utils fileValue plot: Expect BO, ISB bars to match closely to Paper BO, Paper ISB bars.<jupyter_code>def plot_results(df, column, y_label, subtitle=None, paper_data=None, factor = 1., ymin=None, ymax=None, legend=True, plot_baseline=False):
    """Plot data for a
    list of prefetchers and traces.
    """
    traces = sorted(set(df.Trace))
    fig, ax = plt.subplots(dpi=196)
    #ax.grid()
    #ax.set_axisbelow(True) # Plot grid under bars.
    
    sep = 1
    if paper_data:
        step_size = 0.125 #0.1
        width = 0.1 #0.075
    else:
        step_size = 0.2
        width = 0.15
        
    for i, tr in enumerate(traces):
        tr_data = df[df.Trace == tr]
        nsteps = 0 # Tracks where to place each bar
        
        ax.bar(i*sep, tr_data[df.Baseline == 'bo'][column], 
               color='C0', width = width, 
               label='BO' if i == 0 else ''
        )
        nsteps += 1
        
        ax.bar(i + nsteps*step_size, tr_data[df.Baseline == 'sisb'][column], 
               color='C1', width=width, 
               label='SISB' if i == 0 else ''
        )
        nsteps += 1
            
#         ax.bar(i + nsteps*step_size, tr_data[df.Baseline == 'sisb_bo'][column], 
#                color='C2', width=width, 
#                label='ISB + BO' if i == 0 else ''
#         )
#         nsteps += 1
        
        if paper_data and tr in paper_data:
            ax.bar(i*sep + nsteps*step_size, paper_data[tr]['Paper-BO'] * factor, 
                   color='C0', alpha=0.25, width=width, #edgecolor='C0',
                   label='Paper BO' if i == 0 else ''
            )
            nsteps += 1
                
            ax.bar(i*sep + nsteps*step_size, paper_data[tr]['Paper-ISB'] * factor, 
                   color='C1', alpha=0.25, width=width, #edgecolor='C1', 
                   label='Paper ISB' if i == 0 else ''
            )
            nsteps += 1
                
            ax.bar(i*sep + nsteps*step_size, paper_data[tr]['Paper-Voyager'] * factor, 
                   color='C3',  alpha=0.25, width=width, #edgecolor='C3',
                   label='Paper Voyager' if i == 0 else ''
            )
            nsteps += 1
                
        if plot_baseline:
            ax.bar(i*sep + nsteps*step_size, tr_data[df.Baseline == 'no'][column] * factor, 
                   color='gray', alpha=0.25, width=width, #edgecolor='gray',
                   label='No prefetcher' if i == 0 else ''
            )
            nsteps += 1
            
    if legend:
        ax.legend()
    ax.set_xticks(np.arange(0, len(traces)) + (nsteps*width/2))
    ax.set_xticklabels(traces, fontdict={'size': 7})
    if subtitle:
        title += f' ({subtitle})'
    ax.set_title(column)
    ax.set_ylabel(y_label)
    ax.set_ylim(ymin=ymin, ymax=ymax)<jupyter_output><empty_output><jupyter_text>Delta Plot- Expect BO vs. ISB bar to match closely to Paper BO vs. Paper ISB bars (green)- Expect BO vs. Paper BO (blue) and ISB vs. Paper ISB (yellow) bars to be close to zero.<jupyter_code>def plot_deltas(df, column, y_label, subtitle=None, paper_data=None, factor = 1., ymin=None, ymax=None, legend=True, plot_baseline=False):
    """Plot differences in columns for a
    list of prefetchers and traces.
    """
    traces = sorted(set(df.Trace))
    fig, ax = plt.subplots(dpi=196)
    #ax.grid()
    #ax.set_axisbelow(True) # Plot grid under bars.
    
    sep = 1
    if paper_data:
        step_size = 0.15 #0.1
        width = 0.125 #0.075
    else:
        step_size = 0.2
        width = 0.15
        
    for i, tr in enumerate(traces):
        tr_data = df[df.Trace == tr]
        nsteps = 0 # Tracks where to place each bar
        
        sisb = tr_data[df.Baseline == 'sisb'][column].iat[0]
        bo = tr_data[df.Baseline == 'bo'][column].iat[0]
        pisb = paper_data[tr]['Paper-ISB'] * factor
        pbo = paper_data[tr]['Paper-BO'] * factor

        
        ax.bar(i*sep, sisb - bo, 
               color='C2', width = width, 
               label='SISB vs. BO' if i == 0 else ''
        )
        nsteps += 1
        
        if paper_data and tr in paper_data:
            ax.bar(i*sep + nsteps*step_size, pisb - pbo, 
                   color='C2', alpha=0.25, width=width, #edgecolor='C0',
                   label='Paper ISB vs. Paper BO' if i == 0 else ''
            )
            nsteps += 1
                

            ax.bar(i*sep + nsteps*step_size,  bo - pbo,
                   color='C0', alpha=0.25, width=width, #edgecolor='C0',
                   label='BO vs. Paper BO' if i == 0 else ''
            )
            nsteps += 1
  
            ax.bar(i*sep + nsteps*step_size, sisb - pisb,
                   color='C1', alpha=0.25, width=width, #edgecolor='C0',
                   label='SISB vs. Paper SISB' if i == 0 else ''
            )
            nsteps += 1
            
    if legend:
        ax.legend()
    plt.axhline(y = 0, color = 'black', linestyle = '-', lw=0.5)
    ax.set_xticks(np.arange(0, len(traces)) + (nsteps*width/2))
    ax.set_xticklabels(traces, fontdict={'size': 7})
    if subtitle:
        title += f' ({subtitle})'
    ax.set_title(column + ' deltas')
    ax.set_ylabel(y_label)
    ax.set_ylim(ymin=ymin, ymax=ymax)
plot_results(
    stats, 'IPC', 'IPC',
    ymin=0,
    plot_baseline=True,
)<jupyter_output>/tmp/ipykernel_5123/1325702672.py:22: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  ax.bar(i*sep, tr_data[df.Baseline == 'bo'][column],
/tmp/ipykernel_5123/1325702672.py:28: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  ax.bar(i + nsteps*step_size, tr_data[df.Baseline == 'sisb'][column],
/tmp/ipykernel_5123/1325702672.py:60: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  ax.bar(i*sep + nsteps*step_size, tr_data[df.Baseline == 'no'][column] * factor,<jupyter_text>IPC speedup (vs. no prefetcher)<jupyter_code>paper_ipc_speedups = parse_paper_result_file('data/paper-values/paper_ipc_speedups.csv', strip_prefixes=True)

plot_results(
    stats, 'IPC_Improvement', 'IPC speedup (%)', 
    factor=100,
    ymin=0, ymax=100,
    paper_data=paper_ipc_speedups
)
paper_ipc_speedups = parse_paper_result_file('data/paper-values/paper_ipc_speedups.csv', strip_prefixes=True)

plot_deltas(
    stats, 'IPC_Improvement', 'IPC speedup difference (%)', 
    factor=100,
    paper_data=paper_ipc_speedups
)<jupyter_output>/tmp/ipykernel_5123/1035460514.py:22: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  sisb = tr_data[df.Baseline == 'sisb'][column].iat[0]
/tmp/ipykernel_5123/1035460514.py:23: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  bo = tr_data[df.Baseline == 'bo'][column].iat[0]<jupyter_text>--- Accuracy measurementsAccuracy = useful / (useful + useless) prefetches<jupyter_code>paper_accs = parse_paper_result_file('data/paper-values/paper_accuracies.csv', strip_prefixes=True)

plot_results(
    stats, 'Accuracy', 'Accuracy (%)', 
    factor=100,
    ymin=0, ymax=105,
    paper_data=paper_accs,
    legend=False
)
paper_ipc_speedups = parse_paper_result_file('data/paper-values/paper_ipc_speedups.csv', strip_prefixes=True)

plot_deltas(
    stats, 'Accuracy', 'Accuracy difference (%)', 
    factor=100,
    paper_data=paper_ipc_speedups,
    legend=False
)<jupyter_output>/tmp/ipykernel_5123/1035460514.py:22: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  sisb = tr_data[df.Baseline == 'sisb'][column].iat[0]
/tmp/ipykernel_5123/1035460514.py:23: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  bo = tr_data[df.Baseline == 'bo'][column].iat[0]<jupyter_text>--- Coverage measurementsCoverage = Percent decrease of:- \ of LLC load + RFO misses on a prefetcher- vs.  of LLC load + RFO misses on baseline (NoPrefetcher)<jupyter_code>paper_covs = parse_paper_result_file('data/paper-values/paper_coverages.csv', strip_prefixes=True)
paper_covs

plot_results(
    stats, 'Coverage', 'Coverage (%)',
    factor=100.,
    ymin=0, ymax=100,
    paper_data=paper_covs,
    legend=False
)
paper_ipc_speedups = parse_paper_result_file('data/paper-values/paper_ipc_speedups.csv', strip_prefixes=True)

plot_deltas(
    stats, 'Coverage', 'Coverage difference (%)', 
    factor=100,
    paper_data=paper_ipc_speedups,
    legend=False
)
paper_covs = parse_paper_result_file('data/paper-values/paper_coverages.csv', strip_prefixes=True)
paper_covs

plot_results(
    stats, 'Coverage_Old', 'Coverage (%)',
    factor=100.,
    ymin=0, ymax=100,
    paper_data=paper_covs,
    legend=False
)<jupyter_output>/tmp/ipykernel_5123/1325702672.py:22: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  ax.bar(i*sep, tr_data[df.Baseline == 'bo'][column],
/tmp/ipykernel_5123/1325702672.py:28: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  ax.bar(i + nsteps*step_size, tr_data[df.Baseline == 'sisb'][column],<jupyter_text>--- Unified accuracy/coverage Online UACs (calculated using experimental ChampSim feature)<jupyter_code>paper_uacs = parse_paper_result_file('data/paper-values/paper_uacs.csv', strip_prefixes=True)

plot_results(
    stats, 'UAC', 'UAC (%)',
    factor=100.,
    ymin=0, ymax=100,
    paper_data=paper_uacs,
    legend=False
)
paper_ipc_speedups = parse_paper_result_file('data/paper-values/paper_ipc_speedups.csv', strip_prefixes=True)

plot_deltas(
    stats, 'UAC', 'UAC difference (%)', 
    factor=100,
    paper_data=paper_uacs,
    legend=False
)<jupyter_output>/tmp/ipykernel_5123/1035460514.py:22: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  sisb = tr_data[df.Baseline == 'sisb'][column].iat[0]
/tmp/ipykernel_5123/1035460514.py:23: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  bo = tr_data[df.Baseline == 'bo'][column].iat[0]<jupyter_text>Offline UACs (calculated using `prefetch` scripts)<jupyter_code>paper_uacs = parse_paper_result_file('data/paper-values/paper_uacs.csv', strip_prefixes=True)

plot_results(
    stats, 'Offline_UAC', 'UAC (%)',
    factor=100.,
    ymin=0, ymax=100,
    paper_data=paper_uacs,
    legend=False
)
paper_ipc_speedups = parse_paper_result_file('data/paper-values/paper_ipc_speedups.csv', strip_prefixes=True)

plot_deltas(
    stats, 'Offline_UAC', 'UAC difference (%)', 
    factor=100,
    paper_data=paper_uacs,
    legend=False
)<jupyter_output>/tmp/ipykernel_5123/1035460514.py:22: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  sisb = tr_data[df.Baseline == 'sisb'][column].iat[0]
/tmp/ipykernel_5123/1035460514.py:23: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  bo = tr_data[df.Baseline == 'bo'][column].iat[0]
{'fraction_non_alphanumeric': 0.12983909510912856, 'fraction_numerical': 0.033535128245977375, 'mean_word_length': 3.2018072289156625, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '453548', 'license': 'MIT', 'n_tokens_mistral': 4761, 'n_tokens_neox': 4422, 'n_words': 953, 'path': '20211209-zhan-statistics.ipynb', 'repo_name': 'cmolder/voyager-analysis', 'size': '12554'}