<jupyter_start><jupyter_text>Comparison of real HPL executions<jupyter_code>import io
import zipfile
import pandas
import yaml
from plotnine import *
import plotnine
plotnine.options.figure_size = (12, 8)
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning) # removing annoying Pandas warning

def read_result(name):
    archive = zipfile.ZipFile(name)
    result = pandas.read_csv(io.BytesIO(archive.read('results.csv')))
    result['index'] = range(len(result))
    return result

def get_yaml(archive_name, yaml_name):
    archive = zipfile.ZipFile(archive_name)
    return yaml.load(io.BytesIO(archive.read(yaml_name)))

old_execution = [
    'grenoble_2018-10-10_1811810.zip',
    'grenoble_2018-10-10_1811847.zip',
]
new_execution = [
    'grenoble_2019-03-15_1855885.zip'
]
new_execution_fixed = [
    'grenoble_2019-03-18_1856059.zip'
]
dfold = pandas.concat([read_result(f) for f in old_execution])
dfnew = pandas.concat([read_result(f) for f in new_execution])
dfnewfixed = pandas.concat([read_result(f) for f in new_execution_fixed])
dfold['mode'] = 'old'
dfnew['mode'] = 'new'
dfnewfixed['mode'] = 'new:old_env'
df = pandas.concat([dfold, dfnew, dfnewfixed])
sizes = set(dfold.matrix_size) & set(dfnew.matrix_size)
df = df[df.matrix_size.isin(sizes)]
df.head()<jupyter_output><empty_output><jupyter_text>Checking the parameters<jupyter_code>name_exceptions = {'time', 'gflops', 'residual', 'matrix_size', 'mode', 'index'}
colnames = set(df) - name_exceptions
df[list(colnames)].drop_duplicates()<jupyter_output><empty_output><jupyter_text>Checking the OS version<jupyter_code>for version_name in ['version', 'gcc', 'mpi', 'kernel', 'cpu']:
    for exp in [old_execution, new_execution, new_execution_fixed]:
        versions = set()
        for filename in exp:
            info = get_yaml(filename, 'info.yaml')
            hostnames = [key for key in info.keys() if key.startswith('dahu')]
            versions.update([info[h][version_name] for h in hostnames])
        assert len(versions) == 1
        print(versions.pop())
    print()<jupyter_output>Linux version 4.9.0-8-amd64 (<EMAIL>) (gcc version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) ) #1 SMP Debian 4.9.110-3+deb9u4 (2018-08-21)
Linux version 4.9.0-8-amd64 (<EMAIL>) (gcc version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) ) #1 SMP Debian 4.9.144-3.1 (2019-02-19)
Linux version 4.9.0-8-amd64 (<EMAIL>) (gcc version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) ) #1 SMP Debian 4.9.110-3+deb9u4 (2018-08-21)

6.3.0
6.3.0
6.3.0

mpirun (Open MPI) 2.0.2
mpirun (Open MPI) 2.0.2
mpirun (Open MPI) 2.0.2

4.9.0-8-amd64
4.9.0-8-amd64
4.9.0-8-amd64

Intel(R) Xeon(R) Gold 6130 CPU @ 2.10GHz
Intel(R) Xeon(R) Gold 6130 CPU @ 2.10GHz
Intel(R) Xeon(R) Gold 6130 CPU @ 2.10GHz<jupyter_text>Comparing the executions<jupyter_code>import numpy

ggplot(df, aes(x='matrix_size', y='gflops', color='mode')) + stat_summary(fun_y = numpy.mean, geom='line') + geom_point() + theme_bw() + expand_limits(y=0)
ggplot(df, aes(x='matrix_size', y='time', color='mode')) + stat_summary(fun_y = numpy.mean, geom='line') + geom_point() + theme_bw() + expand_limits(y=0)<jupyter_output><empty_output><jupyter_text>What is this weird outlier in the new executions?<jupyter_code>tmp = df[df.matrix_size == 175000]
tmp[['gflops', 'time', 'mode', 'index']]<jupyter_output><empty_output><jupyter_text>Maximum performance<jupyter_code>dfold[dfold.gflops == dfold.gflops.max()]
dfnew[dfnew.gflops == dfnew.gflops.max()]<jupyter_output><empty_output><jupyter_text>Regression<jupyter_code>import statsmodels.formula.api as st

for df in [dfold, dfnew, dfnewfixed]:
    df['N3'] = df['matrix_size'] ** 3
    df['N2'] = df['matrix_size'] ** 2
    reg = st.ols(data=df, formula='time ~ N3+N2').fit()
    p = reg.params
    df['pred_time'] = p.Intercept + df.N2*p.N2 + df.N3*p.N3
    df['nb_flops'] = (2/3*df.N3 + 2*df.N2) * 1e-9
    df['pred_gflops'] = df['nb_flops'] / df['pred_time']
    c = reg.conf_int()
    df['pred_time_min'] = c[0].Intercept + df.N2*c[0].N2 + df.N3*c[0].N3
    df['pred_time_max'] = c[1].Intercept + df.N2*c[1].N2 + df.N3*c[1].N3
    df['pred_gflops_min'] = df['nb_flops'] / df['pred_time_max']
    df['pred_gflops_max'] = df['nb_flops'] / df['pred_time_min']
df = pandas.concat([dfold, dfnew, dfnewfixed])
plot = ggplot(df, aes(x='matrix_size', y='time', color='mode')) + geom_line(aes(x='matrix_size', y='pred_time')) + geom_point() + theme_bw() + expand_limits(y=0)
plot + geom_ribbon(aes(ymin='pred_time_min', ymax='pred_time_max', fill='mode'), alpha=0.1)
plot = ggplot(df, aes(x='matrix_size', y='gflops', color='mode')) + geom_line(aes(x='matrix_size', y='pred_gflops')) + geom_point() + theme_bw() + expand_limits(y=0)
plot + geom_ribbon(aes(ymin='pred_gflops_min', ymax='pred_gflops_max', fill='mode'), alpha=0.1)<jupyter_output><empty_output><jupyter_text>Pretty plot for the slides<jupyter_code>df['date'] = '2019/03'
df.loc[df['mode'] == 'old', 'date'] = '2018/10'
df = df[(df['mode'] == 'old') | (df['index'] > 0)]  # removing the first execution of the recent experiment
def plot_style(plot, font_size=20):
    plot = plot + theme_bw() + expand_limits(y=0) + xlab('Matrix size') + ylab('HPL performance (Gflops)')
    plot = plot + theme(text = element_text(size=font_size))
    return plot

plot = plot_style(ggplot(df[df['mode'] == 'old'], aes(x='matrix_size', y='gflops')))
plot = plot + stat_summary(fun_y = numpy.mean, geom='line') + geom_point()
plot.save('/tmp/hpl_old_perf.pdf')
plot
plot = plot_style(ggplot(aes(x='matrix_size', y='gflops')))
plot = plot + stat_summary(df[df['mode'] == 'old'], fun_y = numpy.mean, geom='line') + geom_point(df[df['mode'] == 'old'])
plot = plot + stat_summary(df[df['mode'] != 'old'], fun_y = numpy.mean, geom='line', color='#F8766D') + geom_point(df[df['mode'] != 'old'], color='#F8766D')
plot = plot + annotate(geom='text', label='2018/10', x=75000, y=20000, size=20)
plot = plot + annotate(geom='text', label='2019/03', x=150000, y=17500, size=20, color='#F8766D')
plot.save('/tmp/hpl_new_perf.pdf')
plot<jupyter_output>/home/tom/.local/share/virtualenvs/mpi_calibration-tfxxshdH/lib/python3.6/site-packages/plotnine/ggplot.py:706: UserWarning: Saving 12 x 8 in image.
  from_inches(height, units), units))
/home/tom/.local/share/virtualenvs/mpi_calibration-tfxxshdH/lib/python3.6/site-packages/plotnine/ggplot.py:707: UserWarning: Filename: /tmp/hpl_new_perf.pdf
  warn('Filename: {}'.format(filename))
{'fraction_non_alphanumeric': 0.18207239397234737, 'fraction_numerical': 0.05685878514836104, 'mean_word_length': 5.58282208588957, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '636010', 'license': 'MIT', 'n_tokens_mistral': 2845, 'n_tokens_neox': 2582, 'n_words': 453, 'path': 'dahu/hpl/hpl_performance.ipynb', 'repo_name': 'Ezibenroc/calibration_analysis', 'size': '6506'}