<jupyter_start><jupyter_text>MPI calibration of a Grid5000 cluster  Data exploration<jupyter_code>import os
import zipfile
import yaml
import pandas
import io
import plotnine
plotnine.options.figure_size = (12, 8)
from plotnine import *

archive_name = 'grenoble_2018-08-29_1808878.zip'

def extract_zip(zip_name):
    '''Taken from https://stackoverflow.com/a/10909016/4110059'''
    input_zip = zipfile.ZipFile(zip_name)
    result = {}
    deployment = yaml.load(input_zip.read('info.yaml'))['deployment']
    experiment = zip_name
    if '/' in experiment:
        experiment = experiment[experiment.index('/')+1:]
    experiment = experiment[:experiment.index('_')]
    for name in input_zip.namelist():
        if name.endswith('.csv'):
            dataframe = pandas.read_csv(io.BytesIO(input_zip.read(name)), names = ['op', 'msg_size', 'start', 'duration'])
            dataframe['experiment'] = experiment
            dataframe['type'] = name
            dataframe['deployment'] = deployment
            dataframe['index'] = range(len(dataframe))
            result[name] = dataframe
        elif name.endswith('.yaml'):
            result[name] = yaml.load(input_zip.read(name))
        else:
            result[name] = input_zip.read(name)
    return result

def aggregate_dataframe(dataframe):
    df = dataframe.groupby('msg_size').mean().reset_index()
    for col in dataframe.columns:
        if col not in df.columns:
            value = list(dataframe[col].unique())
            if len(value) == 1:
                df[col] = value[0]
    return df

threshold = 64000  # "semantic" breakpoint, found with another script

# I really hate that we have to define these two values:
expected_bandwidth = 1e11/8
expected_latency   = 1e-7

result = extract_zip(archive_name)
!unzip -p $archive_name info.yaml
# Special processing for the Pingpong dataset
df_pingpong = result['exp/exp_PingPong.csv']
pingpong_send = df_pingpong[df_pingpong.op=='MPI_Send'].reset_index(drop=True)
pingpong_recv = df_pingpong[df_pingpong.op=='MPI_Recv'].reset_index(drop=True)
df_pingpong = pandas.DataFrame(dict(op = 'PingPong',
                   msg_size = pingpong_send.msg_size,
                   start    = pingpong_send.start,
                   duration = pingpong_recv.duration + pingpong_send.duration
              ))
aggr_pingpong = df_pingpong[df_pingpong.msg_size > 64000].groupby('msg_size')[['duration']].mean().reset_index()
aggr_pingpong['pred'] = 2 * (aggr_pingpong.msg_size / (0.9867951082730274*1e11/8) + (129.24904864615127*0.1e-6))  # values in the XML file

theoretical = []

bandwidth = 1e11/8
for latency in [1e-5, 1.25e-5, 1.5e-5, 1.75e-5, 2e-5]:
    theoretical.append(pandas.DataFrame({'msg_size': aggr_pingpong.msg_size,
                                         'duration': 2*(latency + aggr_pingpong.msg_size/bandwidth),
                                         'latency': latency}))
theoretical = pandas.concat(theoretical)

plot = ggplot(aes(x='msg_size', y='duration')) + geom_point(aggr_pingpong, alpha=0.1) + geom_line(theoretical, aes(color='factor(latency)'))
plot = plot + theme_bw() + geom_line(aggr_pingpong, aes(x='msg_size', y='pred'))
print(plot + scale_x_log10() + scale_y_log10())
print(plot)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13163547100337528, 'fraction_numerical': 0.03804848112918073, 'mean_word_length': 3.4173441734417342, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '450921', 'license': 'MIT', 'n_tokens_mistral': 1194, 'n_tokens_neox': 1092, 'n_words': 201, 'path': 'dahu/mpi/quick_calibration_visualization.ipynb', 'repo_name': 'Ezibenroc/calibration_analysis', 'size': '3259'}